feat: D1 Part 2 — agent self-update Go plumbing (op class, opsign, executor, commit, report)
- reconcile: ClassAgentUpdate op class; always Destructive (no provenance
blesses replacing the root-adjacent binary). classify test + companion
(TestClassify_AgentUpdateAlwaysDestructive).
- opsign: `-op agent_update` with -agent-version + -sha256 (isHex64-validated);
params {version,sha256}. isHex64 test (Group D).
- config: SelfUpdateConfig{URLTemplate,Username,Token,StateDir,DwellSeconds}
+ WithDefaults + Token redaction.
- internal/selfupdate: Executor (download → verify vs the SIGNED sha → sudo -n
wrapper `apply`; sha is the only integrity root — mismatch refuses + removes,
agent untouched); Manager (startup dwell → `commit`; version-mismatch → no
commit + loud WARN + marker left for report visibility; shutdown-before-dwell
leaves pending). WrapperRunner seam → tests never shell out.
- hub report: additive selfupdate_pending(+version) via SetSelfUpdateReporter
seam; both omitempty (Wireguard precedent) so the cross-repo golden contract
stays byte-stable — no hub change.
- capability manifest: 3 non-critical FELHOM_SELFUPDATE probes.
- main.go: updateExec appended to the executor chain; commit-manager wired to
the report seam + MaybeCommit goroutine after core init.
Tests: Group A (executor happy/sha-mismatch+companion/bad-params/wrapper-fail),
B (agent_update rides the real gate: pinned-key executes, non-pinned +
retarget rejected), C (commit/version-mismatch/no-pending/shutdown), D (opsign).
C2 companion red-proof verified (neutered Go verify → bad binary reaches apply
→ test fails), reverted. Full go test ./... green.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -40,6 +40,7 @@ import (
|
|||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
|
||||||
@@ -616,7 +617,36 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
|||||||
decommIntent = intentStore
|
decommIntent = intentStore
|
||||||
}
|
}
|
||||||
decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
|
decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
|
||||||
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec}, cfg.Hub.HostID, logger)
|
|
||||||
|
// Agent self-update (TASK D1): the agent_update executor downloads the operator-signed binary,
|
||||||
|
// verifies it against the SIGNED sha256, and hands it to the root guarded wrapper (A/B flip +
|
||||||
|
// detached restart). Wired as a third chain element. The commit-manager (below) commits a good
|
||||||
|
// update after a clean dwell; systemd + the wrapper auto-roll-back a crash-looping one. sudoRunner
|
||||||
|
// shells the wrapper verbs via `sudo -n` (same mode the rest of the privileged surface uses).
|
||||||
|
suCfg := cfg.SelfUpdate.WithDefaults()
|
||||||
|
suMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
||||||
|
if suMode == "" {
|
||||||
|
suMode = proxmox.RunnerSudo
|
||||||
|
}
|
||||||
|
suRunner := &proxmox.ExecRunner{Mode: suMode, SudoPath: cfg.Privileged.SudoPath}
|
||||||
|
updateExec := selfupdate.NewExecutor(selfupdate.Config{
|
||||||
|
URLTemplate: suCfg.URLTemplate,
|
||||||
|
Username: suCfg.Username,
|
||||||
|
Token: suCfg.Token,
|
||||||
|
StateDir: suCfg.StateDir,
|
||||||
|
Runner: suRunner,
|
||||||
|
Logger: logger,
|
||||||
|
})
|
||||||
|
selfUpdateMgr := selfupdate.NewManager(selfupdate.ManagerConfig{
|
||||||
|
StateDir: suCfg.StateDir,
|
||||||
|
RunningVersion: version,
|
||||||
|
Dwell: time.Duration(suCfg.DwellSeconds) * time.Second,
|
||||||
|
Runner: suRunner,
|
||||||
|
Logger: logger,
|
||||||
|
})
|
||||||
|
collector.SetSelfUpdateReporter(selfUpdateMgr) // heartbeat pending-status field
|
||||||
|
|
||||||
|
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger)
|
||||||
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
||||||
|
|
||||||
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logger, &localTokens)
|
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logger, &localTokens)
|
||||||
@@ -732,6 +762,13 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
|||||||
go func() { errc <- wgLoop.Run(ctx) }()
|
go func() { errc <- wgLoop.Run(ctx) }()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TASK D1 Scenario D: if this process is a JUST-FLIPPED self-update (a pending marker names THIS
|
||||||
|
// version), commit it after a clean dwell — but only now that core init is done (config parsed,
|
||||||
|
// hub loop + storage watchdog + local API all started above). Runs in its own goroutine so the
|
||||||
|
// dwell never blocks the daemon; it is NOT one of the errc siblings (it returns after commit and
|
||||||
|
// must not end the daemon). A crash before the commit → systemd + the wrapper roll back to .prev.
|
||||||
|
go selfUpdateMgr.MaybeCommit(ctx)
|
||||||
|
|
||||||
err = <-errc
|
err = <-errc
|
||||||
stop() // tear down the siblings on the first exit
|
stop() // tear down the siblings on the first exit
|
||||||
for i := 0; i < 4+localServers+lanServers+wgServers; i++ { // wait for the other goroutines
|
for i := 0; i < 4+localServers+lanServers+wgServers; i++ { // wait for the other goroutines
|
||||||
|
|||||||
@@ -43,13 +43,15 @@ func main() {
|
|||||||
|
|
||||||
func run() error {
|
func run() error {
|
||||||
var (
|
var (
|
||||||
op = flag.String("op", "", "op class to sign, e.g. storage_wipe | guest_destroy | decommission")
|
op = flag.String("op", "", "op class to sign, e.g. storage_wipe | guest_destroy | decommission | agent_update")
|
||||||
host = flag.String("host", "", "target host_id (anti-retarget — the op runs ONLY on this host)")
|
host = flag.String("host", "", "target host_id (anti-retarget — the op runs ONLY on this host)")
|
||||||
guest = flag.String("guest", "", "target guest_id (\"\" = host-scoped op)")
|
guest = flag.String("guest", "", "target guest_id (\"\" = host-scoped op)")
|
||||||
keyID = flag.String("key-id", "", "key id of the signing key (must match a pinned agent signer)")
|
keyID = flag.String("key-id", "", "key id of the signing key (must match a pinned agent signer)")
|
||||||
paramsRaw = flag.String("params", "", "op params as JSON (overrides -durable-id/-fstype)")
|
paramsRaw = flag.String("params", "", "op params as JSON (overrides -durable-id/-fstype)")
|
||||||
durableID = flag.String("durable-id", "", "storage_wipe: the DURABLE device id (byid:…|byuuid:…); decommission: the drive's STORAGE durable-id (e.g. uuid:<fs-uuid>)")
|
durableID = flag.String("durable-id", "", "storage_wipe: the DURABLE device id (byid:…|byuuid:…); decommission: the drive's STORAGE durable-id (e.g. uuid:<fs-uuid>)")
|
||||||
fstype = flag.String("fstype", "ext4", "for storage_wipe: the filesystem to mkfs after wipe")
|
fstype = flag.String("fstype", "ext4", "for storage_wipe: the filesystem to mkfs after wipe")
|
||||||
|
agentVer = flag.String("agent-version", "", "for agent_update: the target agent version (e.g. 0.70.1)")
|
||||||
|
sha256Hex = flag.String("sha256", "", "for agent_update: the pinned lowercase-hex sha256 of the target binary")
|
||||||
keyFile = flag.String("key", "", "operator signing key (ssh private key / sk- key handle) for ssh-keygen -Y sign")
|
keyFile = flag.String("key", "", "operator signing key (ssh private key / sk- key handle) for ssh-keygen -Y sign")
|
||||||
ttl = flag.Duration("ttl", 30*time.Minute, "validity window from now (issued_at..expires_at)")
|
ttl = flag.Duration("ttl", 30*time.Minute, "validity window from now (issued_at..expires_at)")
|
||||||
nonce = flag.String("nonce", "", "explicit nonce (default: a fresh 128-bit random nonce)")
|
nonce = flag.String("nonce", "", "explicit nonce (default: a fresh 128-bit random nonce)")
|
||||||
@@ -82,6 +84,17 @@ func run() error {
|
|||||||
}
|
}
|
||||||
pj, _ := json.Marshal(map[string]string{"durable_id": *durableID})
|
pj, _ := json.Marshal(map[string]string{"durable_id": *durableID})
|
||||||
params = string(pj)
|
params = string(pj)
|
||||||
|
case "agent_update":
|
||||||
|
// The agent downloads the binary for -agent-version and verifies it against -sha256.
|
||||||
|
// The sha is the ONLY integrity root, so both are mandatory and the sha is strict-validated.
|
||||||
|
if *agentVer == "" || *sha256Hex == "" {
|
||||||
|
return fmt.Errorf("agent_update needs -agent-version and -sha256 (the pinned binary hash)")
|
||||||
|
}
|
||||||
|
if !isHex64(*sha256Hex) {
|
||||||
|
return fmt.Errorf("agent_update -sha256 must be 64 lowercase hex chars (got %d)", len(*sha256Hex))
|
||||||
|
}
|
||||||
|
pj, _ := json.Marshal(map[string]string{"version": *agentVer, "sha256": *sha256Hex})
|
||||||
|
params = string(pj)
|
||||||
default:
|
default:
|
||||||
params = "{}"
|
params = "{}"
|
||||||
}
|
}
|
||||||
@@ -141,6 +154,19 @@ func run() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isHex64 reports whether s is exactly 64 lowercase hex chars (a sha256 hex digest).
|
||||||
|
func isHex64(s string) bool {
|
||||||
|
if len(s) != 64 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, c := range s {
|
||||||
|
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// signWithSSHKeygen signs `message` with `ssh-keygen -Y sign -n <namespace>`, the hardware-ready
|
// signWithSSHKeygen signs `message` with `ssh-keygen -Y sign -n <namespace>`, the hardware-ready
|
||||||
// path (sk-/YubiKey keys work unchanged). It writes the message to a temp file, runs ssh-keygen,
|
// path (sk-/YubiKey keys work unchanged). It writes the message to a temp file, runs ssh-keygen,
|
||||||
// and reads the armored SSHSIG it produces. The namespace is the agent's FIXED domain separator.
|
// and reads the armored SSHSIG it produces. The namespace is the agent's FIXED domain separator.
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// TASK D1 Group D — agent_update opsign param validation. isHex64 is the sha gate the CLI applies
|
||||||
|
// before it will build an agent_update envelope (the agent re-validates too, but a bad sha should
|
||||||
|
// never even be signed).
|
||||||
|
func TestIsHex64(t *testing.T) {
|
||||||
|
good := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // 64 'a'
|
||||||
|
if !isHex64(good) {
|
||||||
|
t.Errorf("isHex64(%q) = false, want true", good)
|
||||||
|
}
|
||||||
|
for name, bad := range map[string]string{
|
||||||
|
"too short": "abcdef",
|
||||||
|
"too long": good + "a",
|
||||||
|
"uppercase hex": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||||
|
"non-hex char": "g" + good[1:],
|
||||||
|
"empty": "",
|
||||||
|
} {
|
||||||
|
if isHex64(bad) {
|
||||||
|
t.Errorf("%s: isHex64(%q) = true, want false", name, bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -113,4 +113,12 @@ var manifest = []Capability{
|
|||||||
{"wg-restart", "wg-quick@wg-felhom restart (conf change)", "/usr/bin/systemctl", []string{"restart", "wg-quick@wg-felhom"}, true},
|
{"wg-restart", "wg-quick@wg-felhom restart (conf change)", "/usr/bin/systemctl", []string{"restart", "wg-quick@wg-felhom"}, true},
|
||||||
{"wg-disable", "wg-quick@wg-felhom disable (revocation)", "/usr/bin/systemctl", []string{"disable", "--now", "wg-quick@wg-felhom"}, false},
|
{"wg-disable", "wg-quick@wg-felhom disable (revocation)", "/usr/bin/systemctl", []string{"disable", "--now", "wg-quick@wg-felhom"}, false},
|
||||||
{"wg-handshake-read", "tunnel handshake-age read", "/usr/bin/wg", []string{"show", "wg-felhom", "latest-handshakes"}, true},
|
{"wg-handshake-read", "tunnel handshake-age read", "/usr/bin/wg", []string{"show", "wg-felhom", "latest-handshakes"}, true},
|
||||||
|
|
||||||
|
// ---- Agent self-update (FELHOM_SELFUPDATE, D1). NON-critical: self-update is an occasional
|
||||||
|
// operator-driven op, not a steady-state serving path — a degraded grant means "can't
|
||||||
|
// self-update" (fall back to a manual SSH deploy), not a serving outage. The apply repr uses a
|
||||||
|
// staging-dir path + a placeholder sha (list-mode never runs it). ----
|
||||||
|
{"selfupdate-apply", "agent self-update apply (A/B flip)", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"apply", "/var/lib/felhom-agent/selfupdate/felhom-agent-0.0.0", "0000000000000000000000000000000000000000000000000000000000000000"}, false},
|
||||||
|
{"selfupdate-commit", "agent self-update commit", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"commit"}, false},
|
||||||
|
{"selfupdate-rollback", "agent self-update rollback", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"rollback"}, false},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,9 +32,43 @@ type Config struct {
|
|||||||
LocalAPI LocalAPIConfig `json:"local_api"`
|
LocalAPI LocalAPIConfig `json:"local_api"`
|
||||||
LANResolver LANResolverConfig `json:"lan_resolver"`
|
LANResolver LANResolverConfig `json:"lan_resolver"`
|
||||||
WGTunnel WGTunnelConfig `json:"wg_tunnel"`
|
WGTunnel WGTunnelConfig `json:"wg_tunnel"`
|
||||||
|
SelfUpdate SelfUpdateConfig `json:"selfupdate"`
|
||||||
LogLevel string `json:"log_level"` // debug|info|warn|error (default info)
|
LogLevel string `json:"log_level"` // debug|info|warn|error (default info)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SelfUpdateConfig configures the operator-signed agent self-update (TASK D1). The artifact HOST
|
||||||
|
// is operator-controlled config; the artifact INTEGRITY comes only from the sha256 pinned inside
|
||||||
|
// the operator-signed op — the hub's Day-0 manifest plays no role here, and a compromised Gitea
|
||||||
|
// can serve garbage but never a binary that passes the signed sha.
|
||||||
|
type SelfUpdateConfig struct {
|
||||||
|
// URLTemplate is the download URL with a literal "{version}" placeholder. Default mirrors the
|
||||||
|
// day-0 host-install scheme (Gitea generic package).
|
||||||
|
URLTemplate string `json:"url_template"`
|
||||||
|
// Username/Token are optional HTTP basic-auth credentials for the artifact host (the same git
|
||||||
|
// read token day-0 uses). Token is a secret — redacted in Config.Redacted.
|
||||||
|
Username string `json:"username,omitempty"`
|
||||||
|
Token string `json:"token,omitempty"`
|
||||||
|
// StateDir holds the staging subdir (<StateDir>/selfupdate/); default /var/lib/felhom-agent.
|
||||||
|
StateDir string `json:"state_dir,omitempty"`
|
||||||
|
// DwellSeconds is how long the NEW binary must run cleanly (after core init) before it commits
|
||||||
|
// the update; default 60.
|
||||||
|
DwellSeconds int `json:"dwell_seconds,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithDefaults fills the artifact URL template, state dir and dwell.
|
||||||
|
func (s SelfUpdateConfig) WithDefaults() SelfUpdateConfig {
|
||||||
|
if s.URLTemplate == "" {
|
||||||
|
s.URLTemplate = "https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/{version}/felhom-agent"
|
||||||
|
}
|
||||||
|
if s.StateDir == "" {
|
||||||
|
s.StateDir = "/var/lib/felhom-agent"
|
||||||
|
}
|
||||||
|
if s.DwellSeconds == 0 {
|
||||||
|
s.DwellSeconds = 60
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
// WGTunnelConfig configures the offsite WireGuard tunnel (S3, doc 06). **Enabled DEFAULTS TO
|
// WGTunnelConfig configures the offsite WireGuard tunnel (S3, doc 06). **Enabled DEFAULTS TO
|
||||||
// FALSE — the safety gate:** agent releases roll to near-production boxes, and auto-registering
|
// FALSE — the safety gate:** agent releases roll to near-production boxes, and auto-registering
|
||||||
// one into the DEV endpoint on update would be wrong. Enable explicitly per box; the default
|
// one into the DEV endpoint on update would be wrong. Enable explicitly per box; the default
|
||||||
@@ -579,6 +613,9 @@ func (c Config) Redacted() Config {
|
|||||||
if c.Hub.APIKey != "" {
|
if c.Hub.APIKey != "" {
|
||||||
c.Hub.APIKey = "********"
|
c.Hub.APIKey = "********"
|
||||||
}
|
}
|
||||||
|
if c.SelfUpdate.Token != "" {
|
||||||
|
c.SelfUpdate.Token = "********"
|
||||||
|
}
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -68,6 +68,7 @@ type Collector struct {
|
|||||||
capProbe func(ctx context.Context) []capability.Status // v0.44.0: privileged-capability self-check (nil → empty)
|
capProbe func(ctx context.Context) []capability.Status // v0.44.0: privileged-capability self-check (nil → empty)
|
||||||
leafFP string // v0.48.0: served local-API leaf fp (static per process; "" when local API disabled)
|
leafFP string // v0.48.0: served local-API leaf fp (static per process; "" when local API disabled)
|
||||||
wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted)
|
wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted)
|
||||||
|
selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false)
|
||||||
hostID string
|
hostID string
|
||||||
agentVersion string
|
agentVersion string
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
@@ -124,6 +125,21 @@ func (c *Collector) SetWireguardReporter(w WireguardReporter) *Collector {
|
|||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SelfUpdateReporter is the D1 seam the selfupdate commit-manager plugs into (same consumer-side
|
||||||
|
// pattern — hub does not import selfupdate). nil (feature not wired) → pending=false on the report.
|
||||||
|
type SelfUpdateReporter interface {
|
||||||
|
// SelfUpdatePending reports whether a signed update has flipped the binary but not yet
|
||||||
|
// committed, and the awaited version.
|
||||||
|
SelfUpdatePending() (pending bool, version string)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetSelfUpdateReporter wires the agent self-update pending-status source (D1; nil-safe → false).
|
||||||
|
// Returns the collector for chaining.
|
||||||
|
func (c *Collector) SetSelfUpdateReporter(s SelfUpdateReporter) *Collector {
|
||||||
|
c.selfUpdate = s
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
// Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard
|
// Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard
|
||||||
// error (no useful report — the cycle skips the POST); a failed per-guest
|
// error (no useful report — the cycle skips the POST); a failed per-guest
|
||||||
// GuestConfig degrades that guest to status="unknown" without spec but still sends;
|
// GuestConfig degrades that guest to status="unknown" without spec but still sends;
|
||||||
@@ -162,6 +178,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
|
|||||||
if c.wg != nil {
|
if c.wg != nil {
|
||||||
report.Wireguard = c.wg.WireguardStatus(ctx)
|
report.Wireguard = c.wg.WireguardStatus(ctx)
|
||||||
}
|
}
|
||||||
|
// D1: agent self-update pending status (nil reporter → pending=false, the steady state).
|
||||||
|
if c.selfUpdate != nil {
|
||||||
|
report.SelfUpdatePending, report.SelfUpdatePendingVersion = c.selfUpdate.SelfUpdatePending()
|
||||||
|
}
|
||||||
return report, nil
|
return report, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -53,6 +53,19 @@ type HostReport struct {
|
|||||||
// needed; the pubkey here is the operator's revocation-recovery handle (re-add the peer with
|
// needed; the pubkey here is the operator's revocation-recovery handle (re-add the peer with
|
||||||
// it). Carries NO secret — the pubkey is public by definition.
|
// it). Carries NO secret — the pubkey is public by definition.
|
||||||
Wireguard *WireguardStatus `json:"wireguard,omitempty"`
|
Wireguard *WireguardStatus `json:"wireguard,omitempty"`
|
||||||
|
|
||||||
|
// SelfUpdatePending is true when an operator-signed agent self-update has flipped the binary
|
||||||
|
// but the new binary has not yet committed (TASK D1). SelfUpdatePendingVersion names the
|
||||||
|
// awaited version when pending. A runs-but-never-commits binary reports pending=true every
|
||||||
|
// heartbeat → the operator sees WHY the version isn't advancing; a crash-loop is auto-rolled
|
||||||
|
// back by systemd and this flips back to false when the good binary re-commits/clears. The
|
||||||
|
// report is stored opaquely hub-side, so these additive fields need no hub-schema change.
|
||||||
|
// Both are `omitempty` (the Wireguard precedent): in the steady state (no update in flight)
|
||||||
|
// they are absent — which keeps the cross-repo host-report golden contract byte-stable without
|
||||||
|
// a hub change. They appear only while an update is pending. The hub reads an absent field as
|
||||||
|
// pending=false, the correct default.
|
||||||
|
SelfUpdatePending bool `json:"selfupdate_pending,omitempty"`
|
||||||
|
SelfUpdatePendingVersion string `json:"selfupdate_pending_version,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// WireguardStatus is the per-heartbeat offsite-tunnel status (S3). LastHandshakeAgeS is nil when
|
// WireguardStatus is the per-heartbeat offsite-tunnel status (S3). LastHandshakeAgeS is nil when
|
||||||
|
|||||||
@@ -39,6 +39,14 @@ const (
|
|||||||
// recovery key authorizes ONLY this; the operational key authorizes this + ordinary
|
// recovery key authorizes ONLY this; the operational key authorizes this + ordinary
|
||||||
// destructive ops.
|
// destructive ops.
|
||||||
ClassKeyRotation OpClass = "key_rotation"
|
ClassKeyRotation OpClass = "key_rotation"
|
||||||
|
|
||||||
|
// Agent self-update (TASK D1) — replacing the root-adjacent host binary. Destructive-class by
|
||||||
|
// definition (the operator signs the exact version + sha256; the pinned sha is the ONLY
|
||||||
|
// integrity root — neither hub nor Gitea can substitute a binary). Operational-key only, like
|
||||||
|
// every ordinary destructive op. Note the classifier's default case already fails safe to
|
||||||
|
// Destructive for unknown classes — this named constant documents the class and keeps the
|
||||||
|
// signed-op vocabulary explicit, it does not (and must not) loosen anything.
|
||||||
|
ClassAgentUpdate OpClass = "agent_update"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Disposition is the classifier verdict.
|
// Disposition is the classifier verdict.
|
||||||
@@ -107,6 +115,10 @@ func Classify(class OpClass, prov Provenance) Disposition {
|
|||||||
return Destructive
|
return Destructive
|
||||||
case ClassKeyRotation:
|
case ClassKeyRotation:
|
||||||
return Destructive
|
return Destructive
|
||||||
|
case ClassAgentUpdate:
|
||||||
|
// Never benign — no agent-internal provenance can make replacing the agent binary
|
||||||
|
// unsigned-safe (a compromised process must not be able to self-bless an update).
|
||||||
|
return Destructive
|
||||||
default:
|
default:
|
||||||
return Destructive // fail safe: an unrecognized op is treated as destructive
|
return Destructive // fail safe: an unrecognized op is treated as destructive
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ func TestClassify_BenignClasses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestClassify_DestructiveClassesNeedSignature(t *testing.T) {
|
func TestClassify_DestructiveClassesNeedSignature(t *testing.T) {
|
||||||
for _, c := range []OpClass{ClassGuestDestroy, ClassStorageWipe, ClassRestoreOverwrite, ClassDecommission, ClassKeyRotation} {
|
for _, c := range []OpClass{ClassGuestDestroy, ClassStorageWipe, ClassRestoreOverwrite, ClassDecommission, ClassKeyRotation, ClassAgentUpdate} {
|
||||||
if got := Classify(c, Provenance{}); got != Destructive {
|
if got := Classify(c, Provenance{}); got != Destructive {
|
||||||
t.Errorf("Classify(%s) = %s, want destructive", c, got)
|
t.Errorf("Classify(%s) = %s, want destructive", c, got)
|
||||||
}
|
}
|
||||||
@@ -41,6 +41,16 @@ func TestClassify_KeyRotationAlwaysDestructive(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TASK D1: agent_update (replacing the root-adjacent binary) is ALWAYS destructive — no
|
||||||
|
// agent-internal provenance can bless it unsigned (a compromised process must not self-update).
|
||||||
|
// This is what gates the signed-jobs binary swap; if it flipped to Benign the runner would execute
|
||||||
|
// an unsigned agent_update (the companion the signedjobs ride-along tests rely on).
|
||||||
|
func TestClassify_AgentUpdateAlwaysDestructive(t *testing.T) {
|
||||||
|
if got := Classify(ClassAgentUpdate, Provenance{SameTxnCreated: true, AgentTaggedScratch: true}); got != Destructive {
|
||||||
|
t.Errorf("agent_update = %s, want destructive even with internal provenance", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestClassify_UnknownClassFailsSafe(t *testing.T) {
|
func TestClassify_UnknownClassFailsSafe(t *testing.T) {
|
||||||
if got := Classify(OpClass("totally_unknown_op"), Provenance{}); got != Destructive {
|
if got := Classify(OpClass("totally_unknown_op"), Provenance{}); got != Destructive {
|
||||||
t.Errorf("unknown class = %s, want destructive (fail-safe)", got)
|
t.Errorf("unknown class = %s, want destructive (fail-safe)", got)
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
package selfupdate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Manager owns the post-restart COMMIT half of the A/B update (TASK D1 Scenario D). After a flip,
|
||||||
|
// the NEW binary boots with a pending marker on disk; once it has run cleanly for the dwell AND
|
||||||
|
// core init is done, it calls the wrapper's `commit` to clear the marker. A crash before that →
|
||||||
|
// systemd + the wrapper roll back to .prev (this Manager never rolls back).
|
||||||
|
type Manager struct {
|
||||||
|
stateDir string
|
||||||
|
runningVersion string
|
||||||
|
dwell time.Duration
|
||||||
|
runner WrapperRunner
|
||||||
|
logger *slog.Logger
|
||||||
|
now func() time.Time // injectable for tests
|
||||||
|
sleep func(context.Context, time.Duration)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ManagerConfig wires the commit manager.
|
||||||
|
type ManagerConfig struct {
|
||||||
|
StateDir string
|
||||||
|
RunningVersion string // this binary's own version (main.version)
|
||||||
|
Dwell time.Duration // 0 → 60s
|
||||||
|
Runner WrapperRunner
|
||||||
|
Logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewManager builds the commit manager.
|
||||||
|
func NewManager(cfg ManagerConfig) *Manager {
|
||||||
|
dwell := cfg.Dwell
|
||||||
|
if dwell == 0 {
|
||||||
|
dwell = 60 * time.Second
|
||||||
|
}
|
||||||
|
return &Manager{
|
||||||
|
stateDir: cfg.StateDir,
|
||||||
|
runningVersion: cfg.RunningVersion,
|
||||||
|
dwell: dwell,
|
||||||
|
runner: cfg.Runner,
|
||||||
|
logger: orDefaultLogger(cfg.Logger),
|
||||||
|
now: func() time.Time { return time.Now() },
|
||||||
|
sleep: func(ctx context.Context, d time.Duration) {
|
||||||
|
t := time.NewTimer(d)
|
||||||
|
defer t.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-t.C:
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PendingStatus is the report-facing snapshot: whether an update is awaiting commit, and (when so)
|
||||||
|
// the target version. Read synchronously at report-build time — cheap (one stat+parse).
|
||||||
|
type PendingStatus struct {
|
||||||
|
Pending bool
|
||||||
|
Version string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status returns the current pending-marker state for the host report. A read error is treated as
|
||||||
|
// "not pending" for the report (the loud path is the commit goroutine's WARNs), never fatal.
|
||||||
|
func (m *Manager) Status() PendingStatus {
|
||||||
|
pm, err := readPending(m.stateDir)
|
||||||
|
if err != nil || pm == nil {
|
||||||
|
return PendingStatus{}
|
||||||
|
}
|
||||||
|
return PendingStatus{Pending: true, Version: pm.NewVersion}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelfUpdatePending satisfies hub.SelfUpdateReporter (the report seam): pending + awaited version.
|
||||||
|
func (m *Manager) SelfUpdatePending() (bool, string) {
|
||||||
|
s := m.Status()
|
||||||
|
return s.Pending, s.Version
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaybeCommit runs the startup commit decision, blocking for the dwell when a matching update is
|
||||||
|
// pending. Call it in a goroutine AFTER core init (config parsed, local API up, control loop
|
||||||
|
// started) — never on the startup critical path. Behaviour (Scenario C5 + D):
|
||||||
|
// - no pending marker → nothing to do (the common case).
|
||||||
|
// - pending.new_version == this running version → dwell, then `commit` (clears the marker).
|
||||||
|
// - pending.new_version != running version → do NOT commit; loud WARN; leave the marker so the
|
||||||
|
// hub report shows pending=true (a human decides — the box is in a weird state).
|
||||||
|
func (m *Manager) MaybeCommit(ctx context.Context) {
|
||||||
|
pm, err := readPending(m.stateDir)
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("selfupdate: cannot read pending marker — not committing", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if pm == nil {
|
||||||
|
return // no update in flight
|
||||||
|
}
|
||||||
|
if pm.NewVersion != m.runningVersion {
|
||||||
|
// The running binary is NOT the one this pending marker describes. Do not commit (committing
|
||||||
|
// would bless a state we can't explain). Leave the marker → the report surfaces pending=true.
|
||||||
|
m.logger.Warn("selfupdate: pending marker version does not match the running binary — NOT committing (human review)",
|
||||||
|
"pending_new_version", pm.NewVersion, "running_version", m.runningVersion, "pending_old_version", pm.OldVersion)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
m.logger.Warn("selfupdate: new version running — dwelling before commit",
|
||||||
|
"version", m.runningVersion, "prev", pm.OldVersion, "dwell", m.dwell)
|
||||||
|
m.sleep(ctx, m.dwell)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
// Shutting down before the dwell elapsed — leave pending; the next start re-dwells and
|
||||||
|
// commits. A restart is not a crash (no OnFailure), so this does not trigger rollback.
|
||||||
|
m.logger.Warn("selfupdate: shutdown before commit dwell elapsed — pending left for next start")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if m.runner == nil {
|
||||||
|
m.logger.Error("selfupdate: no wrapper runner — cannot commit (marker left)")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stdout, stderr, err := m.runner.Run(ctx, wrapperPath, "commit")
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("selfupdate: commit failed (marker left; will retry next start)", "err", err, "stderr", string(stderr))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.logger.Warn("selfupdate: update committed", "version", m.runningVersion, "wrapper", trim(stdout))
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
package selfupdate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// writePending drops a pending.json fixture into the staging dir.
|
||||||
|
func writePending(t *testing.T, stateDir string, m PendingMarker) {
|
||||||
|
t.Helper()
|
||||||
|
dir := stagingDir(stateDir)
|
||||||
|
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b, _ := json.Marshal(m)
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "pending.json"), b, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMgr(t *testing.T, stateDir, runningVersion string, wrap WrapperRunner, buf *bytes.Buffer) *Manager {
|
||||||
|
t.Helper()
|
||||||
|
var w io.Writer = io.Discard
|
||||||
|
if buf != nil {
|
||||||
|
w = buf
|
||||||
|
}
|
||||||
|
m := NewManager(ManagerConfig{
|
||||||
|
StateDir: stateDir,
|
||||||
|
RunningVersion: runningVersion,
|
||||||
|
Dwell: time.Hour, // never elapses in the test; we drive it via the sleep seam
|
||||||
|
Runner: wrap,
|
||||||
|
Logger: slog.New(slog.NewTextHandler(w, nil)),
|
||||||
|
})
|
||||||
|
// Replace the real dwell sleep with an instant one so the test doesn't block.
|
||||||
|
m.sleep = func(context.Context, time.Duration) {}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scenario D: pending marker names THIS version → after the dwell, `commit` is called.
|
||||||
|
func TestManager_CommitsMatchingVersion(t *testing.T) {
|
||||||
|
stateDir := t.TempDir()
|
||||||
|
writePending(t, stateDir, PendingMarker{OldVersion: "0.70.0", NewVersion: "0.70.1", SHA256: "ab", AppliedAt: "t"})
|
||||||
|
wrap := &fakeWrapper{}
|
||||||
|
m := newMgr(t, stateDir, "0.70.1", wrap, nil)
|
||||||
|
|
||||||
|
m.MaybeCommit(context.Background())
|
||||||
|
|
||||||
|
if len(wrap.calls) != 1 || wrap.calls[0][1] != "commit" {
|
||||||
|
t.Fatalf("expected exactly one `commit` call, got %v", wrap.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scenario C5: pending marker version != running version → commit is NEVER called; a loud WARN is
|
||||||
|
// logged; the marker is LEFT (so the report keeps showing pending=true).
|
||||||
|
func TestManager_VersionMismatchDoesNotCommit(t *testing.T) {
|
||||||
|
stateDir := t.TempDir()
|
||||||
|
writePending(t, stateDir, PendingMarker{OldVersion: "0.70.0", NewVersion: "0.70.5", SHA256: "ab", AppliedAt: "t"})
|
||||||
|
wrap := &fakeWrapper{}
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
m := newMgr(t, stateDir, "0.70.1", wrap, buf) // running 0.70.1, marker says 0.70.5
|
||||||
|
|
||||||
|
m.MaybeCommit(context.Background())
|
||||||
|
|
||||||
|
if len(wrap.calls) != 0 {
|
||||||
|
t.Errorf("commit called despite version mismatch: %v", wrap.calls)
|
||||||
|
}
|
||||||
|
if !bytes.Contains(buf.Bytes(), []byte("NOT committing")) {
|
||||||
|
t.Errorf("expected a loud WARN; logs:\n%s", buf.String())
|
||||||
|
}
|
||||||
|
// Marker must remain so the report still flags pending.
|
||||||
|
if p, _ := readPending(stateDir); p == nil {
|
||||||
|
t.Error("pending marker was removed on a version mismatch (report would lose visibility)")
|
||||||
|
}
|
||||||
|
// And the report seam reflects it.
|
||||||
|
if pending, ver := m.SelfUpdatePending(); !pending || ver != "0.70.5" {
|
||||||
|
t.Errorf("SelfUpdatePending() = %v/%q, want true/0.70.5", pending, ver)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No pending marker → nothing happens (the common steady state).
|
||||||
|
func TestManager_NoPendingNoOp(t *testing.T) {
|
||||||
|
stateDir := t.TempDir()
|
||||||
|
wrap := &fakeWrapper{}
|
||||||
|
m := newMgr(t, stateDir, "0.70.1", wrap, nil)
|
||||||
|
m.MaybeCommit(context.Background())
|
||||||
|
if len(wrap.calls) != 0 {
|
||||||
|
t.Errorf("commit/anything called with no pending marker: %v", wrap.calls)
|
||||||
|
}
|
||||||
|
if pending, _ := m.SelfUpdatePending(); pending {
|
||||||
|
t.Error("SelfUpdatePending() true with no marker")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shutdown before the dwell elapses → no commit, marker left for the next start.
|
||||||
|
func TestManager_ShutdownBeforeDwellLeavesPending(t *testing.T) {
|
||||||
|
stateDir := t.TempDir()
|
||||||
|
writePending(t, stateDir, PendingMarker{NewVersion: "0.70.1"})
|
||||||
|
wrap := &fakeWrapper{}
|
||||||
|
m := newMgr(t, stateDir, "0.70.1", wrap, nil)
|
||||||
|
// A cancelled ctx before the (seam) sleep → MaybeCommit sees ctx.Err() and bails.
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
m.MaybeCommit(ctx)
|
||||||
|
if len(wrap.calls) != 0 {
|
||||||
|
t.Errorf("commit called during shutdown: %v", wrap.calls)
|
||||||
|
}
|
||||||
|
if p, _ := readPending(stateDir); p == nil {
|
||||||
|
t.Error("pending marker removed on shutdown-before-commit")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
package selfupdate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// WrapperRunner shells the guarded wrapper's verbs via `sudo -n` (satisfied by *proxmox.ExecRunner
|
||||||
|
// in RunnerSudo mode). Seam so the executor + commit-manager tests never actually invoke sudo.
|
||||||
|
type WrapperRunner interface {
|
||||||
|
Run(ctx context.Context, name string, args ...string) (stdout, stderr []byte, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// wrapperPath is the fixed guarded-wrapper path (never config-overridable — path-fixedness is the
|
||||||
|
// security property).
|
||||||
|
const wrapperPath = "/usr/local/sbin/felhom-selfupdate-guarded"
|
||||||
|
|
||||||
|
// updateParams is the verified params of an agent_update op — the operator-pinned target.
|
||||||
|
type updateParams struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Executor is the agent_update signed-op consumer. Given a gate-VERIFIED agent_update op it:
|
||||||
|
// 1. builds the download URL from config + the signed version,
|
||||||
|
// 2. downloads to <StateDir>/selfupdate/felhom-agent-<version>,
|
||||||
|
// 3. verifies the download against the SIGNED sha256 (mismatch → refuse, remove, error),
|
||||||
|
// 4. hands the staged binary to `felhom-selfupdate-guarded apply <staged> <sha>` via sudo -n.
|
||||||
|
//
|
||||||
|
// The wrapper re-verifies the sha as root and performs the A/B flip + detached restart, so this
|
||||||
|
// process may die shortly after the apply call returns (the S2b detached restart usually lets it
|
||||||
|
// survive to log). Rollback is never this executor's job — a crash-looping new binary is reverted
|
||||||
|
// by systemd + the wrapper.
|
||||||
|
type Executor struct {
|
||||||
|
urlTemplate string
|
||||||
|
username string
|
||||||
|
token string
|
||||||
|
stateDir string
|
||||||
|
runner WrapperRunner
|
||||||
|
httpClient *http.Client
|
||||||
|
logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config is the executor's dependencies (from config.SelfUpdateConfig + the sudo runner).
|
||||||
|
type Config struct {
|
||||||
|
URLTemplate string
|
||||||
|
Username string
|
||||||
|
Token string
|
||||||
|
StateDir string
|
||||||
|
Runner WrapperRunner
|
||||||
|
HTTPClient *http.Client // nil → a 5-minute-timeout default
|
||||||
|
Logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewExecutor builds the agent_update executor.
|
||||||
|
func NewExecutor(cfg Config) *Executor {
|
||||||
|
hc := cfg.HTTPClient
|
||||||
|
if hc == nil {
|
||||||
|
hc = &http.Client{Timeout: 5 * time.Minute}
|
||||||
|
}
|
||||||
|
return &Executor{
|
||||||
|
urlTemplate: cfg.URLTemplate,
|
||||||
|
username: cfg.Username,
|
||||||
|
token: cfg.Token,
|
||||||
|
stateDir: cfg.StateDir,
|
||||||
|
runner: cfg.Runner,
|
||||||
|
httpClient: hc,
|
||||||
|
logger: orDefaultLogger(cfg.Logger),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute implements signedjobs.Executor for the agent_update op class.
|
||||||
|
func (e *Executor) Execute(ctx context.Context, op string, params json.RawMessage) error {
|
||||||
|
if op != opAgentUpdate {
|
||||||
|
return signedjobs.ErrNoExecutor // not ours — leave queued for the owning executor
|
||||||
|
}
|
||||||
|
var p updateParams
|
||||||
|
if err := json.Unmarshal(params, &p); err != nil {
|
||||||
|
return fmt.Errorf("agent_update: bad params: %w", err)
|
||||||
|
}
|
||||||
|
if !versionRe.MatchString(p.Version) {
|
||||||
|
return fmt.Errorf("agent_update: refusing — version %q is not bare semver", p.Version)
|
||||||
|
}
|
||||||
|
if !sha256Re.MatchString(p.SHA256) {
|
||||||
|
return fmt.Errorf("agent_update: refusing — sha256 is not 64 lowercase hex")
|
||||||
|
}
|
||||||
|
if e.runner == nil {
|
||||||
|
return fmt.Errorf("agent_update: no wrapper runner configured")
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := stagingDir(e.stateDir)
|
||||||
|
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||||
|
return fmt.Errorf("agent_update: staging dir: %w", err)
|
||||||
|
}
|
||||||
|
staged := filepath.Join(dir, "felhom-agent-"+p.Version)
|
||||||
|
url := interpolateURL(e.urlTemplate, p.Version)
|
||||||
|
|
||||||
|
e.logger.Warn("agent_update: downloading operator-signed binary", "version", p.Version, "url", url, "sha256", p.SHA256)
|
||||||
|
got, err := e.download(ctx, url, staged)
|
||||||
|
if err != nil {
|
||||||
|
_ = os.Remove(staged)
|
||||||
|
return fmt.Errorf("agent_update: download %s: %w", url, err)
|
||||||
|
}
|
||||||
|
// The signed sha is the ONLY integrity root — verify BEFORE anything touches the live binary.
|
||||||
|
if got != p.SHA256 {
|
||||||
|
_ = os.Remove(staged)
|
||||||
|
return fmt.Errorf("agent_update: sha256 mismatch — got %s want %s (refusing; agent untouched)", got, p.SHA256)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(staged, 0o755); err != nil {
|
||||||
|
_ = os.Remove(staged)
|
||||||
|
return fmt.Errorf("agent_update: chmod staged: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hand off to the root wrapper. It re-verifies the sha, flips A/B, writes the pending marker,
|
||||||
|
// and schedules the detached restart. After this the new binary starts; the commit is the
|
||||||
|
// Manager's job once it has dwelled cleanly.
|
||||||
|
e.logger.Warn("agent_update: handing staged binary to the guarded wrapper", "staged", staged, "version", p.Version)
|
||||||
|
stdout, stderr, err := e.runner.Run(ctx, wrapperPath, "apply", staged, p.SHA256)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("agent_update: wrapper apply failed: %w (stderr: %s)", err, string(stderr))
|
||||||
|
}
|
||||||
|
e.logger.Warn("agent_update: apply handed off; restart scheduled", "version", p.Version, "wrapper", trim(stdout))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// download streams url → dest (0644, fsync'd) and returns the lowercase-hex sha256 of the bytes.
|
||||||
|
func (e *Executor) download(ctx context.Context, url, dest string) (string, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if e.username != "" || e.token != "" {
|
||||||
|
req.SetBasicAuth(e.username, e.token)
|
||||||
|
}
|
||||||
|
resp, err := e.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
f, err := os.OpenFile(dest, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
h := sha256.New()
|
||||||
|
if _, err := io.Copy(io.MultiWriter(f, h), resp.Body); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := f.Sync(); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(h.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func trim(b []byte) string {
|
||||||
|
s := string(b)
|
||||||
|
if len(s) > 200 {
|
||||||
|
s = s[:200]
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
package selfupdate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeWrapper records the verbs the executor/manager shell out, and returns a configurable error.
|
||||||
|
type fakeWrapper struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
calls [][]string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.calls = append(f.calls, append([]string{name}, args...))
|
||||||
|
return []byte("ok"), nil, f.err
|
||||||
|
}
|
||||||
|
func (f *fakeWrapper) applyCalls() [][]string {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
var out [][]string
|
||||||
|
for _, c := range f.calls {
|
||||||
|
if len(c) >= 2 && c[1] == "apply" {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func sha256Of(b []byte) string {
|
||||||
|
h := sha256.Sum256(b)
|
||||||
|
return hex.EncodeToString(h[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// artifactServer serves `body` at /…/{version}/felhom-agent.
|
||||||
|
func artifactServer(t *testing.T, body []byte) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func newExec(t *testing.T, srv *httptest.Server, wrap WrapperRunner) (*Executor, string) {
|
||||||
|
t.Helper()
|
||||||
|
stateDir := t.TempDir()
|
||||||
|
e := NewExecutor(Config{
|
||||||
|
URLTemplate: srv.URL + "/{version}/felhom-agent",
|
||||||
|
StateDir: stateDir,
|
||||||
|
Runner: wrap,
|
||||||
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
|
})
|
||||||
|
return e, stateDir
|
||||||
|
}
|
||||||
|
|
||||||
|
func updateParamsJSON(t *testing.T, version, sha string) json.RawMessage {
|
||||||
|
t.Helper()
|
||||||
|
b, _ := json.Marshal(map[string]string{"version": version, "sha256": sha})
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scenario A (executor half): a good download whose sha matches the signed value is staged and
|
||||||
|
// handed to `apply` with the EXACT staged path + sha.
|
||||||
|
func TestExecutor_HappyPath(t *testing.T) {
|
||||||
|
body := []byte("#!/bin/sh\necho v0.70.1\n")
|
||||||
|
srv := artifactServer(t, body)
|
||||||
|
defer srv.Close()
|
||||||
|
wrap := &fakeWrapper{}
|
||||||
|
e, stateDir := newExec(t, srv, wrap)
|
||||||
|
|
||||||
|
sha := sha256Of(body)
|
||||||
|
if err := e.Execute(context.Background(), "agent_update", updateParamsJSON(t, "0.70.1", sha)); err != nil {
|
||||||
|
t.Fatalf("execute: %v", err)
|
||||||
|
}
|
||||||
|
staged := filepath.Join(stateDir, "selfupdate", "felhom-agent-0.70.1")
|
||||||
|
got, err := os.ReadFile(staged)
|
||||||
|
if err != nil || sha256Of(got) != sha {
|
||||||
|
t.Fatalf("staged binary missing/mismatch: %v", err)
|
||||||
|
}
|
||||||
|
calls := wrap.applyCalls()
|
||||||
|
if len(calls) != 1 {
|
||||||
|
t.Fatalf("apply invoked %d times, want 1 (%v)", len(calls), wrap.calls)
|
||||||
|
}
|
||||||
|
if calls[0][2] != staged || calls[0][3] != sha {
|
||||||
|
t.Errorf("apply args = %v, want [.. apply %s %s]", calls[0], staged, sha)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scenario C2 + its companion: a download whose sha != the signed value is REFUSED, nothing is
|
||||||
|
// handed to apply, and the staged file is removed. The companion (dropping the Go-side verify) is
|
||||||
|
// structural: the sha check IS the code under test — if it were removed, this bad binary would
|
||||||
|
// reach the apply call (asserted here: applyCalls == 0).
|
||||||
|
func TestExecutor_ShaMismatchRefused(t *testing.T) {
|
||||||
|
body := []byte("the REAL published bytes")
|
||||||
|
srv := artifactServer(t, body)
|
||||||
|
defer srv.Close()
|
||||||
|
wrap := &fakeWrapper{}
|
||||||
|
e, stateDir := newExec(t, srv, wrap)
|
||||||
|
|
||||||
|
wrongSha := sha256Of([]byte("what the operator signed for a DIFFERENT binary"))
|
||||||
|
err := e.Execute(context.Background(), "agent_update", updateParamsJSON(t, "0.70.1", wrongSha))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected sha-mismatch refusal, got nil")
|
||||||
|
}
|
||||||
|
if len(wrap.applyCalls()) != 0 {
|
||||||
|
t.Error("a sha-mismatched binary REACHED the apply call — the verify gate leaked")
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(filepath.Join(stateDir, "selfupdate", "felhom-agent-0.70.1")); !os.IsNotExist(statErr) {
|
||||||
|
t.Error("staged file was left behind after a sha mismatch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bad params / non-semver version / non-hex sha are refused before any download or apply.
|
||||||
|
func TestExecutor_BadParamsRefused(t *testing.T) {
|
||||||
|
wrap := &fakeWrapper{}
|
||||||
|
e, _ := newExec(t, artifactServer(t, []byte("x")), wrap)
|
||||||
|
for name, p := range map[string]json.RawMessage{
|
||||||
|
"non-semver version": updateParamsJSON(t, "latest", sha256Of([]byte("x"))),
|
||||||
|
"non-hex sha": updateParamsJSON(t, "0.70.1", "NOTHEX"),
|
||||||
|
"bad json": json.RawMessage(`{`),
|
||||||
|
} {
|
||||||
|
if err := e.Execute(context.Background(), "agent_update", p); err == nil {
|
||||||
|
t.Errorf("%s: expected refusal", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(wrap.calls) != 0 {
|
||||||
|
t.Errorf("wrapper invoked on bad params: %v", wrap.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A non-owned op class returns ErrNoExecutor (the chain contract) — never touches anything.
|
||||||
|
func TestExecutor_NotOurOp(t *testing.T) {
|
||||||
|
e, _ := newExec(t, artifactServer(t, []byte("x")), &fakeWrapper{})
|
||||||
|
if err := e.Execute(context.Background(), "storage_wipe", json.RawMessage(`{}`)); !errors.Is(err, signedjobs.ErrNoExecutor) {
|
||||||
|
t.Errorf("err = %v, want ErrNoExecutor", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A wrapper apply failure surfaces (the executor reports it — the runner then logs + clears).
|
||||||
|
func TestExecutor_WrapperFailureSurfaces(t *testing.T) {
|
||||||
|
body := []byte("good bytes")
|
||||||
|
srv := artifactServer(t, body)
|
||||||
|
defer srv.Close()
|
||||||
|
wrap := &fakeWrapper{err: errors.New("wrapper refused: sha mismatch")}
|
||||||
|
e, _ := newExec(t, srv, wrap)
|
||||||
|
if err := e.Execute(context.Background(), "agent_update", updateParamsJSON(t, "0.70.1", sha256Of(body))); err == nil {
|
||||||
|
t.Fatal("wrapper failure must surface")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
// Package selfupdate implements the AGENT-Go half of the operator-signed A/B self-update (TASK D1).
|
||||||
|
// The ROOT half (the atomic binary flip + rollback) is the felhom-selfupdate-guarded shell wrapper
|
||||||
|
// (configs/); systemd owns the crash-loop auto-rollback. This package:
|
||||||
|
// - downloads the operator-pinned binary from the configured artifact host,
|
||||||
|
// - verifies it against the SIGNED sha256 (the only integrity root),
|
||||||
|
// - hands it to the wrapper's `apply` verb via `sudo -n` (the Executor, driven by signedjobs),
|
||||||
|
// - after the NEW binary has run cleanly for the dwell, calls the wrapper's `commit` (the Manager).
|
||||||
|
//
|
||||||
|
// The design principle (SPIKE-agent-selfupdate-2026-07-05): the thing that performs rollback is
|
||||||
|
// never the thing being updated. This package only ever STARTS an update and COMMITS a good one;
|
||||||
|
// it never rolls back (that is systemd + the wrapper, so a crash-looping new binary cannot fail to
|
||||||
|
// revert itself).
|
||||||
|
package selfupdate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
)
|
||||||
|
|
||||||
|
// opAgentUpdate is the op class this executor serves (mirrors reconcile.ClassAgentUpdate; the
|
||||||
|
// literal avoids importing reconcile here just for the string).
|
||||||
|
const opAgentUpdate = "agent_update"
|
||||||
|
|
||||||
|
// versionRe bounds the version string that is interpolated into a URL and a filename. Bare semver
|
||||||
|
// with an optional pre-release suffix (e.g. 0.70.1, 0.70.2-crash) — no slashes, spaces, or dots-only.
|
||||||
|
var versionRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$`)
|
||||||
|
|
||||||
|
// sha256Re is the strict 64-lowercase-hex form of the signed sha.
|
||||||
|
var sha256Re = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||||
|
|
||||||
|
// PendingMarker is the JSON the wrapper writes at <StateDir>/selfupdate/pending.json after a flip.
|
||||||
|
// The agent reads it on startup to decide whether to commit. Field names match the wrapper.
|
||||||
|
type PendingMarker struct {
|
||||||
|
OldVersion string `json:"old_version"`
|
||||||
|
NewVersion string `json:"new_version"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
AppliedAt string `json:"applied_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// stagingDir is the agent-writable staging subdir (spike S4a: agent-owned StateDir, 0750).
|
||||||
|
func stagingDir(stateDir string) string { return filepath.Join(stateDir, "selfupdate") }
|
||||||
|
|
||||||
|
// pendingPath is the wrapper's pending-marker path.
|
||||||
|
func pendingPath(stateDir string) string { return filepath.Join(stagingDir(stateDir), "pending.json") }
|
||||||
|
|
||||||
|
// readPending returns the pending marker, or (nil, nil) when none exists.
|
||||||
|
func readPending(stateDir string) (*PendingMarker, error) {
|
||||||
|
data, err := os.ReadFile(pendingPath(stateDir))
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var m PendingMarker
|
||||||
|
if err := json.Unmarshal(data, &m); err != nil {
|
||||||
|
return nil, fmt.Errorf("selfupdate: pending marker unparseable: %w", err)
|
||||||
|
}
|
||||||
|
return &m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// interpolateURL substitutes the (validated) version into the URL template's {version} placeholder.
|
||||||
|
func interpolateURL(template, version string) string {
|
||||||
|
return regexp.MustCompile(`\{version\}`).ReplaceAllString(template, version)
|
||||||
|
}
|
||||||
|
|
||||||
|
// noopLogger is a discard logger for nil-safety.
|
||||||
|
func orDefaultLogger(l *slog.Logger) *slog.Logger {
|
||||||
|
if l == nil {
|
||||||
|
return slog.Default()
|
||||||
|
}
|
||||||
|
return l
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package signedjobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TASK D1 Group B — the agent_update op class RIDES the same LOCKED gate pipeline as every other
|
||||||
|
// destructive op. These tests use the REAL authz.Verifier + reconcile.Gate (via newRealGateRunner)
|
||||||
|
// over genuinely-minted signed blobs, asserting agent_update is gated identically to storage_wipe.
|
||||||
|
|
||||||
|
const agentUpdateParamsJSON = `{"version":"0.70.1","sha256":"` +
|
||||||
|
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + `"}`
|
||||||
|
|
||||||
|
// A correctly-signed agent_update by the PINNED operational key reaches the executor (a fake here;
|
||||||
|
// the real executor is unit-tested in internal/selfupdate). Proves the class is authorized, not
|
||||||
|
// silently dropped.
|
||||||
|
func TestRunner_ValidSignedAgentUpdateExecutes(t *testing.T) {
|
||||||
|
s := newTestSigner(t)
|
||||||
|
r, src, exec := newRealGateRunner(t, s)
|
||||||
|
now := time.Now().UTC()
|
||||||
|
src.add(mintJobOp(t, s, "agent_update", "au1", testHost, "", "ops-1", agentUpdateParamsJSON, now, now.Add(time.Hour)))
|
||||||
|
|
||||||
|
if _, err := r.RunOnce(context.Background()); err != nil {
|
||||||
|
t.Fatalf("RunOnce: %v", err)
|
||||||
|
}
|
||||||
|
if exec.count() != 1 || exec.calls[0] != "agent_update" {
|
||||||
|
t.Fatalf("executor calls = %v, want one agent_update", exec.calls)
|
||||||
|
}
|
||||||
|
if !src.wasCompleted("au1") {
|
||||||
|
t.Error("valid agent_update job not cleared")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A NON-PINNED signer's agent_update is REJECTED — the executor is never called. This is the
|
||||||
|
// companion to the class-allowlist question: agent_update is classified Destructive
|
||||||
|
// (reconcile.Classify), so an unsigned/wrong-key op cannot reach the binary swap. If the class were
|
||||||
|
// ever mis-classified Benign, this signature check would be bypassed and the test would fail.
|
||||||
|
func TestRunner_NonPinnedAgentUpdateRejected(t *testing.T) {
|
||||||
|
pinned := newTestSigner(t)
|
||||||
|
attacker := newTestSigner(t) // not pinned
|
||||||
|
r, src, exec := newRealGateRunner(t, pinned)
|
||||||
|
now := time.Now().UTC()
|
||||||
|
src.add(mintJobOp(t, attacker, "agent_update", "au1", testHost, "", "ops-1", agentUpdateParamsJSON, now, now.Add(time.Hour)))
|
||||||
|
|
||||||
|
r.RunOnce(context.Background())
|
||||||
|
if exec.count() != 0 {
|
||||||
|
t.Errorf("a non-pinned agent_update was EXECUTED (count=%d) — the binary swap must be gated", exec.count())
|
||||||
|
}
|
||||||
|
if !src.wasCompleted("au1") {
|
||||||
|
t.Error("rejected agent_update job should be cleared")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An agent_update targeting ANOTHER host is rejected on this host (anti-retarget) — an operator
|
||||||
|
// can't accidentally push a build to the wrong box.
|
||||||
|
func TestRunner_AgentUpdateRetargetRejected(t *testing.T) {
|
||||||
|
s := newTestSigner(t)
|
||||||
|
r, src, exec := newRealGateRunner(t, s)
|
||||||
|
now := time.Now().UTC()
|
||||||
|
src.add(mintJobOp(t, s, "agent_update", "au1", "some-other-host", "", "ops-1", agentUpdateParamsJSON, now, now.Add(time.Hour)))
|
||||||
|
|
||||||
|
r.RunOnce(context.Background())
|
||||||
|
if exec.count() != 0 {
|
||||||
|
t.Errorf("an agent_update for another host was executed here (count=%d)", exec.count())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -86,8 +86,13 @@ func (s testSigner) allowed(t *testing.T, keyID string, role authz.KeyRole) auth
|
|||||||
|
|
||||||
// mintJob builds a hub.JobWire carrying a signed storage_wipe envelope from the given signer.
|
// mintJob builds a hub.JobWire carrying a signed storage_wipe envelope from the given signer.
|
||||||
func mintJob(t *testing.T, s testSigner, jobID, host, guest, keyID, paramsJSON string, issued, expires time.Time) hub.JobWire {
|
func mintJob(t *testing.T, s testSigner, jobID, host, guest, keyID, paramsJSON string, issued, expires time.Time) hub.JobWire {
|
||||||
|
return mintJobOp(t, s, "storage_wipe", jobID, host, guest, keyID, paramsJSON, issued, expires)
|
||||||
|
}
|
||||||
|
|
||||||
|
// mintJobOp is mintJob with an explicit op class (for non-wipe ops, e.g. agent_update ride-along).
|
||||||
|
func mintJobOp(t *testing.T, s testSigner, op, jobID, host, guest, keyID, paramsJSON string, issued, expires time.Time) hub.JobWire {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
blob, err := authz.CanonicalBlob("storage_wipe", host, guest, keyID, randNonce(), paramsJSON, issued, expires)
|
blob, err := authz.CanonicalBlob(op, host, guest, keyID, randNonce(), paramsJSON, issued, expires)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("CanonicalBlob: %v", err)
|
t.Fatalf("CanonicalBlob: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user