Files
felhom-controller/controller/internal/offsiteapply/seams.go
T
admin cb8bf14599 v0.162.0 — R-71(a): the apply-bridge settle-gate (kills the F10 day-0 race)
The day-0 race (DIAG-f10): a fresh box boots below the operator floor, the
apply-bridge consumes the single-use offsite password, then ~35s later the
managed auto-floor update replaces the container mid-install -> the new process
finds no installed key -> consume -> 404 -> offsite dead until an operator
Re-issue. Recurs on every onboarding whose ISO floor lags the managed floor.

Ordering-only fix (consume/install/persist internals + the 404-no-oracle
contract + the Consumer UNTOUCHED; R-71(b) rejected-by-design):
- New seam offsiteapply.SettleProvider.SettleState() + SettleFunc adapter over
  the self-updater's own GetFloor()/IsUpdateRunning() (no second floor path).
- Bridge.AwaitSettle polls 10s BEFORE the 3-min Reconcile ctx: defers while an
  update runs or the box is below the known floor; GOes at/above floor on the
  first poll with zero added latency (B'). Bounds 90s floor sub-bound / 5min
  overall, both GO+WARN (hub that can't serve a floor can't serve a consume ->
  no burn risk; R-71c is the belt). ReconcileWhenSettled = gate then reconcile.
- main.go: bridge goroutine moved after the updater is built; wired only when an
  updater exists (nil Settle = reconcile immediately, old behavior).

Finding: the floor is in-memory (report-ACK ~5-10s), NOT persisted -> unknown on
any restart until the first ACK; the 90s sub-bound is sized to that.

Tests (injectable clock, fake SettleState, recorded Consumer): A-E + nil-provider
+ cancelled-gate. Four red-proofs all observed FAIL then restored: gate removed /
updateRunning branch / floor sub-bound / overall bound. Deferral paths ship
unit-proven + red-proofed, NOT live-fired -- their precondition is now
structurally prevented by the v1.25.0 build gate. Layering: gate prevents, (a)
defers, (c) heals.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N7Drmtm2RzoqbkJZCNSFNQ
2026-07-24 07:48:45 +02:00

247 lines
8.5 KiB
Go

package offsiteapply
import (
"context"
"crypto/ed25519"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
// --- func adapters (convenient wiring in main.go) ---
type ConsumerFunc func(ctx context.Context) (string, error)
func (f ConsumerFunc) Consume(ctx context.Context) (string, error) { return f(ctx) }
type EnablerFunc func(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error
func (f EnablerFunc) ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error {
return f(ctx, host, user, port, repoPath, privPEM, knownHosts, quotaGB)
}
// SettleFunc adapts a plain func to a SettleProvider (thin adapter over the Updater in main.go —
// the StackDataProvider pattern). It reads the updater's OWN knowledge; the bridge never fetches the
// floor a second way (no second floor path).
type SettleFunc func() (version, floor string, updateRunning, floorKnown bool)
func (f SettleFunc) SettleState() (string, string, bool, bool) { return f() }
// --- HTTPConsumer: POST the hub consume-password endpoint with the per-customer API key ---
type HTTPConsumer struct {
HubURL string
CustomerID string
APIKey string
HC *http.Client
}
func (c HTTPConsumer) Consume(ctx context.Context) (string, error) {
if c.HubURL == "" || c.CustomerID == "" || c.APIKey == "" {
return "", fmt.Errorf("offsite-apply: consume: hub url/customer/apikey not configured")
}
hc := c.HC
if hc == nil {
hc = &http.Client{Timeout: 20 * time.Second}
}
url := strings.TrimRight(c.HubURL, "/") + "/api/v1/offsite/consume-password/" + c.CustomerID
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
if err != nil {
return "", err
}
req.Header.Set("Authorization", "Bearer "+c.APIKey)
resp, err := hc.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
if resp.StatusCode == http.StatusNotFound {
return "", fmt.Errorf("no unconsumed offsite password (already consumed or none provisioned)")
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("consume: HTTP %d", resp.StatusCode)
}
var body struct {
Password string `json:"password"`
}
if err := json.Unmarshal(raw, &body); err != nil || body.Password == "" {
return "", fmt.Errorf("consume: malformed response")
}
return body.Password, nil // NEVER logged
}
// --- KeyscanScanner: capture the box host key (x/crypto/ssh, no binary) → fingerprint + known_hosts line ---
type KeyscanScanner struct {
Timeout time.Duration
}
var errScanCaptured = errors.New("host key captured")
func (s KeyscanScanner) Scan(ctx context.Context, host string, port int) (string, string, error) {
timeout := s.Timeout
if timeout == 0 {
timeout = 10 * time.Second
}
var fp, line string
cfg := &ssh.ClientConfig{
User: "felhom-keyscan",
Timeout: timeout,
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
fp = ssh.FingerprintSHA256(key)
line = knownhosts.Line([]string{knownhosts.Normalize(net.JoinHostPort(host, strconv.Itoa(port)))}, key)
return errScanCaptured
},
}
d := net.Dialer{Timeout: timeout}
conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port)))
if err != nil {
return "", "", fmt.Errorf("dial: %w", err)
}
defer conn.Close()
c, _, _, herr := ssh.NewClientConn(conn, host, cfg)
if c != nil {
c.Close()
}
if fp != "" && line != "" {
return fp, line, nil
}
return "", "", fmt.Errorf("host-key handshake: %w", herr)
}
// --- ED25519KeyGen: a fresh keypair (OpenSSH private PEM + authorized_keys pub line) ---
type ED25519KeyGen struct{}
func (ED25519KeyGen) Generate() (string, string, error) {
pub, priv, err := ed25519.GenerateKey(nil)
if err != nil {
return "", "", err
}
block, err := ssh.MarshalPrivateKey(priv, "felhom-offbox")
if err != nil {
return "", "", err
}
sshPub, err := ssh.NewPublicKey(pub)
if err != nil {
return "", "", err
}
privPEM := string(pem.EncodeToMemory(block))
pubLine := string(ssh.MarshalAuthorizedKey(sshPub)) // includes trailing newline
return privPEM, pubLine, nil
}
// --- SSHCopyIDInstaller: install the pubkey via the proven `sshpass -e ssh-copy-id -p N -s -f`, verify ---
type SSHCopyIDInstaller struct{}
func (SSHCopyIDInstaller) Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized, knownHosts string) error {
if strings.TrimSpace(knownHosts) == "" {
return fmt.Errorf("ssh-copy-id: empty known_hosts — refusing to install without a pinned host key")
}
// ssh-copy-id -s (SFTP mode) mktemp's its batch file under ~/.ssh and dies LOCALLY if the directory
// doesn't exist — the container image ships without /root/.ssh (live finding: the one-time password was
// consumed, then the install failed before ever connecting).
if home, err := os.UserHomeDir(); err == nil {
if err := os.MkdirAll(filepath.Join(home, ".ssh"), 0o700); err != nil {
return fmt.Errorf("ssh-copy-id: ensure ~/.ssh (needed by -s mode): %w", err)
}
}
work, err := os.MkdirTemp("", "felhom-keyinstall-")
if err != nil {
return err
}
defer os.RemoveAll(work)
pubPath := filepath.Join(work, "id.pub")
privPath := filepath.Join(work, "id")
khPath := filepath.Join(work, "known_hosts")
if err := os.WriteFile(pubPath, []byte(pubAuthorized), 0o600); err != nil {
return err
}
if err := os.WriteFile(privPath, []byte(privPEM), 0o600); err != nil {
return err
}
// Pin the scanner-VERIFIED host key: StrictHostKeyChecking=yes against this known_hosts refuses any
// other key (no accept-new/TOFU) — the ssh-copy-id + verify sessions connect ONLY to the box whose
// fingerprint the bridge already matched against the hub descriptor.
if err := os.WriteFile(khPath, []byte(knownHosts+"\n"), 0o600); err != nil {
return err
}
// Install (SSHPASS env is read by `sshpass -e`; the password never appears on argv).
install := exec.CommandContext(ctx, "sshpass", "-e", "ssh-copy-id", "-p", strconv.Itoa(port), "-s", "-f",
"-i", pubPath, "-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile="+khPath, user+"@"+host)
install.Env = append(os.Environ(), "SSHPASS="+password)
if out, err := install.CombinedOutput(); err != nil {
return fmt.Errorf("ssh-copy-id: %w: %s", err, truncate(out))
}
// Verify passwordless key auth (an SFTP no-op; the box's restricted shell only offers SFTP).
verify := exec.CommandContext(ctx, "sftp", "-b", "-", "-P", strconv.Itoa(port),
"-i", privPath, "-oBatchMode=yes", "-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile="+khPath, user+"@"+host)
verify.Stdin = strings.NewReader("pwd\n")
if out, err := verify.CombinedOutput(); err != nil {
return fmt.Errorf("key-auth verify failed after install: %w: %s", err, truncate(out))
}
return nil
}
// --- SFTPKeyAuthProber: does the ALREADY-INSTALLED key still authenticate? (key-auth-first) ---
// SFTPKeyAuthProber probes passwordless auth with the existing installed key (KeyPath), pinned to the
// freshly-verified knownHosts line. No key file → ok=false (fresh guest). The probe never logs secrets.
type SFTPKeyAuthProber struct {
KeyPath string // the installed key, e.g. <dataDir>/offbox/ssh_key
Timeout time.Duration // per-probe budget; 0 → 20s
}
func (p SFTPKeyAuthProber) Probe(ctx context.Context, host, user string, port int, knownHosts string) (string, bool) {
pem, err := os.ReadFile(p.KeyPath)
if err != nil {
return "", false // no existing key — a fresh guest; take the full path
}
timeout := p.Timeout
if timeout == 0 {
timeout = 20 * time.Second
}
pctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
work, err := os.MkdirTemp("", "felhom-keyprobe-")
if err != nil {
return "", false
}
defer os.RemoveAll(work)
khPath := filepath.Join(work, "known_hosts")
if err := os.WriteFile(khPath, []byte(knownHosts+"\n"), 0o600); err != nil {
return "", false
}
probe := exec.CommandContext(pctx, "sftp", "-b", "-", "-P", strconv.Itoa(port),
"-i", p.KeyPath, "-oBatchMode=yes", "-oConnectTimeout=10",
"-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile="+khPath, user+"@"+host)
probe.Stdin = strings.NewReader("pwd\n")
if err := probe.Run(); err != nil {
return "", false // auth refused / unreachable — fall through to the full path
}
return string(pem), true
}
func truncate(b []byte) string {
s := strings.TrimSpace(string(b))
if len(s) > 300 {
return s[:300] + "…"
}
return s
}