cb8bf14599
The day-0 race (DIAG-f10): a fresh box boots below the operator floor, the apply-bridge consumes the single-use offsite password, then ~35s later the managed auto-floor update replaces the container mid-install -> the new process finds no installed key -> consume -> 404 -> offsite dead until an operator Re-issue. Recurs on every onboarding whose ISO floor lags the managed floor. Ordering-only fix (consume/install/persist internals + the 404-no-oracle contract + the Consumer UNTOUCHED; R-71(b) rejected-by-design): - New seam offsiteapply.SettleProvider.SettleState() + SettleFunc adapter over the self-updater's own GetFloor()/IsUpdateRunning() (no second floor path). - Bridge.AwaitSettle polls 10s BEFORE the 3-min Reconcile ctx: defers while an update runs or the box is below the known floor; GOes at/above floor on the first poll with zero added latency (B'). Bounds 90s floor sub-bound / 5min overall, both GO+WARN (hub that can't serve a floor can't serve a consume -> no burn risk; R-71c is the belt). ReconcileWhenSettled = gate then reconcile. - main.go: bridge goroutine moved after the updater is built; wired only when an updater exists (nil Settle = reconcile immediately, old behavior). Finding: the floor is in-memory (report-ACK ~5-10s), NOT persisted -> unknown on any restart until the first ACK; the 90s sub-bound is sized to that. Tests (injectable clock, fake SettleState, recorded Consumer): A-E + nil-provider + cancelled-gate. Four red-proofs all observed FAIL then restored: gate removed / updateRunning branch / floor sub-bound / overall bound. Deferral paths ship unit-proven + red-proofed, NOT live-fired -- their precondition is now structurally prevented by the v1.25.0 build gate. Layering: gate prevents, (a) defers, (c) heals. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01N7Drmtm2RzoqbkJZCNSFNQ
247 lines
8.5 KiB
Go
247 lines
8.5 KiB
Go
package offsiteapply
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"encoding/json"
|
|
"encoding/pem"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/ssh"
|
|
"golang.org/x/crypto/ssh/knownhosts"
|
|
)
|
|
|
|
// --- func adapters (convenient wiring in main.go) ---
|
|
|
|
type ConsumerFunc func(ctx context.Context) (string, error)
|
|
|
|
func (f ConsumerFunc) Consume(ctx context.Context) (string, error) { return f(ctx) }
|
|
|
|
type EnablerFunc func(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error
|
|
|
|
func (f EnablerFunc) ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error {
|
|
return f(ctx, host, user, port, repoPath, privPEM, knownHosts, quotaGB)
|
|
}
|
|
|
|
// SettleFunc adapts a plain func to a SettleProvider (thin adapter over the Updater in main.go —
|
|
// the StackDataProvider pattern). It reads the updater's OWN knowledge; the bridge never fetches the
|
|
// floor a second way (no second floor path).
|
|
type SettleFunc func() (version, floor string, updateRunning, floorKnown bool)
|
|
|
|
func (f SettleFunc) SettleState() (string, string, bool, bool) { return f() }
|
|
|
|
// --- HTTPConsumer: POST the hub consume-password endpoint with the per-customer API key ---
|
|
|
|
type HTTPConsumer struct {
|
|
HubURL string
|
|
CustomerID string
|
|
APIKey string
|
|
HC *http.Client
|
|
}
|
|
|
|
func (c HTTPConsumer) Consume(ctx context.Context) (string, error) {
|
|
if c.HubURL == "" || c.CustomerID == "" || c.APIKey == "" {
|
|
return "", fmt.Errorf("offsite-apply: consume: hub url/customer/apikey not configured")
|
|
}
|
|
hc := c.HC
|
|
if hc == nil {
|
|
hc = &http.Client{Timeout: 20 * time.Second}
|
|
}
|
|
url := strings.TrimRight(c.HubURL, "/") + "/api/v1/offsite/consume-password/" + c.CustomerID
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+c.APIKey)
|
|
resp, err := hc.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return "", fmt.Errorf("no unconsumed offsite password (already consumed or none provisioned)")
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return "", fmt.Errorf("consume: HTTP %d", resp.StatusCode)
|
|
}
|
|
var body struct {
|
|
Password string `json:"password"`
|
|
}
|
|
if err := json.Unmarshal(raw, &body); err != nil || body.Password == "" {
|
|
return "", fmt.Errorf("consume: malformed response")
|
|
}
|
|
return body.Password, nil // NEVER logged
|
|
}
|
|
|
|
// --- KeyscanScanner: capture the box host key (x/crypto/ssh, no binary) → fingerprint + known_hosts line ---
|
|
|
|
type KeyscanScanner struct {
|
|
Timeout time.Duration
|
|
}
|
|
|
|
var errScanCaptured = errors.New("host key captured")
|
|
|
|
func (s KeyscanScanner) Scan(ctx context.Context, host string, port int) (string, string, error) {
|
|
timeout := s.Timeout
|
|
if timeout == 0 {
|
|
timeout = 10 * time.Second
|
|
}
|
|
var fp, line string
|
|
cfg := &ssh.ClientConfig{
|
|
User: "felhom-keyscan",
|
|
Timeout: timeout,
|
|
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
|
|
fp = ssh.FingerprintSHA256(key)
|
|
line = knownhosts.Line([]string{knownhosts.Normalize(net.JoinHostPort(host, strconv.Itoa(port)))}, key)
|
|
return errScanCaptured
|
|
},
|
|
}
|
|
d := net.Dialer{Timeout: timeout}
|
|
conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port)))
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("dial: %w", err)
|
|
}
|
|
defer conn.Close()
|
|
c, _, _, herr := ssh.NewClientConn(conn, host, cfg)
|
|
if c != nil {
|
|
c.Close()
|
|
}
|
|
if fp != "" && line != "" {
|
|
return fp, line, nil
|
|
}
|
|
return "", "", fmt.Errorf("host-key handshake: %w", herr)
|
|
}
|
|
|
|
// --- ED25519KeyGen: a fresh keypair (OpenSSH private PEM + authorized_keys pub line) ---
|
|
|
|
type ED25519KeyGen struct{}
|
|
|
|
func (ED25519KeyGen) Generate() (string, string, error) {
|
|
pub, priv, err := ed25519.GenerateKey(nil)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
block, err := ssh.MarshalPrivateKey(priv, "felhom-offbox")
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
sshPub, err := ssh.NewPublicKey(pub)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
privPEM := string(pem.EncodeToMemory(block))
|
|
pubLine := string(ssh.MarshalAuthorizedKey(sshPub)) // includes trailing newline
|
|
return privPEM, pubLine, nil
|
|
}
|
|
|
|
// --- SSHCopyIDInstaller: install the pubkey via the proven `sshpass -e ssh-copy-id -p N -s -f`, verify ---
|
|
|
|
type SSHCopyIDInstaller struct{}
|
|
|
|
func (SSHCopyIDInstaller) Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized, knownHosts string) error {
|
|
if strings.TrimSpace(knownHosts) == "" {
|
|
return fmt.Errorf("ssh-copy-id: empty known_hosts — refusing to install without a pinned host key")
|
|
}
|
|
// ssh-copy-id -s (SFTP mode) mktemp's its batch file under ~/.ssh and dies LOCALLY if the directory
|
|
// doesn't exist — the container image ships without /root/.ssh (live finding: the one-time password was
|
|
// consumed, then the install failed before ever connecting).
|
|
if home, err := os.UserHomeDir(); err == nil {
|
|
if err := os.MkdirAll(filepath.Join(home, ".ssh"), 0o700); err != nil {
|
|
return fmt.Errorf("ssh-copy-id: ensure ~/.ssh (needed by -s mode): %w", err)
|
|
}
|
|
}
|
|
work, err := os.MkdirTemp("", "felhom-keyinstall-")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.RemoveAll(work)
|
|
pubPath := filepath.Join(work, "id.pub")
|
|
privPath := filepath.Join(work, "id")
|
|
khPath := filepath.Join(work, "known_hosts")
|
|
if err := os.WriteFile(pubPath, []byte(pubAuthorized), 0o600); err != nil {
|
|
return err
|
|
}
|
|
if err := os.WriteFile(privPath, []byte(privPEM), 0o600); err != nil {
|
|
return err
|
|
}
|
|
// Pin the scanner-VERIFIED host key: StrictHostKeyChecking=yes against this known_hosts refuses any
|
|
// other key (no accept-new/TOFU) — the ssh-copy-id + verify sessions connect ONLY to the box whose
|
|
// fingerprint the bridge already matched against the hub descriptor.
|
|
if err := os.WriteFile(khPath, []byte(knownHosts+"\n"), 0o600); err != nil {
|
|
return err
|
|
}
|
|
// Install (SSHPASS env is read by `sshpass -e`; the password never appears on argv).
|
|
install := exec.CommandContext(ctx, "sshpass", "-e", "ssh-copy-id", "-p", strconv.Itoa(port), "-s", "-f",
|
|
"-i", pubPath, "-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile="+khPath, user+"@"+host)
|
|
install.Env = append(os.Environ(), "SSHPASS="+password)
|
|
if out, err := install.CombinedOutput(); err != nil {
|
|
return fmt.Errorf("ssh-copy-id: %w: %s", err, truncate(out))
|
|
}
|
|
// Verify passwordless key auth (an SFTP no-op; the box's restricted shell only offers SFTP).
|
|
verify := exec.CommandContext(ctx, "sftp", "-b", "-", "-P", strconv.Itoa(port),
|
|
"-i", privPath, "-oBatchMode=yes", "-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile="+khPath, user+"@"+host)
|
|
verify.Stdin = strings.NewReader("pwd\n")
|
|
if out, err := verify.CombinedOutput(); err != nil {
|
|
return fmt.Errorf("key-auth verify failed after install: %w: %s", err, truncate(out))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// --- SFTPKeyAuthProber: does the ALREADY-INSTALLED key still authenticate? (key-auth-first) ---
|
|
|
|
// SFTPKeyAuthProber probes passwordless auth with the existing installed key (KeyPath), pinned to the
|
|
// freshly-verified knownHosts line. No key file → ok=false (fresh guest). The probe never logs secrets.
|
|
type SFTPKeyAuthProber struct {
|
|
KeyPath string // the installed key, e.g. <dataDir>/offbox/ssh_key
|
|
Timeout time.Duration // per-probe budget; 0 → 20s
|
|
}
|
|
|
|
func (p SFTPKeyAuthProber) Probe(ctx context.Context, host, user string, port int, knownHosts string) (string, bool) {
|
|
pem, err := os.ReadFile(p.KeyPath)
|
|
if err != nil {
|
|
return "", false // no existing key — a fresh guest; take the full path
|
|
}
|
|
timeout := p.Timeout
|
|
if timeout == 0 {
|
|
timeout = 20 * time.Second
|
|
}
|
|
pctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
work, err := os.MkdirTemp("", "felhom-keyprobe-")
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
defer os.RemoveAll(work)
|
|
khPath := filepath.Join(work, "known_hosts")
|
|
if err := os.WriteFile(khPath, []byte(knownHosts+"\n"), 0o600); err != nil {
|
|
return "", false
|
|
}
|
|
probe := exec.CommandContext(pctx, "sftp", "-b", "-", "-P", strconv.Itoa(port),
|
|
"-i", p.KeyPath, "-oBatchMode=yes", "-oConnectTimeout=10",
|
|
"-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile="+khPath, user+"@"+host)
|
|
probe.Stdin = strings.NewReader("pwd\n")
|
|
if err := probe.Run(); err != nil {
|
|
return "", false // auth refused / unreachable — fall through to the full path
|
|
}
|
|
return string(pem), true
|
|
}
|
|
|
|
func truncate(b []byte) string {
|
|
s := strings.TrimSpace(string(b))
|
|
if len(s) > 300 {
|
|
return s[:300] + "…"
|
|
}
|
|
return s
|
|
}
|