hub v0.61.0 + felhom-tenantsync v1.1.0: Customer RESET (middle lifecycle tier)
One operator action returns a customer to pre-first-install: all operational state dies (offsite repo, PBS namespace+backups, DR recipe, one-time secret, claim state, retained escrow custody); identity + basic config + provenance + events survive. Sits between host delete and customer Delete. - store/customer_reset.go: customer_resets journal, live inventory, ack-gated purge (never touches identity/provenance/events), DeleteClaim. - claim.ResetToUnclaimed: delete claim row -> fresh code next onboarding. - offsite.Deprovision (idempotent) + OffsiteIdentifier + ClearProvisionedDescriptor. - tenantsync.Deprovision + felhom-tenantsync.sh deprovision op (destroys ns + backup groups + token; shared user untouched; idempotent). - web/customer_reset.go: GET reset -> inventory JSON; POST -> orchestration (external teardown FIRST, DB purge LAST; refuse-while-hosts; typed-id + separate escrow ack). Amber RESET card distinct from red Danger-zone Delete. - Red-proofs: ack-gate + partial-failure resumability (both proven red); store ack-gating + journal round-trip; offsite idempotency + descriptor clear; RESET-card render. Green: build + vet + test.
This commit is contained in:
@@ -0,0 +1,181 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Customer RESET (v0.61.0) — the middle lifecycle tier (host delete < RESET < customer delete). The
|
||||
// journal is F-14-style provenance: one row per attempt, per-leg status, resumable. External teardown
|
||||
// runs FIRST and the DB-purge runs LAST (publish-last) so a re-run reads the journal to know what still
|
||||
// needs tearing down. Provenance/events are NEVER wiped — audit outlives every tier.
|
||||
|
||||
// CustomerReset is one reset provenance/journal record.
|
||||
type CustomerReset struct {
|
||||
ID int64
|
||||
CustomerID string
|
||||
StartedAt time.Time
|
||||
CompletedAt *time.Time
|
||||
EscrowAcked bool
|
||||
Legs map[string]string // hetzner | pbs | db_purge → pending|ok|failed|manual
|
||||
}
|
||||
|
||||
// ResetInventory is the live count of what a RESET would destroy (the confirm surface, ruling 4).
|
||||
// Sub-account name + PBS namespace are added by the web layer from the external clients.
|
||||
type ResetInventory struct {
|
||||
HostCount int // ruling 3: RESET refuses while any host row exists
|
||||
SupersededBlobs int // M — retained recovery-key custody destroyed (ack-gated)
|
||||
DRRecipePresent bool
|
||||
OneTimeSecretPresent bool
|
||||
ClaimPresent bool
|
||||
}
|
||||
|
||||
// CustomerResetInventory builds the hub-DB side of the confirm inventory.
|
||||
func (s *Store) CustomerResetInventory(customerID string) (*ResetInventory, error) {
|
||||
inv := &ResetInventory{}
|
||||
hosts, err := s.ListHostsByCustomer(customerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inv.HostCount = len(hosts)
|
||||
queries := []struct {
|
||||
dst *int
|
||||
q string
|
||||
arg string
|
||||
}{
|
||||
{&inv.SupersededBlobs, `SELECT COUNT(*) FROM host_escrow_superseded WHERE host_id IN (SELECT host_id FROM host_deletions WHERE customer_id = ? UNION SELECT host_id FROM hosts WHERE customer_id = ?)`, customerID},
|
||||
}
|
||||
for _, qq := range queries {
|
||||
if err := s.db.QueryRow(qq.q, qq.arg, customerID).Scan(qq.dst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
present := func(q string) (bool, error) {
|
||||
var n int
|
||||
err := s.db.QueryRow(q, customerID).Scan(&n)
|
||||
return n > 0, err
|
||||
}
|
||||
if inv.DRRecipePresent, err = present(`SELECT COUNT(*) FROM dr_recipe WHERE customer_id = ?`); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if inv.OneTimeSecretPresent, err = present(`SELECT COUNT(*) FROM one_time_secrets WHERE customer_id = ?`); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if inv.ClaimPresent, err = present(`SELECT COUNT(*) FROM customer_claims WHERE customer_id = ?`); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
// StartCustomerReset opens a reset journal row and returns its id.
|
||||
func (s *Store) StartCustomerReset(customerID string, escrowAcked bool) (int64, error) {
|
||||
acked := 0
|
||||
if escrowAcked {
|
||||
acked = 1
|
||||
}
|
||||
res, err := s.db.Exec(`INSERT INTO customer_resets (customer_id, escrow_acked, legs_json) VALUES (?, ?, '{}')`, customerID, acked)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.LastInsertId()
|
||||
}
|
||||
|
||||
// UpdateResetLeg merges one leg's status into the journal (read-modify-write; the reset is single-flight
|
||||
// per customer at the handler, so no lock contention).
|
||||
func (s *Store) UpdateResetLeg(resetID int64, leg, status string) error {
|
||||
var raw string
|
||||
if err := s.db.QueryRow(`SELECT legs_json FROM customer_resets WHERE id = ?`, resetID).Scan(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
legs := map[string]string{}
|
||||
if raw != "" {
|
||||
_ = json.Unmarshal([]byte(raw), &legs)
|
||||
}
|
||||
legs[leg] = status
|
||||
out, err := json.Marshal(legs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(`UPDATE customer_resets SET legs_json = ? WHERE id = ?`, string(out), resetID)
|
||||
return err
|
||||
}
|
||||
|
||||
// FinishCustomerReset stamps completion.
|
||||
func (s *Store) FinishCustomerReset(resetID int64) error {
|
||||
_, err := s.db.Exec(`UPDATE customer_resets SET completed_at = datetime('now') WHERE id = ?`, resetID)
|
||||
return err
|
||||
}
|
||||
|
||||
// LatestCustomerReset returns the customer's most recent reset record (nil = never).
|
||||
func (s *Store) LatestCustomerReset(customerID string) (*CustomerReset, error) {
|
||||
var (
|
||||
cr CustomerReset
|
||||
startedAt string
|
||||
completedAt sql.NullString
|
||||
acked int
|
||||
legs string
|
||||
)
|
||||
err := s.db.QueryRow(`
|
||||
SELECT id, customer_id, started_at, completed_at, escrow_acked, legs_json
|
||||
FROM customer_resets WHERE customer_id = ? ORDER BY id DESC LIMIT 1`, customerID).
|
||||
Scan(&cr.ID, &cr.CustomerID, &startedAt, &completedAt, &acked, &legs)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cr.StartedAt = parseSQLiteTime(startedAt)
|
||||
if completedAt.Valid && completedAt.String != "" {
|
||||
t := parseSQLiteTime(completedAt.String)
|
||||
cr.CompletedAt = &t
|
||||
}
|
||||
cr.EscrowAcked = acked != 0
|
||||
cr.Legs = map[string]string{}
|
||||
_ = json.Unmarshal([]byte(legs), &cr.Legs)
|
||||
return &cr, nil
|
||||
}
|
||||
|
||||
// PurgeCustomerResetDBState is the DB-purge phase — runs LAST, after the external teardown legs. In one
|
||||
// tx it removes the customer-scoped operational state a RESET destroys: retained escrow custody
|
||||
// (ack-gated), the one-time repo password, the DR recipe, and customer-scoped log bundles. It does NOT
|
||||
// touch customer_configs identity/config (SURVIVES), provenance (host_deletions/customer_resets), or
|
||||
// events (audit). The claim reset rides the claim engine separately; the offsite descriptor clear rides
|
||||
// the offsite package (config_json). Idempotent — a re-run deletes nothing extra.
|
||||
func (s *Store) PurgeCustomerResetDBState(customerID string, escrowAcked bool) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if escrowAcked {
|
||||
// Retained (superseded) blobs — the hosts are gone by the RESET precondition, so they are found
|
||||
// only via the F-14 host_deletions provenance (UNION current hosts as a belt-and-suspenders).
|
||||
if _, err := tx.Exec(`DELETE FROM host_escrow_superseded WHERE host_id IN (
|
||||
SELECT host_id FROM host_deletions WHERE customer_id = ?
|
||||
UNION SELECT host_id FROM hosts WHERE customer_id = ?)`, customerID, customerID); err != nil {
|
||||
return fmt.Errorf("purge superseded escrow: %w", err)
|
||||
}
|
||||
}
|
||||
for _, q := range []string{
|
||||
`DELETE FROM one_time_secrets WHERE customer_id = ?`,
|
||||
`DELETE FROM dr_recipe WHERE customer_id = ?`,
|
||||
`DELETE FROM log_bundles WHERE scope_id = ?`,
|
||||
`DELETE FROM log_bundle_requests WHERE scope_id = ?`,
|
||||
} {
|
||||
if _, err := tx.Exec(q, customerID); err != nil {
|
||||
return fmt.Errorf("reset purge %q: %w", q, err)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteClaim removes a customer's claim row (RESET → unclaimed / pre-first-install). The claim ENGINE
|
||||
// re-mints a fresh code via EnsureIssued on the next onboarding — this is the store primitive the
|
||||
// engine's ResetToUnclaimed rides (no parallel claimed-flag mechanism).
|
||||
func (s *Store) DeleteClaim(customerID string) error {
|
||||
_, err := s.db.Exec(`DELETE FROM customer_claims WHERE customer_id = ?`, customerID)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func newResetStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
st, err := New(filepath.Join(t.TempDir(), "t.db"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatalf("store.New: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
return st
|
||||
}
|
||||
|
||||
// seedRetainedBlob leaves a superseded escrow blob reachable only via F-14 host_deletions provenance
|
||||
// (host deleted, blob retained) — the customer-RESET escrow-custody target.
|
||||
func seedRetainedBlob(t *testing.T, st *Store, customerID string) {
|
||||
t.Helper()
|
||||
hostID := customerID + "-h1"
|
||||
if err := st.UpsertHost(&Host{HostID: hostID, CustomerID: customerID, APIKey: "h"}); err != nil {
|
||||
t.Fatalf("upsert host: %v", err)
|
||||
}
|
||||
if _, err := st.SaveHostEscrow(hostID, []byte("A"), "fpA", "p", "2026-01-01T00:00:00Z", "shaA"); err != nil {
|
||||
t.Fatalf("escrow A: %v", err)
|
||||
}
|
||||
if _, err := st.SaveHostEscrow(hostID, []byte("B"), "fpB", "p", "2026-01-02T00:00:00Z", "shaB"); err != nil {
|
||||
t.Fatalf("escrow B: %v", err)
|
||||
}
|
||||
if err := st.DeleteHost(hostID, true); err != nil {
|
||||
t.Fatalf("delete host: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The store-level ack contract: PurgeCustomerResetDBState must NEVER destroy retained escrow custody
|
||||
// unless escrowAck is true — even while it clears the rest of the operational state. (The web handler
|
||||
// gates ack separately; this proves the store primitive is itself honest.)
|
||||
func TestPurgeCustomerResetDBState_AckGatesEscrow(t *testing.T) {
|
||||
st := newResetStore(t)
|
||||
if err := st.SaveCustomerConfig(&CustomerConfig{CustomerID: "acme", APIKey: "capi", RetrievalPassword: "pw"}); err != nil {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
seedRetainedBlob(t, st, "acme")
|
||||
if err := st.SaveOneTimeSecret("acme", "ots"); err != nil {
|
||||
t.Fatalf("seed ots: %v", err)
|
||||
}
|
||||
if err := st.SaveDRRecipeHostHalf("acme", "acme-h1", 1, []byte("half")); err != nil {
|
||||
t.Fatalf("seed dr: %v", err)
|
||||
}
|
||||
|
||||
// ack=false: everything else goes, blobs SURVIVE.
|
||||
if err := st.PurgeCustomerResetDBState("acme", false); err != nil {
|
||||
t.Fatalf("purge (no ack): %v", err)
|
||||
}
|
||||
inv, err := st.CustomerResetInventory("acme")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inv.SupersededBlobs == 0 {
|
||||
t.Error("ack=false destroyed retained escrow custody")
|
||||
}
|
||||
if inv.OneTimeSecretPresent || inv.DRRecipePresent {
|
||||
t.Errorf("ack=false failed to clear the non-escrow state: %+v", inv)
|
||||
}
|
||||
|
||||
// ack=true: blobs go too.
|
||||
if err := st.PurgeCustomerResetDBState("acme", true); err != nil {
|
||||
t.Fatalf("purge (ack): %v", err)
|
||||
}
|
||||
if inv, _ := st.CustomerResetInventory("acme"); inv.SupersededBlobs != 0 {
|
||||
t.Errorf("ack=true failed to destroy retained custody: %d", inv.SupersededBlobs)
|
||||
}
|
||||
// Identity SURVIVES the purge.
|
||||
if cfg, _ := st.GetCustomerConfig("acme"); cfg == nil {
|
||||
t.Error("purge destroyed customer_config — identity must survive")
|
||||
}
|
||||
}
|
||||
|
||||
// The journal round-trips: open → per-leg status → finish, and Latest reflects it.
|
||||
func TestCustomerResetJournal_RoundTrip(t *testing.T) {
|
||||
st := newResetStore(t)
|
||||
id, err := st.StartCustomerReset("acme", true)
|
||||
if err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
if err := st.UpdateResetLeg(id, "hetzner", "ok"); err != nil {
|
||||
t.Fatalf("leg: %v", err)
|
||||
}
|
||||
if err := st.UpdateResetLeg(id, "pbs", "failed"); err != nil {
|
||||
t.Fatalf("leg: %v", err)
|
||||
}
|
||||
if err := st.UpdateResetLeg(id, "pbs", "ok"); err != nil { // overwrite on re-run
|
||||
t.Fatalf("leg: %v", err)
|
||||
}
|
||||
cr, err := st.LatestCustomerReset("acme")
|
||||
if err != nil || cr == nil {
|
||||
t.Fatalf("latest: %v / %v", cr, err)
|
||||
}
|
||||
if cr.CompletedAt != nil {
|
||||
t.Error("not finished yet — CompletedAt should be nil")
|
||||
}
|
||||
if !cr.EscrowAcked || cr.Legs["hetzner"] != "ok" || cr.Legs["pbs"] != "ok" {
|
||||
t.Errorf("journal wrong: %+v", cr)
|
||||
}
|
||||
if err := st.FinishCustomerReset(id); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
if cr, _ := st.LatestCustomerReset("acme"); cr.CompletedAt == nil {
|
||||
t.Error("finish did not stamp CompletedAt")
|
||||
}
|
||||
}
|
||||
@@ -596,6 +596,21 @@ func (s *Store) migrate() error {
|
||||
escrow_acked INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_host_deletions_customer ON host_deletions(customer_id, id DESC);
|
||||
|
||||
-- customer_resets (v0.61.0, F-14 style): the RESET provenance + resumable per-leg journal. One
|
||||
-- row per reset attempt; legs_json holds {hetzner,pbs,db_purge → pending|ok|failed|manual}. The
|
||||
-- DB purge runs LAST (publish-last), so a re-run reads the journal to know what still needs
|
||||
-- teardown. escrow_acked records the ruling-1 acknowledgment. NEVER pruned (audit outlives
|
||||
-- every lifecycle tier).
|
||||
CREATE TABLE IF NOT EXISTS customer_resets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
customer_id TEXT NOT NULL,
|
||||
started_at DATETIME NOT NULL DEFAULT (datetime('now')),
|
||||
completed_at DATETIME,
|
||||
escrow_acked INTEGER NOT NULL DEFAULT 0,
|
||||
legs_json TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_customer_resets_customer ON customer_resets(customer_id, id DESC);
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user