diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a2d138..086f3a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,32 @@ All notable changes to **felhom-agent** are recorded here. Update on every code change that gets pushed. +## v0.11.0 — slice 8B: app-consistent backup — /backup/due policy + /backup/status phases (2026-06-10) + +The agent half of slice 8B (doc 03 §8). Turns the 8A thin backup stubs into the real policy the +in-guest controller's quiesce loop drives (controller half: `felhom-controller` v0.36.0). No hub +change. The downtime optimization (`vzdump --mode snapshot` + a `snapshotted` phase) is the 8B.2 +fast-follow; the hub-served per-guest policy is slice 10. + +### Changed (`internal/localapi`) +- **`GET /backup/due`** — real **cadence** policy (replaces the 8A "never backed up" stub): a guest + is due when no **successful** backup is recorded OR the newest one is older than the agent-local + cadence (`backup.backup_cadence_seconds`, default 24h). A successful `POST /backup` flips due to + **false** for the window, so the controller won't re-quiesce in a loop. A failed backup does not + satisfy the cadence. Returns `age_seconds` for diagnosis. +- **`GET /backup/status`** — real **phases** `idle | running | done | failed` + the job id, so the + controller can poll a backup to completion (was: just the latest stored backup). +- **`POST /backup`** — returns a **job id** + `running` phase; tracks the in-flight job and is + **single-flight per guest** (a second POST while one runs returns the same job — no concurrent + vzdump). On completion the job transitions done/failed and the result is recorded to the store. +- Config: `backup.backup_cadence_seconds` + `BackupCadence()`; the local-API server takes the cadence. + +### Tests +- `/backup/due`: due when stale / no backup, **not due within the window after a success**, due again + past the cadence, **a failed backup does not count**. `/backup/status`: running→done and + running→failed (gated fake to observe the running phase). `POST /backup` single-flight (one vzdump + for concurrent POSTs). All still self-scoped (token→guest). + ## v0.10.0 — slice 8A: agent local-API server + provisioning back-half (2026-06-10) The host-agent half of slice 8A (doc 03 §6). Adds the per-guest **local API** the in-guest diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index c2b8cb4..e47f5b2 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -40,7 +40,7 @@ import ( // version is the agent version. Overridable at build time with // -ldflags "-X main.version="; defaults to the in-repo CHANGELOG version. -var version = "0.10.0" +var version = "0.11.0" func main() { var ( @@ -485,14 +485,15 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St logger.Info("local-api leaf ready", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath()) runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom local-api", logger) srv, err := localapi.NewServer(localapi.Options{ - ListenAddr: cfg.LocalAPI.ListenAddr, - Cert: cert, - Guests: px, - Backups: runner, - Store: store, - Storage: observer, - Tokens: tokens, - Logger: logger, + ListenAddr: cfg.LocalAPI.ListenAddr, + Cert: cert, + Guests: px, + Backups: runner, + Store: store, + Storage: observer, + Tokens: tokens, + BackupCadence: cfg.Backup.BackupCadence(), + Logger: logger, }) if err != nil { logger.Warn("daemon: local-api disabled (server build)", "err", err) diff --git a/configs/agent.example.json b/configs/agent.example.json index 422ed48..0d3a5b3 100644 --- a/configs/agent.example.json +++ b/configs/agent.example.json @@ -43,7 +43,8 @@ "scratch_vmid_min": 990000, "scratch_vmid_max": 990009, "pbs_verify_cadence_seconds": 0, - "pbs_secret_dir": "/etc/pve/priv/storage" + "pbs_secret_dir": "/etc/pve/priv/storage", + "backup_cadence_seconds": 0 }, "local_api": { "enable": true, diff --git a/internal/config/config.go b/internal/config/config.go index d4b9462..f91bf5f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -139,6 +139,19 @@ type BackupConfig struct { // PBSSecretDir holds the per-storage PBS token secret files (.pw). Default // /etc/pve/priv/storage (PVE-managed, 0600). The agent reads it at runtime; never logged. PBSSecretDir string `json:"pbs_secret_dir"` + + // BackupCadenceSeconds drives the local-API GET /backup/due (slice 8B): a guest is "due" when + // its newest successful backup is older than this (or none exists). 0 → default (24h). The + // hub-served per-guest policy is slice 10; this is the agent-local cadence. + BackupCadenceSeconds int `json:"backup_cadence_seconds"` +} + +// BackupCadence returns the per-guest /backup/due window: positive as-is, else 24h default. +func (b BackupConfig) BackupCadence() time.Duration { + if b.BackupCadenceSeconds > 0 { + return time.Duration(b.BackupCadenceSeconds) * time.Second + } + return 24 * time.Hour } // Default scratch VMID band + restore-test cadence. diff --git a/internal/localapi/server.go b/internal/localapi/server.go index bc4eb33..90116b2 100644 --- a/internal/localapi/server.go +++ b/internal/localapi/server.go @@ -11,6 +11,7 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" @@ -60,7 +61,33 @@ type Options struct { Store BackupStore Storage StorageView Tokens TokenAuthority - Logger *slog.Logger + // BackupCadence is the per-guest backup interval driving GET /backup/due (slice 8B). A guest + // is "due" when no successful backup is recorded OR the newest one is older than this. 0 → a + // safe default (24h). The hub-served policy is slice 10; this is the agent-local cadence. + BackupCadence time.Duration + Logger *slog.Logger +} + +// defaultBackupCadence is the fallback /backup/due window when none is configured. +const defaultBackupCadence = 24 * time.Hour + +// Backup phase vocabulary reported by GET /backup/status (slice 8B). The 8B.2 fast-follow adds a +// `snapshotted` phase (vzdump --mode snapshot) so the controller can unquiesce at snapshot-taken. +const ( + PhaseIdle = "idle" + PhaseRunning = "running" + PhaseDone = "done" + PhaseFailed = "failed" +) + +// backupJob is the in-flight/last backup job for one guest (drives /backup/status phases). +type backupJob struct { + JobID string + Phase string + StartedAt time.Time + FinishedAt time.Time + Archive string + Error string } // Server is the per-guest local API (doc 03 §6). It serves the agent's pinned self-signed leaf @@ -73,7 +100,12 @@ type Server struct { store BackupStore storage StorageView tokens TokenAuthority + cadence time.Duration logger *slog.Logger + now func() time.Time + + jobsMu sync.Mutex + jobs map[int]*backupJob // per-guest backup job state (slice 8B) baseCtx context.Context // for fire-and-forget backups; set in Run } @@ -89,6 +121,10 @@ func NewServer(o Options) (*Server, error) { if o.Logger == nil { o.Logger = slog.Default() } + cadence := o.BackupCadence + if cadence <= 0 { + cadence = defaultBackupCadence + } return &Server{ addr: o.ListenAddr, cert: o.Cert, @@ -97,7 +133,10 @@ func NewServer(o Options) (*Server, error) { store: o.Store, storage: o.Storage, tokens: o.Tokens, + cadence: cadence, logger: o.Logger, + now: func() time.Time { return time.Now().UTC() }, + jobs: map[int]*backupJob{}, }, nil } @@ -313,6 +352,13 @@ type backupRequest struct { VMID int `json:"vmid"` } +// BackupResponse is POST /backup. The controller polls GET /backup/status on job_id to completion. +type BackupResponse struct { + VMID int `json:"vmid"` + JobID string `json:"job_id"` + Phase string `json:"phase"` +} + func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int) { // Body is optional; if present its vmid must match the token's guest. if r.ContentLength != 0 { @@ -324,10 +370,24 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int) return } } - // Enqueue: a vzdump runs for minutes, so we do not block the request. The backup runs on - // the server's base context (cancelled on daemon shutdown) and records into the store; the - // controller polls GET /backup/status. This is the crash-consistent path (8A); the - // app-consistent quiesce-then-backup loop is 8B. + // Single-flight per guest: if a backup is already running for this guest, return that job + // (don't start a second concurrent vzdump). The controller polls /backup/status on it. + s.jobsMu.Lock() + if cur := s.jobs[vmid]; cur != nil && cur.Phase == PhaseRunning { + job := *cur + s.jobsMu.Unlock() + writeStatus(w, http.StatusAccepted, true, BackupResponse{VMID: vmid, JobID: job.JobID, Phase: job.Phase}, "") + return + } + jobID := "backup-" + strconv.Itoa(vmid) + "-" + strconv.FormatInt(s.now().UnixNano(), 10) + s.jobs[vmid] = &backupJob{JobID: jobID, Phase: PhaseRunning, StartedAt: s.now()} + s.jobsMu.Unlock() + + // Enqueue: a vzdump runs for minutes, so we do not block the request. The backup runs on the + // server's base context (cancelled on daemon shutdown), updates the job phase, and records + // into the store; the controller polls GET /backup/status. This is the host-side half of the + // 8B app-consistent path — the controller quiesces (stops its stacks) BEFORE calling this, so + // the vzdump captures a clean-shutdown-consistent state. base := s.baseCtx if base == nil { base = context.Background() @@ -342,36 +402,94 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int) if b.Error == "" { b.Error = err.Error() } - s.logger.Error("local-api: enqueued backup failed", "vmid", vmid, "err", err) + s.logger.Error("local-api: backup job failed", "vmid", vmid, "job", jobID, "err", err) } else { - s.logger.Info("local-api: enqueued backup complete", "vmid", vmid, "archive", b.Archive) + s.logger.Info("local-api: backup job complete", "vmid", vmid, "job", jobID, "archive", b.Archive) } s.store.RecordBackup(b) + s.finishJob(vmid, jobID, b) }() - writeStatus(w, http.StatusAccepted, true, map[string]any{"vmid": vmid, "enqueued": true}, "") + writeStatus(w, http.StatusAccepted, true, BackupResponse{VMID: vmid, JobID: jobID, Phase: PhaseRunning}, "") } -// BackupDueResponse is GET /backup/due. Thin in 8A: a guest with no successful backup recorded -// is "due"; otherwise not. Policy-scheduled cadence (hub manifest) lands in slice 10, and the -// quiesce-on-due consumer is 8B — both noted in the response. +// finishJob transitions the guest's job to done/failed (only if it is still the current job — a +// later job started after a single-flight gap must not be overwritten by an older one's result). +func (s *Server) finishJob(vmid int, jobID string, b hub.Backup) { + s.jobsMu.Lock() + defer s.jobsMu.Unlock() + cur := s.jobs[vmid] + if cur == nil || cur.JobID != jobID { + return + } + cur.FinishedAt = s.now() + if b.Success { + cur.Phase = PhaseDone + cur.Archive = b.Archive + } else { + cur.Phase = PhaseFailed + cur.Error = b.Error + } +} + +// jobSnapshot returns a copy of the guest's current job (ok=false if none). +func (s *Server) jobSnapshot(vmid int) (backupJob, bool) { + s.jobsMu.Lock() + defer s.jobsMu.Unlock() + if j := s.jobs[vmid]; j != nil { + return *j, true + } + return backupJob{}, false +} + +// BackupDueResponse is GET /backup/due (slice 8B). A guest is due when no successful backup is +// recorded OR the newest successful one is older than the agent-local cadence. A successful +// POST /backup flips this to false for the window, so the controller won't re-quiesce in a loop. +// The hub-served policy is slice 10. type BackupDueResponse struct { - VMID int `json:"vmid"` - Due bool `json:"due"` - Reason string `json:"reason"` + VMID int `json:"vmid"` + Due bool `json:"due"` + Reason string `json:"reason"` + AgeSecs *int64 `json:"age_seconds,omitempty"` // age of the newest successful backup; null if none } func (s *Server) handleBackupDue(w http.ResponseWriter, r *http.Request, vmid int) { - latest := s.latestBackupFor(r.Context(), vmid) - if latest == nil || !latest.Success { + latest := s.latestSuccessfulBackupFor(r.Context(), vmid) + if latest == nil { writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "no successful backup recorded yet"}) return } - writeOK(w, BackupDueResponse{VMID: vmid, Due: false, Reason: "policy-scheduled cadence lands in slice 10"}) + age, ok := backupAge(latest.StartedAt, s.now()) + if !ok { + // Unparseable timestamp: fail safe toward "due" so a backup still happens. + writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "last backup time unparseable — treating as due"}) + return + } + ageSecs := int64(age.Seconds()) + if age >= s.cadence { + writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "older than cadence", AgeSecs: &ageSecs}) + return + } + writeOK(w, BackupDueResponse{VMID: vmid, Due: false, Reason: "within cadence window", AgeSecs: &ageSecs}) +} + +// BackupStatusResponse is GET /backup/status (slice 8B): the current/last job phase + the latest +// recorded backup. Phase is idle when no job has run this process lifetime. +type BackupStatusResponse struct { + VMID int `json:"vmid"` + Phase string `json:"phase"` // idle | running | done | failed + JobID string `json:"job_id,omitempty"` + Error string `json:"error,omitempty"` + Backup *hub.Backup `json:"backup,omitempty"` // latest recorded backup for this guest } func (s *Server) handleBackupStatus(w http.ResponseWriter, r *http.Request, vmid int) { - latest := s.latestBackupFor(r.Context(), vmid) - writeOK(w, map[string]any{"vmid": vmid, "backup": latest}) + resp := BackupStatusResponse{VMID: vmid, Phase: PhaseIdle, Backup: s.latestBackupFor(r.Context(), vmid)} + if job, ok := s.jobSnapshot(vmid); ok { + resp.Phase = job.Phase + resp.JobID = job.JobID + resp.Error = job.Error + } + writeOK(w, resp) } func (s *Server) handleRestoreTestStatus(w http.ResponseWriter, r *http.Request, vmid int) { @@ -389,9 +507,19 @@ func (s *Server) handleRestoreTestStatus(w http.ResponseWriter, r *http.Request, // latestBackupFor returns this guest's most recent backup from the store (nil if none). func (s *Server) latestBackupFor(ctx context.Context, vmid int) *hub.Backup { + return s.pickLatestBackup(ctx, vmid, false) +} + +// latestSuccessfulBackupFor returns this guest's most recent SUCCESSFUL backup (nil if none) — +// the basis for /backup/due (a failed backup must not satisfy the cadence). +func (s *Server) latestSuccessfulBackupFor(ctx context.Context, vmid int) *hub.Backup { + return s.pickLatestBackup(ctx, vmid, true) +} + +func (s *Server) pickLatestBackup(ctx context.Context, vmid int, successOnly bool) *hub.Backup { var latest *hub.Backup for _, b := range s.store.Backups(ctx) { - if b.VMID != vmid { + if b.VMID != vmid || (successOnly && !b.Success) { continue } bb := b @@ -402,6 +530,15 @@ func (s *Server) latestBackupFor(ctx context.Context, vmid int) *hub.Backup { return latest } +// backupAge parses an RFC3339 backup start time and returns its age relative to now. +func backupAge(startedAt string, now time.Time) (time.Duration, bool) { + t, err := time.Parse(time.RFC3339, startedAt) + if err != nil { + return 0, false + } + return now.Sub(t), true +} + // classByStorage builds a storage-id → class(fast|slow|"") map from the host storage view. A // view error is logged and yields an empty map (class falls back to "" — a hint, not load-bearing). func (s *Server) classByStorage(ctx context.Context) map[string]string { diff --git a/internal/localapi/server_test.go b/internal/localapi/server_test.go index a93eb6a..eaaef54 100644 --- a/internal/localapi/server_test.go +++ b/internal/localapi/server_test.go @@ -64,14 +64,24 @@ func (f *fakeGuests) WaitTask(_ context.Context, _ string, _ proxmox.WaitOptions } type fakeBackups struct { - mu sync.Mutex - vmids []int + mu sync.Mutex + vmids []int + gate chan struct{} // if non-nil, Backup blocks until it is closed (observe the running phase) + failErr string // if set, Backup returns this error (drives the failed phase) } func (f *fakeBackups) Backup(_ context.Context, vmid int) (hub.Backup, error) { f.mu.Lock() f.vmids = append(f.vmids, vmid) + gate := f.gate + failErr := f.failErr f.mu.Unlock() + if gate != nil { + <-gate + } + if failErr != "" { + return hub.Backup{VMID: vmid}, fmt.Errorf("%s", failErr) + } return hub.Backup{VMID: vmid, Success: true, Archive: "local:backup/vzdump-x", StartedAt: "2026-06-10T00:00:00Z"}, nil } func (f *fakeBackups) called() []int { f.mu.Lock(); defer f.mu.Unlock(); return append([]int(nil), f.vmids...) } @@ -106,25 +116,35 @@ func (m staticTokens) Lookup(tok string) (int, bool) { v, ok := m[tok]; return v // ---- harness ---------------------------------------------------------------------------- -func newTestServer(t *testing.T, g *fakeGuests, b *fakeBackups, st *fakeStore, sv StorageView) http.Handler { +// testNow is the fixed clock the test server uses, so /backup/due cadence math is deterministic. +var testNow = time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC) + +func newTestServerS(t *testing.T, g *fakeGuests, b *fakeBackups, st *fakeStore, sv StorageView) *Server { t.Helper() if sv == nil { sv = fakeStorage{} } srv, err := NewServer(Options{ - ListenAddr: "127.0.0.1:0", - Guests: g, - Backups: b, - Store: st, - Storage: sv, - Tokens: staticTokens{"A": 8200, "B": 9300}, - Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + ListenAddr: "127.0.0.1:0", + Guests: g, + Backups: b, + Store: st, + Storage: sv, + Tokens: staticTokens{"A": 8200, "B": 9300}, + BackupCadence: 24 * time.Hour, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) if err != nil { t.Fatalf("new server: %v", err) } srv.baseCtx = context.Background() - return srv.Handler() + srv.now = func() time.Time { return testNow } + return srv +} + +func newTestServer(t *testing.T, g *fakeGuests, b *fakeBackups, st *fakeStore, sv StorageView) http.Handler { + t.Helper() + return newTestServerS(t, g, b, st, sv).Handler() } func do(t *testing.T, h http.Handler, method, path, token, body string) *httptest.ResponseRecorder { @@ -304,26 +324,126 @@ func TestBackup_EnqueuesForTokenGuest(t *testing.T) { } } -func TestBackupDue_ThinHeuristic(t *testing.T) { - st := &fakeStore{} - h := newTestServer(t, &fakeGuests{}, &fakeBackups{}, st, nil) - - // no backup recorded → due +func dueOf(t *testing.T, h http.Handler) BackupDueResponse { + t.Helper() w := do(t, h, "GET", "/backup/due", "A", "") var resp struct { Data BackupDueResponse `json:"data"` } - _ = json.Unmarshal(w.Body.Bytes(), &resp) - if !resp.Data.Due { + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode due: %v", err) + } + return resp.Data +} + +// testNow is 2026-06-10T12:00:00Z, cadence 24h. +func TestBackupDue_Cadence(t *testing.T) { + st := &fakeStore{} + h := newTestServer(t, &fakeGuests{}, &fakeBackups{}, st, nil) + + // no backup recorded → due + if d := dueOf(t, h); !d.Due { t.Fatal("expected due=true with no backup recorded") } - // a successful backup for this guest → not due - st.backups = []hub.Backup{{VMID: 8200, Success: true, StartedAt: "2026-06-10T00:00:00Z"}} - w = do(t, h, "GET", "/backup/due", "A", "") - _ = json.Unmarshal(w.Body.Bytes(), &resp) - if resp.Data.Due { - t.Fatal("expected due=false after a successful backup") + // a successful backup 1h ago → NOT due (within the 24h window) + st.backups = []hub.Backup{{VMID: 8200, Success: true, StartedAt: "2026-06-10T11:00:00Z"}} + if d := dueOf(t, h); d.Due { + t.Fatalf("expected due=false 1h after a successful backup; reason=%q", d.Reason) } + // a successful backup >24h ago → due again + st.backups = []hub.Backup{{VMID: 8200, Success: true, StartedAt: "2026-06-08T00:00:00Z"}} + if d := dueOf(t, h); !d.Due { + t.Fatal("expected due=true when the newest backup is older than the cadence") + } + // a FAILED backup 1h ago does NOT satisfy the cadence → still due + st.backups = []hub.Backup{{VMID: 8200, Success: false, StartedAt: "2026-06-10T11:00:00Z"}} + if d := dueOf(t, h); !d.Due { + t.Fatal("expected due=true: a failed backup must not count as a successful one") + } +} + +// POST /backup returns a running phase; the job transitions running→done; a successful backup then +// flips /backup/due to false (the controller won't re-quiesce in a loop). +func TestBackupStatus_RunningToDone(t *testing.T) { + st := &fakeStore{} + b := &fakeBackups{} + srv := newTestServerS(t, &fakeGuests{}, b, st, nil) + h := srv.Handler() + + // gate the backup goroutine so we can observe the running phase deterministically + b.gate = make(chan struct{}) + + w := do(t, h, "POST", "/backup", "A", "") + if w.Code != http.StatusAccepted { + t.Fatalf("POST /backup: got %d want 202", w.Code) + } + var br struct { + Data BackupResponse `json:"data"` + } + _ = json.Unmarshal(w.Body.Bytes(), &br) + if br.Data.Phase != PhaseRunning || br.Data.JobID == "" { + t.Fatalf("expected running phase + job id, got %+v", br.Data) + } + if ph := statusPhase(t, h); ph != PhaseRunning { + t.Fatalf("status while running: got %q want running", ph) + } + close(b.gate) // let the backup complete + // wait for done + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if statusPhase(t, h) == PhaseDone { + break + } + time.Sleep(5 * time.Millisecond) + } + if ph := statusPhase(t, h); ph != PhaseDone { + t.Fatalf("status after completion: got %q want done", ph) + } +} + +func TestBackupStatus_RunningToFailed(t *testing.T) { + b := &fakeBackups{failErr: "vzdump exploded"} + srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil) + h := srv.Handler() + + if do(t, h, "POST", "/backup", "A", "").Code != http.StatusAccepted { + t.Fatal("POST /backup not accepted") + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if statusPhase(t, h) == PhaseFailed { + break + } + time.Sleep(5 * time.Millisecond) + } + if ph := statusPhase(t, h); ph != PhaseFailed { + t.Fatalf("status after a failed backup: got %q want failed", ph) + } +} + +// A second POST /backup while one is running does NOT start a second vzdump (single-flight). +func TestBackup_SingleFlight(t *testing.T) { + b := &fakeBackups{gate: make(chan struct{})} + srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil) + h := srv.Handler() + + do(t, h, "POST", "/backup", "A", "") + do(t, h, "POST", "/backup", "A", "") // should be coalesced onto the running job + close(b.gate) + time.Sleep(30 * time.Millisecond) + if c := b.called(); len(c) != 1 { + t.Fatalf("expected a single vzdump for concurrent POSTs, got %d", len(c)) + } +} + +func statusPhase(t *testing.T, h http.Handler) string { + t.Helper() + w := do(t, h, "GET", "/backup/status", "A", "") + var resp struct { + Data BackupStatusResponse `json:"data"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + return resp.Data.Phase } func TestBackupStatus_FiltersToThisGuest(t *testing.T) {