Files
felhom.eu/hub/internal/tenantsync/client_test.go
T
admin 7f11cfb36c hub v0.65.0 — PBS DR storage visibility (ep0 usage op) + Offsite tab split + dual dashboard gauges (R-5)
Makes PBS DR storage visible like the restic pool box (v0.64.0), differentiated. Scoping
correction: restic = subaccounts on the shared Hetzner Storage Box (Hetzner API); PBS DR =
the felhom-offsite PBS datastore on the ep0 endpoint VM (NO Hetzner API). Option A
(Viktor-ruled): a read-only `usage` op on the felhom-tenantsync ep0 forced command (twin of
fingerprint), polled by a new hub checker on the 15-min throttle. READ-ONLY throughout.

Phase-0 (gate PASSED): on ep0 (PBS 4.2.3), df -B1 --output=size,used,avail <datastore path>
yields bytes (39990112256/7627939840/... ~19%), read-only, existing sudo context, no admin token.

- scripts/felhom-tenantsync.sh -> v1.2.0: read-only `usage` short-circuit (df on the datastore
  path), no customer_id, no admin token, NO mutation. + a bash harness proving zero mutation.
- tenantsync.Client.Usage() + BoxUsage; unknown-op -> typed ErrUsageUnsupported (graceful).
- monitor.PBSDRBoxChecker: OffsiteBoxChecker clone over a usageReader seam; 15-min throttle,
  cached PBSBoxSnapshot, escalation-only pbsdr_box_fill on the "pbsdr-box" scope (operator only,
  no SaveEvent), recovery re-arm. Fill only. THREE states: ok / unavailable (ep0 <=v1.1.0,
  neutral no-alert) / degraded (exec failed, keep last).
- config: Alerting.PBSDRBoxFill{Warn,Crit}Percent (80/90); built with the tenantsync client,
  60s sweep, SetPBSDRBox. Hub deploy INDEPENDENT of the ep0 update (graceful degradation).
- web: /offsite splits into Restic + PBS DR hash tabs (endpoint cards under PBS DR); PBS panel;
  the single dashboard tile becomes two gauges (RESTIC pct.ratio, PBS DR pct / n/a).
- runbook offsite-endpoint.md 10: v1.2.0 update steps (no sudoers/authorized_keys change).

Tests: 10 Go + the harness; 3 red-proofs (usage mutation, escalation-only, unavailable-drives-band)
confirmed red then restored. go build/vet/test + bash -n + hub confirm gate all pass.
2026-07-17 21:13:30 +02:00

333 lines
11 KiB
Go

package tenantsync
// The tenancy SSH client against an IN-PROCESS x/crypto/ssh server (the wgsync client_test
// pattern). Beyond the transport round-trip, these pin the package's two load-bearing contracts:
// the typed token_exists outcome, and the secret-hygiene rule that NO error message ever embeds
// stdout bytes (stdout is the secret channel).
import (
"bytes"
"context"
"encoding/pem"
"errors"
"io"
"log"
"net"
"strings"
"sync"
"testing"
"time"
"crypto/ed25519"
"crypto/rand"
"golang.org/x/crypto/ssh"
)
func testKeys(t *testing.T) ([]byte, ssh.Signer) {
t.Helper()
_, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatalf("ed25519: %v", err)
}
block, err := ssh.MarshalPrivateKey(priv, "")
if err != nil {
t.Fatalf("MarshalPrivateKey: %v", err)
}
signer, err := ssh.NewSignerFromKey(priv)
if err != nil {
t.Fatalf("NewSignerFromKey: %v", err)
}
return pem.EncodeToMemory(block), signer
}
type testServer struct {
addr string
mu sync.Mutex
captured []byte
cmd string
}
func startTestServer(t *testing.T, hostSigner, clientSigner ssh.Signer,
stdoutResp, stderrResp string, exitStatus uint32) *testServer {
t.Helper()
cfg := &ssh.ServerConfig{
PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
if bytes.Equal(key.Marshal(), clientSigner.PublicKey().Marshal()) {
return &ssh.Permissions{}, nil
}
return nil, io.EOF
},
}
cfg.AddHostKey(hostSigner)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
t.Cleanup(func() { ln.Close() })
srv := &testServer{addr: ln.Addr().String()}
go func() {
conn, err := ln.Accept()
if err != nil {
return
}
sconn, chans, reqs, err := ssh.NewServerConn(conn, cfg)
if err != nil {
return
}
defer sconn.Close()
go ssh.DiscardRequests(reqs)
for newCh := range chans {
if newCh.ChannelType() != "session" {
newCh.Reject(ssh.UnknownChannelType, "unsupported")
continue
}
ch, chReqs, err := newCh.Accept()
if err != nil {
continue
}
go func() {
for req := range chReqs {
if req.Type == "exec" {
var p struct{ Command string }
ssh.Unmarshal(req.Payload, &p)
srv.mu.Lock()
srv.cmd = p.Command
srv.mu.Unlock()
req.Reply(true, nil)
data, _ := io.ReadAll(ch)
srv.mu.Lock()
srv.captured = data
srv.mu.Unlock()
if stderrResp != "" {
ch.Stderr().Write([]byte(stderrResp))
}
if stdoutResp != "" {
ch.Write([]byte(stdoutResp))
}
ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{exitStatus}))
ch.Close()
return
}
req.Reply(false, nil)
}
}()
}
}()
return srv
}
func hostKeyLine(t *testing.T, s ssh.Signer) string {
t.Helper()
return strings.TrimSpace(string(ssh.MarshalAuthorizedKey(s.PublicKey())))
}
func newTestClient(t *testing.T, addr, hostKey string, clientPEM []byte) *Client {
t.Helper()
c, err := New(Config{
Addr: addr, User: "felhom-peersync", PrivateKey: clientPEM,
HostKeyLine: hostKey, Timeout: 5 * time.Second,
}, log.New(io.Discard, "", 0))
if err != nil {
t.Fatalf("tenantsync.New: %v", err)
}
return c
}
const okResp = `{"status":"ok","token_id":"felhom@pbs!peti","token_secret":"s3cr3t-uuid-value",` +
`"fingerprint":"aa:bb","datastore":"felhom-offsite","namespace":"peti"}`
func TestProvision_RoundTripParsesResult(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0)
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
res, err := c.Provision(context.Background(), "peti")
if err != nil {
t.Fatalf("Provision: %v", err)
}
if res.TokenID != "felhom@pbs!peti" || res.TokenSecret != "s3cr3t-uuid-value" ||
res.Fingerprint != "aa:bb" || res.Datastore != "felhom-offsite" || res.Namespace != "peti" {
t.Errorf("Result mis-parsed: %+v", res)
}
srv.mu.Lock()
defer srv.mu.Unlock()
if want := `{"customer_id":"peti","op":"provision"}`; string(srv.captured) != want {
t.Errorf("server received %q, want %q", srv.captured, want)
}
if srv.cmd != "felhom-tenantsync" {
t.Errorf("exec command = %q, want felhom-tenantsync", srv.cmd)
}
}
func TestProvision_TokenExistsIsTypedError(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
// The script emits the error JSON on stdout AND exits 1 — both must map to ErrTokenExists.
srv := startTestServer(t, hostSigner, clientSigner,
`{"status":"error","code":"token_exists","error":"token felhom@pbs!peti already exists"}`, "", 1)
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
_, err := c.Provision(context.Background(), "peti")
if !errors.Is(err, ErrTokenExists) {
t.Fatalf("err = %v, want ErrTokenExists", err)
}
}
func TestReissue_SendsReissueOp(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0)
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
if _, err := c.Reissue(context.Background(), "peti"); err != nil {
t.Fatalf("Reissue: %v", err)
}
srv.mu.Lock()
defer srv.mu.Unlock()
if want := `{"customer_id":"peti","op":"reissue"}`; string(srv.captured) != want {
t.Errorf("server received %q, want %q", srv.captured, want)
}
}
func TestFingerprint_Op(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
srv := startTestServer(t, hostSigner, clientSigner, `{"status":"ok","fingerprint":"cc:dd"}`, "", 0)
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
fp, err := c.Fingerprint(context.Background())
if err != nil {
t.Fatalf("Fingerprint: %v", err)
}
if fp != "cc:dd" {
t.Errorf("fingerprint = %q, want cc:dd", fp)
}
}
// TestUsage_Op (v0.65.0): the read-only usage op parses total/used/avail (bytes).
func TestUsage_Op(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
srv := startTestServer(t, hostSigner, clientSigner,
`{"status":"ok","total":39990112256,"used":7628091392,"avail":30686175232}`, "", 0)
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
u, err := c.Usage(context.Background())
if err != nil {
t.Fatalf("Usage: %v", err)
}
if u.Total != 39990112256 || u.Used != 7628091392 || u.Avail != 30686175232 {
t.Errorf("usage = %+v, want total/used/avail 39990112256/7628091392/30686175232", u)
}
}
// TestUsage_UnknownOpTypedUnsupported (v0.65.0, graceful degradation): an ep0 still on tenantsync
// ≤ v1.1.0 answers `{"op":"usage"}` with its generic `bad_request "unknown op"` (exit 1). That must
// map to the typed ErrUsageUnsupported so the PBS checker records "unavailable", not an error.
func TestUsage_UnknownOpTypedUnsupported(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
srv := startTestServer(t, hostSigner, clientSigner,
`{"status":"error","code":"bad_request","error":"unknown op"}`, "", 1)
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
if _, err := c.Usage(context.Background()); !errors.Is(err, ErrUsageUnsupported) {
t.Fatalf("an ep0 without the usage op must map to ErrUsageUnsupported, got %v", err)
}
}
// TestErrors_NeverEmbedStdout is the secret-hygiene contract: stdout may carry the token secret,
// so NO error path may quote it. A malformed-but-secret-bearing stdout must yield an error that
// does not contain the marker bytes. (wgsync quotes stdout in its malformed error — this package
// deliberately must not; that divergence is the point of this test.)
func TestErrors_NeverEmbedStdout(t *testing.T) {
cases := []struct {
name string
stdout string
exitStatus uint32
}{
{"malformed-ok-exit", `garbage SECRET-MARKER garbage`, 0},
{"malformed-fail-exit", `partial {"token_secret":"SECRET-MARKER"`, 1},
{"wrong-status", `{"status":"weird","token_secret":"SECRET-MARKER"}`, 0},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
srv := startTestServer(t, hostSigner, clientSigner, tc.stdout, "some stderr", tc.exitStatus)
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
_, err := c.Provision(context.Background(), "peti")
if err == nil {
t.Fatal("Provision accepted a malformed response")
}
if strings.Contains(err.Error(), "SECRET-MARKER") {
t.Errorf("error %q embeds stdout bytes — the secret channel leaked into logs", err)
}
})
}
}
func TestOkResponseMissingFieldsRefused(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
srv := startTestServer(t, hostSigner, clientSigner,
`{"status":"ok","token_id":"felhom@pbs!peti"}`, "", 0) // no secret/ns/fp/ds
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
if _, err := c.Provision(context.Background(), "peti"); err == nil {
t.Fatal("Provision accepted an ok-response with missing fields")
}
}
func TestWrongHostKeyRefused(t *testing.T) {
clientPEM, clientSigner := testKeys(t)
_, hostSigner := testKeys(t)
_, otherSigner := testKeys(t)
srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0)
c := newTestClient(t, srv.addr, hostKeyLine(t, otherSigner), clientPEM)
_, err := c.Provision(context.Background(), "peti")
if err == nil {
t.Fatal("Provision succeeded against a server with the WRONG host key — the pin is dead")
}
srv.mu.Lock()
defer srv.mu.Unlock()
if len(srv.captured) != 0 {
t.Errorf("payload leaked to a mis-keyed server: %q", srv.captured)
}
}
func TestInvalidCustomerIDRefusedClientSide(t *testing.T) {
clientPEM, _ := testKeys(t)
_, hostSigner := testKeys(t)
// No server needed — the refusal must happen before any dial.
c := newTestClient(t, "127.0.0.1:1", hostKeyLine(t, hostSigner), clientPEM)
for _, bad := range []string{"", "-leading-dash", ".dot", "has space", strings.Repeat("x", 40)} {
if _, err := c.Provision(context.Background(), bad); err == nil {
t.Errorf("customer_id %q accepted", bad)
}
}
}
func TestNew_BadInputsFailEarly(t *testing.T) {
clientPEM, _ := testKeys(t)
_, hostSigner := testKeys(t)
hk := hostKeyLine(t, hostSigner)
logger := log.New(io.Discard, "", 0)
if _, err := New(Config{Addr: "x:22", User: "u", PrivateKey: []byte("not-a-key"), HostKeyLine: hk}, logger); err == nil {
t.Error("bad private key accepted")
}
if _, err := New(Config{Addr: "x:22", User: "u", PrivateKey: clientPEM, HostKeyLine: "not a key line"}, logger); err == nil {
t.Error("bad host key line accepted")
}
if _, err := New(Config{User: "u", PrivateKey: clientPEM, HostKeyLine: hk}, logger); err == nil {
t.Error("missing addr accepted")
}
}