7f11cfb36c
Makes PBS DR storage visible like the restic pool box (v0.64.0), differentiated. Scoping
correction: restic = subaccounts on the shared Hetzner Storage Box (Hetzner API); PBS DR =
the felhom-offsite PBS datastore on the ep0 endpoint VM (NO Hetzner API). Option A
(Viktor-ruled): a read-only `usage` op on the felhom-tenantsync ep0 forced command (twin of
fingerprint), polled by a new hub checker on the 15-min throttle. READ-ONLY throughout.
Phase-0 (gate PASSED): on ep0 (PBS 4.2.3), df -B1 --output=size,used,avail <datastore path>
yields bytes (39990112256/7627939840/... ~19%), read-only, existing sudo context, no admin token.
- scripts/felhom-tenantsync.sh -> v1.2.0: read-only `usage` short-circuit (df on the datastore
path), no customer_id, no admin token, NO mutation. + a bash harness proving zero mutation.
- tenantsync.Client.Usage() + BoxUsage; unknown-op -> typed ErrUsageUnsupported (graceful).
- monitor.PBSDRBoxChecker: OffsiteBoxChecker clone over a usageReader seam; 15-min throttle,
cached PBSBoxSnapshot, escalation-only pbsdr_box_fill on the "pbsdr-box" scope (operator only,
no SaveEvent), recovery re-arm. Fill only. THREE states: ok / unavailable (ep0 <=v1.1.0,
neutral no-alert) / degraded (exec failed, keep last).
- config: Alerting.PBSDRBoxFill{Warn,Crit}Percent (80/90); built with the tenantsync client,
60s sweep, SetPBSDRBox. Hub deploy INDEPENDENT of the ep0 update (graceful degradation).
- web: /offsite splits into Restic + PBS DR hash tabs (endpoint cards under PBS DR); PBS panel;
the single dashboard tile becomes two gauges (RESTIC pct.ratio, PBS DR pct / n/a).
- runbook offsite-endpoint.md 10: v1.2.0 update steps (no sudoers/authorized_keys change).
Tests: 10 Go + the harness; 3 red-proofs (usage mutation, escalation-only, unavailable-drives-band)
confirmed red then restored. go build/vet/test + bash -n + hub confirm gate all pass.
333 lines
11 KiB
Go
333 lines
11 KiB
Go
package tenantsync
|
|
|
|
// The tenancy SSH client against an IN-PROCESS x/crypto/ssh server (the wgsync client_test
|
|
// pattern). Beyond the transport round-trip, these pin the package's two load-bearing contracts:
|
|
// the typed token_exists outcome, and the secret-hygiene rule that NO error message ever embeds
|
|
// stdout bytes (stdout is the secret channel).
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/pem"
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"crypto/ed25519"
|
|
"crypto/rand"
|
|
|
|
"golang.org/x/crypto/ssh"
|
|
)
|
|
|
|
func testKeys(t *testing.T) ([]byte, ssh.Signer) {
|
|
t.Helper()
|
|
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
|
if err != nil {
|
|
t.Fatalf("ed25519: %v", err)
|
|
}
|
|
block, err := ssh.MarshalPrivateKey(priv, "")
|
|
if err != nil {
|
|
t.Fatalf("MarshalPrivateKey: %v", err)
|
|
}
|
|
signer, err := ssh.NewSignerFromKey(priv)
|
|
if err != nil {
|
|
t.Fatalf("NewSignerFromKey: %v", err)
|
|
}
|
|
return pem.EncodeToMemory(block), signer
|
|
}
|
|
|
|
type testServer struct {
|
|
addr string
|
|
mu sync.Mutex
|
|
captured []byte
|
|
cmd string
|
|
}
|
|
|
|
func startTestServer(t *testing.T, hostSigner, clientSigner ssh.Signer,
|
|
stdoutResp, stderrResp string, exitStatus uint32) *testServer {
|
|
t.Helper()
|
|
cfg := &ssh.ServerConfig{
|
|
PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
|
|
if bytes.Equal(key.Marshal(), clientSigner.PublicKey().Marshal()) {
|
|
return &ssh.Permissions{}, nil
|
|
}
|
|
return nil, io.EOF
|
|
},
|
|
}
|
|
cfg.AddHostKey(hostSigner)
|
|
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen: %v", err)
|
|
}
|
|
t.Cleanup(func() { ln.Close() })
|
|
srv := &testServer{addr: ln.Addr().String()}
|
|
|
|
go func() {
|
|
conn, err := ln.Accept()
|
|
if err != nil {
|
|
return
|
|
}
|
|
sconn, chans, reqs, err := ssh.NewServerConn(conn, cfg)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer sconn.Close()
|
|
go ssh.DiscardRequests(reqs)
|
|
for newCh := range chans {
|
|
if newCh.ChannelType() != "session" {
|
|
newCh.Reject(ssh.UnknownChannelType, "unsupported")
|
|
continue
|
|
}
|
|
ch, chReqs, err := newCh.Accept()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
go func() {
|
|
for req := range chReqs {
|
|
if req.Type == "exec" {
|
|
var p struct{ Command string }
|
|
ssh.Unmarshal(req.Payload, &p)
|
|
srv.mu.Lock()
|
|
srv.cmd = p.Command
|
|
srv.mu.Unlock()
|
|
req.Reply(true, nil)
|
|
data, _ := io.ReadAll(ch)
|
|
srv.mu.Lock()
|
|
srv.captured = data
|
|
srv.mu.Unlock()
|
|
if stderrResp != "" {
|
|
ch.Stderr().Write([]byte(stderrResp))
|
|
}
|
|
if stdoutResp != "" {
|
|
ch.Write([]byte(stdoutResp))
|
|
}
|
|
ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{exitStatus}))
|
|
ch.Close()
|
|
return
|
|
}
|
|
req.Reply(false, nil)
|
|
}
|
|
}()
|
|
}
|
|
}()
|
|
return srv
|
|
}
|
|
|
|
func hostKeyLine(t *testing.T, s ssh.Signer) string {
|
|
t.Helper()
|
|
return strings.TrimSpace(string(ssh.MarshalAuthorizedKey(s.PublicKey())))
|
|
}
|
|
|
|
func newTestClient(t *testing.T, addr, hostKey string, clientPEM []byte) *Client {
|
|
t.Helper()
|
|
c, err := New(Config{
|
|
Addr: addr, User: "felhom-peersync", PrivateKey: clientPEM,
|
|
HostKeyLine: hostKey, Timeout: 5 * time.Second,
|
|
}, log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatalf("tenantsync.New: %v", err)
|
|
}
|
|
return c
|
|
}
|
|
|
|
const okResp = `{"status":"ok","token_id":"felhom@pbs!peti","token_secret":"s3cr3t-uuid-value",` +
|
|
`"fingerprint":"aa:bb","datastore":"felhom-offsite","namespace":"peti"}`
|
|
|
|
func TestProvision_RoundTripParsesResult(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0)
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
|
|
|
|
res, err := c.Provision(context.Background(), "peti")
|
|
if err != nil {
|
|
t.Fatalf("Provision: %v", err)
|
|
}
|
|
if res.TokenID != "felhom@pbs!peti" || res.TokenSecret != "s3cr3t-uuid-value" ||
|
|
res.Fingerprint != "aa:bb" || res.Datastore != "felhom-offsite" || res.Namespace != "peti" {
|
|
t.Errorf("Result mis-parsed: %+v", res)
|
|
}
|
|
srv.mu.Lock()
|
|
defer srv.mu.Unlock()
|
|
if want := `{"customer_id":"peti","op":"provision"}`; string(srv.captured) != want {
|
|
t.Errorf("server received %q, want %q", srv.captured, want)
|
|
}
|
|
if srv.cmd != "felhom-tenantsync" {
|
|
t.Errorf("exec command = %q, want felhom-tenantsync", srv.cmd)
|
|
}
|
|
}
|
|
|
|
func TestProvision_TokenExistsIsTypedError(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
// The script emits the error JSON on stdout AND exits 1 — both must map to ErrTokenExists.
|
|
srv := startTestServer(t, hostSigner, clientSigner,
|
|
`{"status":"error","code":"token_exists","error":"token felhom@pbs!peti already exists"}`, "", 1)
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
|
|
|
|
_, err := c.Provision(context.Background(), "peti")
|
|
if !errors.Is(err, ErrTokenExists) {
|
|
t.Fatalf("err = %v, want ErrTokenExists", err)
|
|
}
|
|
}
|
|
|
|
func TestReissue_SendsReissueOp(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0)
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
|
|
|
|
if _, err := c.Reissue(context.Background(), "peti"); err != nil {
|
|
t.Fatalf("Reissue: %v", err)
|
|
}
|
|
srv.mu.Lock()
|
|
defer srv.mu.Unlock()
|
|
if want := `{"customer_id":"peti","op":"reissue"}`; string(srv.captured) != want {
|
|
t.Errorf("server received %q, want %q", srv.captured, want)
|
|
}
|
|
}
|
|
|
|
func TestFingerprint_Op(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
srv := startTestServer(t, hostSigner, clientSigner, `{"status":"ok","fingerprint":"cc:dd"}`, "", 0)
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
|
|
|
|
fp, err := c.Fingerprint(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Fingerprint: %v", err)
|
|
}
|
|
if fp != "cc:dd" {
|
|
t.Errorf("fingerprint = %q, want cc:dd", fp)
|
|
}
|
|
}
|
|
|
|
// TestUsage_Op (v0.65.0): the read-only usage op parses total/used/avail (bytes).
|
|
func TestUsage_Op(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
srv := startTestServer(t, hostSigner, clientSigner,
|
|
`{"status":"ok","total":39990112256,"used":7628091392,"avail":30686175232}`, "", 0)
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
|
|
|
|
u, err := c.Usage(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Usage: %v", err)
|
|
}
|
|
if u.Total != 39990112256 || u.Used != 7628091392 || u.Avail != 30686175232 {
|
|
t.Errorf("usage = %+v, want total/used/avail 39990112256/7628091392/30686175232", u)
|
|
}
|
|
}
|
|
|
|
// TestUsage_UnknownOpTypedUnsupported (v0.65.0, graceful degradation): an ep0 still on tenantsync
|
|
// ≤ v1.1.0 answers `{"op":"usage"}` with its generic `bad_request "unknown op"` (exit 1). That must
|
|
// map to the typed ErrUsageUnsupported so the PBS checker records "unavailable", not an error.
|
|
func TestUsage_UnknownOpTypedUnsupported(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
srv := startTestServer(t, hostSigner, clientSigner,
|
|
`{"status":"error","code":"bad_request","error":"unknown op"}`, "", 1)
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
|
|
|
|
if _, err := c.Usage(context.Background()); !errors.Is(err, ErrUsageUnsupported) {
|
|
t.Fatalf("an ep0 without the usage op must map to ErrUsageUnsupported, got %v", err)
|
|
}
|
|
}
|
|
|
|
// TestErrors_NeverEmbedStdout is the secret-hygiene contract: stdout may carry the token secret,
|
|
// so NO error path may quote it. A malformed-but-secret-bearing stdout must yield an error that
|
|
// does not contain the marker bytes. (wgsync quotes stdout in its malformed error — this package
|
|
// deliberately must not; that divergence is the point of this test.)
|
|
func TestErrors_NeverEmbedStdout(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
stdout string
|
|
exitStatus uint32
|
|
}{
|
|
{"malformed-ok-exit", `garbage SECRET-MARKER garbage`, 0},
|
|
{"malformed-fail-exit", `partial {"token_secret":"SECRET-MARKER"`, 1},
|
|
{"wrong-status", `{"status":"weird","token_secret":"SECRET-MARKER"}`, 0},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
srv := startTestServer(t, hostSigner, clientSigner, tc.stdout, "some stderr", tc.exitStatus)
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
|
|
|
|
_, err := c.Provision(context.Background(), "peti")
|
|
if err == nil {
|
|
t.Fatal("Provision accepted a malformed response")
|
|
}
|
|
if strings.Contains(err.Error(), "SECRET-MARKER") {
|
|
t.Errorf("error %q embeds stdout bytes — the secret channel leaked into logs", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOkResponseMissingFieldsRefused(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
srv := startTestServer(t, hostSigner, clientSigner,
|
|
`{"status":"ok","token_id":"felhom@pbs!peti"}`, "", 0) // no secret/ns/fp/ds
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM)
|
|
|
|
if _, err := c.Provision(context.Background(), "peti"); err == nil {
|
|
t.Fatal("Provision accepted an ok-response with missing fields")
|
|
}
|
|
}
|
|
|
|
func TestWrongHostKeyRefused(t *testing.T) {
|
|
clientPEM, clientSigner := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
_, otherSigner := testKeys(t)
|
|
srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0)
|
|
c := newTestClient(t, srv.addr, hostKeyLine(t, otherSigner), clientPEM)
|
|
|
|
_, err := c.Provision(context.Background(), "peti")
|
|
if err == nil {
|
|
t.Fatal("Provision succeeded against a server with the WRONG host key — the pin is dead")
|
|
}
|
|
srv.mu.Lock()
|
|
defer srv.mu.Unlock()
|
|
if len(srv.captured) != 0 {
|
|
t.Errorf("payload leaked to a mis-keyed server: %q", srv.captured)
|
|
}
|
|
}
|
|
|
|
func TestInvalidCustomerIDRefusedClientSide(t *testing.T) {
|
|
clientPEM, _ := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
// No server needed — the refusal must happen before any dial.
|
|
c := newTestClient(t, "127.0.0.1:1", hostKeyLine(t, hostSigner), clientPEM)
|
|
for _, bad := range []string{"", "-leading-dash", ".dot", "has space", strings.Repeat("x", 40)} {
|
|
if _, err := c.Provision(context.Background(), bad); err == nil {
|
|
t.Errorf("customer_id %q accepted", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNew_BadInputsFailEarly(t *testing.T) {
|
|
clientPEM, _ := testKeys(t)
|
|
_, hostSigner := testKeys(t)
|
|
hk := hostKeyLine(t, hostSigner)
|
|
logger := log.New(io.Discard, "", 0)
|
|
|
|
if _, err := New(Config{Addr: "x:22", User: "u", PrivateKey: []byte("not-a-key"), HostKeyLine: hk}, logger); err == nil {
|
|
t.Error("bad private key accepted")
|
|
}
|
|
if _, err := New(Config{Addr: "x:22", User: "u", PrivateKey: clientPEM, HostKeyLine: "not a key line"}, logger); err == nil {
|
|
t.Error("bad host key line accepted")
|
|
}
|
|
if _, err := New(Config{User: "u", PrivateKey: clientPEM, HostKeyLine: hk}, logger); err == nil {
|
|
t.Error("missing addr accepted")
|
|
}
|
|
}
|