v0.109.0: soft-quota gate + usage bar + offsite report status (SLICE 4)
QuotaGB rides the descriptor into OffboxTarget; RepoSizeBytes persisted from restic stats. Pre-run gate: >=100% refuses NEW backups (Hungarian notice + operator alert) but prune STILL runs (red-proofed) and restore is never gated; >=80% warns. /backups usage bar (quota>0 only). The hub report gains the non-secret offsite status object for the OffsiteChecker. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -346,7 +346,21 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
start := time.Now()
|
||||
_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "running"; o.LastError = "" })
|
||||
|
||||
backedUp, missing, runErr := m.runOffboxInternal(ctx, apps, base, env)
|
||||
var backedUp int
|
||||
var missing []string
|
||||
var runErr error
|
||||
if usedGB, quota, over := offboxQuotaState(t); over {
|
||||
// SLICE 4 soft-quota gate (pre-run): NEW backups are refused at ≥100% of the shared-model quota —
|
||||
// but the retention/prune step STILL RUNS (pruning is the customer's only way back under quota;
|
||||
// gating it too would deadlock them over-quota) and restore paths are untouched. The CURRENT run's
|
||||
// gate uses the last-known repo size; a run that crosses 100% mid-flight finishes and the NEXT
|
||||
// run refuses.
|
||||
m.offboxPruneOnly(ctx, base, env)
|
||||
m.offboxRecordStats(ctx, base, env) // the prune may have brought the size back down — refresh
|
||||
runErr = fmt.Errorf("A NAS-mentés túllépte a tárhelykeretet (%d/%d GB) — törölj régi mentéseket vagy kérj nagyobb keretet.", usedGB, quota)
|
||||
} else {
|
||||
backedUp, missing, runErr = m.runOffboxInternal(ctx, apps, base, env)
|
||||
}
|
||||
|
||||
// No-silent-success: apps were toggled but NOTHING was captured (every unit missing) → promote to a
|
||||
// hard error so the run reports "error" and the operator is alerted, instead of a misleading ok/0.
|
||||
@@ -371,12 +385,16 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
o.LastStatus = "ok"
|
||||
o.LastError = ""
|
||||
o.SnapshotCount = snapshots
|
||||
if len(missing) == 0 {
|
||||
o.LastWarning = ""
|
||||
} else {
|
||||
o.LastWarning = fmt.Sprintf("Figyelmeztetés: %d alkalmazásnak nincs elérhető mentése, ezek kimaradtak: %s",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
var warns []string
|
||||
if len(missing) > 0 {
|
||||
warns = append(warns, fmt.Sprintf("Figyelmeztetés: %d alkalmazásnak nincs elérhető mentése, ezek kimaradtak: %s",
|
||||
len(missing), strings.Join(missing, ", ")))
|
||||
}
|
||||
// SLICE 4: approaching the soft quota (≥80%, <100%) — warn on an otherwise-OK run.
|
||||
if qw := offboxQuotaWarning(o); qw != "" {
|
||||
warns = append(warns, qw)
|
||||
}
|
||||
o.LastWarning = strings.Join(warns, " ")
|
||||
}
|
||||
})
|
||||
if m.offboxNotify != nil {
|
||||
@@ -515,6 +533,88 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
|
||||
return backedUp, missing, nil
|
||||
}
|
||||
|
||||
// offboxGiB is the soft-quota unit: QuotaGB counts binary gigabytes (GiB) of restic restore-size.
|
||||
const offboxGiB = int64(1) << 30
|
||||
|
||||
// OffboxReportStatus is the NON-SECRET offsite summary carried on the hub report (SLICE 4) — the input
|
||||
// to the hub's OffsiteChecker (fill + staleness alerts). nil when no offbox target is configured.
|
||||
type OffboxReportStatus struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
EscrowState string `json:"escrow_state"`
|
||||
LastRun string `json:"last_run,omitempty"` // RFC3339
|
||||
LastStatus string `json:"last_status,omitempty"` // "ok" | "error" | "running"
|
||||
SnapshotCount int `json:"snapshot_count"`
|
||||
RepoSizeBytes int64 `json:"repo_size_bytes"`
|
||||
QuotaGB int `json:"quota_gb"`
|
||||
}
|
||||
|
||||
// OffboxReportStatus returns the offsite summary for the hub report (nil = not configured; the hub's
|
||||
// checker treats absence as "nothing to watch" — pre-v0.109 reports look the same).
|
||||
func (m *Manager) OffboxReportStatus() *OffboxReportStatus {
|
||||
t := m.settings.GetOffboxTarget()
|
||||
if t == nil || !t.Enabled {
|
||||
return nil
|
||||
}
|
||||
return &OffboxReportStatus{
|
||||
Enabled: true, EscrowState: t.EscrowState, LastRun: t.LastRun, LastStatus: t.LastStatus,
|
||||
SnapshotCount: t.SnapshotCount, RepoSizeBytes: t.RepoSizeBytes, QuotaGB: t.QuotaGB,
|
||||
}
|
||||
}
|
||||
|
||||
// offboxQuotaState evaluates the SLICE 4 soft-quota gate against the LAST-KNOWN repo size (a failed stats
|
||||
// call keeps the previous value — stale-but-safe). quota<=0 = no soft limit (dedicated/manual targets).
|
||||
func offboxQuotaState(t *settings.OffboxTarget) (usedGB, quotaGB int, over bool) {
|
||||
if t == nil || t.QuotaGB <= 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
usedGB = int(t.RepoSizeBytes / offboxGiB)
|
||||
return usedGB, t.QuotaGB, t.RepoSizeBytes >= int64(t.QuotaGB)*offboxGiB
|
||||
}
|
||||
|
||||
// offboxQuotaWarning returns the Hungarian ≥80% (<100%) usage notice, or "" (quota unset / usage fine /
|
||||
// already over — over-quota is the run-refusal error, not a warning).
|
||||
func offboxQuotaWarning(t *settings.OffboxTarget) string {
|
||||
if t == nil || t.QuotaGB <= 0 || t.RepoSizeBytes <= 0 {
|
||||
return ""
|
||||
}
|
||||
limit := int64(t.QuotaGB) * offboxGiB
|
||||
pct := t.RepoSizeBytes * 100 / limit
|
||||
if pct < 80 || t.RepoSizeBytes >= limit {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("A NAS-mentés a keret %d%%-át használja (%d/%d GB).", pct, t.RepoSizeBytes/offboxGiB, t.QuotaGB)
|
||||
}
|
||||
|
||||
// OffboxQuotaPercent returns the usage percentage for the /backups usage bar (0 when no quota/size).
|
||||
func OffboxQuotaPercent(t *settings.OffboxTarget) int {
|
||||
if t == nil || t.QuotaGB <= 0 || t.RepoSizeBytes <= 0 {
|
||||
return 0
|
||||
}
|
||||
pct := int(t.RepoSizeBytes * 100 / (int64(t.QuotaGB) * offboxGiB))
|
||||
if pct > 100 {
|
||||
pct = 100
|
||||
}
|
||||
return pct
|
||||
}
|
||||
|
||||
// offboxPruneOnly runs ONLY the retention/prune step (the over-quota path: new backups are refused but
|
||||
// pruning must stay available — it is the only way back under the quota). Repo-ensure first so a fresh
|
||||
// target still fails loudly; errors are non-fatal (same as the regular run's prune).
|
||||
func (m *Manager) offboxPruneOnly(ctx context.Context, base, env []string) {
|
||||
if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] over-quota prune: repo unreachable: %v", rerr)
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
fargs := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
|
||||
if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] over-quota prune failed: %v: %s", ferr, truncate(out))
|
||||
} else {
|
||||
m.logger.Printf("[INFO] [offbox] over-quota: prune executed (new backups refused until under quota)")
|
||||
}
|
||||
}
|
||||
|
||||
// offboxRecordStats reads the snapshot count (best-effort) for the UI; also fills repo size when stats works.
|
||||
func (m *Manager) offboxRecordStats(ctx context.Context, base, env []string) int {
|
||||
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
@@ -529,14 +629,18 @@ func (m *Manager) offboxRecordStats(ctx context.Context, base, env []string) int
|
||||
if json.Unmarshal(out, &snaps) != nil {
|
||||
return 0
|
||||
}
|
||||
// Repo size (best-effort, restore-size).
|
||||
// Repo size (best-effort, restore-size). Bytes feed the soft-quota gate (SLICE 4); a failed stats
|
||||
// call keeps the last-known value (stale-but-safe).
|
||||
if so, serr := m.runner()(sctx, env, append(append([]string{}, base...), "stats", "--json")...); serr == nil {
|
||||
var st struct {
|
||||
TotalSize int64 `json:"total_size"`
|
||||
}
|
||||
if json.Unmarshal(so, &st) == nil && st.TotalSize > 0 {
|
||||
human := humanizeBytes(st.TotalSize)
|
||||
_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.RepoSizeHuman = human })
|
||||
_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
|
||||
o.RepoSizeHuman = human
|
||||
o.RepoSizeBytes = st.TotalSize
|
||||
})
|
||||
}
|
||||
}
|
||||
return len(snaps)
|
||||
|
||||
Reference in New Issue
Block a user