feat(v0.156.0): dead-primary alerting (R-51) + boot desired-state reconciliation (R-52)
R-51: aggregateState's mixed branch returned StateRunning ("partial"), so a stack whose
MAIN container was dead behind live helpers alerted on nothing — immich-server sat Exited
for 18 h, 100 % unreachable, no banner and no app_start_failed (audit F4). New
StateDegraded: a DOWN member whose docker restart policy is always/unless-stopped is a
fault (degraded, a down state); no/on-failure is a finished one-shot and stays benign; an
unreadable policy fails CLOSED. The unhealthy/restarting/paused/unknown exclusions are
byte-identical — folding unhealthy into down is the flapping fix-3 avoided.
R-52: new internal/bootrecon — one bounded start-once sweep at startup (2 attempts, 30 s
apart) for apps an interrupted boot left behind, inside the 90 s boot grace so a success
is silent and a failure still alerts. A zero-container stack is NEVER touched: the UI's
Stop is compose down, so a deliberate stop survives a reboot.
Both features carry a production-path wiring test (the v0.154.0 / v0.91.0 inert-seam
class). The main() assertion is an AST walk, not strings.Contains — the substring version
passed its own red-proof, because a commented-out call still contains the string.
Red-proofs run and restored: mix branch reverted -> "running" on the immich fixture;
boot hook commented out -> wiring test fails; zero-container gate dropped -> the
user-stopped app gets started.
NOTE: controller/cmd/controller/ is matched by .gitignore's `controller` entry, so new
files there need `git add -f` (and ripgrep silently skips main.go without --no-ignore).
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
// Package bootrecon implements R-52: the bounded, start-ONCE recovery of apps that were left
|
||||
// behind by an interrupted boot.
|
||||
//
|
||||
// The live failure it closes (AUDIT-vacation-remote-ops-2026-07-20, finding F5): a pre-transport
|
||||
// shutdown left `immich-server` and `calibre-web` Exited; ten sibling containers came back and
|
||||
// those two did not, and they were still down ~18 hours later. The controller REPORTED them (the
|
||||
// 30 s deadapp-check) but never started them — deployed-but-stopped was an alarm with no recovery.
|
||||
//
|
||||
// Two deliberate boundaries, both load-bearing:
|
||||
//
|
||||
// - **Bounded, never a loop.** At most `attempts` tries, `retryDelay` apart, then it stops and the
|
||||
// alarm owns the problem. A restart loop would paper over a genuinely broken app forever and
|
||||
// hammer docker while doing it.
|
||||
// - **A user's Stop survives a reboot.** The UI's Stop is `docker compose down`, which REMOVES the
|
||||
// containers; a boot interruption leaves them behind as Exited. So "has containers on disk that
|
||||
// are down" is the boot-orphan signature, and a stack with ZERO containers is deliberately never
|
||||
// touched. This distinction is the whole safety argument — see TestReconcile_UserStoppedAppIsNeverStarted.
|
||||
//
|
||||
// It runs inside the notifier's boot grace (cmd/controller/main.go `deadAppBootGrace`), so a
|
||||
// successful recovery never fires an alert and a failed one alerts honestly once the grace expires.
|
||||
package bootrecon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// StackProvider is the slice of *stacks.Manager this package needs. Declared consumer-side so the
|
||||
// tests can count StartStack calls without a docker anywhere near them.
|
||||
type StackProvider interface {
|
||||
GetStacks() []stacks.Stack
|
||||
StartStack(name string) error
|
||||
RefreshStatus() error
|
||||
}
|
||||
|
||||
const (
|
||||
// DefaultAttempts is the total number of start attempts per boot (not per app per retry-forever).
|
||||
DefaultAttempts = 2
|
||||
// DefaultRetryDelay spaces the attempts. 2 × 30 s fits comfortably inside the 90 s boot grace,
|
||||
// so a recovery that works is silent and one that does not is honest.
|
||||
DefaultRetryDelay = 30 * time.Second
|
||||
)
|
||||
|
||||
// Reconciler performs the start-once sweep. Zero value is not usable — use New.
|
||||
type Reconciler struct {
|
||||
stacks StackProvider
|
||||
logger *log.Logger
|
||||
attempts int
|
||||
retryDelay time.Duration
|
||||
|
||||
// sleep is the inter-attempt wait; injectable so tests never spend 30 real seconds.
|
||||
sleep func(context.Context, time.Duration)
|
||||
}
|
||||
|
||||
// Result is the outcome, returned for logging/testing (the hub learns about failures only through
|
||||
// the existing app_start_failed alarm — this package deliberately pushes no events of its own).
|
||||
type Result struct {
|
||||
Candidates []string // boot-orphaned apps found
|
||||
Recovered []string // running again by the end
|
||||
StillDown []string // still down after the last attempt — the alarm's problem now
|
||||
Attempts int // attempts actually made (0 when there was nothing to do)
|
||||
}
|
||||
|
||||
// New builds a Reconciler with the shipped defaults.
|
||||
func New(p StackProvider, logger *log.Logger) *Reconciler {
|
||||
return &Reconciler{
|
||||
stacks: p,
|
||||
logger: logger,
|
||||
attempts: DefaultAttempts,
|
||||
retryDelay: DefaultRetryDelay,
|
||||
sleep: sleepCtx,
|
||||
}
|
||||
}
|
||||
|
||||
func sleepCtx(ctx context.Context, d time.Duration) {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
|
||||
// isBootOrphan reports whether a stack is an app the boot left behind.
|
||||
//
|
||||
// The gate, term by term:
|
||||
// - Deployed — an app the customer asked to have running.
|
||||
// - not Protected — traefik/cloudflared/felhom-controller have their own supervision; this must
|
||||
// never race the base-stack self-heal.
|
||||
// - not Deploying — mid-deploy is not a fault.
|
||||
// - has containers — the D-case guard: a UI Stop removes them, and a deliberate stop must survive
|
||||
// a reboot.
|
||||
// - IsDownState — stopped/exited/degraded (R-51 included: a boot that half-started a stack is the
|
||||
// same interrupted-boot shape).
|
||||
func isBootOrphan(s stacks.Stack) bool {
|
||||
return s.Deployed && !s.Protected && !s.Deploying &&
|
||||
len(s.Containers) > 0 && stacks.IsDownState(s.State)
|
||||
}
|
||||
|
||||
// Run performs the sweep once and returns what happened. It is safe to call with no boot orphans
|
||||
// (the quiet path logs one DEBUG-free INFO-free line — see below) and it never returns an error:
|
||||
// a failure to start is an app-level fact the alarm reports, not a controller startup failure.
|
||||
func (r *Reconciler) Run(ctx context.Context) Result {
|
||||
var res Result
|
||||
|
||||
pending := map[string]bool{}
|
||||
for _, s := range r.stacks.GetStacks() {
|
||||
if isBootOrphan(s) {
|
||||
pending[s.Name] = true
|
||||
res.Candidates = append(res.Candidates, s.Name)
|
||||
}
|
||||
}
|
||||
sortStrings(res.Candidates)
|
||||
|
||||
if len(pending) == 0 {
|
||||
// The healthy path must be observable — "no alarms" and "never ran" have to be
|
||||
// distinguishable in a log (the v0.91.2 lesson).
|
||||
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: no boot-orphaned apps (nothing to start)")
|
||||
return res
|
||||
}
|
||||
|
||||
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: %d boot-orphaned app(s) found: %v — up to %d attempt(s)",
|
||||
len(res.Candidates), res.Candidates, r.attempts)
|
||||
|
||||
for attempt := 1; attempt <= r.attempts && len(pending) > 0; attempt++ {
|
||||
res.Attempts = attempt
|
||||
for _, name := range sortedKeys(pending) {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
start := time.Now()
|
||||
if err := r.stacks.StartStack(name); err != nil {
|
||||
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation attempt %d/%d: start %q failed after %.1fs: %v",
|
||||
attempt, r.attempts, name, time.Since(start).Seconds(), err)
|
||||
continue
|
||||
}
|
||||
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation attempt %d/%d: started %q (took %.1fs)",
|
||||
attempt, r.attempts, name, time.Since(start).Seconds())
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// Re-read reality rather than trusting a nil error: `compose up -d` exits 0 on a crash-loop
|
||||
// (a session-critical invariant of this repo), so only a fresh docker ps can say whether the
|
||||
// app is actually up.
|
||||
if err := r.stacks.RefreshStatus(); err != nil {
|
||||
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation: status refresh failed: %v", err)
|
||||
}
|
||||
for _, s := range r.stacks.GetStacks() {
|
||||
if pending[s.Name] && !stacks.IsDownState(s.State) {
|
||||
delete(pending, s.Name)
|
||||
res.Recovered = append(res.Recovered, s.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if len(pending) > 0 && attempt < r.attempts {
|
||||
r.sleep(ctx, r.retryDelay)
|
||||
}
|
||||
}
|
||||
|
||||
res.StillDown = sortedKeys(pending)
|
||||
sortStrings(res.Recovered)
|
||||
|
||||
if len(res.StillDown) == 0 {
|
||||
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation complete: %d app(s) recovered in %d attempt(s): %v",
|
||||
len(res.Recovered), res.Attempts, res.Recovered)
|
||||
} else {
|
||||
// Deliberately no hub event here: the app_start_failed alarm fires on its own once the boot
|
||||
// grace expires, and two events for one dead app is how an operator inbox becomes noise.
|
||||
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation gave up after %d attempt(s): recovered=%v still down=%v (the dead-app alarm now owns these)",
|
||||
res.Attempts, res.Recovered, res.StillDown)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// --- tiny local helpers (no dependency on sort ordering semantics elsewhere) ---
|
||||
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sortStrings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortStrings(s []string) {
|
||||
for i := 1; i < len(s); i++ {
|
||||
for j := i; j > 0 && s[j] < s[j-1]; j-- {
|
||||
s[j], s[j-1] = s[j-1], s[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package bootrecon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// fakeStacks counts StartStack calls per app — the assertion that matters in BOTH directions:
|
||||
// scenario C needs exactly-bounded starts, scenario D needs a start count of ZERO.
|
||||
type fakeStacks struct {
|
||||
list []stacks.Stack
|
||||
starts map[string]int
|
||||
failWith map[string]error
|
||||
// onStart mutates the world the way a real successful start would (fresh docker ps).
|
||||
onStart func(f *fakeStacks, name string)
|
||||
refreshN int
|
||||
}
|
||||
|
||||
func (f *fakeStacks) GetStacks() []stacks.Stack { return f.list }
|
||||
func (f *fakeStacks) RefreshStatus() error { f.refreshN++; return nil }
|
||||
|
||||
func (f *fakeStacks) StartStack(name string) error {
|
||||
if f.starts == nil {
|
||||
f.starts = map[string]int{}
|
||||
}
|
||||
f.starts[name]++
|
||||
if err := f.failWith[name]; err != nil {
|
||||
return err
|
||||
}
|
||||
if f.onStart != nil {
|
||||
f.onStart(f, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStacks) setState(name string, st stacks.ContainerState) {
|
||||
for i := range f.list {
|
||||
if f.list[i].Name == name {
|
||||
f.list[i].State = st
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// comesUp is the ordinary success behaviour: the app is running after the start.
|
||||
func comesUp(f *fakeStacks, name string) { f.setState(name, stacks.StateRunning) }
|
||||
|
||||
func newTestReconciler(f *fakeStacks) (*Reconciler, *int) {
|
||||
slept := 0
|
||||
r := New(f, log.New(io.Discard, "", 0))
|
||||
r.sleep = func(context.Context, time.Duration) { slept++ }
|
||||
return r, &slept
|
||||
}
|
||||
|
||||
// bootOrphan is the F5 shape: deployed, containers still present on disk, all Exited — the boot
|
||||
// interrupted them, nobody stopped them.
|
||||
func bootOrphan(name string) stacks.Stack {
|
||||
return stacks.Stack{
|
||||
Name: name,
|
||||
Deployed: true,
|
||||
State: stacks.StateExited,
|
||||
Containers: []stacks.ContainerInfo{
|
||||
{Name: name + "-app", State: stacks.StateExited, Status: "Exited (0) 3 minutes ago"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// userStopped is the UI-Stop shape: `docker compose down` REMOVED the containers.
|
||||
func userStopped(name string) stacks.Stack {
|
||||
return stacks.Stack{Name: name, Deployed: true, State: stacks.StateStopped, Containers: nil}
|
||||
}
|
||||
|
||||
// --- Scenario C -------------------------------------------------------------------------------
|
||||
|
||||
func TestReconcile_BootOrphanGetsExactlyOneRecovery(t *testing.T) {
|
||||
f := &fakeStacks{
|
||||
list: []stacks.Stack{bootOrphan("immich"), bootOrphan("calibre-web")},
|
||||
onStart: comesUp,
|
||||
}
|
||||
r, slept := newTestReconciler(f)
|
||||
|
||||
res := r.Run(context.Background())
|
||||
|
||||
for _, name := range []string{"immich", "calibre-web"} {
|
||||
if f.starts[name] != 1 {
|
||||
t.Fatalf("StartStack(%q) called %d times, want exactly 1", name, f.starts[name])
|
||||
}
|
||||
}
|
||||
if len(res.Recovered) != 2 || len(res.StillDown) != 0 {
|
||||
t.Fatalf("recovered=%v stillDown=%v, want both apps recovered", res.Recovered, res.StillDown)
|
||||
}
|
||||
if res.Attempts != 1 {
|
||||
t.Fatalf("attempts = %d, want 1 — a success must not retry", res.Attempts)
|
||||
}
|
||||
if *slept != 0 {
|
||||
t.Fatalf("slept %d times after a first-attempt success, want 0", *slept)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario D (the WRONG case: this must assert the negative) ---------------------------------
|
||||
|
||||
func TestReconcile_UserStoppedAppIsNeverStarted(t *testing.T) {
|
||||
f := &fakeStacks{
|
||||
list: []stacks.Stack{userStopped("jellyfin"), bootOrphan("immich")},
|
||||
onStart: comesUp,
|
||||
}
|
||||
r, _ := newTestReconciler(f)
|
||||
|
||||
res := r.Run(context.Background())
|
||||
|
||||
if n := f.starts["jellyfin"]; n != 0 {
|
||||
t.Fatalf("StartStack(\"jellyfin\") called %d times, want 0 — a deliberate Stop must survive a reboot", n)
|
||||
}
|
||||
if f.starts["immich"] != 1 {
|
||||
t.Fatalf("the real boot orphan was not started: %v", f.starts)
|
||||
}
|
||||
for _, c := range res.Candidates {
|
||||
if c == "jellyfin" {
|
||||
t.Fatalf("a zero-container stack must never be a reconciliation candidate: %v", res.Candidates)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Bounded, never a loop ----------------------------------------------------------------------
|
||||
|
||||
func TestReconcile_StopsAfterTwoAttemptsAndHandsOverToTheAlarm(t *testing.T) {
|
||||
f := &fakeStacks{
|
||||
list: []stacks.Stack{bootOrphan("immich")},
|
||||
failWith: map[string]error{"immich": errors.New("compose up: exit status 1")},
|
||||
}
|
||||
r, slept := newTestReconciler(f)
|
||||
|
||||
res := r.Run(context.Background())
|
||||
|
||||
if f.starts["immich"] != DefaultAttempts {
|
||||
t.Fatalf("StartStack called %d times, want exactly %d (bounded, never a loop)", f.starts["immich"], DefaultAttempts)
|
||||
}
|
||||
if *slept != DefaultAttempts-1 {
|
||||
t.Fatalf("slept %d times, want %d (one wait BETWEEN attempts, never after the last)", *slept, DefaultAttempts-1)
|
||||
}
|
||||
if len(res.StillDown) != 1 || res.StillDown[0] != "immich" {
|
||||
t.Fatalf("stillDown = %v, want [immich] — the alarm must inherit the failure", res.StillDown)
|
||||
}
|
||||
}
|
||||
|
||||
// `compose up -d` exits 0 on a crash-loop, so a nil error is not proof the app is up. Only a
|
||||
// re-read of docker ps can retire a candidate.
|
||||
func TestReconcile_NilErrorIsNotProofTheAppCameUp(t *testing.T) {
|
||||
f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}} // onStart nil → stays Exited
|
||||
r, _ := newTestReconciler(f)
|
||||
|
||||
res := r.Run(context.Background())
|
||||
|
||||
if f.starts["immich"] != DefaultAttempts {
|
||||
t.Fatalf("StartStack called %d times, want %d — a still-down app must be retried", f.starts["immich"], DefaultAttempts)
|
||||
}
|
||||
if len(res.StillDown) != 1 {
|
||||
t.Fatalf("stillDown = %v, want the app still listed despite StartStack returning nil", res.StillDown)
|
||||
}
|
||||
if f.refreshN < 1 {
|
||||
t.Fatalf("RefreshStatus was never called — the outcome was taken on trust")
|
||||
}
|
||||
}
|
||||
|
||||
// A second-attempt success must still end clean (and must not alert).
|
||||
func TestReconcile_SecondAttemptSucceeds(t *testing.T) {
|
||||
f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}}
|
||||
f.onStart = func(fs *fakeStacks, name string) {
|
||||
if fs.starts[name] >= 2 {
|
||||
comesUp(fs, name)
|
||||
}
|
||||
}
|
||||
r, slept := newTestReconciler(f)
|
||||
|
||||
res := r.Run(context.Background())
|
||||
|
||||
if f.starts["immich"] != 2 {
|
||||
t.Fatalf("StartStack called %d times, want 2", f.starts["immich"])
|
||||
}
|
||||
if *slept != 1 {
|
||||
t.Fatalf("slept %d times, want 1", *slept)
|
||||
}
|
||||
if len(res.StillDown) != 0 || len(res.Recovered) != 1 {
|
||||
t.Fatalf("recovered=%v stillDown=%v, want a clean recovery on attempt 2", res.Recovered, res.StillDown)
|
||||
}
|
||||
}
|
||||
|
||||
// --- The gate, term by term ---------------------------------------------------------------------
|
||||
|
||||
func TestIsBootOrphan_Gate(t *testing.T) {
|
||||
base := bootOrphan("app")
|
||||
cases := []struct {
|
||||
name string
|
||||
mut func(s *stacks.Stack)
|
||||
want bool
|
||||
}{
|
||||
{"boot orphan", func(*stacks.Stack) {}, true},
|
||||
{"degraded counts (R-51 half-started boot)", func(s *stacks.Stack) { s.State = stacks.StateDegraded }, true},
|
||||
{"not deployed", func(s *stacks.Stack) { s.Deployed = false }, false},
|
||||
{"protected infra", func(s *stacks.Stack) { s.Protected = true }, false},
|
||||
{"mid-deploy", func(s *stacks.Stack) { s.Deploying = true }, false},
|
||||
{"no containers (UI Stop)", func(s *stacks.Stack) { s.Containers = nil }, false},
|
||||
{"running", func(s *stacks.Stack) { s.State = stacks.StateRunning }, false},
|
||||
{"unhealthy is not down", func(s *stacks.Stack) { s.State = stacks.StateUnhealthy }, false},
|
||||
{"restarting recovers itself", func(s *stacks.Stack) { s.State = stacks.StateRestarting }, false},
|
||||
{"paused is deliberate", func(s *stacks.Stack) { s.State = stacks.StatePaused }, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := base
|
||||
s.Containers = append([]stacks.ContainerInfo(nil), base.Containers...)
|
||||
tc.mut(&s)
|
||||
if got := isBootOrphan(s); got != tc.want {
|
||||
t.Fatalf("isBootOrphan = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A cancelled context (controller shutting down mid-boot) must abandon the sweep, not soldier on.
|
||||
func TestReconcile_ContextCancellationStops(t *testing.T) {
|
||||
f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}}
|
||||
r, _ := newTestReconciler(f)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
res := r.Run(ctx)
|
||||
|
||||
if f.starts["immich"] != 0 {
|
||||
t.Fatalf("StartStack called %d times on a cancelled context, want 0", f.starts["immich"])
|
||||
}
|
||||
if len(res.Candidates) != 1 {
|
||||
t.Fatalf("candidates = %v, want the app still identified", res.Candidates)
|
||||
}
|
||||
}
|
||||
|
||||
// The quiet path must be observable — "nothing to do" and "never ran" must not look identical.
|
||||
func TestReconcile_QuietPathLogsAndStartsNothing(t *testing.T) {
|
||||
f := &fakeStacks{list: []stacks.Stack{{Name: "immich", Deployed: true, State: stacks.StateRunning,
|
||||
Containers: []stacks.ContainerInfo{{Name: "immich-app", State: stacks.StateRunning}}}}}
|
||||
var buf logCapture
|
||||
r := New(f, log.New(&buf, "", 0))
|
||||
r.sleep = func(context.Context, time.Duration) {}
|
||||
|
||||
res := r.Run(context.Background())
|
||||
|
||||
if len(f.starts) != 0 {
|
||||
t.Fatalf("a healthy box must produce zero starts, got %v", f.starts)
|
||||
}
|
||||
if len(res.Candidates) != 0 {
|
||||
t.Fatalf("candidates = %v, want none", res.Candidates)
|
||||
}
|
||||
if !buf.contains("no boot-orphaned apps") {
|
||||
t.Fatalf("the quiet path logged nothing identifiable: %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
type logCapture struct{ b []byte }
|
||||
|
||||
func (l *logCapture) Write(p []byte) (int, error) { l.b = append(l.b, p...); return len(p), nil }
|
||||
func (l *logCapture) String() string { return string(l.b) }
|
||||
func (l *logCapture) contains(s string) bool {
|
||||
return len(l.b) > 0 && bytesContains(l.b, []byte(s))
|
||||
}
|
||||
|
||||
func bytesContains(hay, needle []byte) bool {
|
||||
for i := 0; i+len(needle) <= len(hay); i++ {
|
||||
if string(hay[i:i+len(needle)]) == string(needle) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var _ io.Writer = (*logCapture)(nil)
|
||||
Reference in New Issue
Block a user