fix(sync): 120s deadline on git subprocesses — hung remote no longer wedges sync (campaign F3)
runGitInDir had no context: a hanging remote parked the sync goroutine in cmd.Run(), the doSync defer never ran, `syncing` stayed true, and every manual + periodic sync was refused with "Szinkronizálás már folyamatban" until a controller restart. Each git command now runs under exec.CommandContext with a fresh per-command gitCmdTimeout (120s); the deadline error names the timeout and the (masked) git args. Debounce and failed-sync-arms-debounce unchanged. Tests: T-C1 cancelled-context kills the subprocess promptly (red-proof: pre-fix exec.Command shape runs to completion → test FAILS); T-C2 failed sync releases `syncing` and a post-debounce retry EXECUTES. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -2,6 +2,7 @@ package sync
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -18,6 +19,13 @@ import (
|
|||||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// gitCmdTimeout bounds each individual git subprocess (campaign finding F3, 2026-07-06):
|
||||||
|
// without a deadline, a hung remote parks the sync goroutine inside cmd.Run() — the doSync
|
||||||
|
// defer never runs, `syncing` stays true, and every manual + periodic sync is refused with
|
||||||
|
// "Szinkronizálás már folyamatban" until a controller restart. 120s is generous for the
|
||||||
|
// shallow catalog clone/fetch.
|
||||||
|
const gitCmdTimeout = 120 * time.Second
|
||||||
|
|
||||||
// Syncer handles periodic git sync of the app catalog to the local stacks directory.
|
// Syncer handles periodic git sync of the app catalog to the local stacks directory.
|
||||||
type Syncer struct {
|
type Syncer struct {
|
||||||
cfg *config.Config
|
cfg *config.Config
|
||||||
@@ -270,10 +278,10 @@ func (s *Syncer) gitCloneOrPull() error {
|
|||||||
if s.isDebug() {
|
if s.isDebug() {
|
||||||
s.logger.Printf("[DEBUG] [sync] git fetch --depth 1 origin %s in %s", s.cfg.Git.Branch, s.cacheDir)
|
s.logger.Printf("[DEBUG] [sync] git fetch --depth 1 origin %s in %s", s.cfg.Git.Branch, s.cacheDir)
|
||||||
}
|
}
|
||||||
if err := s.runGitInDir(s.cacheDir, "fetch", "--depth", "1", "origin", s.cfg.Git.Branch); err != nil {
|
if err := s.gitCmd(s.cacheDir, "fetch", "--depth", "1", "origin", s.cfg.Git.Branch); err != nil {
|
||||||
return fmt.Errorf("git fetch: %w", err)
|
return fmt.Errorf("git fetch: %w", err)
|
||||||
}
|
}
|
||||||
if err := s.runGitInDir(s.cacheDir, "reset", "--hard", "origin/"+s.cfg.Git.Branch); err != nil {
|
if err := s.gitCmd(s.cacheDir, "reset", "--hard", "origin/"+s.cfg.Git.Branch); err != nil {
|
||||||
return fmt.Errorf("git reset: %w", err)
|
return fmt.Errorf("git reset: %w", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -414,14 +422,22 @@ func copyIfChanged(src, dst string) (bool, error) {
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// runGit executes a git command with the given args.
|
// runGit executes a git command with the given args under the standard per-command deadline.
|
||||||
func (s *Syncer) runGit(args ...string) error {
|
func (s *Syncer) runGit(args ...string) error {
|
||||||
return s.runGitInDir("", args...)
|
return s.gitCmd("", args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// runGitInDir executes a git command in the specified directory.
|
// gitCmd runs one git command in dir under a fresh gitCmdTimeout deadline (per-command,
|
||||||
func (s *Syncer) runGitInDir(dir string, args ...string) error {
|
// not per-sync — a multi-step sync gets a full budget for each subprocess).
|
||||||
cmd := exec.Command("git", args...)
|
func (s *Syncer) gitCmd(dir string, args ...string) error {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), gitCmdTimeout)
|
||||||
|
defer cancel()
|
||||||
|
return s.runGitInDir(ctx, dir, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runGitInDir executes a git command in the specified directory, killed when ctx expires.
|
||||||
|
func (s *Syncer) runGitInDir(ctx context.Context, dir string, args ...string) error {
|
||||||
|
cmd := exec.CommandContext(ctx, "git", args...)
|
||||||
if dir != "" {
|
if dir != "" {
|
||||||
cmd.Dir = dir
|
cmd.Dir = dir
|
||||||
}
|
}
|
||||||
@@ -433,7 +449,13 @@ func (s *Syncer) runGitInDir(dir string, args ...string) error {
|
|||||||
s.logger.Printf("[DEBUG] [sync] Running: git %s", maskRepoURL(strings.Join(args, " ")))
|
s.logger.Printf("[DEBUG] [sync] Running: git %s", maskRepoURL(strings.Join(args, " ")))
|
||||||
|
|
||||||
if err := cmd.Run(); err != nil {
|
if err := cmd.Run(); err != nil {
|
||||||
return fmt.Errorf("git %s: %w\nstderr: %s", strings.Join(args, " "), err, stderr.String())
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
// Subprocess was killed by the deadline/cancellation — name it explicitly so the
|
||||||
|
// failure is diagnosable from SyncResult/logs (never a silent hang).
|
||||||
|
return fmt.Errorf("git %s: %v (subprocess killed at the %s deadline): %w\nstderr: %s",
|
||||||
|
maskRepoURL(strings.Join(args, " ")), ctxErr, gitCmdTimeout, err, stderr.String())
|
||||||
|
}
|
||||||
|
return fmt.Errorf("git %s: %w\nstderr: %s", maskRepoURL(strings.Join(args, " ")), err, stderr.String())
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package sync
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newTestSyncer(t *testing.T, repoURL string) *Syncer {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := &config.Config{}
|
||||||
|
cfg.Git.RepoURL = repoURL
|
||||||
|
cfg.Git.Branch = "main"
|
||||||
|
cfg.Git.SyncInterval = "15m"
|
||||||
|
cfg.Paths.DataDir = filepath.Join(dir, "data")
|
||||||
|
cfg.Paths.StacksDir = filepath.Join(dir, "stacks")
|
||||||
|
if err := os.MkdirAll(cfg.Paths.StacksDir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
logger := log.New(os.Stderr, "", 0)
|
||||||
|
return New(cfg, logger, func() error { return nil }, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// T-C1 (campaign F3): a git subprocess must die with its context — the pre-fix
|
||||||
|
// exec.Command shape ignored the context entirely, so a hung remote parked the sync
|
||||||
|
// goroutine in cmd.Run() forever, holding `syncing=true` until a controller restart.
|
||||||
|
// Effect asserted: the returned error NAMES the context cause (not a generic git error),
|
||||||
|
// and the call returns promptly instead of running the command to completion.
|
||||||
|
func TestRunGitInDir_CancelledContextKillsSubprocess(t *testing.T) {
|
||||||
|
s := newTestSyncer(t, "https://example.invalid/repo.git")
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel() // already cancelled — the subprocess must not be allowed to run to completion
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
err := s.runGitInDir(ctx, "", "--version")
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected an error from a cancelled context, got nil (git --version ran to completion — the pre-fix behavior)")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), context.Canceled.Error()) {
|
||||||
|
t.Fatalf("error must name the context cause; got: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "deadline") && !strings.Contains(err.Error(), "killed") {
|
||||||
|
t.Fatalf("error must say the subprocess was killed at the deadline; got: %v", err)
|
||||||
|
}
|
||||||
|
if elapsed > 5*time.Second {
|
||||||
|
t.Fatalf("cancelled-context git call took %v — subprocess not killed promptly", elapsed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// T-C2 (campaign F3): a failed sync must release the single-flight `syncing` flag and a
|
||||||
|
// later TriggerSync (past the debounce) must EXECUTE — the campaign symptom was every
|
||||||
|
// subsequent sync refused with "Szinkronizálás már folyamatban". Uses a nonexistent local
|
||||||
|
// repo_url so git fails fast with no network.
|
||||||
|
func TestTriggerSync_FailureReleasesSyncingAndAllowsRetry(t *testing.T) {
|
||||||
|
s := newTestSyncer(t, filepath.Join(t.TempDir(), "no-such-repo"))
|
||||||
|
|
||||||
|
res := s.TriggerSync()
|
||||||
|
if res.OK {
|
||||||
|
t.Fatalf("sync against a nonexistent repo must fail; got OK with message %q", res.Message)
|
||||||
|
}
|
||||||
|
if !strings.Contains(res.Message, "Git hiba") {
|
||||||
|
t.Fatalf("failure must carry the git error to the caller (no silent swallow); got %q", res.Message)
|
||||||
|
}
|
||||||
|
|
||||||
|
st := s.Status()
|
||||||
|
if st.Syncing {
|
||||||
|
t.Fatalf("`syncing` still true after a failed sync — the single-flight flag leaked (the F3 lockup)")
|
||||||
|
}
|
||||||
|
if st.LastStatus != "error" || st.LastError == "" {
|
||||||
|
t.Fatalf("failed sync must be recorded: LastStatus=%q LastError=%q", st.LastStatus, st.LastError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Debounce window: an immediate retry is refused (existing behavior, unchanged).
|
||||||
|
res2 := s.TriggerSync()
|
||||||
|
if res2.OK || !strings.Contains(res2.Message, "Túl gyakori") {
|
||||||
|
t.Fatalf("immediate retry should hit the 30s debounce; got OK=%v message=%q", res2.OK, res2.Message)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Past the debounce (rewind lastSync directly — same-package seam), the retry must
|
||||||
|
// EXECUTE: it reaches git again and fails with the git error, not a refusal message.
|
||||||
|
s.mu.Lock()
|
||||||
|
s.lastSync = time.Now().Add(-time.Minute)
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
res3 := s.TriggerSync()
|
||||||
|
if res3.OK {
|
||||||
|
t.Fatalf("retry against the same nonexistent repo should still fail: %q", res3.Message)
|
||||||
|
}
|
||||||
|
if !strings.Contains(res3.Message, "Git hiba") {
|
||||||
|
t.Fatalf("retry past the debounce must EXECUTE (git error expected); got refusal/other: %q", res3.Message)
|
||||||
|
}
|
||||||
|
if s.Status().Syncing {
|
||||||
|
t.Fatal("`syncing` leaked after the retry")
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user