fix(collector): retire deleted worktrees instead of failing the git source

internal/collector/git_failure_test.go

@@ -0,0 +1,153 @@
+package collector
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/labstack/driga/internal/cleanup"
+	"github.com/labstack/driga/internal/gitscan"
+)
+
+func TestGitWorktreeFailuresAreIsolated(t *testing.T) {
+	for _, kind := range []string{"deleted", "no-marker", "corrupt"} {
+		t.Run(kind, func(t *testing.T) {
+			d, st, plat, a := setup(t)
+			a.obs = nil
+			plat.procs = nil
+			t.Setenv("GIT_CONFIG_NOSYSTEM", "1")
+			t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull)
+			for _, key := range []string{"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR"} {
+				t.Setenv(key, "")
+				if err := os.Unsetenv(key); err != nil {
+					t.Fatal(err)
+				}
+			}
+			repo := t.TempDir()
+			linked := filepath.Join(t.TempDir(), "linked")
+			git := func(args ...string) {
+				t.Helper()
+				c := exec.Command("git", args...)
+				c.Dir = repo
+				if out, err := c.CombinedOutput(); err != nil {
+					t.Fatalf("git %v: %v: %s", args, err, out)
+				}
+			}
+			git("init", "-b", "main")
+			git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "--allow-empty", "-m", "initial")
+			git("worktree", "add", "-b", "topic", linked)
+			d.Cfg.Roots = []string{repo}
+			pass(t, d, st, Plan{Git: true, GitFull: true})
+			marker := filepath.Join(linked, ".git")
+			original, err := os.ReadFile(marker)
+			if err != nil {
+				t.Fatal(err)
+			}
+			switch kind {
+			case "deleted":
+				err = os.RemoveAll(linked)
+			case "no-marker":
+				err = os.Remove(marker)
+			case "corrupt":
+				err = os.WriteFile(marker, []byte("gitdir: missing-gitdir\n"), 0600)
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "--allow-empty", "-m", "healthy update")
+			var logs bytes.Buffer
+			old := slog.Default()
+			slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
+			defer slog.SetDefault(old)
+			at := d.Now().Add(time.Hour)
+			d.Now = func() time.Time { return at }
+			for i := 0; i < 2; i++ {
+				// A fresh scanner also proves diagnostics survive service restarts.
+				d.Git = gitscan.NewScanner(2)
+				r, err := Pass(context.Background(), d, st, Plan{Git: true, GitFull: true})
+				if err != nil || r.SourceErrors["git"] != "" {
+					t.Fatalf("git source failed: %v %+v", err, r.SourceErrors)
+				}
+				ws, err := st.Worktrees(0)
+				if err != nil {
+					t.Fatal(err)
+				}
+				want := 1
+				if kind == "corrupt" {
+					want = 2
+				}
+				if len(ws) != want {
+					t.Fatalf("inventory: %+v", ws)
+				}
+				for _, w := range ws {
+					if w.Path == gitscan.NormalizePath(repo) && w.LastCommitSubject != "healthy update" {
+						t.Fatalf("healthy checkout stale: %+v", w)
+					}
+					if w.Path == gitscan.NormalizePath(linked) {
+						if w.ScanError == "" || w.LastCommitSubject != "initial" {
+							t.Fatalf("missing failure or lost previous observation: %+v", w)
+						}
+					}
+				}
+				sources, err := st.Sources()
+				if err != nil {
+					t.Fatal(err)
+				}
+				for _, s := range sources {
+					if s.Name == "git" && !s.LastOKAt.Equal(at) {
+						t.Fatalf("freshness: %+v", s)
+					}
+				}
+				candidates, err := cleanup.List(context.Background(), cleanup.Env{Store: st, Plat: plat, Sigs: d.Sigs, Home: os.Getenv("HOME"), DataDir: t.TempDir(), Now: at, Cfg: d.Cfg.Cleanup, Why: true, TempRoots: []string{t.TempDir()}})
+				if err != nil {
+					t.Fatal(err)
+				}
+				for _, c := range candidates {
+					if kind != "corrupt" && c.Path == gitscan.NormalizePath(linked) {
+						t.Fatalf("retired cleanup candidate: %+v", c)
+					}
+				}
+			}
+			count := strings.Count(logs.String(), "git_worktree_failed")
+			want := 0
+			if kind == "corrupt" {
+				want = 1
+			}
+			if count != want {
+				t.Fatalf("failure logs=%d want=%d: %s", count, want, logs.String())
+			}
+			if kind == "corrupt" {
+				if err := os.WriteFile(marker, original, 0600); err != nil {
+					t.Fatal(err)
+				}
+				pass(t, d, st, Plan{Git: true, GitFull: true})
+				ws, err := st.Worktrees(0)
+				if err != nil {
+					t.Fatal(err)
+				}
+				for _, w := range ws {
+					if w.ScanError != "" {
+						t.Fatalf("recovery retained error: %+v", w)
+					}
+				}
+				// A new diagnostic logs once; recurrence of a previously seen one
+				// stays quiet even after recovery.
+				for _, contents := range []string{"invalid gitfile\n", "invalid gitfile\n", "gitdir: missing-gitdir\n"} {
+					if err := os.WriteFile(marker, []byte(contents), 0600); err != nil {
+						t.Fatal(err)
+					}
+					pass(t, d, st, Plan{Git: true, GitFull: true})
+				}
+				if count := strings.Count(logs.String(), "git_worktree_failed"); count != 2 {
+					t.Fatalf("distinct failure logs=%d: %s", count, logs.String())
+				}
+			}
+		})
+	}
+}

internal/collector/sources.go

@@ -2,8 +2,10 @@ package collector
 
 import (
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
+	"log/slog"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -17,6 +19,7 @@ import (
 	"github.com/labstack/driga/internal/gitscan"
 	"github.com/labstack/driga/internal/model"
 	"github.com/labstack/driga/internal/platform"
+	"github.com/labstack/driga/internal/scrub"
 	"github.com/labstack/driga/internal/store"
 )
 
@@ -477,14 +480,38 @@ func writeGit(tx *store.Tx, repos []repoResult, scanErr error, source func(strin
 			return err
 		}
 		report.Repos++
+		previous, err := tx.Worktrees(id)
+		if err != nil {
+			return err
+		}
+		byPath := make(map[string]model.Worktree, len(previous))
+		for _, w := range previous {
+			byPath[w.Path] = w
+		}
 		for _, w := range repo.worktrees {
+			if w.ScanError != "" {
+				message := scrub.Scrub(w.ScanError)
+				if old, ok := byPath[w.Path]; ok {
+					w = old
+				}
+				w.ScanError = message
+				key := fmt.Sprintf("git_worktree_error:%x", sha256.Sum256([]byte(w.Path+"\x00"+message)))
+				if _, err := tx.Meta(key); errors.Is(err, store.ErrNotFound) {
+					if err := tx.SetMeta(key, "1"); err != nil {
+						return err
+					}
+					slog.Warn("git worktree scan failed", "event", "git_worktree_failed", "path", w.Path, "err", message)
+				} else if err != nil {
+					return err
+				}
+			}
 			w.RepoID = id
 			if _, err := tx.UpsertWorktree(w); err != nil {
 				return err
 			}
 		}
-		// Only a complete scan can prove absence. A failed status/log read
-		// returns a partial list and must retain the last known inventory.
+		// Checkout failures remain in inventory with their diagnostics. Only
+		// a repository-level inventory failure prevents reconciliation.
 		if repo.err == nil {
 			previous, err := tx.Worktrees(id)
 			if err != nil {

internal/gitscan/scan.go

@@ -284,6 +284,10 @@ func (s *Scanner) git(ctx context.Context, dir string, args ...string) ([]byte,
 		return nil, ctx.Err()
 	}
 	if err != nil {
+		var exit *exec.ExitError
+		if errors.As(err, &exit) && len(exit.Stderr) != 0 {
+			return nil, fmt.Errorf("git %s in %s: %w: %s", strings.Join(args, " "), dir, err, strings.TrimSpace(string(exit.Stderr)))
+		}
 		return nil, fmt.Errorf("git %s in %s: %w", strings.Join(args, " "), dir, err)
 	}
 	return out, nil
@@ -328,7 +332,9 @@ const (
 )
 
 // Scan reads the repository's non-bare worktrees. A full scan bypasses metadata
-// caching; its caller schedules one at least every five minutes.
+// caching; its caller schedules one at least every five minutes. Missing checkouts
+// are omitted; status/log failures remain in inventory with ScanError populated.
+// Repository inventory failures and cancellation are returned as errors.
 func (s *Scanner) Scan(ctx context.Context, repoPath string, mode ScanMode) ([]model.Worktree, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -391,12 +397,13 @@ func (s *Scanner) Scan(ctx context.Context, repoPath string, mode ScanMode) ([]m
 		if entry.Bare {
 			continue
 		}
-		// A linked worktree whose directory was deleted stays listed until
-		// `git worktree prune`; it has nothing to scan.
-		if _, err := os.Stat(entry.Path); errors.Is(err, fs.ErrNotExist) {
+		// Git retains registrations after directories or their markers disappear.
+		// Lstat distinguishes an absent marker from a damaged gitdir/symlink.
+		if goneWorktree(entry.Path) {
+			s.mu.Lock()
+			delete(s.cache, entry.Path)
+			s.mu.Unlock()
 			continue
-		} else if err != nil {
-			return nil, err
 		}
 		entries = append(entries, entry)
 	}
@@ -431,14 +438,40 @@ func (s *Scanner) Scan(ctx context.Context, repoPath string, mode ScanMode) ([]m
 	}
 	close(jobs)
 	wg.Wait()
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
 	var worktrees []model.Worktree
 	for i, w := range results {
-		if errs[i] == nil {
-			worktrees = append(worktrees, w)
+		if errs[i] != nil {
+			// A checkout may disappear between inventory and status.
+			if goneWorktree(entries[i].Path) {
+				continue
+			}
+			s.mu.Lock()
+			w = s.cache[entries[i].Path].Worktree
+			// Retry failures even when metadata timestamps have not changed.
+			delete(s.cache, entries[i].Path)
+			s.mu.Unlock()
+			w.Path = entries[i].Path
+			w.ScanError = errs[i].Error()
 		}
+		worktrees = append(worktrees, w)
 	}
 	sort.Slice(worktrees, func(i, j int) bool { return worktrees[i].Path < worktrees[j].Path })
-	return worktrees, errors.Join(errs...)
+	return worktrees, nil
+}
+
+func goneWorktree(path string) bool {
+	info, err := os.Stat(path)
+	if errors.Is(err, fs.ErrNotExist) || (err == nil && !info.IsDir()) {
+		return true
+	}
+	if err != nil {
+		return false
+	}
+	_, err = os.Lstat(filepath.Join(path, ".git"))
+	return errors.Is(err, fs.ErrNotExist)
 }
 
 func (s *Scanner) scanWorktree(ctx context.Context, entry WorktreeEntry, full bool) (model.Worktree, error) {

internal/gitscan/scanner_pruning_test.go

@@ -24,7 +24,10 @@ func TestScannerPrunesRemovedRepositoriesAndWorktrees(t *testing.T) {
 	var paths []string
 	for i := 0; i < 32; i++ {
 		p := filepath.Join(base, fmt.Sprint(i))
-		if err := os.MkdirAll(filepath.Join(p, "linked"), 0700); err != nil {
+		if err := os.MkdirAll(filepath.Join(p, "linked", ".git"), 0700); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.MkdirAll(filepath.Join(p, ".git"), 0700); err != nil {
 			t.Fatal(err)
 		}
 		paths = append(paths, p)

internal/gitscan/worktree_failure_test.go

@@ -0,0 +1,78 @@
+package gitscan
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+)
+
+func TestScanRetiresMissingMarkersAndRetriesCorruption(t *testing.T) {
+	gitEnv(t)
+	repo := initRepo(t, filepath.Join(t.TempDir(), "repo"))
+	linked := filepath.Join(t.TempDir(), "linked")
+	gitTest(t, repo, "worktree", "add", "-b", "topic", linked)
+	linked = NormalizePath(linked)
+	scanner := NewScanner(2)
+	if _, err := scanner.Scan(context.Background(), repo, Full); err != nil {
+		t.Fatal(err)
+	}
+	marker := filepath.Join(linked, ".git")
+	original, err := os.ReadFile(marker)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Remove(marker); err != nil {
+		t.Fatal(err)
+	}
+	var polls atomic.Int32
+	run := scanner.runGit
+	scanner.runGit = func(ctx context.Context, dir string, args ...string) ([]byte, error) {
+		if dir == linked {
+			polls.Add(1)
+		}
+		return run(ctx, dir, args...)
+	}
+	for _, mode := range []ScanMode{Cached, Full, Identity} {
+		ws, err := scanner.Scan(context.Background(), repo, mode)
+		if err != nil || len(ws) != 1 || ws[0].Path != repo {
+			t.Fatalf("retired inventory: %+v %v", ws, err)
+		}
+	}
+	if polls.Load() != 0 {
+		t.Fatalf("retired checkout polled %d times", polls.Load())
+	}
+	if err := os.WriteFile(marker, []byte("gitdir: missing\n"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 2; i++ {
+		ws, err := scanner.Scan(context.Background(), repo, Cached)
+		if err != nil || len(ws) != 2 {
+			t.Fatalf("partial scan: %+v %v", ws, err)
+		}
+		for _, w := range ws {
+			if w.Path == linked && w.ScanError == "" {
+				t.Fatalf("missing diagnostic: %+v", w)
+			}
+			if w.Path == repo && (w.ScanError != "" || w.Head == "") {
+				t.Fatalf("healthy checkout: %+v", w)
+			}
+		}
+	}
+	if polls.Load() != 2 {
+		t.Fatalf("failed checkout was not retried: %d", polls.Load())
+	}
+	if err := os.WriteFile(marker, original, 0600); err != nil {
+		t.Fatal(err)
+	}
+	ws, err := scanner.Scan(context.Background(), repo, Cached)
+	if err != nil || len(ws) != 2 {
+		t.Fatalf("recovery: %+v %v", ws, err)
+	}
+	for _, w := range ws {
+		if w.ScanError != "" || w.LastCommitSubject == "" {
+			t.Fatalf("failed recovery: %+v", w)
+		}
+	}
+}

internal/model/model.go

@@ -104,6 +104,7 @@ type Repo struct {
 
 // Worktree is one checkout of a repo.
 type Worktree struct {
+	ScanError         string    `json:"scan_error,omitempty"`
 	MachineID         string    `json:"machine_id"`
 	ID                int64     `json:"id"`
 	RepoID            int64     `json:"repo_id"`