diff --git a/internal/indexer/go_module_content_stamp_test.go b/internal/indexer/go_module_content_stamp_test.go new file mode 100644 index 000000000..8b1e49064 --- /dev/null +++ b/internal/indexer/go_module_content_stamp_test.go @@ -0,0 +1,222 @@ +package indexer + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +// Given cached source facts, when equal-size bytes change with restored mtime, +// then provider recovery must re-extract the current calls and match cold. +func TestGoModuleSameStampSourceRecoveryColdEquivalent(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + main := filepath.Join(repos[0].Path, "main.go") + lib := filepath.Join(repos[1].Path, "lib.go") + before := "package caller\nimport library \"example.org/library\"\nfunc Caller(){ library.Hello(library.Missing()) }\n" + after := strings.Replace(before, "Missing", "Present", 1) + require.Equal(t, len(before), len(after)) + writeFile(t, main, before) + // The cache records its read at the file's own mtime, so the + // stat stays inside the racy window however slow the setup is. + recent := time.Unix(1700000000, 0) + require.NoError(t, os.Chtimes(main, recent, recent)) + pinGoModuleSourceClock(t, recent) + writeFile(t, lib, "package library\nfunc Other() {}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) + require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") + info, err := os.Stat(main) + require.NoError(t, err) + writeFile(t, main, after) + stamp := info.ModTime() + require.NoError(t, os.Chtimes(main, stamp, stamp)) + changed, err := os.Stat(main) + require.NoError(t, err) + require.Equal(t, info.Size(), changed.Size()) + require.Equal(t, info.ModTime().UnixNano(), changed.ModTime().UnixNano()) + writeFile(t, lib, "package library\nfunc Present() int{return 1}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + cold := revisionColdGraph(t, backend, repos[:2]) + require.ElementsMatch(t, []string{"external-call::dep::example.org/library", "b/lib.go::Present"}, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls)) + assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) + }) + } +} + +func TestGoModuleSnapshotRejectsFailedInputs(t *testing.T) { + for _, failure := range []string{"stat", "read"} { + t.Run(failure, func(t *testing.T) { + root := t.TempDir() + if failure == "read" { + require.NoError(t, os.Mkdir(filepath.Join(root, "go.mod"), 0o755)) + } else { + root = "\x00" + } + core, logs := observer.New(zap.WarnLevel) + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"b": {RootPath: root}}, logger: zap.New(core)} + // Equal failed stat/read results must not be valid cache evidence. + modules, _ := mi.goModuleSnapshot() + require.Empty(t, modules) + require.Len(t, logs.All(), 1) + mi.goModuleChecked = time.Time{} + modules, _ = mi.goModuleSnapshot() + assert.Empty(t, modules) + assert.Len(t, logs.All(), 2, "failed inputs must rebuild/refuse on each eligible check") + }) + } +} + +func TestGoModuleSourceFailureDoesNotRestoreCachedCalls(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, failure := range []string{"stat", "read", "size"} { + t.Run(backend+"/"+failure, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + main := filepath.Join(repos[0].Path, "main.go") + // Keep the caller's stat racy, so a cached fact can only be + // reused after a read, independent of runner speed. + recent := time.Unix(1700000000, 0) + require.NoError(t, os.Chtimes(main, recent, recent)) + pinGoModuleSourceClock(t, recent) + writeFile(t, filepath.Join(repos[1].Path, "lib.go"), "package library\nfunc Other(){}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) + require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") + // Publish a recoverable target without retrying the terminal. + g.AddNode(&graph.Node{ID: "b/lib.go::Hello", Name: "Hello", Kind: graph.KindFunction, RepoPrefix: "b", FilePath: "b/lib.go", WorkspaceID: "shared", Language: "go"}) + cr := mi.newCrossRepoResolver() + before := revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls) + original := readGoModuleCallerFile + reads := 0 + readGoModuleCallerFile = func(string) ([]byte, error) { reads++; return nil, os.ErrPermission } + t.Cleanup(func() { readGoModuleCallerFile = original }) + switch failure { + case "stat": + require.NoError(t, os.Remove(main)) + case "size": + mi.GetIndexer("a").config.MaxFileSize = 1 + } + // A cached fact is not evidence for unreadable current source. + mi.restoreGoModuleSourceFile(cr, "a/main.go") + if failure == "read" { + assert.Equal(t, 1, reads) + } else { + assert.Zero(t, reads) + } + assert.ElementsMatch(t, before, revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) + }) + } + } +} + +func TestGoModuleContentStampReusesMixedRootSnapshot(t *testing.T) { + goRoot, nonGoRoot := t.TempDir(), t.TempDir() + manifest := filepath.Join(goRoot, "go.mod") + writeFile(t, manifest, "module example.org/library\n") + original := readGoModuleFile + reads := map[string]int{} + readGoModuleFile = func(path string) ([]byte, error) { reads[path]++; return original(path) } + t.Cleanup(func() { readGoModuleFile = original }) + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"go": {RootPath: goRoot}, "js": {RootPath: nonGoRoot}}, logger: zap.NewNop()} + // Given one module and one valid absence, when an eligible unchanged + // snapshot check runs, then only the content-verification read is needed. + first, generation := mi.goModuleSnapshot() + require.Equal(t, map[string]string{"go": "example.org/library"}, first) + require.Equal(t, 1, reads[manifest], "verification, module identity, and replacements share one read") + clear(reads) + mi.goModuleChecked = time.Time{} + second, nextGeneration := mi.goModuleSnapshot() + assert.Equal(t, first, second) + assert.Equal(t, generation, nextGeneration) + assert.Equal(t, map[string]int{manifest: 1, filepath.Join(nonGoRoot, "go.mod"): 1}, reads, "one read attempt per root per eligible check") +} + +func TestGoModuleContentStampAndParsedEvidenceShareBytes(t *testing.T) { + root := t.TempDir() + x := []byte("module example.org/library\nreplace example.org/dep => example.org/other v1.2.3\n") + y := []byte("module example.org/renamed\n") + writeFile(t, filepath.Join(root, "go.mod"), string(x)) + original := readGoModuleFile + reads := 0 + readGoModuleFile = func(string) ([]byte, error) { + reads++ + if reads == 1 { + return x, nil + } + return y, nil + } + t.Cleanup(func() { readGoModuleFile = original }) + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"lib": {RootPath: root}}, logger: zap.NewNop()} + modules, _ := mi.goModuleSnapshot() + assert.Equal(t, 1, reads, "one captured read must supply hash, identity and replacements") + assert.Equal(t, "example.org/library", modules["lib"]) + assert.Equal(t, contentHashForSource(x), mi.goModuleFiles["lib"].content) + require.Len(t, mi.goModuleReplacements["lib"], 1) + assert.Equal(t, "example.org/dep", mi.goModuleReplacements["lib"][0].Module) +} + +// pinGoModuleSourceClock records caller-cache reads at a fixed time, so each +// test takes one branch of the racy-file rule regardless of runner speed. +func pinGoModuleSourceClock(t *testing.T, at time.Time) { + t.Helper() + original := goModuleSourceNow + goModuleSourceNow = func() time.Time { return at } + t.Cleanup(func() { goModuleSourceNow = original }) +} + +// Given a recovery that reads a caller before a save of that caller is known, +// when it stores its result, then it keeps the facts but not the stat trust: +// the next recovery reads again instead of reusing pre-save bytes. +func TestGoModuleSourceStoreDropsTrustAfterConcurrentSave(t *testing.T) { + mi, _, repos := goModuleLifecycleFixture(t, "memory") + main := filepath.Join(repos[0].Path, "main.go") + now := time.Unix(1700000000, 0) + old := now.Add(-time.Hour) + require.NoError(t, os.Chtimes(main, old, old)) + pinGoModuleSourceClock(t, now) + writeFile(t, filepath.Join(repos[1].Path, "lib.go"), "package library\nfunc Other(){}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) + idx := mi.GetIndexer("a") + require.Contains(t, idx.goModuleSourceCache, "main.go") + // Touch the stat so the next recovery verifies by reading. + touched := old.Add(-time.Hour) + require.NoError(t, os.Chtimes(main, touched, touched)) + original := readGoModuleCallerFile + reads := 0 + readGoModuleCallerFile = func(path string) ([]byte, error) { + reads++ + src, err := original(path) + if reads == 1 { + idx.invalidateGoModuleSourceStamp("main.go") // a save lands during the read + } + return src, err + } + t.Cleanup(func() { readGoModuleCallerFile = original }) + cr := mi.newCrossRepoResolver() + mi.restoreGoModuleSourceFile(cr, "a/main.go") + require.Equal(t, 1, reads) + assert.True(t, idx.goModuleSourceCache["main.go"].recordedAt.IsZero(), "a result read before a known save must not be trusted by stat") + mi.restoreGoModuleSourceFile(cr, "a/main.go") + assert.Equal(t, 2, reads, "the next recovery verifies content again") + mi.restoreGoModuleSourceFile(cr, "a/main.go") + assert.Equal(t, 2, reads, "an undisturbed verification restores stat trust") +} diff --git a/internal/indexer/go_module_extern_test.go b/internal/indexer/go_module_extern_test.go index 01baee2f7..a7151757a 100644 --- a/internal/indexer/go_module_extern_test.go +++ b/internal/indexer/go_module_extern_test.go @@ -257,7 +257,10 @@ func Caller() { library.Hello(); child.Nested() } } } -func TestReadGoModModule(t *testing.T) { +func TestGoModuleIdentities(t *testing.T) { + identity := func(root string) string { + return goModuleIdentities(readGoModuleInputs(map[string]string{"r": root}))["r"] + } for _, source := range []string{ "module example.org/library\n", "module\texample.org/library\n", @@ -266,15 +269,15 @@ func TestReadGoModModule(t *testing.T) { t.Run(source, func(t *testing.T) { root := t.TempDir() writeFile(t, filepath.Join(root, "go.mod"), source) - assert.Equal(t, "example.org/library", readGoModModule(root)) + assert.Equal(t, "example.org/library", identity(root)) }) } for _, source := range []string{"", "module", "module \"unterminated"} { root := t.TempDir() writeFile(t, filepath.Join(root, "go.mod"), source) - assert.Empty(t, readGoModModule(root)) + assert.Empty(t, identity(root)) } - assert.Empty(t, readGoModModule(t.TempDir())) + assert.Empty(t, identity(t.TempDir())) } func TestTrackedGoModuleCacheRevision(t *testing.T) { @@ -298,6 +301,7 @@ func TestTrackedGoModuleCacheRevision(t *testing.T) { cached, cachedGeneration := mi.goModuleSnapshot() assert.Equal(t, "example.org/new", cached["b"], "root go.mod edits refresh without config reload") assert.Greater(t, cachedGeneration, generation) + mi.goModuleChecked = time.Time{} // Exercise unchanged content after the throttle expires. unchanged, unchangedGeneration := mi.goModuleSnapshot() assert.Equal(t, cached, unchanged) assert.Equal(t, cachedGeneration, unchangedGeneration) diff --git a/internal/indexer/go_module_replacements.go b/internal/indexer/go_module_replacements.go index d93371bca..00dab29b3 100644 --- a/internal/indexer/go_module_replacements.go +++ b/internal/indexer/go_module_replacements.go @@ -21,10 +21,15 @@ func (mi *MultiIndexer) trackedGoModuleReplacements() map[string][]resolver.GoMo } func (mi *MultiIndexer) readGoModuleReplacements(roots map[string]string) map[string][]resolver.GoModuleReplacement { + return mi.goModuleReplacementsFromInputs(roots, readGoModuleInputs(roots)) +} + +func (mi *MultiIndexer) goModuleReplacementsFromInputs(roots map[string]string, inputs map[string]goModuleInput) map[string][]resolver.GoModuleReplacement { replacements := make(map[string][]resolver.GoModuleReplacement) for repo, root := range roots { filename := filepath.Join(root, "go.mod") - data, err := os.ReadFile(filename) + input := inputs[repo] + data, err := input.data, input.err if os.IsNotExist(err) { continue } diff --git a/internal/indexer/go_module_revision_test.go b/internal/indexer/go_module_revision_test.go index 5b9f1b012..ec6d29603 100644 --- a/internal/indexer/go_module_revision_test.go +++ b/internal/indexer/go_module_revision_test.go @@ -86,9 +86,19 @@ func TestGoModuleSameLineCallsColdEquivalent(t *testing.T) { if mode == "rename_back" { manifest := filepath.Join(repos[1].Path, "go.mod") writeFile(t, manifest, "module example.org/renamed\n") + frozen := time.Now().Truncate(time.Second) + require.NoError(t, os.Chtimes(manifest, frozen, frozen)) + renamed, err := os.Stat(manifest) + require.NoError(t, err) mi.goModuleChecked = time.Time{} mi.RunGlobalResolve() + require.ElementsMatch(t, []string{"external-call::dep::example.org/library"}, revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) writeFile(t, manifest, "module example.org/library\n") + require.NoError(t, os.Chtimes(manifest, renamed.ModTime(), renamed.ModTime())) + restored, err := os.Stat(manifest) + require.NoError(t, err) + require.Equal(t, renamed.Size(), restored.Size()) + require.Equal(t, renamed.ModTime(), restored.ModTime()) mi.goModuleChecked = time.Time{} mi.RunGlobalResolve() } @@ -416,35 +426,44 @@ func TestGoModuleSameSizeSourceStampColdEquivalent(t *testing.T) { // the caller, then a provider save must recover C2's calls rather than C1's. func TestGoModuleSameStampCallerReindexColdEquivalent(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { - t.Run(backend, func(t *testing.T) { - mi, g, repos := goModuleLifecycleFixture(t, backend) - main := filepath.Join(repos[0].Path, "main.go") - lib := filepath.Join(repos[1].Path, "lib.go") - before := "package caller\nimport library \"example.org/library\"\nfunc Caller(){ library.Hello(library.Missing()) }\n" - after := strings.Replace(before, "Missing", "Present", 1) - require.Equal(t, len(before), len(after)) - writeFile(t, main, before) - writeFile(t, lib, "package library\nfunc Other(){}\n") - for _, entry := range repos[:2] { - _, err := mi.TrackRepoCtx(t.Context(), entry) + for _, mode := range []string{"recent", "old_point"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + main := filepath.Join(repos[0].Path, "main.go") + lib := filepath.Join(repos[1].Path, "lib.go") + before := "package caller\nimport library \"example.org/library\"\nfunc Caller(){ library.Hello(library.Missing()) }\n" + after := strings.Replace(before, "Missing", "Present", 1) + require.Equal(t, len(before), len(after)) + writeFile(t, main, before) + now := time.Unix(1700000000, 0) + pinGoModuleSourceClock(t, now) + stamp := now // inside the racy window + if mode != "recent" { + stamp = now.Add(-time.Hour) + } + require.NoError(t, os.Chtimes(main, stamp, stamp)) + writeFile(t, lib, "package library\nfunc Other(){}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) + require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") + info, err := os.Stat(main) require.NoError(t, err) - } - mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) - require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") - info, err := os.Stat(main) - require.NoError(t, err) - writeFile(t, main, after) - require.NoError(t, os.Chtimes(main, info.ModTime(), info.ModTime())) - changed, err := os.Stat(main) - require.NoError(t, err) - require.Equal(t, info.Size(), changed.Size()) - require.Equal(t, info.ModTime().UnixNano(), changed.ModTime().UnixNano()) - require.NoError(t, mi.GetIndexer("a").IndexFile(main)) - writeFile(t, lib, "package library\nfunc Missing() int{return 0}\nfunc Present() int{return 1}\n") - require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - cold := revisionColdGraph(t, backend, repos[:2]) - require.ElementsMatch(t, []string{"external-call::dep::example.org/library", "b/lib.go::Present"}, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls)) - assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) - }) + writeFile(t, main, after) + require.NoError(t, os.Chtimes(main, info.ModTime(), info.ModTime())) + changed, err := os.Stat(main) + require.NoError(t, err) + require.Equal(t, info.Size(), changed.Size()) + require.Equal(t, info.ModTime().UnixNano(), changed.ModTime().UnixNano()) + require.NoError(t, mi.GetIndexer("a").IndexFile(main)) + writeFile(t, lib, "package library\nfunc Missing() int{return 0}\nfunc Present() int{return 1}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + cold := revisionColdGraph(t, backend, repos[:2]) + require.ElementsMatch(t, []string{"external-call::dep::example.org/library", "b/lib.go::Present"}, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls)) + assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) + }) + } } } diff --git a/internal/indexer/go_module_source_calls.go b/internal/indexer/go_module_source_calls.go index 773eab4b6..13b256f5b 100644 --- a/internal/indexer/go_module_source_calls.go +++ b/internal/indexer/go_module_source_calls.go @@ -1,12 +1,14 @@ package indexer import ( + "context" "crypto/sha256" "github.com/zzet/gortex/internal/parser/crashpool" "maps" "os" "path/filepath" "strings" + "time" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/resolver" @@ -14,16 +16,24 @@ import ( ) // goModuleSourceCacheEntry keeps source facts, not resolution decisions. -// Matching size and mtime reuse those facts. If either differs, the digest -// decides whether to reuse them. +// Equal stats are trusted only when the recorded read was outside the racy window. type goModuleSourceCacheEntry struct { digest [32]byte edges []*graph.Edge size, modified int64 + recordedAt time.Time } +// FAT timestamps can cover two seconds (HFS+ one). Entries recorded within +// that granularity must verify content even when size and mtime match. +const goModuleSourceRacyWindow = 2 * time.Second + var readGoModuleCallerFile = os.ReadFile +// goModuleSourceNow stamps the start of a verifying read. Tests pin it so each +// case takes one branch of the racy-file rule regardless of runner speed. +var goModuleSourceNow = time.Now + func cloneGoModuleSourceEdges(edges []*graph.Edge) []*graph.Edge { out := make([]*graph.Edge, 0, len(edges)) for _, e := range edges { @@ -37,17 +47,33 @@ func cloneGoModuleSourceEdges(edges []*graph.Edge) []*graph.Edge { return out } -// A successful replacement can change bytes without changing the stat. -// Keep source facts, but make recovery validate them with the digest. +// Known saves must verify again even when an editor restores an old mtime. +// The epoch also tells a recovery that read before this save not to trust its +// stat when it stores its result. func (idx *Indexer) invalidateGoModuleSourceStamp(rel string) { idx.goModuleSourceMu.Lock() defer idx.goModuleSourceMu.Unlock() + idx.goModuleSourceEpoch++ if cached, ok := idx.goModuleSourceCache[rel]; ok { - cached.size, cached.modified = 0, 0 + cached.recordedAt = time.Time{} idx.goModuleSourceCache[rel] = cached } } +// storeGoModuleSource keeps the facts but drops stat trust when a save was +// invalidated after epoch was read, since the bytes may predate that save. +func (idx *Indexer) storeGoModuleSource(rel string, entry goModuleSourceCacheEntry, epoch uint64) { + idx.goModuleSourceMu.Lock() + defer idx.goModuleSourceMu.Unlock() + if idx.goModuleSourceEpoch != epoch { + entry.recordedAt = time.Time{} + } + if idx.goModuleSourceCache == nil { + idx.goModuleSourceCache = make(map[string]goModuleSourceCacheEntry) + } + idx.goModuleSourceCache[rel] = entry +} + func (mi *MultiIndexer) finishGoModuleCalls(cr *resolver.CrossRepoResolver) { for _, file := range cr.TakeGoModuleSourceFiles() { mi.restoreGoModuleSourceFile(cr, file) @@ -68,19 +94,24 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver mi.logger.Warn("stat Go module caller", zap.String("file", file), zap.Error(err)) return } + if idx.config.MaxFileSize > 0 && info.Size() > idx.config.MaxFileSize { + return + } idx.goModuleSourceMu.Lock() cached, hit := idx.goModuleSourceCache[rel] + epoch := idx.goModuleSourceEpoch idx.goModuleSourceMu.Unlock() - if hit && cached.size == info.Size() && cached.modified == info.ModTime().UnixNano() { + if hit && cached.size == info.Size() && cached.modified == info.ModTime().UnixNano() && info.ModTime().Before(cached.recordedAt.Add(-goModuleSourceRacyWindow)) { cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) return } - parseLease, err := idx.acquireSharedParsePath(abs) + parseLease, err := acquireParseAdmission(context.Background(), info.Size(), 0, nil, idx.parseAdmission.Load()) if err != nil { mi.logger.Warn("admit Go module caller", zap.String("file", file), zap.Error(err)) return } defer parseLease.Release() + recordedAt := goModuleSourceNow() // Record wall time before reading the bytes being verified. src, err := readGoModuleCallerFile(abs) if err != nil { mi.logger.Warn("read Go module caller", zap.String("file", file), zap.Error(err)) @@ -96,10 +127,8 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver src = idx.transforms.run(rel, src) digest := sha256.Sum256(src) if hit && cached.digest == digest { - cached.size, cached.modified = info.Size(), info.ModTime().UnixNano() - idx.goModuleSourceMu.Lock() - idx.goModuleSourceCache[rel] = cached - idx.goModuleSourceMu.Unlock() + cached.size, cached.modified, cached.recordedAt = info.Size(), info.ModTime().UnixNano(), recordedAt + idx.storeGoModuleSource(rel, cached, epoch) cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) return } @@ -120,12 +149,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver } if err == nil && !skipped { idx.applyRepoPrefix(result.Nodes, result.Edges) - idx.goModuleSourceMu.Lock() - if idx.goModuleSourceCache == nil { - idx.goModuleSourceCache = make(map[string]goModuleSourceCacheEntry) - } - idx.goModuleSourceCache[rel] = goModuleSourceCacheEntry{digest: digest, edges: cloneGoModuleSourceEdges(result.Edges), size: info.Size(), modified: info.ModTime().UnixNano()} - idx.goModuleSourceMu.Unlock() + idx.storeGoModuleSource(rel, goModuleSourceCacheEntry{digest: digest, edges: cloneGoModuleSourceEdges(result.Edges), size: info.Size(), modified: info.ModTime().UnixNano(), recordedAt: recordedAt}, epoch) cr.RestoreGoModuleSourceCalls(result.Edges) } result.ReleaseTree() diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 4548e1748..54fb83a31 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -506,6 +506,7 @@ type Indexer struct { goModuleSourceMu sync.Mutex goModuleSourceCache map[string]goModuleSourceCacheEntry + goModuleSourceEpoch uint64 // bumped by each known save; guarded by goModuleSourceMu // deferResolve, when set, makes IndexCtx skip the cross-cutting passes // (per-repo ResolveAll / semantic enrichment / contract extraction + diff --git a/internal/indexer/module_work_count_test.go b/internal/indexer/module_work_count_test.go index 7a2ef74e6..5e843a5a9 100644 --- a/internal/indexer/module_work_count_test.go +++ b/internal/indexer/module_work_count_test.go @@ -273,30 +273,43 @@ func TestGoModuleSourceCacheEviction(t *testing.T) { } // Given cached unbindable callers, when a second provider IndexFile save keeps -// the same definition set, then it reads zero unchanged caller files. -func TestGoModuleProviderSaveSkipsCachedSourceReads(t *testing.T) { - mi, _, repos := goModuleLifecycleFixture(t, "memory") - lib := filepath.Join(repos[1].Path, "lib.go") - writeFile(t, lib, "package library\nfunc Other() {}\n") - for _, entry := range repos[:2] { - _, err := mi.TrackRepoCtx(t.Context(), entry) - require.NoError(t, err) - } - original := readGoModuleCallerFile - reads := 0 - readGoModuleCallerFile = func(path string) ([]byte, error) { - if filepath.Base(path) == "main.go" { - reads++ - } - return original(path) +// the same definition set, then old-mtime source facts cost no read or parse. +func TestGoModuleProviderSaveReusesCachedSourceFacts(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, _, repos := goModuleLifecycleFixture(t, backend) + lib := filepath.Join(repos[1].Path, "lib.go") + main := filepath.Join(repos[0].Path, "main.go") + old := time.Now().Add(-time.Hour) + require.NoError(t, os.Chtimes(main, old, old)) + writeFile(t, lib, "package library\nfunc Other() {}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + original := readGoModuleCallerFile + reads := 0 + readGoModuleCallerFile = func(path string) ([]byte, error) { + if filepath.Base(path) == "main.go" { + reads++ + } + return original(path) + } + t.Cleanup(func() { readGoModuleCallerFile = original }) + writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + assert.Zero(t, reads, "the first provider save must also trust unchanged old source") + ext, ok := mi.registry.GetByLanguage("go") + require.True(t, ok) + spy := &goModuleSourceExtractionSpy{Extractor: ext} + mi.registry.Register(spy) + reads = 0 + writeFile(t, lib, "package library\nfunc Other(){println(2)}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + assert.Zero(t, reads, "unchanged old-mtime callers must reuse safely verified facts without a read") + assert.Zero(t, spy.calls, "identical content must not re-extract the caller") + }) } - t.Cleanup(func() { readGoModuleCallerFile = original }) - writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") - require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - reads = 0 - writeFile(t, lib, "package library\nfunc Other(){println(2)}\n") - require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - assert.Zero(t, reads, "unchanged caller stat must avoid source reads and transforms") } // Given a shared parse lease, when source reading fails or panics, then the @@ -336,9 +349,9 @@ func TestGoModuleSourceRecoveryReleasesLease(t *testing.T) { } } -// Given cached unchanged bytes, when only the caller mtime changes, then one -// provider save reads once to refresh the stamp and the next save reads zero. -func TestGoModuleSourceCacheRefreshesTouchedStamp(t *testing.T) { +// Given equal bytes with a touched old mtime, the first save verifies content; +// later saves trust the refreshed safe stamp without reading or parsing. +func TestGoModuleSourceCacheVerifiesTouchedContent(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { t.Run(backend, func(t *testing.T) { mi, _, repos := goModuleLifecycleFixture(t, backend) @@ -352,9 +365,7 @@ func TestGoModuleSourceCacheRefreshesTouchedStamp(t *testing.T) { } mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") - info, err := os.Stat(main) - require.NoError(t, err) - stamp := info.ModTime().Add(2 * time.Second) + stamp := time.Now().Add(-time.Hour) require.NoError(t, os.Chtimes(main, stamp, stamp)) original := readGoModuleCallerFile reads := 0 @@ -365,13 +376,19 @@ func TestGoModuleSourceCacheRefreshesTouchedStamp(t *testing.T) { return original(path) } t.Cleanup(func() { readGoModuleCallerFile = original }) + ext, ok := mi.registry.GetByLanguage("go") + require.True(t, ok) + spy := &goModuleSourceExtractionSpy{Extractor: ext} + mi.registry.Register(spy) writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - assert.Equal(t, 1, reads, "new stamp with identical bytes must refresh once") + assert.Equal(t, 1, reads, "changed metadata requires one content verification") + assert.Zero(t, spy.calls, "the touched-mtime save must reuse equal-content source facts") reads = 0 writeFile(t, lib, "package library\nfunc Other(){println(2)}\nfunc More(){}\n") require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - assert.Zero(t, reads, "the refreshed stamp must make the next provider save read-free") + assert.Zero(t, reads, "verified old-mtime content is now outside the racy window") + assert.Zero(t, spy.calls, "equal content must reuse source facts") }) } } diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 8209a8339..6604fbcfb 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2156,7 +2156,7 @@ func (idx *Indexer) installGoResolverContext() { } // trackedGoModules caches root identities by config revision and root go.mod -// stat. Configured roots keep cold-shadow imports recognizable before publish. +// content. Configured roots keep cold-shadow imports recognizable before publish. func (mi *MultiIndexer) trackedGoModules() map[string]string { modules, _ := mi.goModuleSnapshot() return modules @@ -2186,19 +2186,22 @@ func (mi *MultiIndexer) goModuleSnapshot() (map[string]string, uint64) { roots[prefix] = meta.RootPath } } + inputs := readGoModuleInputs(roots) files := make(map[string]goModuleFileStamp, len(roots)) for prefix, root := range roots { - stamp := goModuleFileStamp{root: root} - if info, err := os.Stat(filepath.Join(root, "go.mod")); err == nil { - stamp.size, stamp.modified = info.Size(), info.ModTime().UnixNano() + input := inputs[prefix] + stamp := goModuleFileStamp{root: root, valid: os.IsNotExist(input.err), present: !os.IsNotExist(input.err)} + if input.err == nil { + stamp.content = contentHashForSource(input.data) + stamp.valid = true } files[prefix] = stamp } - if mi.goModuleCache != nil && mi.goModuleRevision == revision && mi.goModuleRepoCount == count && maps.Equal(mi.goModuleFiles, files) { + if mi.goModuleCache != nil && mi.goModuleRevision == revision && mi.goModuleRepoCount == count && maps.EqualFunc(mi.goModuleFiles, files, func(a, b goModuleFileStamp) bool { return a.valid && a == b }) { return mi.goModuleCache, mi.goModuleGeneration } - modules := readTrackedGoModules(roots) - replacements := mi.readGoModuleReplacements(roots) + modules := goModuleIdentities(inputs) + replacements := mi.goModuleReplacementsFromInputs(roots, inputs) sameReplacements := maps.EqualFunc(replacements, mi.goModuleReplacements, slices.Equal[[]resolver.GoModuleReplacement]) mi.goModuleReplacements = replacements mi.goModuleFiles = files @@ -2221,28 +2224,42 @@ func (mi *MultiIndexer) goModuleSnapshot() (map[string]string, uint64) { return modules, mi.goModuleGeneration } +var readGoModuleFile = os.ReadFile + type goModuleFileStamp struct { - root string - size, modified int64 + root, content string + valid bool + present bool // Unreadable inputs remain conservative teardown dependencies. } -func readTrackedGoModules(roots map[string]string) map[string]string { - modules := make(map[string]string, len(roots)) +type goModuleInput struct { + data []byte + err error +} + +func readGoModuleInputs(roots map[string]string) map[string]goModuleInput { + inputs := make(map[string]goModuleInput, len(roots)) for prefix, root := range roots { - if module := readGoModModule(root); module != "" { - modules[prefix] = module - } + data, err := readGoModuleFile(filepath.Join(root, "go.mod")) + inputs[prefix] = goModuleInput{data: data, err: err} } - return modules + return inputs } -// readGoModModule reads the module path from a go.mod file. -func readGoModModule(repoPath string) string { - data, err := os.ReadFile(filepath.Join(repoPath, "go.mod")) - if err != nil { - return "" +func readTrackedGoModules(roots map[string]string) map[string]string { + return goModuleIdentities(readGoModuleInputs(roots)) +} + +func goModuleIdentities(inputs map[string]goModuleInput) map[string]string { + modules := make(map[string]string, len(inputs)) + for prefix, input := range inputs { + if input.err == nil { + if module := modfile.ModulePath(input.data); module != "" { + modules[prefix] = module + } + } } - return modfile.ModulePath(data) + return modules } // rebuildColdRefFacts seeds the durable resolved-reference sidecar once for diff --git a/internal/indexer/repository_untrack.go b/internal/indexer/repository_untrack.go index 3915be809..041aa9204 100644 --- a/internal/indexer/repository_untrack.go +++ b/internal/indexer/repository_untrack.go @@ -110,7 +110,7 @@ func (mi *MultiIndexer) untrackRepoCheckedRetainingAdmission( mi.goModuleMu.Lock() stamp := mi.goModuleFiles[repoPrefix] - hadGoModule := mi.goModuleCache[repoPrefix] != "" || stamp.modified != 0 + hadGoModule := mi.goModuleCache[repoPrefix] != "" || stamp.present mi.goModuleMu.Unlock() hadRules := mi.configMgr != nil && len(mi.configMgr.GetRepoConfig(repoPrefix).CrossWorkspaceDeps) > 0 goModuleAffected := hadGoModule || hadRules || len(graph.ReadRepoFilePaths(mi.graph, repoPrefix, "", []string{"go"}, nil)) > 0