From ae6bc2a93f67f8788fe9205c02c697010e074bbe Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 00:47:53 -0700 Subject: [PATCH 1/8] fix(indexer): compare go.mod and Go caller contents before cache reuse The tracked go.mod snapshot and the Go caller source cache treated a file as unchanged when its size and mtime matched. A same-size edit within one clock tick, such as renaming a module to a path of equal length and back on Windows, kept the old module path or the old caller facts. The go.mod stamp now carries a content hash, and a missing go.mod is a valid absent state. Other stat or read errors no longer compare equal. The one-second check throttle is unchanged. Caller recovery always reads the file and reuses cached facts only for an equal digest, so unchanged files are still not parsed again. The stamp invalidation calls on the save paths are removed, since the digest check now covers them. Tests hold size and mtime equal with os.Chtimes for a module rename and rename back and for a caller edit on the in-memory and SQLite stores, and cover failed reads and snapshot reuse with Go and non-Go roots. --- .../indexer/go_module_content_stamp_test.go | 134 ++++++++++++++++++ internal/indexer/go_module_extern_test.go | 1 + internal/indexer/go_module_replacements.go | 2 +- internal/indexer/go_module_revision_test.go | 10 ++ internal/indexer/go_module_source_calls.go | 31 +--- internal/indexer/incremental_batch.go | 3 - internal/indexer/indexer.go | 1 - internal/indexer/module_work_count_test.go | 24 +++- internal/indexer/multi.go | 20 ++- 9 files changed, 184 insertions(+), 42 deletions(-) create mode 100644 internal/indexer/go_module_content_stamp_test.go diff --git a/internal/indexer/go_module_content_stamp_test.go b/internal/indexer/go_module_content_stamp_test.go new file mode 100644 index 000000000..b03b1b425 --- /dev/null +++ b/internal/indexer/go_module_content_stamp_test.go @@ -0,0 +1,134 @@ +package indexer + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +// Given cached source facts, when equal-size bytes change with restored mtime, +// then provider recovery must re-extract the current calls and match cold. +func TestGoModuleSameStampSourceRecoveryColdEquivalent(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + main := filepath.Join(repos[0].Path, "main.go") + lib := filepath.Join(repos[1].Path, "lib.go") + before := "package caller\nimport library \"example.org/library\"\nfunc Caller(){ library.Hello(library.Missing()) }\n" + after := strings.Replace(before, "Missing", "Present", 1) + require.Equal(t, len(before), len(after)) + writeFile(t, main, before) + writeFile(t, lib, "package library\nfunc Other() {}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) + require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") + info, err := os.Stat(main) + require.NoError(t, err) + writeFile(t, main, after) + stamp := info.ModTime() + require.NoError(t, os.Chtimes(main, stamp, stamp)) + changed, err := os.Stat(main) + require.NoError(t, err) + require.Equal(t, info.Size(), changed.Size()) + require.Equal(t, info.ModTime().UnixNano(), changed.ModTime().UnixNano()) + writeFile(t, lib, "package library\nfunc Present() int{return 1}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + cold := revisionColdGraph(t, backend, repos[:2]) + require.ElementsMatch(t, []string{"external-call::dep::example.org/library", "b/lib.go::Present"}, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls)) + assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) + }) + } +} + +func TestGoModuleSnapshotRejectsFailedInputs(t *testing.T) { + for _, failure := range []string{"stat", "read"} { + t.Run(failure, func(t *testing.T) { + root := t.TempDir() + if failure == "read" { + require.NoError(t, os.Mkdir(filepath.Join(root, "go.mod"), 0o755)) + } else { + root = "\x00" + } + core, logs := observer.New(zap.WarnLevel) + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"b": {RootPath: root}}, logger: zap.New(core)} + // Equal failed stat/read results must not be valid cache evidence. + modules, _ := mi.goModuleSnapshot() + require.Empty(t, modules) + require.Len(t, logs.All(), 1) + mi.goModuleChecked = time.Time{} + modules, _ = mi.goModuleSnapshot() + assert.Empty(t, modules) + assert.Len(t, logs.All(), 2, "failed inputs must rebuild/refuse on each eligible check") + }) + } +} + +func TestGoModuleSourceFailureDoesNotRestoreCachedCalls(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, failure := range []string{"stat", "read"} { + t.Run(backend+"/"+failure, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + writeFile(t, filepath.Join(repos[1].Path, "lib.go"), "package library\nfunc Other(){}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) + require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") + // Publish a recoverable target without retrying the terminal. + g.AddNode(&graph.Node{ID: "b/lib.go::Hello", Name: "Hello", Kind: graph.KindFunction, RepoPrefix: "b", FilePath: "b/lib.go", WorkspaceID: "shared", Language: "go"}) + cr := mi.newCrossRepoResolver() + before := revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls) + main := filepath.Join(repos[0].Path, "main.go") + original := readGoModuleCallerFile + reads := 0 + readGoModuleCallerFile = func(string) ([]byte, error) { reads++; return nil, os.ErrPermission } + t.Cleanup(func() { readGoModuleCallerFile = original }) + if failure == "stat" { + require.NoError(t, os.Remove(main)) + } + // A cached fact is not evidence for unreadable current source. + mi.restoreGoModuleSourceFile(cr, "a/main.go") + if failure == "read" { + assert.Equal(t, 1, reads) + } else { + assert.Zero(t, reads) + } + assert.ElementsMatch(t, before, revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) + }) + } + } +} + +func TestGoModuleContentStampReusesMixedRootSnapshot(t *testing.T) { + goRoot, nonGoRoot := t.TempDir(), t.TempDir() + manifest := filepath.Join(goRoot, "go.mod") + writeFile(t, manifest, "module example.org/library\n") + original := readGoModuleFile + reads := map[string]int{} + readGoModuleFile = func(path string) ([]byte, error) { reads[path]++; return original(path) } + t.Cleanup(func() { readGoModuleFile = original }) + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"go": {RootPath: goRoot}, "js": {RootPath: nonGoRoot}}, logger: zap.NewNop()} + // Given one module and one valid absence, when an eligible unchanged + // snapshot check runs, then only the content-verification read is needed. + first, generation := mi.goModuleSnapshot() + require.Equal(t, map[string]string{"go": "example.org/library"}, first) + require.Equal(t, 3, reads[manifest], "initial verification, module identity, and replacement reads") + clear(reads) + mi.goModuleChecked = time.Time{} + second, nextGeneration := mi.goModuleSnapshot() + assert.Equal(t, first, second) + assert.Equal(t, generation, nextGeneration) + assert.Equal(t, map[string]int{manifest: 1}, reads, "valid absence must not force module/replacement rebuild reads") +} diff --git a/internal/indexer/go_module_extern_test.go b/internal/indexer/go_module_extern_test.go index 01baee2f7..05fca2da5 100644 --- a/internal/indexer/go_module_extern_test.go +++ b/internal/indexer/go_module_extern_test.go @@ -298,6 +298,7 @@ func TestTrackedGoModuleCacheRevision(t *testing.T) { cached, cachedGeneration := mi.goModuleSnapshot() assert.Equal(t, "example.org/new", cached["b"], "root go.mod edits refresh without config reload") assert.Greater(t, cachedGeneration, generation) + mi.goModuleChecked = time.Time{} // Exercise unchanged content after the throttle expires. unchanged, unchangedGeneration := mi.goModuleSnapshot() assert.Equal(t, cached, unchanged) assert.Equal(t, cachedGeneration, unchangedGeneration) diff --git a/internal/indexer/go_module_replacements.go b/internal/indexer/go_module_replacements.go index d93371bca..9c344fa00 100644 --- a/internal/indexer/go_module_replacements.go +++ b/internal/indexer/go_module_replacements.go @@ -24,7 +24,7 @@ func (mi *MultiIndexer) readGoModuleReplacements(roots map[string]string) map[st replacements := make(map[string][]resolver.GoModuleReplacement) for repo, root := range roots { filename := filepath.Join(root, "go.mod") - data, err := os.ReadFile(filename) + data, err := readGoModuleFile(filename) if os.IsNotExist(err) { continue } diff --git a/internal/indexer/go_module_revision_test.go b/internal/indexer/go_module_revision_test.go index 5b9f1b012..76cddeaa5 100644 --- a/internal/indexer/go_module_revision_test.go +++ b/internal/indexer/go_module_revision_test.go @@ -86,9 +86,19 @@ func TestGoModuleSameLineCallsColdEquivalent(t *testing.T) { if mode == "rename_back" { manifest := filepath.Join(repos[1].Path, "go.mod") writeFile(t, manifest, "module example.org/renamed\n") + frozen := time.Unix(1700000000, 0) + require.NoError(t, os.Chtimes(manifest, frozen, frozen)) + renamed, err := os.Stat(manifest) + require.NoError(t, err) mi.goModuleChecked = time.Time{} mi.RunGlobalResolve() + require.ElementsMatch(t, []string{"external-call::dep::example.org/library"}, revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) writeFile(t, manifest, "module example.org/library\n") + require.NoError(t, os.Chtimes(manifest, renamed.ModTime(), renamed.ModTime())) + restored, err := os.Stat(manifest) + require.NoError(t, err) + require.Equal(t, renamed.Size(), restored.Size()) + require.Equal(t, renamed.ModTime(), restored.ModTime()) mi.goModuleChecked = time.Time{} mi.RunGlobalResolve() } diff --git a/internal/indexer/go_module_source_calls.go b/internal/indexer/go_module_source_calls.go index 773eab4b6..c99ed17a8 100644 --- a/internal/indexer/go_module_source_calls.go +++ b/internal/indexer/go_module_source_calls.go @@ -14,12 +14,10 @@ import ( ) // goModuleSourceCacheEntry keeps source facts, not resolution decisions. -// Matching size and mtime reuse those facts. If either differs, the digest -// decides whether to reuse them. +// Every recovery reads current bytes; only an equal digest reuses those facts. type goModuleSourceCacheEntry struct { - digest [32]byte - edges []*graph.Edge - size, modified int64 + digest [32]byte + edges []*graph.Edge } var readGoModuleCallerFile = os.ReadFile @@ -37,17 +35,6 @@ func cloneGoModuleSourceEdges(edges []*graph.Edge) []*graph.Edge { return out } -// A successful replacement can change bytes without changing the stat. -// Keep source facts, but make recovery validate them with the digest. -func (idx *Indexer) invalidateGoModuleSourceStamp(rel string) { - idx.goModuleSourceMu.Lock() - defer idx.goModuleSourceMu.Unlock() - if cached, ok := idx.goModuleSourceCache[rel]; ok { - cached.size, cached.modified = 0, 0 - idx.goModuleSourceCache[rel] = cached - } -} - func (mi *MultiIndexer) finishGoModuleCalls(cr *resolver.CrossRepoResolver) { for _, file := range cr.TakeGoModuleSourceFiles() { mi.restoreGoModuleSourceFile(cr, file) @@ -63,7 +50,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver } rel := strings.TrimPrefix(file, prefix+"/") abs := filepath.Join(idx.rootPath, filepath.FromSlash(rel)) - info, err := os.Stat(abs) + _, err := os.Stat(abs) if err != nil { mi.logger.Warn("stat Go module caller", zap.String("file", file), zap.Error(err)) return @@ -71,10 +58,6 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver idx.goModuleSourceMu.Lock() cached, hit := idx.goModuleSourceCache[rel] idx.goModuleSourceMu.Unlock() - if hit && cached.size == info.Size() && cached.modified == info.ModTime().UnixNano() { - cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) - return - } parseLease, err := idx.acquireSharedParsePath(abs) if err != nil { mi.logger.Warn("admit Go module caller", zap.String("file", file), zap.Error(err)) @@ -96,10 +79,6 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver src = idx.transforms.run(rel, src) digest := sha256.Sum256(src) if hit && cached.digest == digest { - cached.size, cached.modified = info.Size(), info.ModTime().UnixNano() - idx.goModuleSourceMu.Lock() - idx.goModuleSourceCache[rel] = cached - idx.goModuleSourceMu.Unlock() cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) return } @@ -124,7 +103,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver if idx.goModuleSourceCache == nil { idx.goModuleSourceCache = make(map[string]goModuleSourceCacheEntry) } - idx.goModuleSourceCache[rel] = goModuleSourceCacheEntry{digest: digest, edges: cloneGoModuleSourceEdges(result.Edges), size: info.Size(), modified: info.ModTime().UnixNano()} + idx.goModuleSourceCache[rel] = goModuleSourceCacheEntry{digest: digest, edges: cloneGoModuleSourceEdges(result.Edges)} idx.goModuleSourceMu.Unlock() cr.RestoreGoModuleSourceCalls(result.Edges) } diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 5ac14cf98..b447317af 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -810,9 +810,6 @@ func (idx *Indexer) commitIncrementalStages( ) idx.applyLap("classify") - for _, stage := range stages { - idx.invalidateGoModuleSourceStamp(stage.relPath) - } idx.replaceIncrementalContentBatch(stages) idx.applyLap("content") diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 4548e1748..4e47d72a2 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -5145,7 +5145,6 @@ func (idx *Indexer) indexFile( // so it must be captured before evictExisting runs. var oldFuncIDs []string evictExisting := func() { - idx.invalidateGoModuleSourceStamp(relPath) var oldFTSNodeIDs []string for _, n := range idx.graph.GetFileNodes(graphPath) { if n == nil { diff --git a/internal/indexer/module_work_count_test.go b/internal/indexer/module_work_count_test.go index 7a2ef74e6..75d4cf6ae 100644 --- a/internal/indexer/module_work_count_test.go +++ b/internal/indexer/module_work_count_test.go @@ -273,8 +273,8 @@ func TestGoModuleSourceCacheEviction(t *testing.T) { } // Given cached unbindable callers, when a second provider IndexFile save keeps -// the same definition set, then it reads zero unchanged caller files. -func TestGoModuleProviderSaveSkipsCachedSourceReads(t *testing.T) { +// the same definition set, then it reads once but reuses unchanged source facts. +func TestGoModuleProviderSaveReusesCachedSourceFacts(t *testing.T) { mi, _, repos := goModuleLifecycleFixture(t, "memory") lib := filepath.Join(repos[1].Path, "lib.go") writeFile(t, lib, "package library\nfunc Other() {}\n") @@ -293,10 +293,15 @@ func TestGoModuleProviderSaveSkipsCachedSourceReads(t *testing.T) { t.Cleanup(func() { readGoModuleCallerFile = original }) writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + ext, ok := mi.registry.GetByLanguage("go") + require.True(t, ok) + spy := &goModuleSourceExtractionSpy{Extractor: ext} + mi.registry.Register(spy) reads = 0 writeFile(t, lib, "package library\nfunc Other(){println(2)}\n") require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - assert.Zero(t, reads, "unchanged caller stat must avoid source reads and transforms") + assert.Equal(t, 1, reads, "unchanged caller bytes must be verified before cached facts are reused") + assert.Zero(t, spy.calls, "identical content must not re-extract the caller") } // Given a shared parse lease, when source reading fails or panics, then the @@ -336,9 +341,9 @@ func TestGoModuleSourceRecoveryReleasesLease(t *testing.T) { } } -// Given cached unchanged bytes, when only the caller mtime changes, then one -// provider save reads once to refresh the stamp and the next save reads zero. -func TestGoModuleSourceCacheRefreshesTouchedStamp(t *testing.T) { +// Given cached unchanged bytes, when only the caller mtime changes, then +// provider saves verify bytes each time but never re-parse the unchanged caller. +func TestGoModuleSourceCacheVerifiesTouchedContent(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { t.Run(backend, func(t *testing.T) { mi, _, repos := goModuleLifecycleFixture(t, backend) @@ -368,10 +373,15 @@ func TestGoModuleSourceCacheRefreshesTouchedStamp(t *testing.T) { writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) assert.Equal(t, 1, reads, "new stamp with identical bytes must refresh once") + ext, ok := mi.registry.GetByLanguage("go") + require.True(t, ok) + spy := &goModuleSourceExtractionSpy{Extractor: ext} + mi.registry.Register(spy) reads = 0 writeFile(t, lib, "package library\nfunc Other(){println(2)}\nfunc More(){}\n") require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - assert.Zero(t, reads, "the refreshed stamp must make the next provider save read-free") + assert.Equal(t, 1, reads, "equal stat still requires content verification") + assert.Zero(t, spy.calls, "equal content must reuse source facts") }) } } diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 8209a8339..0f3325b95 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2156,7 +2156,7 @@ func (idx *Indexer) installGoResolverContext() { } // trackedGoModules caches root identities by config revision and root go.mod -// stat. Configured roots keep cold-shadow imports recognizable before publish. +// content. Configured roots keep cold-shadow imports recognizable before publish. func (mi *MultiIndexer) trackedGoModules() map[string]string { modules, _ := mi.goModuleSnapshot() return modules @@ -2189,12 +2189,20 @@ func (mi *MultiIndexer) goModuleSnapshot() (map[string]string, uint64) { files := make(map[string]goModuleFileStamp, len(roots)) for prefix, root := range roots { stamp := goModuleFileStamp{root: root} - if info, err := os.Stat(filepath.Join(root, "go.mod")); err == nil { + manifest := filepath.Join(root, "go.mod") + info, err := os.Stat(manifest) + if os.IsNotExist(err) { + stamp.valid = true + } else if err == nil { stamp.size, stamp.modified = info.Size(), info.ModTime().UnixNano() + if src, err := readGoModuleFile(manifest); err == nil { + stamp.content = contentHashForSource(src) + stamp.valid = true + } } files[prefix] = stamp } - if mi.goModuleCache != nil && mi.goModuleRevision == revision && mi.goModuleRepoCount == count && maps.Equal(mi.goModuleFiles, files) { + if mi.goModuleCache != nil && mi.goModuleRevision == revision && mi.goModuleRepoCount == count && maps.EqualFunc(mi.goModuleFiles, files, func(a, b goModuleFileStamp) bool { return a.valid && a == b }) { return mi.goModuleCache, mi.goModuleGeneration } modules := readTrackedGoModules(roots) @@ -2221,9 +2229,13 @@ func (mi *MultiIndexer) goModuleSnapshot() (map[string]string, uint64) { return modules, mi.goModuleGeneration } +var readGoModuleFile = os.ReadFile + type goModuleFileStamp struct { root string size, modified int64 + content string + valid bool } func readTrackedGoModules(roots map[string]string) map[string]string { @@ -2238,7 +2250,7 @@ func readTrackedGoModules(roots map[string]string) map[string]string { // readGoModModule reads the module path from a go.mod file. func readGoModModule(repoPath string) string { - data, err := os.ReadFile(filepath.Join(repoPath, "go.mod")) + data, err := readGoModuleFile(filepath.Join(repoPath, "go.mod")) if err != nil { return "" } From f9c4496b0a5fd649a905181f43cce0ec7a6bee34 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 03:29:53 -0700 Subject: [PATCH 2/8] fix(indexer): skip caller reads when the stat is safely old Reading and hashing every queued Go caller on each provider save made unchanged callers cost one read each. The caller cache now keeps size, mtime and the wall time taken before its read. It trusts equal stats without a read only when the file's mtime is older than that time minus a two-second window (Git's racy-file rule; FAT timestamps can cover two seconds). Recent stats, such as a same-size edit in one clock tick, still verify the digest. Known saves clear the trust again, so an editor that restores an old mtime after a save is verified. Limit, as in Git: an unobserved same-size edit that restores an mtime older than the window is trusted until the next save of that file. --- .../indexer/go_module_content_stamp_test.go | 2 + internal/indexer/go_module_revision_test.go | 67 ++++++++++--------- internal/indexer/go_module_source_calls.go | 36 ++++++++-- internal/indexer/incremental_batch.go | 3 + internal/indexer/indexer.go | 1 + internal/indexer/module_work_count_test.go | 62 +++++++++-------- 6 files changed, 109 insertions(+), 62 deletions(-) diff --git a/internal/indexer/go_module_content_stamp_test.go b/internal/indexer/go_module_content_stamp_test.go index b03b1b425..db1a69260 100644 --- a/internal/indexer/go_module_content_stamp_test.go +++ b/internal/indexer/go_module_content_stamp_test.go @@ -26,6 +26,8 @@ func TestGoModuleSameStampSourceRecoveryColdEquivalent(t *testing.T) { after := strings.Replace(before, "Missing", "Present", 1) require.Equal(t, len(before), len(after)) writeFile(t, main, before) + recent := time.Now().Truncate(time.Second) + require.NoError(t, os.Chtimes(main, recent, recent)) writeFile(t, lib, "package library\nfunc Other() {}\n") for _, entry := range repos[:2] { _, err := mi.TrackRepoCtx(t.Context(), entry) diff --git a/internal/indexer/go_module_revision_test.go b/internal/indexer/go_module_revision_test.go index 76cddeaa5..f83171d14 100644 --- a/internal/indexer/go_module_revision_test.go +++ b/internal/indexer/go_module_revision_test.go @@ -86,7 +86,7 @@ func TestGoModuleSameLineCallsColdEquivalent(t *testing.T) { if mode == "rename_back" { manifest := filepath.Join(repos[1].Path, "go.mod") writeFile(t, manifest, "module example.org/renamed\n") - frozen := time.Unix(1700000000, 0) + frozen := time.Now().Truncate(time.Second) require.NoError(t, os.Chtimes(manifest, frozen, frozen)) renamed, err := os.Stat(manifest) require.NoError(t, err) @@ -426,35 +426,42 @@ func TestGoModuleSameSizeSourceStampColdEquivalent(t *testing.T) { // the caller, then a provider save must recover C2's calls rather than C1's. func TestGoModuleSameStampCallerReindexColdEquivalent(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { - t.Run(backend, func(t *testing.T) { - mi, g, repos := goModuleLifecycleFixture(t, backend) - main := filepath.Join(repos[0].Path, "main.go") - lib := filepath.Join(repos[1].Path, "lib.go") - before := "package caller\nimport library \"example.org/library\"\nfunc Caller(){ library.Hello(library.Missing()) }\n" - after := strings.Replace(before, "Missing", "Present", 1) - require.Equal(t, len(before), len(after)) - writeFile(t, main, before) - writeFile(t, lib, "package library\nfunc Other(){}\n") - for _, entry := range repos[:2] { - _, err := mi.TrackRepoCtx(t.Context(), entry) + for _, mode := range []string{"recent", "old_point"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + main := filepath.Join(repos[0].Path, "main.go") + lib := filepath.Join(repos[1].Path, "lib.go") + before := "package caller\nimport library \"example.org/library\"\nfunc Caller(){ library.Hello(library.Missing()) }\n" + after := strings.Replace(before, "Missing", "Present", 1) + require.Equal(t, len(before), len(after)) + writeFile(t, main, before) + stamp := time.Now().Truncate(time.Second) + if mode != "recent" { + stamp = time.Now().Add(-time.Hour) + } + require.NoError(t, os.Chtimes(main, stamp, stamp)) + writeFile(t, lib, "package library\nfunc Other(){}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) + require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") + info, err := os.Stat(main) require.NoError(t, err) - } - mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) - require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") - info, err := os.Stat(main) - require.NoError(t, err) - writeFile(t, main, after) - require.NoError(t, os.Chtimes(main, info.ModTime(), info.ModTime())) - changed, err := os.Stat(main) - require.NoError(t, err) - require.Equal(t, info.Size(), changed.Size()) - require.Equal(t, info.ModTime().UnixNano(), changed.ModTime().UnixNano()) - require.NoError(t, mi.GetIndexer("a").IndexFile(main)) - writeFile(t, lib, "package library\nfunc Missing() int{return 0}\nfunc Present() int{return 1}\n") - require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - cold := revisionColdGraph(t, backend, repos[:2]) - require.ElementsMatch(t, []string{"external-call::dep::example.org/library", "b/lib.go::Present"}, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls)) - assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) - }) + writeFile(t, main, after) + require.NoError(t, os.Chtimes(main, info.ModTime(), info.ModTime())) + changed, err := os.Stat(main) + require.NoError(t, err) + require.Equal(t, info.Size(), changed.Size()) + require.Equal(t, info.ModTime().UnixNano(), changed.ModTime().UnixNano()) + require.NoError(t, mi.GetIndexer("a").IndexFile(main)) + writeFile(t, lib, "package library\nfunc Missing() int{return 0}\nfunc Present() int{return 1}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + cold := revisionColdGraph(t, backend, repos[:2]) + require.ElementsMatch(t, []string{"external-call::dep::example.org/library", "b/lib.go::Present"}, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls)) + assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) + }) + } } } diff --git a/internal/indexer/go_module_source_calls.go b/internal/indexer/go_module_source_calls.go index c99ed17a8..e661931c0 100644 --- a/internal/indexer/go_module_source_calls.go +++ b/internal/indexer/go_module_source_calls.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/resolver" @@ -14,12 +15,18 @@ import ( ) // goModuleSourceCacheEntry keeps source facts, not resolution decisions. -// Every recovery reads current bytes; only an equal digest reuses those facts. +// Equal stats are trusted only when the recorded read was outside the racy window. type goModuleSourceCacheEntry struct { - digest [32]byte - edges []*graph.Edge + digest [32]byte + edges []*graph.Edge + size, modified int64 + recordedAt time.Time } +// FAT timestamps can cover two seconds (HFS+ one). Entries recorded within +// that granularity must verify content even when size and mtime match. +const goModuleSourceRacyWindow = 2 * time.Second + var readGoModuleCallerFile = os.ReadFile func cloneGoModuleSourceEdges(edges []*graph.Edge) []*graph.Edge { @@ -35,6 +42,16 @@ func cloneGoModuleSourceEdges(edges []*graph.Edge) []*graph.Edge { return out } +// Known saves must verify again even when an editor restores an old mtime. +func (idx *Indexer) invalidateGoModuleSourceStamp(rel string) { + idx.goModuleSourceMu.Lock() + defer idx.goModuleSourceMu.Unlock() + if cached, ok := idx.goModuleSourceCache[rel]; ok { + cached.recordedAt = time.Time{} + idx.goModuleSourceCache[rel] = cached + } +} + func (mi *MultiIndexer) finishGoModuleCalls(cr *resolver.CrossRepoResolver) { for _, file := range cr.TakeGoModuleSourceFiles() { mi.restoreGoModuleSourceFile(cr, file) @@ -50,7 +67,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver } rel := strings.TrimPrefix(file, prefix+"/") abs := filepath.Join(idx.rootPath, filepath.FromSlash(rel)) - _, err := os.Stat(abs) + info, err := os.Stat(abs) if err != nil { mi.logger.Warn("stat Go module caller", zap.String("file", file), zap.Error(err)) return @@ -58,12 +75,17 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver idx.goModuleSourceMu.Lock() cached, hit := idx.goModuleSourceCache[rel] idx.goModuleSourceMu.Unlock() + if hit && cached.size == info.Size() && cached.modified == info.ModTime().UnixNano() && info.ModTime().Before(cached.recordedAt.Add(-goModuleSourceRacyWindow)) { + cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) + return + } parseLease, err := idx.acquireSharedParsePath(abs) if err != nil { mi.logger.Warn("admit Go module caller", zap.String("file", file), zap.Error(err)) return } defer parseLease.Release() + recordedAt := time.Now() // Record wall time before reading the bytes being verified. src, err := readGoModuleCallerFile(abs) if err != nil { mi.logger.Warn("read Go module caller", zap.String("file", file), zap.Error(err)) @@ -79,6 +101,10 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver src = idx.transforms.run(rel, src) digest := sha256.Sum256(src) if hit && cached.digest == digest { + cached.size, cached.modified, cached.recordedAt = info.Size(), info.ModTime().UnixNano(), recordedAt + idx.goModuleSourceMu.Lock() + idx.goModuleSourceCache[rel] = cached + idx.goModuleSourceMu.Unlock() cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) return } @@ -103,7 +129,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver if idx.goModuleSourceCache == nil { idx.goModuleSourceCache = make(map[string]goModuleSourceCacheEntry) } - idx.goModuleSourceCache[rel] = goModuleSourceCacheEntry{digest: digest, edges: cloneGoModuleSourceEdges(result.Edges)} + idx.goModuleSourceCache[rel] = goModuleSourceCacheEntry{digest: digest, edges: cloneGoModuleSourceEdges(result.Edges), size: info.Size(), modified: info.ModTime().UnixNano(), recordedAt: recordedAt} idx.goModuleSourceMu.Unlock() cr.RestoreGoModuleSourceCalls(result.Edges) } diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index b447317af..5ac14cf98 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -810,6 +810,9 @@ func (idx *Indexer) commitIncrementalStages( ) idx.applyLap("classify") + for _, stage := range stages { + idx.invalidateGoModuleSourceStamp(stage.relPath) + } idx.replaceIncrementalContentBatch(stages) idx.applyLap("content") diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 4e47d72a2..4548e1748 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -5145,6 +5145,7 @@ func (idx *Indexer) indexFile( // so it must be captured before evictExisting runs. var oldFuncIDs []string evictExisting := func() { + idx.invalidateGoModuleSourceStamp(relPath) var oldFTSNodeIDs []string for _, n := range idx.graph.GetFileNodes(graphPath) { if n == nil { diff --git a/internal/indexer/module_work_count_test.go b/internal/indexer/module_work_count_test.go index 75d4cf6ae..58e952fbb 100644 --- a/internal/indexer/module_work_count_test.go +++ b/internal/indexer/module_work_count_test.go @@ -273,35 +273,43 @@ func TestGoModuleSourceCacheEviction(t *testing.T) { } // Given cached unbindable callers, when a second provider IndexFile save keeps -// the same definition set, then it reads once but reuses unchanged source facts. +// the same definition set, then old-mtime source facts cost no read or parse. func TestGoModuleProviderSaveReusesCachedSourceFacts(t *testing.T) { - mi, _, repos := goModuleLifecycleFixture(t, "memory") - lib := filepath.Join(repos[1].Path, "lib.go") - writeFile(t, lib, "package library\nfunc Other() {}\n") - for _, entry := range repos[:2] { - _, err := mi.TrackRepoCtx(t.Context(), entry) - require.NoError(t, err) - } - original := readGoModuleCallerFile - reads := 0 - readGoModuleCallerFile = func(path string) ([]byte, error) { - if filepath.Base(path) == "main.go" { - reads++ - } - return original(path) + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, _, repos := goModuleLifecycleFixture(t, backend) + lib := filepath.Join(repos[1].Path, "lib.go") + main := filepath.Join(repos[0].Path, "main.go") + old := time.Now().Add(-time.Hour) + require.NoError(t, os.Chtimes(main, old, old)) + writeFile(t, lib, "package library\nfunc Other() {}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + original := readGoModuleCallerFile + reads := 0 + readGoModuleCallerFile = func(path string) ([]byte, error) { + if filepath.Base(path) == "main.go" { + reads++ + } + return original(path) + } + t.Cleanup(func() { readGoModuleCallerFile = original }) + writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + assert.Zero(t, reads, "the first provider save must also trust unchanged old source") + ext, ok := mi.registry.GetByLanguage("go") + require.True(t, ok) + spy := &goModuleSourceExtractionSpy{Extractor: ext} + mi.registry.Register(spy) + reads = 0 + writeFile(t, lib, "package library\nfunc Other(){println(2)}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + assert.Zero(t, reads, "unchanged old-mtime callers must reuse safely verified facts without a read") + assert.Zero(t, spy.calls, "identical content must not re-extract the caller") + }) } - t.Cleanup(func() { readGoModuleCallerFile = original }) - writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") - require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - ext, ok := mi.registry.GetByLanguage("go") - require.True(t, ok) - spy := &goModuleSourceExtractionSpy{Extractor: ext} - mi.registry.Register(spy) - reads = 0 - writeFile(t, lib, "package library\nfunc Other(){println(2)}\n") - require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - assert.Equal(t, 1, reads, "unchanged caller bytes must be verified before cached facts are reused") - assert.Zero(t, spy.calls, "identical content must not re-extract the caller") } // Given a shared parse lease, when source reading fails or panics, then the From aff4e78a5da9aeaf9007f15aee3c3c3f4ca8ffa0 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 03:30:20 -0700 Subject: [PATCH 3/8] fix(indexer): build the Go module snapshot from one go.mod read The snapshot hashed one read of each root's go.mod, and the module path and replacements came from two later reads. An edit between those reads could pair the hash of one version with the parse of another, and later checks would keep the stale identity. Each check now reads every root's go.mod once and takes the hash, module path and replacements from those bytes. A missing go.mod is still a valid absent state, and a failed read still blocks reuse. Untrack now asks whether go.mod was present instead of whether a stored mtime was set. --- .../indexer/go_module_content_stamp_test.go | 28 +++++++++- internal/indexer/go_module_replacements.go | 7 ++- internal/indexer/multi.go | 54 ++++++++++++------- internal/indexer/repository_untrack.go | 2 +- 4 files changed, 67 insertions(+), 24 deletions(-) diff --git a/internal/indexer/go_module_content_stamp_test.go b/internal/indexer/go_module_content_stamp_test.go index db1a69260..bc437b0c0 100644 --- a/internal/indexer/go_module_content_stamp_test.go +++ b/internal/indexer/go_module_content_stamp_test.go @@ -126,11 +126,35 @@ func TestGoModuleContentStampReusesMixedRootSnapshot(t *testing.T) { // snapshot check runs, then only the content-verification read is needed. first, generation := mi.goModuleSnapshot() require.Equal(t, map[string]string{"go": "example.org/library"}, first) - require.Equal(t, 3, reads[manifest], "initial verification, module identity, and replacement reads") + require.Equal(t, 1, reads[manifest], "verification, module identity, and replacements share one read") clear(reads) mi.goModuleChecked = time.Time{} second, nextGeneration := mi.goModuleSnapshot() assert.Equal(t, first, second) assert.Equal(t, generation, nextGeneration) - assert.Equal(t, map[string]int{manifest: 1}, reads, "valid absence must not force module/replacement rebuild reads") + assert.Equal(t, map[string]int{manifest: 1, filepath.Join(nonGoRoot, "go.mod"): 1}, reads, "one read attempt per root per eligible check") +} + +func TestGoModuleContentStampAndParsedEvidenceShareBytes(t *testing.T) { + root := t.TempDir() + x := []byte("module example.org/library\nreplace example.org/dep => example.org/other v1.2.3\n") + y := []byte("module example.org/renamed\n") + writeFile(t, filepath.Join(root, "go.mod"), string(x)) + original := readGoModuleFile + reads := 0 + readGoModuleFile = func(string) ([]byte, error) { + reads++ + if reads == 1 { + return x, nil + } + return y, nil + } + t.Cleanup(func() { readGoModuleFile = original }) + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"lib": {RootPath: root}}, logger: zap.NewNop()} + modules, _ := mi.goModuleSnapshot() + assert.Equal(t, 1, reads, "one captured read must supply hash, identity and replacements") + assert.Equal(t, "example.org/library", modules["lib"]) + assert.Equal(t, contentHashForSource(x), mi.goModuleFiles["lib"].content) + require.Len(t, mi.goModuleReplacements["lib"], 1) + assert.Equal(t, "example.org/dep", mi.goModuleReplacements["lib"][0].Module) } diff --git a/internal/indexer/go_module_replacements.go b/internal/indexer/go_module_replacements.go index 9c344fa00..00dab29b3 100644 --- a/internal/indexer/go_module_replacements.go +++ b/internal/indexer/go_module_replacements.go @@ -21,10 +21,15 @@ func (mi *MultiIndexer) trackedGoModuleReplacements() map[string][]resolver.GoMo } func (mi *MultiIndexer) readGoModuleReplacements(roots map[string]string) map[string][]resolver.GoModuleReplacement { + return mi.goModuleReplacementsFromInputs(roots, readGoModuleInputs(roots)) +} + +func (mi *MultiIndexer) goModuleReplacementsFromInputs(roots map[string]string, inputs map[string]goModuleInput) map[string][]resolver.GoModuleReplacement { replacements := make(map[string][]resolver.GoModuleReplacement) for repo, root := range roots { filename := filepath.Join(root, "go.mod") - data, err := readGoModuleFile(filename) + input := inputs[repo] + data, err := input.data, input.err if os.IsNotExist(err) { continue } diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 0f3325b95..c241ce2ed 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2186,27 +2186,22 @@ func (mi *MultiIndexer) goModuleSnapshot() (map[string]string, uint64) { roots[prefix] = meta.RootPath } } + inputs := readGoModuleInputs(roots) files := make(map[string]goModuleFileStamp, len(roots)) for prefix, root := range roots { - stamp := goModuleFileStamp{root: root} - manifest := filepath.Join(root, "go.mod") - info, err := os.Stat(manifest) - if os.IsNotExist(err) { + input := inputs[prefix] + stamp := goModuleFileStamp{root: root, valid: os.IsNotExist(input.err), present: !os.IsNotExist(input.err)} + if input.err == nil { + stamp.content = contentHashForSource(input.data) stamp.valid = true - } else if err == nil { - stamp.size, stamp.modified = info.Size(), info.ModTime().UnixNano() - if src, err := readGoModuleFile(manifest); err == nil { - stamp.content = contentHashForSource(src) - stamp.valid = true - } } files[prefix] = stamp } if mi.goModuleCache != nil && mi.goModuleRevision == revision && mi.goModuleRepoCount == count && maps.EqualFunc(mi.goModuleFiles, files, func(a, b goModuleFileStamp) bool { return a.valid && a == b }) { return mi.goModuleCache, mi.goModuleGeneration } - modules := readTrackedGoModules(roots) - replacements := mi.readGoModuleReplacements(roots) + modules := goModuleIdentities(inputs) + replacements := mi.goModuleReplacementsFromInputs(roots, inputs) sameReplacements := maps.EqualFunc(replacements, mi.goModuleReplacements, slices.Equal[[]resolver.GoModuleReplacement]) mi.goModuleReplacements = replacements mi.goModuleFiles = files @@ -2232,17 +2227,36 @@ func (mi *MultiIndexer) goModuleSnapshot() (map[string]string, uint64) { var readGoModuleFile = os.ReadFile type goModuleFileStamp struct { - root string - size, modified int64 - content string - valid bool + root, content string + valid bool + present bool // Unreadable inputs remain conservative teardown dependencies. } -func readTrackedGoModules(roots map[string]string) map[string]string { - modules := make(map[string]string, len(roots)) +type goModuleInput struct { + data []byte + err error +} + +func readGoModuleInputs(roots map[string]string) map[string]goModuleInput { + inputs := make(map[string]goModuleInput, len(roots)) for prefix, root := range roots { - if module := readGoModModule(root); module != "" { - modules[prefix] = module + data, err := readGoModuleFile(filepath.Join(root, "go.mod")) + inputs[prefix] = goModuleInput{data: data, err: err} + } + return inputs +} + +func readTrackedGoModules(roots map[string]string) map[string]string { + return goModuleIdentities(readGoModuleInputs(roots)) +} + +func goModuleIdentities(inputs map[string]goModuleInput) map[string]string { + modules := make(map[string]string, len(inputs)) + for prefix, input := range inputs { + if input.err == nil { + if module := modfile.ModulePath(input.data); module != "" { + modules[prefix] = module + } } } return modules diff --git a/internal/indexer/repository_untrack.go b/internal/indexer/repository_untrack.go index 3915be809..041aa9204 100644 --- a/internal/indexer/repository_untrack.go +++ b/internal/indexer/repository_untrack.go @@ -110,7 +110,7 @@ func (mi *MultiIndexer) untrackRepoCheckedRetainingAdmission( mi.goModuleMu.Lock() stamp := mi.goModuleFiles[repoPrefix] - hadGoModule := mi.goModuleCache[repoPrefix] != "" || stamp.modified != 0 + hadGoModule := mi.goModuleCache[repoPrefix] != "" || stamp.present mi.goModuleMu.Unlock() hadRules := mi.configMgr != nil && len(mi.configMgr.GetRepoConfig(repoPrefix).CrossWorkspaceDeps) > 0 goModuleAffected := hadGoModule || hadRules || len(graph.ReadRepoFilePaths(mi.graph, repoPrefix, "", []string{"go"}, nil)) > 0 From 3a12e389ad695030822c2b55c8fbe4124461a979 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 03:30:30 -0700 Subject: [PATCH 4/8] perf(indexer): reuse the caller stat for parse admission Caller recovery stat'ed the file, and shared parse admission stat'ed it again. Recovery now passes the size it already has to admission, and skips an oversized caller before admission and before the read. The check after the read for a file that grew stays as it was. --- internal/indexer/go_module_content_stamp_test.go | 7 +++++-- internal/indexer/go_module_source_calls.go | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/indexer/go_module_content_stamp_test.go b/internal/indexer/go_module_content_stamp_test.go index bc437b0c0..46ea38348 100644 --- a/internal/indexer/go_module_content_stamp_test.go +++ b/internal/indexer/go_module_content_stamp_test.go @@ -78,7 +78,7 @@ func TestGoModuleSnapshotRejectsFailedInputs(t *testing.T) { func TestGoModuleSourceFailureDoesNotRestoreCachedCalls(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { - for _, failure := range []string{"stat", "read"} { + for _, failure := range []string{"stat", "read", "size"} { t.Run(backend+"/"+failure, func(t *testing.T) { mi, g, repos := goModuleLifecycleFixture(t, backend) writeFile(t, filepath.Join(repos[1].Path, "lib.go"), "package library\nfunc Other(){}\n") @@ -97,8 +97,11 @@ func TestGoModuleSourceFailureDoesNotRestoreCachedCalls(t *testing.T) { reads := 0 readGoModuleCallerFile = func(string) ([]byte, error) { reads++; return nil, os.ErrPermission } t.Cleanup(func() { readGoModuleCallerFile = original }) - if failure == "stat" { + switch failure { + case "stat": require.NoError(t, os.Remove(main)) + case "size": + mi.GetIndexer("a").config.MaxFileSize = 1 } // A cached fact is not evidence for unreadable current source. mi.restoreGoModuleSourceFile(cr, "a/main.go") diff --git a/internal/indexer/go_module_source_calls.go b/internal/indexer/go_module_source_calls.go index e661931c0..7b48321ea 100644 --- a/internal/indexer/go_module_source_calls.go +++ b/internal/indexer/go_module_source_calls.go @@ -1,6 +1,7 @@ package indexer import ( + "context" "crypto/sha256" "github.com/zzet/gortex/internal/parser/crashpool" "maps" @@ -72,6 +73,9 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver mi.logger.Warn("stat Go module caller", zap.String("file", file), zap.Error(err)) return } + if idx.config.MaxFileSize > 0 && info.Size() > idx.config.MaxFileSize { + return + } idx.goModuleSourceMu.Lock() cached, hit := idx.goModuleSourceCache[rel] idx.goModuleSourceMu.Unlock() @@ -79,7 +83,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) return } - parseLease, err := idx.acquireSharedParsePath(abs) + parseLease, err := acquireParseAdmission(context.Background(), info.Size(), 0, nil, idx.parseAdmission.Load()) if err != nil { mi.logger.Warn("admit Go module caller", zap.String("file", file), zap.Error(err)) return From c8fcc221212684a28cc3aca2938c218efd5f2f11 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 03:30:44 -0700 Subject: [PATCH 5/8] test(indexer): count caller work from the first touched save TestGoModuleSourceCacheVerifiesTouchedContent registered its extraction spy after the touched-mtime save, so that save's "never re-parsed" claim was not checked, and its message still described the removed stamp refresh. The spy now starts before the first save. The test checks one read and zero parses for the touched save, then no read once the stat is outside the racy window. --- internal/indexer/module_work_count_test.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/indexer/module_work_count_test.go b/internal/indexer/module_work_count_test.go index 58e952fbb..5e843a5a9 100644 --- a/internal/indexer/module_work_count_test.go +++ b/internal/indexer/module_work_count_test.go @@ -349,8 +349,8 @@ func TestGoModuleSourceRecoveryReleasesLease(t *testing.T) { } } -// Given cached unchanged bytes, when only the caller mtime changes, then -// provider saves verify bytes each time but never re-parse the unchanged caller. +// Given equal bytes with a touched old mtime, the first save verifies content; +// later saves trust the refreshed safe stamp without reading or parsing. func TestGoModuleSourceCacheVerifiesTouchedContent(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { t.Run(backend, func(t *testing.T) { @@ -365,9 +365,7 @@ func TestGoModuleSourceCacheVerifiesTouchedContent(t *testing.T) { } mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) require.Contains(t, mi.GetIndexer("a").goModuleSourceCache, "main.go") - info, err := os.Stat(main) - require.NoError(t, err) - stamp := info.ModTime().Add(2 * time.Second) + stamp := time.Now().Add(-time.Hour) require.NoError(t, os.Chtimes(main, stamp, stamp)) original := readGoModuleCallerFile reads := 0 @@ -378,17 +376,18 @@ func TestGoModuleSourceCacheVerifiesTouchedContent(t *testing.T) { return original(path) } t.Cleanup(func() { readGoModuleCallerFile = original }) - writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") - require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - assert.Equal(t, 1, reads, "new stamp with identical bytes must refresh once") ext, ok := mi.registry.GetByLanguage("go") require.True(t, ok) spy := &goModuleSourceExtractionSpy{Extractor: ext} mi.registry.Register(spy) + writeFile(t, lib, "package library\nfunc Other(){println(1)}\n") + require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) + assert.Equal(t, 1, reads, "changed metadata requires one content verification") + assert.Zero(t, spy.calls, "the touched-mtime save must reuse equal-content source facts") reads = 0 writeFile(t, lib, "package library\nfunc Other(){println(2)}\nfunc More(){}\n") require.NoError(t, mi.GetIndexer("b").IndexFile(lib)) - assert.Equal(t, 1, reads, "equal stat still requires content verification") + assert.Zero(t, reads, "verified old-mtime content is now outside the racy window") assert.Zero(t, spy.calls, "equal content must reuse source facts") }) } From f65a06d184986681ecde91a4559ca605fe8f352d Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 03:45:58 -0700 Subject: [PATCH 6/8] refactor(indexer): remove readGoModModule The snapshot now takes module paths from goModuleIdentities, so readGoModModule had no production caller and repeated its parse. Its test now checks goModuleIdentities on the same go.mod inputs. --- internal/indexer/go_module_extern_test.go | 11 +++++++---- internal/indexer/multi.go | 9 --------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/internal/indexer/go_module_extern_test.go b/internal/indexer/go_module_extern_test.go index 05fca2da5..a7151757a 100644 --- a/internal/indexer/go_module_extern_test.go +++ b/internal/indexer/go_module_extern_test.go @@ -257,7 +257,10 @@ func Caller() { library.Hello(); child.Nested() } } } -func TestReadGoModModule(t *testing.T) { +func TestGoModuleIdentities(t *testing.T) { + identity := func(root string) string { + return goModuleIdentities(readGoModuleInputs(map[string]string{"r": root}))["r"] + } for _, source := range []string{ "module example.org/library\n", "module\texample.org/library\n", @@ -266,15 +269,15 @@ func TestReadGoModModule(t *testing.T) { t.Run(source, func(t *testing.T) { root := t.TempDir() writeFile(t, filepath.Join(root, "go.mod"), source) - assert.Equal(t, "example.org/library", readGoModModule(root)) + assert.Equal(t, "example.org/library", identity(root)) }) } for _, source := range []string{"", "module", "module \"unterminated"} { root := t.TempDir() writeFile(t, filepath.Join(root, "go.mod"), source) - assert.Empty(t, readGoModModule(root)) + assert.Empty(t, identity(root)) } - assert.Empty(t, readGoModModule(t.TempDir())) + assert.Empty(t, identity(t.TempDir())) } func TestTrackedGoModuleCacheRevision(t *testing.T) { diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index c241ce2ed..6604fbcfb 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2262,15 +2262,6 @@ func goModuleIdentities(inputs map[string]goModuleInput) map[string]string { return modules } -// readGoModModule reads the module path from a go.mod file. -func readGoModModule(repoPath string) string { - data, err := readGoModuleFile(filepath.Join(repoPath, "go.mod")) - if err != nil { - return "" - } - return modfile.ModulePath(data) -} - // rebuildColdRefFacts seeds the durable resolved-reference sidecar once for // the complete successful repository set. The backend implementation is a // single set-oriented transaction; keeping the capability probe here avoids a From 5bc8f569d68f5d6dd5cd90db595cd633d55d3e6f Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 03:47:12 -0700 Subject: [PATCH 7/8] test(indexer): pin the caller cache clock in racy-window tests Three tests set a caller's mtime from the wall clock and expected the recovery that follows to stay inside the two-second racy window. On a slow or loaded runner, setup could take longer, and the recovery would trust the stat and skip the read the test depends on. The time a verifying read is recorded now comes from goModuleSourceNow, which these tests pin to the caller's fixed mtime (or an hour after it for the old-stat case), so each case takes one branch of the rule. --- .../indexer/go_module_content_stamp_test.go | 21 +++++++++++++++++-- internal/indexer/go_module_revision_test.go | 6 ++++-- internal/indexer/go_module_source_calls.go | 6 +++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/internal/indexer/go_module_content_stamp_test.go b/internal/indexer/go_module_content_stamp_test.go index 46ea38348..1a57b7049 100644 --- a/internal/indexer/go_module_content_stamp_test.go +++ b/internal/indexer/go_module_content_stamp_test.go @@ -26,8 +26,11 @@ func TestGoModuleSameStampSourceRecoveryColdEquivalent(t *testing.T) { after := strings.Replace(before, "Missing", "Present", 1) require.Equal(t, len(before), len(after)) writeFile(t, main, before) - recent := time.Now().Truncate(time.Second) + // The cache records its read at the file's own mtime, so the + // stat stays inside the racy window however slow the setup is. + recent := time.Unix(1700000000, 0) require.NoError(t, os.Chtimes(main, recent, recent)) + pinGoModuleSourceClock(t, recent) writeFile(t, lib, "package library\nfunc Other() {}\n") for _, entry := range repos[:2] { _, err := mi.TrackRepoCtx(t.Context(), entry) @@ -81,6 +84,12 @@ func TestGoModuleSourceFailureDoesNotRestoreCachedCalls(t *testing.T) { for _, failure := range []string{"stat", "read", "size"} { t.Run(backend+"/"+failure, func(t *testing.T) { mi, g, repos := goModuleLifecycleFixture(t, backend) + main := filepath.Join(repos[0].Path, "main.go") + // Keep the caller's stat racy, so a cached fact can only be + // reused after a read, independent of runner speed. + recent := time.Unix(1700000000, 0) + require.NoError(t, os.Chtimes(main, recent, recent)) + pinGoModuleSourceClock(t, recent) writeFile(t, filepath.Join(repos[1].Path, "lib.go"), "package library\nfunc Other(){}\n") for _, entry := range repos[:2] { _, err := mi.TrackRepoCtx(t.Context(), entry) @@ -92,7 +101,6 @@ func TestGoModuleSourceFailureDoesNotRestoreCachedCalls(t *testing.T) { g.AddNode(&graph.Node{ID: "b/lib.go::Hello", Name: "Hello", Kind: graph.KindFunction, RepoPrefix: "b", FilePath: "b/lib.go", WorkspaceID: "shared", Language: "go"}) cr := mi.newCrossRepoResolver() before := revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls) - main := filepath.Join(repos[0].Path, "main.go") original := readGoModuleCallerFile reads := 0 readGoModuleCallerFile = func(string) ([]byte, error) { reads++; return nil, os.ErrPermission } @@ -161,3 +169,12 @@ func TestGoModuleContentStampAndParsedEvidenceShareBytes(t *testing.T) { require.Len(t, mi.goModuleReplacements["lib"], 1) assert.Equal(t, "example.org/dep", mi.goModuleReplacements["lib"][0].Module) } + +// pinGoModuleSourceClock records caller-cache reads at a fixed time, so each +// test takes one branch of the racy-file rule regardless of runner speed. +func pinGoModuleSourceClock(t *testing.T, at time.Time) { + t.Helper() + original := goModuleSourceNow + goModuleSourceNow = func() time.Time { return at } + t.Cleanup(func() { goModuleSourceNow = original }) +} diff --git a/internal/indexer/go_module_revision_test.go b/internal/indexer/go_module_revision_test.go index f83171d14..ec6d29603 100644 --- a/internal/indexer/go_module_revision_test.go +++ b/internal/indexer/go_module_revision_test.go @@ -435,9 +435,11 @@ func TestGoModuleSameStampCallerReindexColdEquivalent(t *testing.T) { after := strings.Replace(before, "Missing", "Present", 1) require.Equal(t, len(before), len(after)) writeFile(t, main, before) - stamp := time.Now().Truncate(time.Second) + now := time.Unix(1700000000, 0) + pinGoModuleSourceClock(t, now) + stamp := now // inside the racy window if mode != "recent" { - stamp = time.Now().Add(-time.Hour) + stamp = now.Add(-time.Hour) } require.NoError(t, os.Chtimes(main, stamp, stamp)) writeFile(t, lib, "package library\nfunc Other(){}\n") diff --git a/internal/indexer/go_module_source_calls.go b/internal/indexer/go_module_source_calls.go index 7b48321ea..c074ad44f 100644 --- a/internal/indexer/go_module_source_calls.go +++ b/internal/indexer/go_module_source_calls.go @@ -30,6 +30,10 @@ const goModuleSourceRacyWindow = 2 * time.Second var readGoModuleCallerFile = os.ReadFile +// goModuleSourceNow stamps the start of a verifying read. Tests pin it so each +// case takes one branch of the racy-file rule regardless of runner speed. +var goModuleSourceNow = time.Now + func cloneGoModuleSourceEdges(edges []*graph.Edge) []*graph.Edge { out := make([]*graph.Edge, 0, len(edges)) for _, e := range edges { @@ -89,7 +93,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver return } defer parseLease.Release() - recordedAt := time.Now() // Record wall time before reading the bytes being verified. + recordedAt := goModuleSourceNow() // Record wall time before reading the bytes being verified. src, err := readGoModuleCallerFile(abs) if err != nil { mi.logger.Warn("read Go module caller", zap.String("file", file), zap.Error(err)) From 784e92229e4a04aa891169d20f240a85ebe81c52 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 03:47:24 -0700 Subject: [PATCH 8/8] fix(indexer): drop caller stat trust when a save overlaps recovery Caller recovery reads and hashes a file without the cache lock, then stores the result with a fresh trusted stamp. Repository lanes can run a provider-lane recovery while the caller's own repository saves that file. If the save kept the size and an old mtime, the stored pre-save digest was then trusted by stat, and later recoveries reused stale calls until the caller was saved again. Each known save now bumps a per-indexer epoch under the cache lock. Recovery reads the epoch with the entry and, when it stores its result, keeps the facts but drops the stat trust if the epoch moved. The next recovery then verifies the bytes. A test runs a save during the read and checks that the next recovery reads again and the one after does not. --- .../indexer/go_module_content_stamp_test.go | 42 +++++++++++++++++++ internal/indexer/go_module_source_calls.go | 29 +++++++++---- internal/indexer/indexer.go | 1 + 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/internal/indexer/go_module_content_stamp_test.go b/internal/indexer/go_module_content_stamp_test.go index 1a57b7049..8b1e49064 100644 --- a/internal/indexer/go_module_content_stamp_test.go +++ b/internal/indexer/go_module_content_stamp_test.go @@ -178,3 +178,45 @@ func pinGoModuleSourceClock(t *testing.T, at time.Time) { goModuleSourceNow = func() time.Time { return at } t.Cleanup(func() { goModuleSourceNow = original }) } + +// Given a recovery that reads a caller before a save of that caller is known, +// when it stores its result, then it keeps the facts but not the stat trust: +// the next recovery reads again instead of reusing pre-save bytes. +func TestGoModuleSourceStoreDropsTrustAfterConcurrentSave(t *testing.T) { + mi, _, repos := goModuleLifecycleFixture(t, "memory") + main := filepath.Join(repos[0].Path, "main.go") + now := time.Unix(1700000000, 0) + old := now.Add(-time.Hour) + require.NoError(t, os.Chtimes(main, old, old)) + pinGoModuleSourceClock(t, now) + writeFile(t, filepath.Join(repos[1].Path, "lib.go"), "package library\nfunc Other(){}\n") + for _, entry := range repos[:2] { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + mi.runCrossRepoResolveFiles([]string{"b/lib.go"}) + idx := mi.GetIndexer("a") + require.Contains(t, idx.goModuleSourceCache, "main.go") + // Touch the stat so the next recovery verifies by reading. + touched := old.Add(-time.Hour) + require.NoError(t, os.Chtimes(main, touched, touched)) + original := readGoModuleCallerFile + reads := 0 + readGoModuleCallerFile = func(path string) ([]byte, error) { + reads++ + src, err := original(path) + if reads == 1 { + idx.invalidateGoModuleSourceStamp("main.go") // a save lands during the read + } + return src, err + } + t.Cleanup(func() { readGoModuleCallerFile = original }) + cr := mi.newCrossRepoResolver() + mi.restoreGoModuleSourceFile(cr, "a/main.go") + require.Equal(t, 1, reads) + assert.True(t, idx.goModuleSourceCache["main.go"].recordedAt.IsZero(), "a result read before a known save must not be trusted by stat") + mi.restoreGoModuleSourceFile(cr, "a/main.go") + assert.Equal(t, 2, reads, "the next recovery verifies content again") + mi.restoreGoModuleSourceFile(cr, "a/main.go") + assert.Equal(t, 2, reads, "an undisturbed verification restores stat trust") +} diff --git a/internal/indexer/go_module_source_calls.go b/internal/indexer/go_module_source_calls.go index c074ad44f..13b256f5b 100644 --- a/internal/indexer/go_module_source_calls.go +++ b/internal/indexer/go_module_source_calls.go @@ -48,15 +48,32 @@ func cloneGoModuleSourceEdges(edges []*graph.Edge) []*graph.Edge { } // Known saves must verify again even when an editor restores an old mtime. +// The epoch also tells a recovery that read before this save not to trust its +// stat when it stores its result. func (idx *Indexer) invalidateGoModuleSourceStamp(rel string) { idx.goModuleSourceMu.Lock() defer idx.goModuleSourceMu.Unlock() + idx.goModuleSourceEpoch++ if cached, ok := idx.goModuleSourceCache[rel]; ok { cached.recordedAt = time.Time{} idx.goModuleSourceCache[rel] = cached } } +// storeGoModuleSource keeps the facts but drops stat trust when a save was +// invalidated after epoch was read, since the bytes may predate that save. +func (idx *Indexer) storeGoModuleSource(rel string, entry goModuleSourceCacheEntry, epoch uint64) { + idx.goModuleSourceMu.Lock() + defer idx.goModuleSourceMu.Unlock() + if idx.goModuleSourceEpoch != epoch { + entry.recordedAt = time.Time{} + } + if idx.goModuleSourceCache == nil { + idx.goModuleSourceCache = make(map[string]goModuleSourceCacheEntry) + } + idx.goModuleSourceCache[rel] = entry +} + func (mi *MultiIndexer) finishGoModuleCalls(cr *resolver.CrossRepoResolver) { for _, file := range cr.TakeGoModuleSourceFiles() { mi.restoreGoModuleSourceFile(cr, file) @@ -82,6 +99,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver } idx.goModuleSourceMu.Lock() cached, hit := idx.goModuleSourceCache[rel] + epoch := idx.goModuleSourceEpoch idx.goModuleSourceMu.Unlock() if hit && cached.size == info.Size() && cached.modified == info.ModTime().UnixNano() && info.ModTime().Before(cached.recordedAt.Add(-goModuleSourceRacyWindow)) { cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) @@ -110,9 +128,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver digest := sha256.Sum256(src) if hit && cached.digest == digest { cached.size, cached.modified, cached.recordedAt = info.Size(), info.ModTime().UnixNano(), recordedAt - idx.goModuleSourceMu.Lock() - idx.goModuleSourceCache[rel] = cached - idx.goModuleSourceMu.Unlock() + idx.storeGoModuleSource(rel, cached, epoch) cr.RestoreGoModuleSourceCalls(cloneGoModuleSourceEdges(cached.edges)) return } @@ -133,12 +149,7 @@ func (mi *MultiIndexer) restoreGoModuleSourceFile(cr *resolver.CrossRepoResolver } if err == nil && !skipped { idx.applyRepoPrefix(result.Nodes, result.Edges) - idx.goModuleSourceMu.Lock() - if idx.goModuleSourceCache == nil { - idx.goModuleSourceCache = make(map[string]goModuleSourceCacheEntry) - } - idx.goModuleSourceCache[rel] = goModuleSourceCacheEntry{digest: digest, edges: cloneGoModuleSourceEdges(result.Edges), size: info.Size(), modified: info.ModTime().UnixNano(), recordedAt: recordedAt} - idx.goModuleSourceMu.Unlock() + idx.storeGoModuleSource(rel, goModuleSourceCacheEntry{digest: digest, edges: cloneGoModuleSourceEdges(result.Edges), size: info.Size(), modified: info.ModTime().UnixNano(), recordedAt: recordedAt}, epoch) cr.RestoreGoModuleSourceCalls(result.Edges) } result.ReleaseTree() diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 4548e1748..54fb83a31 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -506,6 +506,7 @@ type Indexer struct { goModuleSourceMu sync.Mutex goModuleSourceCache map[string]goModuleSourceCacheEntry + goModuleSourceEpoch uint64 // bumped by each known save; guarded by goModuleSourceMu // deferResolve, when set, makes IndexCtx skip the cross-cutting passes // (per-repo ResolveAll / semantic enrichment / contract extraction +