From 30d2e311ce282ad1720f8a92d60319c74ae3bdef Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Sun, 11 Oct 2026 22:14:08 +0200 Subject: [PATCH 1/2] fix(scanner): bound scans by file count; refuse the dependency scan above it and say so Measured on 2026-10-11 with synthetic TypeScript repos (one import per file): 60k files, --deps 5.3 s and 421 MB RSS; 200k files, --deps 29.6 s and 1.57 GB RSS, --importers 1.63 GB. Growth is linear in files, so two concurrent scans of large repos exhaust a laptop and codemap never said anything (#186). The walk now keeps at most limits.DefaultMaxScanFiles (50000) files and counts the rest, so the tree reports "bounded: scanned N of M files". ast-grep walks the tree itself and its whole JSON output is held in memory, so the dependency scan is bounded before it starts: every graph build routes through scanForGraphOutcomeWithFilters, which counts first and above the bound records a scan-bound source with status unavailable and the same note, and runs nothing. --importers then says the answer is unknown instead of "No files import", and --deps says the scan did not run instead of "No source files found". --max-files and config max_files raise the bound; -1 removes it. Hooks and the watch daemon read the same filters, so they degrade to the bounded result inside the 8 s budget. Not added: an edge cap (edges are bounded by files once files are) and a memory sampler (the bound is what prevents the allocation). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NRaBgKRAZMX9hjbkkefA7T --- README.md | 2 + blast_radius.go | 10 ++++- collide.go | 2 +- config/config.go | 1 + find_cmd.go | 2 +- limits/limits.go | 20 +++++++++ main.go | 18 ++++++-- mcp/main.go | 5 +++ render/depgraph.go | 22 +++++++++- scan_bound_cli_test.go | 29 +++++++++++++ scanner/cargofallback.go | 24 ++++++++++ scanner/filegraph.go | 2 +- scanner/scan_bound_test.go | 89 ++++++++++++++++++++++++++++++++++++++ scanner/types.go | 18 ++++++++ scanner/walker.go | 60 ++++++++++++++++++++++--- watch/daemon.go | 4 +- 16 files changed, 292 insertions(+), 16 deletions(-) create mode 100644 scan_bound_cli_test.go create mode 100644 scanner/scan_bound_test.go diff --git a/README.md b/README.md index 135d844..490bc5b 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,7 @@ subdirectory; relative setup paths resolve from the project root. | `--depth, -d ` | Limit tree depth (0 = unlimited) | | `--only ` | Only include files with these extensions | | `--exclude ` | Exclude files matching patterns | +| `--max-files ` | Scan bound: files kept by the tree, and the largest repo a dependency scan runs on (default 50000, or config `max_files`; `-1` = unbounded). Above it the output says `bounded: scanned N of M files` instead of presenting a prefix as the repo | | `--diff` | Show files changed vs main branch | | `--ref ` | Branch to compare against (with `--diff`) | | `--deps` | Dependency flow mode | @@ -403,6 +404,7 @@ codemap config show # display current config "only": ["rs", "sh", "sql", "toml", "yml"], "exclude": ["docs/reference", "docs/research"], "depth": 4, + "max_files": 50000, "mode": "auto", "guidance": { "missing_extension_hints": true, diff --git a/blast_radius.go b/blast_radius.go index 7ce6e82..63e70df 100644 --- a/blast_radius.go +++ b/blast_radius.go @@ -445,7 +445,7 @@ func buildBlastRadiusBundle(absRoot, ref string, limits blastRadiusLimits) (blas cmd.EnsureProjectConfig(absRoot) cfg := config.Load(absRoot) - filters := scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude} + filters := scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude, MaxFiles: cfg.MaxFiles} gitCache := scanner.NewGitIgnoreCache(absRoot) allFiles, err := scanner.ScanFiles(context.Background(), absRoot, gitCache, filters.Only, filters.Exclude) if err != nil { @@ -1422,6 +1422,14 @@ func renderImportersReportCLI(w io.Writer, report scanner.ImportersReport) { renderCoverage(w, report.CoverageStatus, report.CoverageNotes) return } + if report.CoverageStatus == string(analysis.CoverageUnavailable) { + // No dependency scan ran (above the file bound, ast-grep missing), + // so there is no answer, and "No files import" would be a + // confident zero with nothing behind it (#186, #138). + fmt.Fprintf(w, "Importers of %s: unknown, no dependency scan ran.\n", report.File) + renderCoverage(w, report.CoverageStatus, report.CoverageNotes) + return + } if report.Package != "" && report.PackageSiblings > 0 { // A Go file's importers are the files that import its package, so // a zero here says nothing about the package's own files: those diff --git a/collide.go b/collide.go index e536b13..f7a566c 100644 --- a/collide.go +++ b/collide.go @@ -235,7 +235,7 @@ func runCollideSubcommand(args []string, launchDir string) int { cmd.EnsureProjectConfig(resolvedRoot) cfg := config.Load(resolvedRoot) - filters := scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude} + filters := scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude, MaxFiles: cfg.MaxFiles} // The scan outcome is kept rather than discarded: a package-resolved // language needs the raw import strings, which the file graph does not // preserve. BuildFileGraphFromOutcome reuses this same scan, so keeping it diff --git a/config/config.go b/config/config.go index 655fefb..6628de5 100644 --- a/config/config.go +++ b/config/config.go @@ -17,6 +17,7 @@ type ProjectConfig struct { Only []string `json:"only,omitempty"` Exclude []string `json:"exclude,omitempty"` Depth int `json:"depth,omitempty"` + MaxFiles int `json:"max_files,omitempty"` // scan bound; 0 = default, negative = unbounded Mode string `json:"mode,omitempty"` Budgets HookBudgets `json:"budgets,omitempty"` Routing RoutingConfig `json:"routing,omitempty"` diff --git a/find_cmd.go b/find_cmd.go index e5d1c08..97653ac 100644 --- a/find_cmd.go +++ b/find_cmd.go @@ -73,7 +73,7 @@ func runFindSubcommand(args []string, launchDir string) int { cmd.EnsureProjectConfig(absRoot) cfg := config.Load(absRoot) - report, err := find.Run(context.Background(), absRoot, scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude}, query, *limit) + report, err := find.Run(context.Background(), absRoot, scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude, MaxFiles: cfg.MaxFiles}, query, *limit) if err != nil { fmt.Fprintf(os.Stderr, "Error scanning project: %v\n", err) return 1 diff --git a/limits/limits.go b/limits/limits.go index 7eb2a96..b4e5e10 100644 --- a/limits/limits.go +++ b/limits/limits.go @@ -19,8 +19,28 @@ const ( const ( MediumRepoFileCount = 2000 LargeRepoFileCount = 5000 + + // DefaultMaxScanFiles bounds the files a scan keeps and the size of repo + // a dependency scan is attempted on. Measured on 2026-10-11: --deps on a + // synthetic 60k-file TypeScript repo peaked at 421 MB RSS, so two + // concurrent scans of 200k-file repos exhaust a laptop (#186). Raised + // per run with --max-files or per repo with config max_files; a + // negative value removes the bound. + DefaultMaxScanFiles = 50000 ) +// ScanFileBound resolves a configured bound: zero means the default, +// negative means unbounded. +func ScanFileBound(configured int) int { + if configured == 0 { + return DefaultMaxScanFiles + } + if configured < 0 { + return 0 + } + return configured +} + // AdaptiveDepth returns a safe tree depth based on repository size. // Unknown file count (<=0) defaults to a conservative depth. func AdaptiveDepth(fileCount int) int { diff --git a/main.go b/main.go index 7f2e8e5..74c0be8 100644 --- a/main.go +++ b/main.go @@ -237,6 +237,7 @@ func main() { diffMode := flag.Bool("diff", false, "Only show files changed vs main (or use --ref to specify branch)") diffRef := flag.String("ref", "main", "Branch/ref to compare against (use with --diff)") depthLimit := flag.Int("depth", 0, "Limit tree depth (0 = unlimited)") + maxFiles := flag.Int("max-files", 0, "Scan bound: files kept and the largest repo a dependency scan runs on (0 = config max_files or 50000, -1 = unbounded)") onlyExts := flag.String("only", "", "Only show files with these extensions (comma-separated, e.g., 'swift,go')") excludePatterns := flag.String("exclude", "", "Exclude files matching patterns (comma-separated, e.g., '.xcassets,Fonts')") jsonMode := flag.Bool("json", false, "Output JSON (for Python renderer compatibility)") @@ -422,7 +423,10 @@ func main() { if *depthLimit == 0 && projCfg.Depth > 0 { *depthLimit = projCfg.Depth } - filters := scanner.Filters{Only: only, Exclude: exclude} + if *maxFiles == 0 { + *maxFiles = projCfg.MaxFiles + } + filters := scanner.Filters{Only: only, Exclude: exclude, MaxFiles: *maxFiles} if *debugMode { fmt.Fprintf(os.Stderr, "[debug] Root path: %s\n", root) @@ -500,12 +504,16 @@ func main() { mode = "skyline" } - // Scan files - files, err := scanner.ScanFiles(context.Background(), root, gitCache, only, exclude) + // Scan files, keeping at most the bound and counting the rest (#186). + files, totalFiles, err := scanner.ScanFilesBounded(context.Background(), root, gitCache, filters) if err != nil { fmt.Fprintf(os.Stderr, "Error walking tree: %v\n", err) os.Exit(1) } + var scanBound *scanner.ScanBound + if totalFiles > len(files) { + scanBound = &scanner.ScanBound{Scanned: len(files), Total: totalFiles, Limit: limits.ScanFileBound(*maxFiles)} + } // Filter to changed files if --diff specified (with diff info annotations) var impact []scanner.ImpactInfo @@ -532,6 +540,7 @@ func main() { Depth: *depthLimit, Only: only, Exclude: exclude, + ScanBound: scanBound, } // Render or output JSON @@ -542,6 +551,9 @@ func main() { } else { render.Tree(os.Stdout, project) } + if !*jsonMode && scanBound != nil { + fmt.Printf(" Coverage: partial — %s\n", scanBound.Note()) + } } func applyGlobalRootOptions(args []string) ([]string, bool, error) { diff --git a/mcp/main.go b/mcp/main.go index 7d873a4..266c3c0 100644 --- a/mcp/main.go +++ b/mcp/main.go @@ -18,6 +18,7 @@ import ( "time" "unicode/utf8" + "codemap/analysis" "codemap/config" "codemap/find" "codemap/handoff" @@ -940,6 +941,10 @@ func handleGetImporters(ctx context.Context, req *mcp.CallToolRequest, input Imp structured.NotIndexedReason = why.Reason return textResult(notIndexedText(file, why, fg)), structured, nil } + if fg.Coverage.EffectiveStatus() == analysis.CoverageUnavailable { + // No scan ran (file bound, ast-grep missing): unknown, not zero. + empty = fmt.Sprintf("Importers of '%s': unknown, no dependency scan ran", file) + } return textResult(empty + mcpCoverageText(fg)), structured, nil } diff --git a/render/depgraph.go b/render/depgraph.go index 5d8c3dd..dc3d0bd 100644 --- a/render/depgraph.go +++ b/render/depgraph.go @@ -63,7 +63,13 @@ func Depgraph(ctx context.Context, w io.Writer, project scanner.DepsProject) { projectName := filepath.Base(project.Root) if len(files) == 0 { - fmt.Fprintln(w, " No source files found.") + if refusedByScanBound(project.Coverage) { + // The repo is above the file bound, so nothing was scanned and + // "no source files" is not what happened (#186). + fmt.Fprintln(w, " Dependency scan not run: repository is above the file bound.") + } else { + fmt.Fprintln(w, " No source files found.") + } renderCoverageLine(w, project.Coverage) return } @@ -459,6 +465,20 @@ func pathDir(path string) string { // renderCoverageLine surfaces degraded scan coverage in text output so a // fail-closed or partial scan never reads as a complete, empty graph. +// refusedByScanBound reports whether coverage is unavailable because the +// scanner refused to run above the file bound (its note starts "bounded: "). +func refusedByScanBound(coverage analysis.Coverage) bool { + if coverage.Status != analysis.CoverageUnavailable { + return false + } + for _, source := range coverage.Sources { + if source.Name == "scan-bound" { + return true + } + } + return false +} + func renderCoverageLine(w io.Writer, coverage analysis.Coverage) { if coverage.Status == "" || coverage.Status == analysis.CoverageComplete { return diff --git a/scan_bound_cli_test.go b/scan_bound_cli_test.go new file mode 100644 index 0000000..c687eba --- /dev/null +++ b/scan_bound_cli_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "bytes" + "strings" + "testing" + + "codemap/analysis" + "codemap/scanner" +) + +// Above the file bound no dependency scan runs, so --importers has no answer. +// It used to print "No files import X" above an unavailable coverage line, +// which reads as a confident zero (#186, #138). +func TestImportersReportUnavailableSaysUnknown(t *testing.T) { + var out bytes.Buffer + renderImportersReportCLI(&out, scanner.ImportersReport{ + File: "src/a.ts", + CoverageStatus: string(analysis.CoverageUnavailable), + CoverageNotes: []string{scanner.ScanBoundNote(0, 60001, 50000)}, + }) + got := out.String() + if strings.Contains(got, "No files import") { + t.Fatalf("printed a confident zero with no scan:\n%s", got) + } + if !strings.Contains(got, "unknown, no dependency scan ran") || !strings.Contains(got, "scanned 0 of 60001 files (limit 50000") { + t.Fatalf("output = %q, want the unknown wording and the bound note", got) + } +} diff --git a/scanner/cargofallback.go b/scanner/cargofallback.go index 2f77372..928a080 100644 --- a/scanner/cargofallback.go +++ b/scanner/cargofallback.go @@ -7,6 +7,8 @@ import ( "path/filepath" "sort" "strings" + + "codemap/limits" ) var errCargoFallbackUnavailable = errors.New("cargo dependency fallback unavailable") @@ -73,6 +75,28 @@ func scanForGraphOutcomeWithFilters(ctx context.Context, root string, filters Fi } root = absRoot + // ast-grep walks the tree itself and its full JSON output is held in + // memory, so the only place to bound a dependency scan is before it + // starts. Every graph build routes through here (ScanForDeps and + // BuildFileGraph both). Count first, a stat walk that keeps one entry, + // and refuse the scan above the bound with the reason on the coverage + // line; --importers, blast-radius, find, hooks and MCP all read it. + if bound := limits.ScanFileBound(filters.MaxFiles); bound > 0 { + // A walk error (missing root, permission) is the scanner's to report + // with its own precedence; only a cancelled context stops here. + _, total, countErr := ScanFilesBounded(ctx, root, NewGitIgnoreCache(root), Filters{Only: filters.Only, Exclude: filters.Exclude, MaxFiles: 1}) + if ctxErr := ctx.Err(); ctxErr != nil { + return ScanOutcome{}, false, ctxErr + } + if countErr == nil && total > bound { + return ScanOutcome{Sources: []ScanSourceOutcome{{ + Name: scanBoundSource, + Status: ScanSourceUnavailable, + Detail: ScanBoundNote(0, total, bound), + }}}, true, nil + } + } + outcome, err := scan(root) degraded := false if err == nil { diff --git a/scanner/filegraph.go b/scanner/filegraph.go index 14ee665..4d3759d 100644 --- a/scanner/filegraph.go +++ b/scanner/filegraph.go @@ -159,7 +159,7 @@ func BuildFileGraphFromAnalyses(ctx context.Context, root string, analyses []Fil func buildFileGraphFromAnalysesWithCargoMetadata(ctx context.Context, root string, analyses []FileAnalysis, loader cargoMetadataLoader) (*FileGraph, error) { cfg := config.Load(root) - filters := Filters{Only: cfg.Only, Exclude: cfg.Exclude} + filters := Filters{Only: cfg.Only, Exclude: cfg.Exclude, MaxFiles: cfg.MaxFiles} filtered, err := filterAnalysesContext(ctx, analyses, filters) if err != nil { return nil, err diff --git a/scanner/scan_bound_test.go b/scanner/scan_bound_test.go new file mode 100644 index 0000000..30881c2 --- /dev/null +++ b/scanner/scan_bound_test.go @@ -0,0 +1,89 @@ +package scanner + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "codemap/analysis" +) + +func writeChainRepo(t *testing.T, n int) string { + t.Helper() + dir := t.TempDir() + for i := 0; i < n; i++ { + body := fmt.Sprintf("import { x } from './f%d';\nexport const x = %d;\n", (i+1)%n, i) + if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("f%d.ts", i)), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +// A repository above the file bound used to be walked and graphed in full, +// and a machine running two such scans ran out of memory (#186). The walk +// now keeps the first N files and reports the whole count. +func TestScanFilesBoundedStopsAtLimit(t *testing.T) { + dir := writeChainRepo(t, 5) + files, total, err := ScanFilesBounded(context.Background(), dir, nil, Filters{MaxFiles: 3}) + if err != nil { + t.Fatal(err) + } + if len(files) != 3 || total != 5 { + t.Fatalf("files = %d, total = %d, want 3 of 5", len(files), total) + } + files, total, err = ScanFilesBounded(context.Background(), dir, nil, Filters{MaxFiles: -1}) + if err != nil { + t.Fatal(err) + } + if len(files) != 5 || total != 5 { + t.Fatalf("unbounded: files = %d, total = %d, want 5 of 5", len(files), total) + } +} + +// Above the bound the dependency scan is not attempted at all, so no graph is +// allocated, and every consumer of the coverage line sees why. +func TestScanForDepsRefusesAboveBound(t *testing.T) { + dir := writeChainRepo(t, 3) + outcome, err := ScanForDeps(context.Background(), dir, Filters{MaxFiles: 2}) + if err != nil { + t.Fatal(err) + } + if len(outcome.Analyses) != 0 { + t.Fatalf("analyses = %d, want none above the bound", len(outcome.Analyses)) + } + coverage := CoverageFromSources(outcome.Sources) + if coverage.Status != analysis.CoverageUnavailable { + t.Fatalf("status = %v, want unavailable (sources %v)", coverage.Status, outcome.Sources) + } + want := "bounded: scanned 0 of 3 files (limit 2; raise with --max-files or config max_files)" + found := false + for _, source := range outcome.Sources { + if source.Name == "scan-bound" && source.Detail == want { + found = true + } + } + if !found { + t.Fatalf("sources = %v, want scan-bound with detail %q", outcome.Sources, want) + } + + graph, err := BuildFileGraph(context.Background(), dir, Filters{MaxFiles: 2}) + if err != nil { + t.Fatal(err) + } + if len(graph.Imports) != 0 || graph.Coverage.EffectiveStatus() != analysis.CoverageUnavailable || !strings.Contains(strings.Join(graph.Coverage.Notes, "\n"), "bounded: scanned 0 of 3 files") { + t.Fatalf("graph above bound: imports = %d, status = %v, notes = %v", len(graph.Imports), graph.Coverage.EffectiveStatus(), graph.Coverage.Notes) + } + + // Under the bound nothing changes. + outcome, err = ScanForDeps(context.Background(), dir, Filters{MaxFiles: 3}) + if err != nil { + t.Fatal(err) + } + if len(outcome.Analyses) != 3 || CoverageFromSources(outcome.Sources).Status != analysis.CoverageComplete { + t.Fatalf("at bound: analyses = %d, status = %v", len(outcome.Analyses), CoverageFromSources(outcome.Sources).Status) + } +} diff --git a/scanner/types.go b/scanner/types.go index 39c7a6b..331922c 100644 --- a/scanner/types.go +++ b/scanner/types.go @@ -35,6 +35,24 @@ type Project struct { // DiffTotals describes the whole diff when Files holds only a shown // subset, so a renderer never presents the subset's counts as the total. DiffTotals *DiffTotals `json:"diff_totals,omitempty"` + // ScanBound is set when the walk kept fewer files than matched, so Files + // is a prefix of the repository, not the repository (#186). + ScanBound *ScanBound `json:"scan_bound,omitempty"` +} + +// ScanBound records a walk that stopped keeping files at Limit. +type ScanBound struct { + Scanned int `json:"scanned"` + Total int `json:"total"` + Limit int `json:"limit"` +} + +// Note renders the bound as the coverage line every mode prints. +func (b *ScanBound) Note() string { + if b == nil { + return "" + } + return ScanBoundNote(b.Scanned, b.Total, b.Limit) } // DiffTotals are the counts over every changed file, shown or not. diff --git a/scanner/walker.go b/scanner/walker.go index a69bbf9..9d5795a 100644 --- a/scanner/walker.go +++ b/scanner/walker.go @@ -3,6 +3,7 @@ package scanner import ( "bufio" "context" + "fmt" "io/fs" "os" "path/filepath" @@ -11,6 +12,7 @@ import ( "codemap/config" "codemap/internal/projectpath" + "codemap/limits" ignore "github.com/sabhiram/go-gitignore" ) @@ -325,10 +327,24 @@ func LoadGitignore(root string) *ignore.GitIgnore { // only: list of extensions to include (empty = all) // exclude: list of patterns to exclude func ScanFiles(ctx context.Context, root string, cache *GitIgnoreCache, only []string, exclude []string) ([]FileInfo, error) { + files, _, err := ScanFilesBounded(ctx, root, cache, Filters{Only: only, Exclude: exclude, MaxFiles: -1}) + return files, err +} + +// ScanFilesBounded walks like ScanFiles but keeps at most the configured +// number of files (limits.ScanFileBound) and reports how many matched in +// total, so a caller can say "scanned N of M" instead of presenting the +// prefix as the repository. The walk continues past the bound to count, +// which is a stat per file and no allocation; a repo that is too large to +// graph is still cheap to measure. +func ScanFilesBounded(ctx context.Context, root string, cache *GitIgnoreCache, filters Filters) ([]FileInfo, int, error) { if err := ctx.Err(); err != nil { - return nil, err + return nil, 0, err } + only, exclude := filters.Only, filters.Exclude + bound := limits.ScanFileBound(filters.MaxFiles) var files []FileInfo + total := 0 absRoot := projectpath.CanonicalPath(root) err := filepath.Walk(absRoot, func(path string, info os.FileInfo, err error) error { @@ -392,6 +408,10 @@ func ScanFiles(ctx context.Context, root string, cache *GitIgnoreCache, only []s return nil } + total++ + if bound > 0 && total > bound { + return nil + } files = append(files, FileInfo{ Path: relPath, Size: info.Size(), @@ -404,34 +424,59 @@ func ScanFiles(ctx context.Context, root string, cache *GitIgnoreCache, only []s err = ctx.Err() } - return files, err + return files, total, err +} + +// scanBoundSource names the coverage source recorded when a dependency scan +// was refused because the repository is above the file bound. +const scanBoundSource = "scan-bound" + +// refusedByBound reports whether this outcome is the bound refusal rather +// than a scan, so callers skip the passes that would walk the tree again. +func (o ScanOutcome) refusedByBound() bool { + for _, source := range o.Sources { + if source.Name == scanBoundSource { + return true + } + } + return false +} + +// ScanBoundNote is the coverage line printed when a scan stopped at the file +// bound. scanned is the number kept, total the number that matched. +func ScanBoundNote(scanned, total, bound int) string { + return fmt.Sprintf("bounded: scanned %d of %d files (limit %d; raise with --max-files or config max_files)", scanned, total, bound) } // Filters controls which files scanner operations include. type Filters struct { Only []string Exclude []string + // MaxFiles bounds the scan: 0 = limits.DefaultMaxScanFiles, negative = + // unbounded (see limits.ScanFileBound). + MaxFiles int } // ConfiguredFilters returns the project's configured dependency filters. func ConfiguredFilters(root string) Filters { cfg := config.Load(root) - return Filters{Only: cfg.Only, Exclude: cfg.Exclude} + return Filters{Only: cfg.Only, Exclude: cfg.Exclude, MaxFiles: cfg.MaxFiles} } // ScanConfiguredFiles scans using the active setup root's project filters // while honoring caller cancellation. func ScanConfiguredFiles(ctx context.Context, root string, cache *GitIgnoreCache) ([]FileInfo, error) { - cfg := config.Load(root) - return ScanConfiguredFilesWithFilters(ctx, root, cache, Filters{Only: cfg.Only, Exclude: cfg.Exclude}) + return ScanConfiguredFilesWithFilters(ctx, root, cache, ConfiguredFilters(root)) } // ScanConfiguredFilesWithFilters scans files using already resolved filters. +// The result honours the file bound; hooks and the watch daemon therefore +// degrade to a prefix on a repo above it instead of holding every file. func ScanConfiguredFilesWithFilters(ctx context.Context, root string, cache *GitIgnoreCache, filters Filters) ([]FileInfo, error) { if err := ctx.Err(); err != nil { return nil, err } - files, err := ScanFiles(ctx, root, cache, filters.Only, filters.Exclude) + files, _, err := ScanFilesBounded(ctx, root, cache, filters) if err != nil { return nil, err } @@ -480,6 +525,9 @@ func ScanForDeps(ctx context.Context, root string, filters Filters) (ScanOutcome if err != nil { return outcome, err } + if outcome.refusedByBound() { + return outcome, nil + } outcome, err = appendCUEOutcome(ctx, root, filters, outcome) if err != nil { return outcome, err diff --git a/watch/daemon.go b/watch/daemon.go index 38b4d6e..117ba6a 100644 --- a/watch/daemon.go +++ b/watch/daemon.go @@ -384,7 +384,7 @@ func (d *Daemon) refreshConfiguredFiles(resetIgnoreCache bool) error { d.gitCache = gitCache } cfg := d.loadConfig() - files, err := scanner.ScanConfiguredFilesWithFilters(context.Background(), d.root, gitCache, scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude}) + files, err := scanner.ScanConfiguredFilesWithFilters(context.Background(), d.root, gitCache, scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude, MaxFiles: cfg.MaxFiles}) if err != nil { return err } @@ -474,7 +474,7 @@ func (d *Daemon) startDependencyWorker() { return case snapshot := <-d.dependencyRequests: started := time.Now() - filters := scanner.Filters{Only: snapshot.config.Only, Exclude: snapshot.config.Exclude} + filters := scanner.Filters{Only: snapshot.config.Only, Exclude: snapshot.config.Exclude, MaxFiles: snapshot.config.MaxFiles} graph, err := buildDependencyGraph(ctx, d.root, filters, buildFileGraph) result := dependencyGraphResult{snapshot: snapshot, graph: graph, err: err, started: started} select { From 07a8cae2e1d32f9a6c41e3ee51abc4ad66a5a80b Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Sun, 11 Oct 2026 22:16:28 +0200 Subject: [PATCH 2/2] limits: default scan bound 100000 files (about 800 MB, 10 s on the 2026-10-11 measurements) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NRaBgKRAZMX9hjbkkefA7T --- README.md | 4 ++-- limits/limits.go | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2d6451c..20a41ea 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,7 @@ subdirectory; relative setup paths resolve from the project root. | `--depth, -d ` | Limit tree depth (0 = unlimited) | | `--only ` | Only include files with these extensions | | `--exclude ` | Exclude files matching patterns | -| `--max-files ` | Scan bound: files kept by the tree, and the largest repo a dependency scan runs on (default 50000, or config `max_files`; `-1` = unbounded). Above it the output says `bounded: scanned N of M files` instead of presenting a prefix as the repo | +| `--max-files ` | Scan bound: files kept by the tree, and the largest repo a dependency scan runs on (default 100000, or config `max_files`; `-1` = unbounded). Above it the output says `bounded: scanned N of M files` instead of presenting a prefix as the repo | | `--diff` | Show files changed vs main branch | | `--ref ` | Branch to compare against (with `--diff`) | | `--deps` | Dependency flow mode | @@ -409,7 +409,7 @@ codemap config show # display current config "only": ["rs", "sh", "sql", "toml", "yml"], "exclude": ["docs/reference", "docs/research"], "depth": 4, - "max_files": 50000, + "max_files": 100000, "mode": "auto", "guidance": { "missing_extension_hints": true, diff --git a/limits/limits.go b/limits/limits.go index b4e5e10..086fd49 100644 --- a/limits/limits.go +++ b/limits/limits.go @@ -21,12 +21,14 @@ const ( LargeRepoFileCount = 5000 // DefaultMaxScanFiles bounds the files a scan keeps and the size of repo - // a dependency scan is attempted on. Measured on 2026-10-11: --deps on a - // synthetic 60k-file TypeScript repo peaked at 421 MB RSS, so two - // concurrent scans of 200k-file repos exhaust a laptop (#186). Raised - // per run with --max-files or per repo with config max_files; a - // negative value removes the bound. - DefaultMaxScanFiles = 50000 + // a dependency scan is attempted on. Measured on 2026-10-11 with + // synthetic TypeScript repos: --deps peaked at 421 MB RSS on 60k files + // and 1,565 MB on 200k, about 8 MB per 1,000 files, linear. 100k is + // therefore roughly 800 MB and 10 s: the point where a default should + // stop and ask rather than let two concurrent scans exhaust a laptop + // (#186). Raised per run with --max-files or per repo with config + // max_files; a negative value removes the bound. + DefaultMaxScanFiles = 100000 ) // ScanFileBound resolves a configured bound: zero means the default,