diff --git a/.gitignore b/.gitignore index 079d2cc..0b551b8 100644 --- a/.gitignore +++ b/.gitignore @@ -34,8 +34,19 @@ Thumbs.db # Private/local files .private/ .env +.env.local +.env.*.local *.local +# Credentials and keys. A scanner that publishes findings about other people's +# secrets should not be able to commit its own by accident. +secrets.json +credentials.json +*.pem +*.key +*.p12 +*.pfx + # Downloaded package sources (cache) .cache/ diff --git a/.goreleaser.yml b/.goreleaser.yml index 4216275..012e2cf 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -38,7 +38,7 @@ builds: archives: - id: default - format: tar.gz + formats: [tar.gz] name_template: >- {{ .ProjectName }}_ {{- .Version }}_ @@ -49,7 +49,7 @@ archives: {{- if .Arm }}v{{ .Arm }}{{ end }} format_overrides: - goos: windows - format: zip + formats: [zip] files: - README.md - LICENSE diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cab54d..51dde2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,52 @@ All notable changes to QRAMM CryptoDeps will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.3.0] - 2026-07-30 -Release-blocking defects found by a fresh-user release test on the 1.3.0 -candidate. 1.3.0 was never tagged. +Fixes both open community issues, and the release-blocking defects a fresh-user +release test then found while reproducing them. An earlier 1.3.0 candidate +carrying only the community fixes was prepared on 2026-07-27 but never tagged, +so everything below ships together in this release. ### Fixed +- **`requirements-*.txt` files were skipped** + ([#1](https://github.com/csnp/cryptodeps/issues/1)). Manifest discovery + matched the exact filename `requirements.txt`, so the + `requirements-dev.txt` and `requirements-prod.txt` split that most Python + projects use was never scanned. The `requirements*.txt` family and the + `requirements/*.txt` directory layout are now discovered and parsed. + +- **CBOM did not identify which dependency an algorithm came from** + ([#2](https://github.com/csnp/cryptodeps/issues/2)). Output emitted only the + algorithm, carrying the dependency's version but never its name, so a + component reading `{"name": "RSA", "version": "1.3.2"}` was unattributable. + Each dependency is now emitted as its own `library` component with a + `bom-ref`, and the CycloneDX `dependencies` graph links each algorithm to the + library that provides it. + +- **`pyproject.toml` and `Pipfile` produced fabricated dependencies.** Both were + advertised as supported, but parsing fell through to the `requirements.txt` + line parser behind a `TODO`. A `pyproject.toml` yielded entries named after + TOML keys (`name`, `dependencies`, `requires-python`), and because the real + packages were never identified, a project depending on `cryptography` reported + "No cryptographic usage detected". Both formats now have real TOML parsers + covering PEP 621, Poetry, and Pipfile layouts. + +- **Every CBOM shared one serial number.** `generateUUID` returned a hardcoded + all-zero UUID, which also failed the CycloneDX `urn:uuid` pattern. Serial + numbers are now random version 4 UUIDs. + +- **Unresolved build properties were emitted as versions.** A Maven dependency + declared as `${java-jwt.version}` appeared in the CBOM with that literal as + its version. Version ranges such as `>=2.0` were likewise emitted where + CycloneDX expects a concrete version. Only pinned versions are now reported as + versions; the declared constraint is preserved in the component description. + +- **CBOM `primitive` values were outside the CycloneDX enum**, which failed + schema validation for the whole document. Categories now map onto the + permitted enum, resolving `encryption` by algorithm where possible. + - **Every machine-readable output reported the wrong tool version.** One binary gave four answers: `version` said 1.3.0 while SARIF and CBOM both claimed 1.0.0 and JSON carried no version at all. SARIF and CBOM are provenance @@ -35,13 +74,293 @@ candidate. 1.3.0 was never tagged. worded differently, and the `--deep` hints that the analyzer had always generated are finally printed. +- **`--deep` results were then ignored by that same verdict, and the report sent + the user back to `--deep`.** Found while release-testing this version, and a + regression against 1.2.2 rather than a pre-existing defect: `analyze + --deep`, where the database covers none of the dependencies and source + analysis reads all of them, printed "Not analyzed. All 3 dependencies are + absent from the crypto database, so no conclusion about cryptographic usage + can be drawn from this scan. Run with `--deep`". Every clause of that was + false for the run that produced it, the JSON for the same invocation reported + `deepAnalyzed` on all three, and 1.2.2 printed a correct clean verdict. The + fix above asked whether the database covered a package in order to answer + whether anything had examined it, which were the same question until `--deep` + became a second way to examine one. Coverage is now counted where examination + happens, as `summary.notExamined`, and every format reads it. Reports also + distinguish "not examined because you did not ask for source analysis", where + the next step is `--deep`, from "not examined because the package could not be + fetched", where it is not; `summary.deepAttempted` records which. + +- **A cached package that had never been unpacked was counted as analyzed.** + The npm and PyPI fetchers returned one directory after extracting an archive + and a different one on a cache hit, and the cache-hit check asked only whether + the version directory existed. A directory holding nothing but the downloaded + tarball answers yes, so its contents were walked, no source file was found, + and the package was recorded as read by source analysis with no findings and + no warning. The two paths disagreed for real packages too: an npm tarball that + does not unpack to `package/`, as `ejs` does not, failed on the first run and + was silently analyzed from the wrong root on every run after it, so the same + project reported 43 packages analyzed with a warning and then 44 without one. + Both paths now resolve the extracted source through one helper, and a cache + entry that holds none is refetched rather than accepted. The Maven fetcher + accepted an `extracted/` directory that a failed unzip had left empty; that is + covered by the same check. + +- **A cache entry the fetcher could not identify was deleted rather than + reported.** Where an entry held more than one candidate directory, the + resolver could not say which was the package and returned an error saying so. + All three cache-hit sites discarded that error and cleared the entry, so a + cached tree that may well have held the findings was destroyed by a scan that + had only failed to identify it, the refetch loop was unbounded, and the + diagnostic written for the case was unreachable: the user was shown whatever + the refetch failed with, typically `npm pack failed`. Such an entry is now + kept and the reason reported. + +- **A deep scan of a package with no readable source was counted as an + examination.** `--deep` marked a dependency analyzed whenever the fetch and + the walk both returned without an error, and a walk over a tree holding no + file the analyzer understands returns nothing rather than an error. A Maven + artifact whose sources JAR does not exist falls back to the main JAR, which + carries compiled classes only, so no `.java` file was ever read and the report + still said `No cryptographic usage detected in the 1 of 1 dependencies that + were examined`. The walkers now report how many files they parsed, the count + travels with the analysis as `analysis.filesAnalyzed`, and the claim of an + examination is made from it. A package whose archive holds nothing readable is + reported as not examined, with the reason on stderr and in the `error` field + of the document, so a consumer can tell it from a package that was skipped. + 1.2.2 answers the same scan with a plain clean verdict, so this is not a + regression; the wording introduced in this release asserted an examination + that had not happened. + + The count is of files **parsed**, not files opened, which is a distinction the + first version of this fix did not make. Three of the four analyzers are line + scanners whose only failure was `os.Open`, so any openable file with a + matching extension counted: a zero-byte `Empty.java`, or compiled class bytes + under a `.java` name, each produced `filesAnalyzed: 1` and restored the exact + verdict above. An extension is a descriptor rather than the thing itself, so a + file now counts only if it holds text the analyzer can read. + + "Text the analyzer can read" then had to be corrected twice more. It first + meant "the head is valid UTF-8", which accepted every NUL-free binary over the + head size, because the allowance for a rune straddling the boundary shrank the + head one byte at a time and `utf8.Valid` reports true for an empty slice: 2000 + bytes of `0xff` counted as a parsed file. Bounding that allowance then left the + UTF-8 requirement itself, which was never the right question. These analyzers + are line scanners matching ASCII identifiers, so they neither need nor check + valid UTF-8, and requiring it refused legitimate source in any single-byte + encoding: a `.java` or `.js` file holding one Latin-1 accent in a comment was + reported as unreadable and the cryptography beside the accent was never looked + for, where 1.2.2 had found it. A file is now text if it holds no NUL byte and + its head is either valid UTF-8 or predominantly printable ASCII, which is the + property a line scanner actually depends on. The threshold is measured: across + 140 real npm source files carrying non-ASCII characters, re-encoded to Latin-1, + the lowest printable-ASCII fraction was 0.904 and the median 0.999, while + random NUL-free bytes reached at most 0.453 and real binaries with their NUL + bytes stripped reached 0.286. + +- **A source file the analyzer refused was dropped silently, and the package was + still reported as examined.** The count of failed files was incremented inside + the directory walk and went no further, so a dependency holding one file that + parsed and one that did not was described as examined and clean, on every + stream and in every format, with zero bytes on stderr. Combined with the + encoding defect above it made an evasion rather than an inconvenience: a + dependency could hide its cryptography from this scanner by encoding the file + that calls MD5 in Latin-1, and the report said the package had been examined + and no cryptography was detected. The earlier coverage fix does not reach this + state, because it asks its question of a dependency nothing examined, and a + dependency read in part is counted as examined. The count now travels with the + analysis as `analysis.filesUnreadable`, sums into the summary as + `sourceFilesUnreadable`, is named on stderr per package, and reaches all five + formats through the one classifier they share. + +- **Source analysis of a PyPI dependency could execute code the scanned manifest + chose.** `npm pack` runs with `--ignore-scripts` in this release, which closes + that half of the class; pip had no equivalent. Resolving a PyPI name and version + with no matching wheel makes pip install the project's build dependencies and + run its build backend, which for a `setup.py` sdist is arbitrary code on the + scanning host, from a package named by a manifest the operator did not write. + Verified directly: `pip3 download --no-deps psycopg2==2.9.9` reports + "Installing build dependencies" and "Preparing metadata". Fetching source in + order to read it must not be a way to run it, so the fetch is now restricted to + built wheels with `--only-binary=:all:`. Pre-existing rather than a regression; + it is fixed here because the release would otherwise state that this class was + closed while half of it was open. The cost is recorded under known limitations. + +- **The npm name grammar refused 1,054 real, installable packages.** Introduced by + this release. Holding a name to its registry's grammar was the right layer, and + the pattern encoded the rules npm applies to a name from a NEW publisher: a + scope and a name each beginning with a letter or a digit. npm grandfathered + every name that predates those rules. Swept against the complete registry + (4,240,864 names), the pattern refused 1,054 that are published and installable + now, nine of them above 100,000 downloads a month, including `@lingo.dev/_spec` + at 212,000, `@-xun/fs`, `@_sh/strapi-plugin-ckeditor` and `@~39/empty`. A + refused dependency is silently never analyzed, so the scan reports a coverage it + never had, which is this release's own false-clean defect arrived at from the + other direction. The grammar now excludes what lets a name be read as something + other than a name, which is the question a fetch guard exists to answer: an at + sign separates a name from a spec, a slash or a backslash makes it a path, a + colon makes it a scheme, and whitespace separates arguments. A leading dot is + refused separately, so that loosening the grammar cannot quietly admit it. The + same sweep over the other three ecosystems refuses nothing real: 0 of 860,284 + PyPI names, 0 of 16,279 Go module paths, 0 of 1,800 Maven coordinates. + +- **A Maven property in a dependency's groupId or artifactId was never + resolved.** Properties were expanded in the version alone, so + `${project.groupId}`, which is how a multi-module build names a sibling module, + survived into the coordinate and was then refused as a name that could steer a + fetch. Across 2,099 real poms from Maven Central, 81 coordinates in 16 published + artifacts use it. On one real pom the tool analyzed 6 of its dependencies where + the same pom with the property expanded analyzed 11. Properties are now resolved + in all three fields, and the project's own `groupId` and `artifactId` are + available as properties alongside its version. + +- **A manifest-declared version traversed out of the source cache behind any + scheme.** The screen that refuses a local path skipped its traversal check + whenever the value contained a colon, on the reasoning that a colon meant a + remote reference such as `github:owner/repo`. An invented scheme defeats that: + a version of `a1:../../../../../../../victim` carries a colon, so the check + never ran, `npm pack` resolved a directory outside the cache, and the analyzer + read it and published its absolute paths as that dependency's source. + + Reproduced against the candidate binary: `npm pack` resolves such a spec as a + directory, consuming two `../` per level, and at eight the target sat two + levels above the cache root. The scan exited 1 having copied the victim's + source into the cache and published its `SECRET.js` as MD5 and RSA findings + attributed to the declared dependency, with zero bytes on stderr. Present in + 1.2.2, where it additionally ran the target's `prepare` script, so the fourth + spelling of this class was until now closed only in its execution half. The + traversal check now runs on every value: no legitimate version carries a `..` + path element, and whether a value is remote is decided by the scheme rather + than by the presence of a punctuation mark. + +- **A failed archive extraction was reported as an examined package on the next + run.** The three download sites discard what a failed fetch left behind; the + four extraction sites returned a bare error instead. Both `tar` and `unzip` + extract partially before failing, so the cache entry was left holding a + populated directory, which the cache-hit check accepts. The first scan reported + the package as not examined and the second reported it as examined, with + findings, from a partially extracted archive, with nothing on any stream saying + so. `unzip` exits non-zero on warnings alone, so a benign real wheel could take + this path too. All four sites now discard the entry. + +- **A file with one line longer than 64 KiB contributed nothing, silently.** + `bufio.Scanner`'s default token limit stops the scan and reports an error, the + three line-scanning analyzers return it, and the walk discards every usage they + had already found. Bundled and minified output is normally one very long line, + so a `dist/*.js` of 70 KB on one line calling `crypto.createHash('md5')` + produced no finding, no warning and a clean verdict, on this candidate and on + 1.2.2. The scanner is now given the same bound the file already has, and the + per-file cap moves from 8 MiB to 32 MiB: `aws-sdk-go` v1.55.5 ships + `service/ec2/api.go` at 7,771,273 bytes, within 8 percent of the old cap, and + that file has grown every release. This does not reach a file named + `*.min.js`, which the JavaScript walker skips by name before the analyzer sees + it; see known limitations. What the larger cap costs was measured: a + 30 MiB single-line file dense with cryptographic calls peaks at about 309 MB of + resident memory and takes 2.4 seconds. Files are read one at a time, so that is + the bound for a scan rather than a per-archive total. + +- **A Poetry or Pipfile dependency declared by location was fetched from PyPI + under its bare name.** An inline table with no `version` key returned no + version at all, discarding the `path`, `git`, `url` or `file` it was actually + declared by, and the fetcher then downloaded whatever PyPI serves under that + name. So `internal-lib = {path = "../internal-lib"}` was replaced by a public + package of the same name, whose source was analyzed and reported as this + project's. That is dependency confusion performed by the scanner: registering + the name of an organisation's local package is enough to be handed the + attribution, and because `pip` builds sdists, execution with it. New in this + release, which is where these TOML parsers were added. The locator is now kept + so the fetcher's own guards see what was declared. + +- **`--fail-on` was validated one way and read another.** The validation added in + this release trims and lowercases before deciding; the code that turns the value + into an exit code only lowercased. So `" partial "` passed validation, matched + no policy, and fell through to the default vulnerable-only gate: a project whose + findings are partial risk exited 3 for `partial` and 0 for `" partial "`, with + nothing on either stream. Whitespace around a value is the ordinary result of a + YAML block scalar or a workflow expression, which is exactly where this flag is + used, and the new validation is what made the padded value look accepted. Both + readers now canonicalise through one function. + +- **A symlink in an extracted archive was followed out of the source cache.** + The directory walk lstats, so a symlink is not a directory and fell through to + the file analyzer, which opened whatever it pointed at. Both `tar` and `unzip` + restore absolute symlink targets, so a package could ship `Leak.java` pointing + at any file the scanning process can read and have this tool read it, count it + as evidence that the package was examined, and publish its contents as that + dependency's source in JSON, SARIF and CBOM. Symlinks are no longer read: an + extracted package is analyzed from its own contents. + +- **CBOM and SARIF omitted the dependencies a scan could not examine, whenever + it had findings for the others.** The coverage question was asked only of + reports that produced nothing, and a partially examined project produces + something. So a Maven scan that told the operator on three separate streams + that it could not read one of its dependencies handed GitHub code scanning a + SARIF run reporting success with no notification, and produced a CBOM listing + the libraries it had findings for with no coverage property. The two formats + that omitted it are the two that get uploaded. Incomplete coverage is now + reported as a property of the scan, in the same place a withheld-findings + filter already was, so it reaches every format whether or not findings were + also produced. + +- **`--fail-on` accepted any value and silently loosened the gate.** It was the + one enum flag of the three that did not validate, and the only one that + decides an exit code: a project exiting 3 under `--fail-on partial` exited 0 + under `--fail-on partail`, with nothing on either stream, because an + unrecognised value fell through to the default policy. It now rejects an + unknown value, naming the legal ones. + + It rejects an **empty** value too, which `--risk` and `--min-severity` + deliberately still accept, so this is not the parity an earlier draft of this + entry claimed. For those two, blank means "do not filter", which is a real + state they can express. `--fail-on` already has a default of `vulnerable` and + cannot express one by being blank, so a blank value there was indistinguishable + from an unset flag while silently selecting the vulnerable-only policy: a + partial-risk project exited 3 for `partial` and 0 for `""`. An unset workflow + input is exactly how a CI gate arrives here empty. **This is a behaviour change + for anyone forwarding a possibly-unset input:** `--fail-on ""` and + `--fail-on " "` now exit 2 without scanning, where they previously ran a full + scan under the vulnerable policy. It fails closed. The bundled Action is + unaffected, since it defaults the input to `vulnerable`. + +- **`pyproject.toml` and `Pipfile` kept the comparison operator in the version.** + The same two packages gave `pycryptodome@3.20.0` through `requirements.txt` + and `pycryptodome@==3.20.0` through the other two formats, and within one scan + the CBOM purl normalised it while JSON and SARIF did not, so a consumer could + not join the two documents by version. It also reached the fetcher, which + builds a pip spec as `name==version`. New in 1.3.0, alongside the parsers that + made those formats work at all. + - **`--risk` and `--min-severity` did nothing.** Both were stored and never read, so every value, including a misspelt one, produced byte-identical - output. They now filter, and the summary and exit code are computed from what - survives so that every number describes the same set of findings. Unknown + output. They now filter, and the summary is computed from what survives so + that every number in the report describes the same set of findings. Unknown values are rejected instead of ignored. When a filter removes every finding, the report says so rather than reporting a clean scan. +- **A reporting filter could then open the CI gate, which was a regression + introduced by making those filters work.** Found in this release's own final + review. The exit code was computed from the same filtered summary as the + report, so `analyze . --fail-on vulnerable` exited 1 and + `analyze . --fail-on vulnerable --risk safe` exited 0 on the same project, + with nine vulnerable findings withheld. The same run printed "This is not a + clean result. Re-run without the filter to see them." to stdout: the tool told + the operator it had hidden findings and told CI the project was clean. + `--min-severity` reaches it the same way, which matters more in practice, + because `--min-severity high --fail-on vulnerable` is a plausible CI line that + silently stops failing on anything below HIGH. Released 1.2.2 exits 1 for all + of these, because the filters did nothing at all there, so this was a + regression in the one flag that decides CI outcomes, in the release that + hardened that flag three separate times against silent loosening. + + A view flag does not answer a gate. The scan now records what the filters + withheld, broken down by the risk levels `--fail-on` asks about, and the gate + reads found-not-shown alongside shown. `--fail-on none` still means none, an + unfiltered clean project still exits 0, and no output format changes: the + withheld breakdown is deliberately not serialized. The two copies of the + exit-code switch, one for a single project and one for a workspace, are now + one function with two callers, since carrying the same logic twice is how they + could disagree in the first place. + - **Every SARIF result pointed at a literal path `"multiple"`.** Multi-project runs flattened all projects into one synthetic result, discarding the real manifest paths, so every alert landed on a file that does not exist. Results @@ -157,13 +476,105 @@ candidate. 1.3.0 was never tagged. so `("/repo", "/repository/go.mod")` gave `./sitory/go.mod`. - **`cryptodeps status` reported roughly twice the packages the database holds.** - It announced 1731 packages, and every per-ecosystem number was wrong the same - way, for a database of 901. The index files each package under two keys, + On a 901-record database it announced 1731, and every per-ecosystem number was + wrong the same way. The index files each package under two keys, `name@version` and `name`, so a lookup succeeds with or without a version, and - the count was of index entries rather than packages. `status` now reports 901, - matching the database's own stats block and a direct count of its records, and - it lists the ecosystems in a fixed order instead of the order the map happened - to iterate in. + the count was of index entries rather than packages. `status` now reports the + record count itself, matching the database's own stats block and a direct + count of its records, and it lists the ecosystems in a fixed order instead of + the order the map happened to iterate in. + +### Security + +- **A scanned manifest could point `--deep` at any directory the process could + read.** The source cache path was built from the dependency name and version + exactly as declared, so a `package.json` carrying + `{"ejs": "../../../somewhere"}` resolved the cache entry outside the cache + directory. That directory existed, the cache-hit check accepted it, and the + analyzer walked it and published its absolute file paths and line numbers as + that dependency's source. Scanning an untrusted repository in CI could + therefore put the contents of an unrelated directory into the SARIF the run + uploads. Present in 1.2.2 and in every earlier release with `--deep`. + Manifest-supplied names and versions are now reduced to a single safe path + segment before they are joined to the cache path, and a name or a version that + names a local path is refused with a message saying so rather than passed to + `npm pack` or `pip download`, which would resolve it. Registry and VCS + references such as `github:owner/repo` are unaffected. + +- **The same class reached the dependency name, not only the version, and + through the name it reached code execution.** Reducing the name to a safe path + segment kept the cache entry in place, which made the name look handled, but + the name is also given to the package manager as a spec, and + `npm pack ../../../../victim` resolves a directory. A manifest declaring + `{"../../../../victim": ""}` packed a tree outside the project, walked it and + published its file names and line numbers. + + Screening the name for the spellings of a local path did not close this, + because the package manager parses a string the scanner treated as opaque. + `npm pack` splits `name@spec` on the first `@` after index 0, so a dependency + named `x@/path/to/victim` presented a passing string to the guard and a + directory to npm. Reproduced end to end: npm packed the victim directory, + **ran its `prepare` and `prepack` scripts on the scanning host**, and the + analyzer published the victim's files as that dependency's findings, with + nothing on stderr. The relative form needs no knowledge of absolute paths. + The same shape reaches `pip download` as a PEP 508 direct reference, which + arrives as a Poetry or Pipfile table key (`victimpkg @ file:///path`), and + `git+file://` reached `npm pack` through the version, since it neither begins + with `file:` nor contains a `..` segment. + + Names are now held to their own registry's grammar (npm's scope-and-name + form, PEP 503 for PyPI, module-path form for Go, and Maven coordinates as + before), which is a question with a single answer rather than a list of the + ways a path can be spelled. `npm pack` additionally runs with + `--ignore-scripts` and `pip download` with `--only-binary=:all:`, so neither the + remote VCS references that remain fetchable nor a source distribution can + execute anything during a fetch. The suite asserts that real npm, PyPI and Go names + pass the grammars, including scoped npm names and versioned Go module paths. + Two limits on that claim, both stated because an earlier draft of this entry + overstated it: the assertion is that a grammar accepts a name, which is not the + same as a completed fetch, and a dotted PyPI name reaches the grammar intact + only through `pyproject.toml`. Declared in `requirements.txt`, + `zope.interface` is split at the first dot by the requirements parser long + before the grammar sees it, which is a separate pre-existing defect, recorded + under known limitations below. An earlier draft of this sentence said it was + recorded there when it was not. + +- **A scanned `pom.xml` could write any file the scanning process could write.** + This one is not a read. `fetchMavenArtifact` joined the `artifactId` from the + manifest straight onto the download path handed to `curl -o`, and + `filepath.Join` resolves `..` lexically, so the target escaped the cache. A + `?` in the `artifactId` split the Maven URL so that its path portion still + named a real artifact while the file portion traversed, which supplies the 200 + that `curl -f` needs in order to write at all. Reproduced against 1.2.2 and + against the 1.3.0 candidate: a 26-byte file outside the cache was replaced + with 234540 bytes of an unrelated archive by one `analyze --deep`. The + attacker chooses the path; the content is any artifact on Maven Central. + Coordinates are now validated as coordinates, which closes the URL and the + path at once, and the file name the fetcher writes is derived from sanitized + segments regardless. + + Present in 1.2.2 and in every earlier release with `--deep`, alongside the + read above. All three require the operator to run `--deep` over a manifest + they do not control, which is what a CI scan of an untrusted repository does. + +- **The local-path guard did not cover the Windows spellings.** A colon was read + as evidence that a reference was remote, so a drive letter passed: `C:\victim`, + `\victim` and `\\host\share` reached the package manager as folder specs on a + target this tool ships binaries for. They are now refused with the other local + paths, together with the Yarn and pnpm `link:` and `portal:` spellings, which + `npm pack` rejects today and which should not depend on another tool's parser + staying strict. Maven coordinates and npm aliases, which legitimately carry a + colon, are unaffected. + +- **The source cache could delete more than the entry it meant to.** The only + destructive operation in the fetcher took whatever path it was handed, and its + argument is built from manifest-supplied text. Nothing stated its bounds: + four separate mutations of it, up to and including removing the entire cache + root, left the test suite passing. It now refuses anything that is not exactly + one package entry inside the cache directory, and the failed-download paths + route through the same guard. No input was found that reached the wider + deletion, since the segments are sanitized before they are joined; this bounds + the operation rather than closing a known route to it. ### Changed @@ -181,44 +592,9 @@ candidate. 1.3.0 was never tagged. message that explains the failure was being pushed off the top of the terminal. Usage is still shown for genuine flag mistakes, where it helps. -## [1.3.0] - 2026-07-27 - -Fixes both open community issues, plus a silent false negative found while -reproducing them. - -### Fixed - -- **`requirements-*.txt` files were skipped** - ([#1](https://github.com/csnp/cryptodeps/issues/1)). Manifest discovery - matched the exact filename `requirements.txt`, so the - `requirements-dev.txt` and `requirements-prod.txt` split that most Python - projects use was never scanned. The `requirements*.txt` family and the - `requirements/*.txt` directory layout are now discovered and parsed. -- **CBOM did not identify which dependency an algorithm came from** - ([#2](https://github.com/csnp/cryptodeps/issues/2)). Output emitted only the - algorithm, carrying the dependency's version but never its name, so a - component reading `{"name": "RSA", "version": "1.3.2"}` was unattributable. - Each dependency is now emitted as its own `library` component with a - `bom-ref`, and the CycloneDX `dependencies` graph links each algorithm to the - library that provides it. -- **`pyproject.toml` and `Pipfile` produced fabricated dependencies.** Both were - advertised as supported, but parsing fell through to the `requirements.txt` - line parser behind a `TODO`. A `pyproject.toml` yielded entries named after - TOML keys (`name`, `dependencies`, `requires-python`), and because the real - packages were never identified, a project depending on `cryptography` reported - "No cryptographic usage detected". Both formats now have real TOML parsers - covering PEP 621, Poetry, and Pipfile layouts. -- **Every CBOM shared one serial number.** `generateUUID` returned a hardcoded - all-zero UUID, which also failed the CycloneDX `urn:uuid` pattern. Serial - numbers are now random version 4 UUIDs. -- **Unresolved build properties were emitted as versions.** A Maven dependency - declared as `${java-jwt.version}` appeared in the CBOM with that literal as - its version. Version ranges such as `>=2.0` were likewise emitted where - CycloneDX expects a concrete version. Only pinned versions are now reported as - versions; the declared constraint is preserved in the component description. -- **CBOM `primitive` values were outside the CycloneDX enum**, which failed - schema validation for the whole document. Categories now map onto the - permitted enum, resolving `encryption` by algorithm where possible. +- The root help no longer advertises "Full dependency tree analysis", and the + README no longer claims transitive coverage. Neither was true: only declared + dependencies are read. See known limitations. ### Added @@ -226,10 +602,333 @@ reproducing them. requirement parsing, CBOM dependency attribution, version resolution, serial number uniqueness, and primitive enum conformance. +- `analysis.filesAnalyzed` on every deep-analyzed record, and `error` on any + dependency source analysis could not examine, in JSON output. A + document that claims a package was read now carries the count it was claimed + from, and one that skips a package says why. + +- `analysis.filesUnreadable` per dependency and `summary.sourceFilesUnreadable` + per project, in JSON output, with a warning on stderr naming each + package and a coverage note in the table, markdown, SARIF and CBOM. The + evidence for an examination now carries its exceptions as well as its count, + so a partial reading cannot be read as a complete one. + +- `summary.notExamined` and `summary.deepAttempted` in JSON output, and + a **Not Examined** row in the markdown summary table. How much of a tree a + scan actually covered was previously only derivable, and only wrongly, from + the count of packages missing from the database. + +- Regression tests for the coverage verdict driven end to end through a real + `--deep` scan against a pre-populated source cache, for cache entries that + hold no extracted source, for an archive that carries no readable source at + all, for the bounds of the cache reset, for a spec smuggled through a + dependency name in each ecosystem, for `--ignore-scripts` reaching the npm + process, for a symlink out of an extracted archive, for incomplete coverage in + every output format, and for manifest-supplied paths in their Unix and Windows + spellings. Each was confirmed to fail at runtime against the code it guards, + never merely to fail compilation. + + Mutation matrices were run over these guards, and they are reported with their + survivors rather than as a blanket claim, because an earlier draft of this entry + said each guard had been mutation-tested while several mutations survived. Four + rounds were run and each round's survivors were closed by a further test before + the next. The ones that mattered: a threshold no fixture could pin from below, a + bound whose removal no fixture could detect, three of the four + archive-extraction sites with no test at all, and a disclosure asserted in only + one of the five output formats. Real npm, PyPI and Go names are asserted to pass + the grammars, so a guard tightened far enough to refuse a legitimate name fails + in the suite rather than in a user's CI; that assertion covers the grammar + rather than a completed fetch. + ### Dependencies - Added `github.com/BurntSushi/toml` for pyproject.toml and Pipfile parsing. +### Known limitations + +Present in 1.2.2 as well unless noted. Each was reproduced by hand against both +the 1.2.2 and the 1.3.0 binary during the release test, and each is tracked. + +- **NEW in 1.3.0: a PyPI package that publishes no wheel is no longer analyzed.** + This is the cost of the fix above: `pip download` now runs with + `--only-binary=:all:`, so a package distributed only as an sdist is reported as + not examined rather than built. The reason is named on stderr and carried in + every format. Preferring an unexamined dependency to an executed one is the + trade this release makes deliberately; a source-build mode behind an explicit + opt-in is the shape of the fix if the coverage turns out to matter. + +- **How much of a dependency tree a scan covers depends on the ecosystem, and + the documentation stated it wrongly in both directions.** Measured per + ecosystem against this binary: + + - **Go: indirect requirements ARE read.** A `go.mod` declaring one direct + require and a `// indirect` block of two reports + `totalDependencies: 3, directDependencies: 1`, and the indirect + `golang.org/x/crypto` contributes five findings of its own. For Go 1.17 and + later the indirect block is the transitive closure, so a tidied `go.mod` + gives full coverage. + - **npm, Python and Maven: declared dependencies only.** Adding a + `package-lock.json` that declares a transitive dependency leaves both + counters at 1, and the same holds for `poetry.lock`. + + Lock files are **not** reported as unsupported, which is a distinction this + release cares about: `package-lock.json` and `poetry.lock` leave the + document's `skipped` field `null`, so they are invisible rather than refused. + A `Cargo.toml`, which genuinely has no parser, IS reported with + `"reason": "no parser for this manifest type"`. So a lock file is a + skip-by-omission of exactly the kind disclosed two entries below. + + The README previously claimed "Scans all transitive dependencies, not just + direct ones" and the root help listed "Full dependency tree analysis", which + overstated coverage. A first correction in this release then understated it, + by generalising an npm measurement to all four ecosystems and telling Go users + their indirect dependencies were out of scope when they are read and reported. + Both are now stated per ecosystem, which is the only form of this claim that + is true. Parsing lock files for npm, Python and Maven is the fix, and it is + not in this release. + +- **A version can name any URL, and the scanner will fetch it.** npm's own + semantics allow a dependency version to be a tarball URL or an alias + (`npm:other@1.0.0`), and neither is screened beyond the local-path checks. So a + manifest under scan can direct the fetch at a host of its choosing, including + one on the scanning machine's own network, and the response bytes are then + handed to `tar`. An alias additionally makes the report key a different + package's findings to the declared name. Pre-existing. Closing it needs a host + policy rather than a path check. + +- **`--deep` on a Go project writes to the tree under scan.** `go mod download` + is run without a working directory of its own, so it inherits the invocation + directory and can write `go.sum` into the project being scanned. A read-only + scanner that dirties the working tree breaks a `git diff --exit-code` check in + CI. Pre-existing. + +- **A database record can answer for a different version than the one declared, + and nothing says so.** `github.com/cloudflare/circl v1.3.9` is reported with + `ML-KEM` and `ML-DSA` as SAFE from a record built for v1.6.4. A full-text scan + of the real v1.3.9 module finds no ML-KEM or ML-DSA at all: it ships round-3 + Kyber and Dilithium (`kem/kyber/kyber768`, `sign/dilithium/mode2`), which are + not interoperable with the FIPS standards. `org.bouncycastle:bcprov-jdk18on + 1.78.1` is the same case: no `pqc.crypto.mlkem` package exists in it, only + `pqc.crypto.crystals.kyber`. This is the worst direction for this tool to be + wrong in, because a project appears post-quantum ready while shipping + pre-standard primitives, and it reaches the README's own example, whose + `go.mod` declares `circl v1.3.7`. Present identically in 1.2.2, and in the + embedded database as well as the downloadable one, so `--offline` does not + avoid it. Only `--format json` reveals the version a record was built for. + +- **A file skipped by name is not analyzed and not counted.** The JavaScript + walker skips `*.min.js` and `test/`, `tests/` and `__tests__/` directories + before the analyzer sees them, so unlike a file that is refused, they appear in + neither `filesAnalyzed` nor `filesUnreadable`, and nothing on any stream or in + any format mentions them. A `bundle.min.js` of 162,025 bytes calling + `crypto.createHash('md5')` produces no finding and no trace, while a file of + byte-identical content named `dist/app.js` is analyzed and its MD5 reported. + The difference is the filename alone. Present in 1.2.2, which additionally has + no counts at all, so this is not a regression; it is disclosed here because it + is the same silent-skip shape this release exists to close, and because + renaming a file is a cheaper evasion than choosing an encoding. Counting a + skip-by-policy alongside a refusal is the fix. + +- **`requirements.txt` splits a dotted PyPI name at the first dot.** + `zope.interface==6.1` parses as name `zope` and version `.interface==6.1`, and + the version guard then refuses that as a local path, which is a misleading + message for a parse defect upstream of it. Both `zope` and `ruamel` are real + PyPI packages, so the wrong name reaches every output format. 323 of the top + 15,000 PyPI packages have a dotted canonical spelling. The same packages parse + correctly through `pyproject.toml` and `Pipfile`, which normalise the name + first, so only the `requirements.txt` path is affected. Present identically in + 1.2.2. + +- **NEW in 1.3.0: a source file whose first 1024 bytes are mostly non-ASCII is + not read.** The text check judges a file on its head, and accepts a head that + is not valid UTF-8 only if it is predominantly printable ASCII. That covers + source in a single-byte encoding, where the code is ASCII even when its + comments are not, and it does not cover a file that opens with a long comment + in a non-Latin single-byte encoding, such as a cp1251 licence header. UTF-16 + source is refused for a separate reason: it carries NUL bytes. In every case + the file is now counted in `analysis.filesUnreadable`, named on stderr, and + disclosed as a coverage note in all five formats, so the gap is stated rather + than silent. Reading whole files, or detecting the encoding, is the larger fix. + +- **A package that provides post-quantum cryptography is reported as + quantum-vulnerable, from algorithms it does not contain.** A project whose only + dependency is `@noble/post-quantum` is reported with four HIGH findings and + advised to migrate to ML-KEM and ML-DSA, which is what the package implements. + + The records must be read against the version they were BUILT for, not the + version a manifest declares, which is the limitation stated immediately above + and which is easy to fall into while investigating this one. A manifest + declaring `@noble/post-quantum@0.2.0` is answered by a record built for + `0.6.1`, and `@noble/hashes@1.4.0`, `@noble/ciphers@1.0.0` and + `@noble/curves@1.4.0` are all answered by records built for `2.2.0`. Checked + against those published artifacts: + + - **`@noble/post-quantum@0.6.1`: mostly correct, and one fabrication.** + `ECDSA`, `Ed25519` and `X25519` are all genuinely present, in `hybrid.js`, + whose own comment describes a preset combining ML-KEM-768 with X25519, and + `AES` and `ChaCha` are present in `falcon.js`. So these findings really are + the classical halves of hybrid constructions, and reporting them as + standalone quantum-vulnerable usage is a classification defect rather than a + fabricated one. `RSA` is fabricated: it occurs nowhere in the package. + - **`@noble/hashes@2.2.0`, a hashing library: four fabrications.** It is + credited with `RSA`, `ECDSA`, `Ed25519` and `X25519`, none of which occur + anywhere in its source. `@noble/ciphers@2.2.0` is credited with the same + four, likewise absent. `@noble/curves@2.2.0` is credited with `RSA`, `AES` + and `ChaCha20-Poly1305`, all absent, while its `ECDSA`, `X25519` and + `Ed25519` are real. + + So there are two distinct defects behind one symptom: a real hybrid component + classified as if it were standalone classical usage, and primitives inferred + into records that do not contain them. The second comes from name inference, + which supplies 780 of the 849 records in the database this release publishes, + and 832 of the 901 in the newer one a machine may already have cached; the two + are distinguished below. Correct classification of a + declared hybrid component, and a verification pass over the inferred records, + are the fixes, and neither is in this release. + +- **Following the tool's own remediation advice does not change the verdict.** + A five-dependency project that replaces `node-forge` and `elliptic` with the + `@noble` libraries the report names in its own `Libraries:` field, removing + DES, 3DES, MD5, SHA-1, secp256k1 and ECDH, produces a byte-identical summary + and the same exit 1: `5 deps | 4 with crypto | 10 vulnerable | 3 partial` + before and after. The cause is the inferred attribution above, which gives + four single-purpose `@noble` libraries the same classical core, so the + migration target carries the same findings as the thing being migrated away + from. Identical on 1.2.2. Until the inferred records are corrected, treat the + remediation list as a pointer to the right family of libraries rather than as + a step that a rescan will confirm. + +- **`--offline` does not prevent network access for a GitHub-shaped argument.** + The flag is documented as "Only use local database, no downloads", and it does + gate the database download, but the GitHub fetch path is not behind it: + `analyze owner/repo --offline` clones the repository and scans it. A mistyped + local path with exactly one slash is read as `owner/repo`, so a typo becomes an + outbound request to api.github.com. Identical on 1.2.2. Anyone relying on + `--offline` to mean no egress should not pass a repository argument. + +- **Maven coverage in the downloadable database is unstable.** The weekly + refresh has published between 15 and 356 Maven packages over the last seven + runs, because a partial result from the upstream search is committed as though + it were a complete one. The 15 that are always present are a curated seed list. + This affects `cryptodeps update` only; the binary's built-in database is + unchanged by it. + +- **Findings carry no line number.** JSON `location.file` is empty and + `location.line` is zero for every finding, and SARIF results carry no + `region`, so an alert lands on the manifest rather than on the line that + declares the dependency. + +- **A `GITHUB_TOKEN` in the environment breaks `analyze `.** The GitHub API + is called with whatever token is present, so an expired or wrongly-scoped one + fails with `401 Unauthorized` against a public repository that needs no + authentication at all. `GITHUB_TOKEN` is set by default in GitHub Actions. + Clearing it for the command is the workaround. + +- **The database download is unverified, and it is not opt-in.** `update --url` + accepts any URL, and the database file carries no signature or checksum, so + the update path is unauthenticated end to end. The download is also automatic + and silent: the first `analyze` on a machine with no `~/.cryptodeps` fetches + the database over the network and writes it there without printing anything, + so a user who never runs `update` is still scanning against downloaded data. + That is why the same project can be reported differently on two machines. Pass + `--offline` to use only the database built into the binary, which is smaller + (72 packages against 849 in the asset this release publishes) and is curated + rather than inferred: it does not carry `@noble/post-quantum` at all, and its + `@noble/hashes` record lists only real hash algorithms (BLAKE2b, BLAKE2s, + BLAKE3, SHA-256, SHA-384, SHA-512, SHA3-256) with none of the fabrications the + inferred record carries. + + **Two different databases report the same version label.** The asset published + with this release holds 849 records, 780 of them inferred, and calls itself + version 1.1.0; a database downloaded on 2026-07-27 holds 901 records, 832 of + them inferred, and also calls itself version 1.1.0. The label does not identify + the knowledge base, and no scan output records which one answered, so two runs + that disagree cannot be told apart from their reports. Both numbers appear in + this changelog and each is stated with the database it describes. + +- **`--deep` requires `pip` on `PATH` for Python packages**, not `pip3`, so it + fails on a default Homebrew macOS with `exec: "pip": executable file not + found`. The failure is reported on stderr and the report now says the packages + could not be read and points at those warnings, rather than suggesting the + command that just failed, but the missing `pip3` fallback itself is not fixed. + +- **A repository whose only manifests are of an unsupported type exits 2**, with + the same status as a genuine analysis error. 1.3.0 now names the file and the + reason rather than reporting that no manifest was found. + +- **`--fail-on any` fails a project whose cryptography is entirely quantum + safe, and reports it as a partial-risk exit.** The flag is described as + exiting non-zero "when risk found", but it is implemented as any cryptographic + usage at all, so a project whose only dependency is `bcrypt` exits 3 with + `0 vulnerable | 0 partial` in its own summary and nothing on either stream + explaining the failure. Exit 3 is documented as partial-risk findings, which + this is not. Identical on 1.2.2. The default, `--fail-on vulnerable`, is + unaffected and behaves as documented. + +- **Two dependency names that differ only in characters the source cache + replaces share one cache entry.** The cache path is built by replacing + anything outside `[A-Za-z0-9._-]` with `_`, so `@scope/pkg` and `_scope_pkg` + both give `_scope_pkg`, and the second package declared in a manifest is + analyzed from the first one's source and reported under its own name. It + needs a manifest that declares both spellings, and it misattributes findings + rather than reaching anything outside the cache. Present in 1.2.2. Closing it + changes the cache path of every Maven coordinate and every scoped npm name, + and those paths appear in reported findings, so it is held for 1.3.1 rather + than changed in a release whose output has already been verified. + +- **The bundled GitHub Action scans against the built-in database and reports + its version as `dev`.** `action.yml` passes `--offline` at both of its scan + steps and a fresh runner has no `~/.cryptodeps`, so the Action sees the 72 + packages built into the binary rather than the 849 in the downloadable + database. The coverage note fires, so the scan is not silently narrower, but + nothing states that the Action's database is a twelfth of the one a local + install downloads. Separately, the Action installs with + `go install ...@latest`, which injects no ldflags, so the binary it runs + reports version `dev` and the SARIF it uploads carries that as its driver + version. That is the version-provenance defect this release fixes, + reintroduced by the install method rather than by the binary. + +- **The CBOM component list is not a complete bill of materials.** It carries + the libraries a scan has findings for, so a dependency that was examined and + found clean is absent from the components alongside one that could not be + examined at all. Incomplete coverage is now reported as a property of the + scan, so the gap is stated rather than silent, but a reader who takes the + component list as the set of dependencies will undercount. Making the list + complete is an output change held for a later release. + +- **Some CBOM primitive values are schema-legal but not the closest available + term.** `ML-KEM` is mapped to `key-agree` where the CycloneDX enum offers + `kem`, AEAD ciphers to `other` where it offers `ae`, HMAC to `signature` where + it offers `mac`, and bcrypt, scrypt and Argon2 to `hash` where it offers + `kdf`. The document validates, and mapping the flagship post-quantum KEM + imprecisely in a document whose purpose is post-quantum readiness is the one + that matters. + +- **The per-analysis provenance block carries zero values.** Every deep-analyzed + record emits `"date": "0001-01-01T00:00:00Z"`, `"method": ""`, `"tool": ""` + and `"toolVersion": ""` inside `analysis.analysis`. The tool version fixed in + this release is the top-level `tool` object, the SARIF driver and the CBOM + metadata, which are correct; this separate per-record block is unchanged and + is a placeholder rendered where a reader looks for provenance. Identical in + 1.2.2. Either populate it at the point of analysis or omit it. + +- **`--offline` silently disables `--deep`, and the report then suggests + `--deep`.** Source analysis fetches package archives, so it cannot run with + downloads refused, but passing both prints no warning that one was ignored. + Because no source analysis was attempted, a scan whose dependencies are all + absent from the database ends at "Run with `--deep` to analyze package source + code directly", which is the flag that was just passed. 1.2.2 answered the + same invocation with "No cryptographic usage detected in dependencies", a + clean verdict for a scan that examined nothing, so the verdict itself is + fixed and the suggestion that follows it is not. + +## [1.2.2] - 2025-12-27 + +### Fixed + +- Docker image name in the release workflow now matches the repository, so the + GHCR login succeeds and the container image publishes. + ## [1.2.1] - 2025-12-27 ### Added @@ -304,7 +1003,9 @@ reproducing them. - Maps findings to CNSA 2.0 compliance requirements - Supports OMB M-23-02 cryptographic inventory requirements -[1.2.1]: https://github.com/csnp/qramm-cryptodeps/compare/v1.2.0...v1.2.1 -[1.2.0]: https://github.com/csnp/qramm-cryptodeps/compare/v1.1.0...v1.2.0 -[1.1.0]: https://github.com/csnp/qramm-cryptodeps/compare/v1.0.0...v1.1.0 -[1.0.0]: https://github.com/csnp/qramm-cryptodeps/releases/tag/v1.0.0 +[1.3.0]: https://github.com/csnp/cryptodeps/compare/v1.2.2...v1.3.0 +[1.2.2]: https://github.com/csnp/cryptodeps/compare/v1.2.1...v1.2.2 +[1.2.1]: https://github.com/csnp/cryptodeps/compare/v1.2.0...v1.2.1 +[1.2.0]: https://github.com/csnp/cryptodeps/compare/v1.1.0...v1.2.0 +[1.1.0]: https://github.com/csnp/cryptodeps/compare/v1.0.0...v1.1.0 +[1.0.0]: https://github.com/csnp/cryptodeps/releases/tag/v1.0.0 diff --git a/README.md b/README.md index abe8cf3..21edca3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **Quantum-Safe Dependency Analysis for Your Software Supply Chain** -Find every cryptographic vulnerability in your dependencies. Know your quantum risk. Focus on what matters. +Find the cryptography your dependencies bring in, see which of it quantum computers will break, and tell apart what your code actually calls from what is merely available to it. [![CI](https://github.com/csnp/qramm-cryptodeps/actions/workflows/ci.yml/badge.svg)](https://github.com/csnp/qramm-cryptodeps/actions/workflows/ci.yml) [![Go Report Card](https://img.shields.io/badge/go%20report-A-brightgreen)](https://goreportcard.com/report/github.com/csnp/qramm-cryptodeps) @@ -15,13 +15,15 @@ Find every cryptographic vulnerability in your dependencies. Know your quantum r ## The Quantum Computing Challenge -Quantum computers will break RSA, ECDSA, and Diffie-Hellman within the next decade. This isn't speculation—the NSA, NIST, and major technology companies are already migrating to post-quantum cryptography (PQC). +Quantum computers will break RSA, ECDSA, and Diffie-Hellman within the next decade. This isn't speculation: the NSA, NIST, and major technology companies are already migrating to post-quantum cryptography (PQC). The challenge? **You can't migrate what you can't find.** -Your code might be quantum-safe, but what about your **dependencies**? The average software project has 300-1000+ transitive dependencies. Each one potentially uses cryptographic algorithms that quantum computers will break. Traditional security scanners miss this—they focus on CVEs, not cryptographic readiness. +Your code might be quantum-safe, but what about your **dependencies**? The average software project has 300-1000+ transitive dependencies. Each one potentially uses cryptographic algorithms that quantum computers will break. Traditional security scanners miss this: they focus on CVEs, not cryptographic readiness. -CryptoDeps solves this by analyzing your entire dependency tree and using **reachability analysis** to show exactly which crypto your code actually uses versus what's merely present in libraries. +CryptoDeps reads the dependencies your manifests declare, identifies the cryptography each one provides, and uses **reachability analysis** to show which of it your code actually calls rather than merely has available. + +How much of the tree that covers depends on the ecosystem, so it is worth stating plainly. In Go, `go.mod` lists indirect requirements, and CryptoDeps reads and analyzes them, so a tidied `go.mod` gives you the full closure. In npm, Python and Maven it reads only what the manifest itself declares: lock files are not parsed, so a package pulled in only by another package is outside what those scans report. --- @@ -31,7 +33,7 @@ CryptoDeps is purpose-built for quantum readiness assessment: | Capability | CryptoDeps | grep/ripgrep | Commercial Tools | |------------|------------|--------------|------------------| -| Dependency tree analysis | Yes | No | Some | +| Declared dependency analysis | Yes | No | Some | | Reachability analysis | Yes (Go) | No | Rarely | | Quantum risk classification | Yes | No | Some | | Context-aware confidence | Yes | No | Varies | @@ -46,7 +48,7 @@ CryptoDeps is purpose-built for quantum readiness assessment:
What These Capabilities Mean -- **Dependency tree analysis**: Scans all transitive dependencies, not just direct ones +- **Declared dependency analysis**: Scans what your manifests declare. Go `// indirect` requirements are included, so a tidied `go.mod` covers the closure; npm, Python and Maven are direct-only because lock files are not parsed - **Reachability analysis**: Traces call graphs to find crypto your code actually invokes - **Quantum risk classification**: Categorizes by threat level (VULNERABLE, PARTIAL, SAFE) - **Context-aware confidence**: Distinguishes confirmed usage from mere availability @@ -281,7 +283,10 @@ cryptodeps status ## Sample Output ``` -[*] Scanning go.mod... found 2 dependencies + +Scanning /home/dev/myproject... + +[*] Scanning ./go.mod... found 2 dependencies [!] CONFIRMED - Actually used by your code (requires action): ────────────────────────────────────────────────────────────────────────────────────────── @@ -386,7 +391,7 @@ cryptodeps: |------|---------|---------| | `0` | Success | No findings matching `--fail-on` threshold | | `1` | Vulnerable | Quantum-vulnerable crypto detected | -| `2` | Error | Analysis failed (invalid manifest, network error) | +| `2` | Error | Analysis failed, including any manifest that was found but could not be read | | `3` | Partial | Partial-risk crypto detected (with `--fail-on partial`) | --- @@ -458,7 +463,7 @@ qramm-cryptodeps/ │ ├── crypto/ # Algorithm patterns & remediation │ ├── output/ # Formatters (table, JSON, CBOM, SARIF) │ └── types/ # Shared type definitions -├── data/ # Crypto database (901 packages) +├── data/ # Crypto database (downloaded by `cryptodeps update`) └── examples/ # Sample projects for testing ``` @@ -474,7 +479,8 @@ qramm-cryptodeps/ - [x] Quantum risk classification with CNSA 2.0 timeline - [x] Smart remediation guidance with NIST references - [x] GitHub repository URL scanning -- [x] Crypto database of 901 packages (69 verified, 832 inferred) +- [x] Crypto database of curated and inferred package entries, refreshed weekly + (run `cryptodeps status` for the counts your install currently holds) - [x] Workspace & monorepo support (npm, pnpm, Go workspaces) - [x] Multi-project aggregated results @@ -568,28 +574,28 @@ Learn more at [qramm.org](https://qramm.org) and [csnp.org](https://csnp.org). ### NIST Post-Quantum Cryptography Standards -- [FIPS 203 - ML-KEM](https://csrc.nist.gov/pubs/fips/203/final) — Module-Lattice-Based Key-Encapsulation Mechanism (replaces RSA/ECDH) -- [FIPS 204 - ML-DSA](https://csrc.nist.gov/pubs/fips/204/final) — Module-Lattice-Based Digital Signature Algorithm (replaces RSA/ECDSA) -- [FIPS 205 - SLH-DSA](https://csrc.nist.gov/pubs/fips/205/final) — Stateless Hash-Based Digital Signature Algorithm -- [NIST SP 800-131A Rev 2](https://csrc.nist.gov/publications/detail/sp/800-131a/rev-2/final) — Transitioning cryptographic algorithms and key lengths +- [FIPS 203 - ML-KEM](https://csrc.nist.gov/pubs/fips/203/final): Module-Lattice-Based Key-Encapsulation Mechanism (replaces RSA/ECDH) +- [FIPS 204 - ML-DSA](https://csrc.nist.gov/pubs/fips/204/final): Module-Lattice-Based Digital Signature Algorithm (replaces RSA/ECDSA) +- [FIPS 205 - SLH-DSA](https://csrc.nist.gov/pubs/fips/205/final): Stateless Hash-Based Digital Signature Algorithm +- [NIST SP 800-131A Rev 2](https://csrc.nist.gov/publications/detail/sp/800-131a/rev-2/final): Transitioning cryptographic algorithms and key lengths ### Additional Resources -- [NSA CNSA 2.0](https://media.defense.gov/2022/Sep/07/2003071834/-1/-1/0/CSA_CNSA_2.0_ALGORITHMS_.PDF) — Commercial National Security Algorithm Suite -- [OMB M-23-02](https://www.whitehouse.gov/wp-content/uploads/2022/11/M-23-02-M-Memo-on-Migrating-to-Post-Quantum-Cryptography.pdf) — Federal PQC Migration Requirements +- [NSA CNSA 2.0](https://media.defense.gov/2022/Sep/07/2003071834/-1/-1/0/CSA_CNSA_2.0_ALGORITHMS_.PDF): Commercial National Security Algorithm Suite +- [OMB M-23-02](https://www.whitehouse.gov/wp-content/uploads/2022/11/M-23-02-M-Memo-on-Migrating-to-Post-Quantum-Cryptography.pdf): Federal PQC Migration Requirements - [CISA Post-Quantum Cryptography Initiative](https://www.cisa.gov/quantum) -- [CycloneDX CBOM](https://cyclonedx.org/capabilities/cbom/) — Cryptographic Bill of Materials +- [CycloneDX CBOM](https://cyclonedx.org/capabilities/cbom/): Cryptographic Bill of Materials --- ## License -Apache License 2.0 — see [LICENSE](LICENSE) for details. +Apache License 2.0. See [LICENSE](LICENSE) for details. Copyright 2025 Cyber Security Non-Profit (CSNP) --- -Built with purpose by [CSNP](https://csnp.org) — Advancing cybersecurity for everyone +Built with purpose by [CSNP](https://csnp.org). Advancing cybersecurity for everyone. [QRAMM](https://qramm.org) | [CSNP](https://csnp.org) | [Issues](https://github.com/csnp/qramm-cryptodeps/issues) | [Twitter](https://twitter.com/caborgsec) diff --git a/cmd/cryptodeps/exit_code_test.go b/cmd/cryptodeps/exit_code_test.go new file mode 100644 index 0000000..fca8e2b --- /dev/null +++ b/cmd/cryptodeps/exit_code_test.go @@ -0,0 +1,104 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/csnp/qramm-cryptodeps/internal/analyzer" + "github.com/csnp/qramm-cryptodeps/pkg/types" +) + +// partialRiskScan is a project whose only findings are partial risk, which is +// the one shape in which --fail-on partial and the default policy disagree. A +// project with a vulnerable finding exits 1 under every threshold, so it cannot +// detect a threshold that was ignored. +func partialRiskScan() *types.ScanResult { + return &types.ScanResult{ + Summary: types.ScanSummary{ + TotalDependencies: 1, + WithCrypto: 1, + QuantumPartial: 1, + }, + } +} + +// TestFailOnIsReadTheSameWayItWasValidated is the regression test for a silent +// loosening of the CI gate by whitespace. +// +// ValidateFailOn trimmed and lowercased before deciding, and determineExitCode +// only lowercased. So " partial " passed validation, matched no case, and fell +// through to the default vulnerable-only policy: a project with partial-risk +// findings exited 3 for "partial" and 0 for " partial ", with nothing on either +// stream. Whitespace around a value is the ordinary result of a YAML block +// scalar or a workflow expression, which is exactly where this flag is used, and +// the validation added in this release is what made the value look accepted. +func TestFailOnIsReadTheSameWayItWasValidated(t *testing.T) { + result := partialRiskScan() + + // Guard the fixture: the two policies have to disagree on it, or every + // assertion below passes for the wrong reason. + if got := determineExitCode(result, "partial"); got != ExitPartial { + t.Fatalf("determineExitCode(partial) = %d, want %d; the fixture does not exercise "+ + "the partial policy", got, ExitPartial) + } + if got := determineExitCode(result, "vulnerable"); got != ExitSuccess { + t.Fatalf("determineExitCode(vulnerable) = %d, want %d; the fixture does not "+ + "distinguish the two policies", got, ExitSuccess) + } + + for _, threshold := range []string{" partial", "partial ", " partial ", "\tpartial", "PARTIAL", " Partial\n"} { + t.Run(threshold, func(t *testing.T) { + // Whatever the validator accepts, the exit code must honour. + if err := analyzer.ValidateFailOn(threshold); err != nil { + t.Fatalf("ValidateFailOn(%q) rejected it, so this case is not the one under "+ + "test: %v", threshold, err) + } + if got := determineExitCode(result, threshold); got != ExitPartial { + t.Errorf("determineExitCode(%q) = %d, want %d: the value was accepted as valid "+ + "and then ignored, reverting the gate to the default policy", + threshold, got, ExitPartial) + } + }) + } +} + +// TestFailOnRejectsWhatItCannotHonour is the inverse: canonicalising must not +// turn a typo into a valid value, because a gate that silently accepts "partail" +// is the defect the validation exists to prevent. +func TestFailOnRejectsWhatItCannotHonour(t *testing.T) { + for _, threshold := range []string{"partail", "vulnerble", "maybe", "partial risk", "1"} { + if err := analyzer.ValidateFailOn(threshold); err == nil { + t.Errorf("ValidateFailOn(%q) accepted a value that determineExitCode reads as the "+ + "default policy", threshold) + } + } +} + +// TestFailOnRefusesAnEmptyValue closes the last spelling of the silent gate +// loosening. +// +// The padded value was fixed by canonicalising; the empty string still fell +// through, because "" was listed as a legal value and matched no policy. A +// project with partial-risk findings exited 3 for "partial" and 0 for "", with +// nothing on either stream, and an unset workflow input is exactly how a CI gate +// arrives here empty. This flag has a default, so a blank value expresses +// nothing: unlike --risk and --min-severity, where empty means "do not filter". +func TestFailOnRefusesAnEmptyValue(t *testing.T) { + result := partialRiskScan() + + // Guard the fixture: the two policies must disagree on it, or the assertion + // below is answered by a project no threshold would fail. + if determineExitCode(result, "partial") == determineExitCode(result, "vulnerable") { + t.Fatalf("the fixture does not distinguish the partial policy from the default") + } + + for _, empty := range []string{"", " ", "\t", "\n"} { + if err := analyzer.ValidateFailOn(empty); err == nil { + t.Errorf("ValidateFailOn(%q) accepted a value that determineExitCode reads as the "+ + "default policy, so a gate asking for partial risk passes a partial-risk "+ + "project", empty) + } + } +} diff --git a/cmd/cryptodeps/gate_filter_test.go b/cmd/cryptodeps/gate_filter_test.go new file mode 100644 index 0000000..294b972 --- /dev/null +++ b/cmd/cryptodeps/gate_filter_test.go @@ -0,0 +1,216 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/csnp/qramm-cryptodeps/internal/analyzer" + "github.com/csnp/qramm-cryptodeps/internal/database" +) + +// The gate must describe the scan, not the view. +// +// Making --risk and --min-severity actually filter, in this release, let them +// decide the exit code, because the gate read a summary that counted only +// survivors. `analyze . --fail-on vulnerable` exited 1 and the same command with +// `--risk safe` exited 0, while stdout printed "This is not a clean result". +// Released 1.2.2 exits 1 for both, because its filters did nothing at all, so +// this was a regression in the one flag that decides CI outcomes. +// +// The invariant asserted here is the whole fix in one line: **a reporting filter +// never changes the exit code.** It is asserted across both code paths, because +// the single-project and workspace gates read different summaries and the +// workspace one is what the CLI uses by default. An earlier version of these +// tests covered the workspace path with a struct literal, which meant the three +// aggregation lines that carry the withheld counts could all be deleted with the +// suite still green, restoring the exact bug on the default path. +// +// These tests deliberately use database.NewEmbedded() rather than +// NewWithCachedData(): the latter reads the developer's real ~/.cryptodeps, so a +// cache that happened to classify these packages differently would turn the only +// runtime protection for a CI gate into a silent skip. + +// gateFixture is a project whose findings sit at known risk levels in the +// embedded database, so that each filter has something to withhold. +type gateFixture struct { + name string + deps map[string]string +} + +var gateFixtures = []gateFixture{ + // Vulnerable AND partial: the shape where --risk safe withholds everything + // and the default policy must still fail. + {"vulnerable-and-partial", map[string]string{ + "crypto-js": "4.2.0", "node-forge": "1.3.1"}}, + // Partial only: the shape that distinguishes --fail-on partial from the + // default, and the one that detects a gate ignoring withheld PARTIAL counts. + {"partial-only", map[string]string{ + "aes-js": "3.1.2", "pbkdf2": "3.1.2"}}, + // Safe only: --fail-on any must still fail here, which is the only shape + // that detects a gate ignoring the withheld with-crypto count. + {"safe-only", map[string]string{ + "argon2": "0.31.2", "bcrypt": "5.1.1"}}, + // Mixed within one dependency, so a partially-filtered package is covered. + {"mixed", map[string]string{ + "tweetnacl": "1.0.3", "aes-js": "3.1.2"}}, +} + +// filters are the reporting filters, each of which withholds a different slice +// of the findings. The empty entry is the unfiltered baseline. +var gateFilters = []struct { + name string + opts analyzer.Options +}{ + {"none", analyzer.Options{}}, + {"risk=safe", analyzer.Options{RiskFilter: "safe"}}, + {"risk=vulnerable", analyzer.Options{RiskFilter: "vulnerable"}}, + {"risk=partial", analyzer.Options{RiskFilter: "partial"}}, + {"minSeverity=critical", analyzer.Options{MinSeverity: "critical"}}, + {"minSeverity=high", analyzer.Options{MinSeverity: "high"}}, +} + +var gateThresholds = []string{"vulnerable", "partial", "any", "none"} + +func writeManifest(t *testing.T, dir string, deps map[string]string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + body := `{"name":"gate","version":"1.0.0","dependencies":{` + first := true + for name, version := range deps { + if !first { + body += "," + } + body += `"` + name + `":"` + version + `"` + first = false + } + body += `}}` + if err := os.WriteFile(filepath.Join(dir, "package.json"), []byte(body), 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } +} + +func newGateAnalyzer(opts analyzer.Options) *analyzer.Analyzer { + opts.Offline = true + return analyzer.New(database.NewEmbedded(), opts) +} + +// TestReportingFilterNeverChangesTheExitCode covers the single-project path. +func TestReportingFilterNeverChangesTheExitCode(t *testing.T) { + for _, fx := range gateFixtures { + t.Run(fx.name, func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "proj") + writeManifest(t, dir, fx.deps) + + base, err := newGateAnalyzer(analyzer.Options{}).Analyze(dir) + if err != nil { + t.Fatalf("baseline analyze: %v", err) + } + // Guard the fixture: it must actually produce findings, or every + // comparison below is between two empty scans. + if base.Summary.WithCrypto == 0 { + t.Fatalf("fixture %q produced no findings from the embedded database "+ + "(summary %+v); it cannot exercise a filter", fx.name, base.Summary) + } + + for _, f := range gateFilters { + result, err := newGateAnalyzer(f.opts).Analyze(dir) + if err != nil { + t.Fatalf("analyze with %s: %v", f.name, err) + } + // Guard again: a filter that withheld nothing tests nothing. + if f.name != "none" && result.Summary.FilteredOut == 0 && + base.Summary.WithCrypto > 0 { + t.Logf("note: %s withheld nothing on %s", f.name, fx.name) + } + for _, th := range gateThresholds { + want := determineExitCode(base, th) + got := determineExitCode(result, th) + if got != want { + t.Errorf("--fail-on %s with %s: exit %d, want %d (unfiltered). "+ + "A reporting filter changed the CI verdict; the scan withheld "+ + "%d finding(s)", th, f.name, got, want, result.Summary.FilteredOut) + } + } + } + }) + } +} + +// TestWorkspaceReportingFilterNeverChangesTheExitCode covers the workspace path, +// which is what the CLI runs by default and which reads TotalSummary rather than +// a project summary. Without this, the three lines in AggregateResults that carry +// the withheld counts can each be deleted with the suite still green. +func TestWorkspaceReportingFilterNeverChangesTheExitCode(t *testing.T) { + root := t.TempDir() + // A workspace of several projects, so the aggregate has something to sum and + // so no single project's summary can stand in for the total. + for _, fx := range gateFixtures { + writeManifest(t, filepath.Join(root, fx.name), fx.deps) + } + + base, err := newGateAnalyzer(analyzer.Options{}).AnalyzeAll(root) + if err != nil { + t.Fatalf("baseline AnalyzeAll: %v", err) + } + if len(base.Projects) < 2 { + t.Fatalf("workspace discovered %d project(s); the aggregate path is not "+ + "exercised by fewer than 2", len(base.Projects)) + } + if base.TotalSummary.WithCrypto == 0 { + t.Fatalf("workspace baseline found no cryptography (total %+v)", base.TotalSummary) + } + + for _, f := range gateFilters { + result, err := newGateAnalyzer(f.opts).AnalyzeAll(root) + if err != nil { + t.Fatalf("AnalyzeAll with %s: %v", f.name, err) + } + for _, th := range gateThresholds { + want := determineExitCodeMulti(base, th) + got := determineExitCodeMulti(result, th) + if got != want { + t.Errorf("workspace --fail-on %s with %s: exit %d, want %d (unfiltered). "+ + "The aggregate gate read the filtered totals; %d finding(s) withheld", + th, f.name, got, want, result.TotalSummary.FilteredOut) + } + } + } +} + +// TestFailOnNoneStaysOff guards the other direction: the fix must not switch on +// a gate the operator switched off, and must not invent findings on a clean tree. +func TestFailOnNoneStaysOff(t *testing.T) { + dir := filepath.Join(t.TempDir(), "proj") + writeManifest(t, dir, map[string]string{"crypto-js": "4.2.0"}) + for _, f := range gateFilters { + result, err := newGateAnalyzer(f.opts).Analyze(dir) + if err != nil { + t.Fatalf("analyze: %v", err) + } + if got := determineExitCode(result, "none"); got != ExitSuccess { + t.Errorf("--fail-on none with %s: exit %d, want %d", f.name, got, ExitSuccess) + } + } + + clean := filepath.Join(t.TempDir(), "clean") + writeManifest(t, clean, map[string]string{"left-pad": "1.3.0"}) + result, err := newGateAnalyzer(analyzer.Options{}).Analyze(clean) + if err != nil { + t.Fatalf("analyze clean: %v", err) + } + if result.Summary.WithCrypto != 0 { + t.Fatalf("the clean fixture is not clean (summary %+v)", result.Summary) + } + for _, th := range gateThresholds { + if got := determineExitCode(result, th); got != ExitSuccess { + t.Errorf("clean project --fail-on %s: exit %d, want %d; the fix invented a "+ + "finding", th, got, ExitSuccess) + } + } +} diff --git a/cmd/cryptodeps/gate_withheld_counts_test.go b/cmd/cryptodeps/gate_withheld_counts_test.go new file mode 100644 index 0000000..b0428d8 --- /dev/null +++ b/cmd/cryptodeps/gate_withheld_counts_test.go @@ -0,0 +1,60 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + + "github.com/csnp/qramm-cryptodeps/internal/analyzer" +) + +// This file is separate from gate_filter_test.go on purpose. It names the +// withheld counters directly, so it cannot compile against the code before they +// existed. gate_filter_test.go deliberately names none of them: it asserts the +// BEHAVIOUR (a reporting filter never changes the exit code), so it compiles +// against the pre-fix commit and fails there at runtime, which is the only form +// of proof that a test detects the defect it was written for. Keeping the two in +// one file made the behavioural test unprovable by a plain revert. + +// TestWithheldCountsReconcile pins the counters themselves rather than only +// their effect, so that a counter which stops being populated fails here with a +// reason instead of somewhere downstream. +func TestWithheldCountsReconcile(t *testing.T) { + root := t.TempDir() + for _, fx := range gateFixtures { + writeManifest(t, filepath.Join(root, fx.name), fx.deps) + } + base, err := newGateAnalyzer(analyzer.Options{}).AnalyzeAll(root) + if err != nil { + t.Fatalf("baseline: %v", err) + } + + for _, f := range gateFilters { + result, err := newGateAnalyzer(f.opts).AnalyzeAll(root) + if err != nil { + t.Fatalf("AnalyzeAll with %s: %v", f.name, err) + } + checks := []struct { + label string + shown, hid int + unfiltered int + }{ + {"withCrypto", result.TotalSummary.WithCrypto, + result.TotalSummary.WithheldWithCrypto, base.TotalSummary.WithCrypto}, + {"quantumVulnerable", result.TotalSummary.QuantumVulnerable, + result.TotalSummary.WithheldVulnerable, base.TotalSummary.QuantumVulnerable}, + {"quantumPartial", result.TotalSummary.QuantumPartial, + result.TotalSummary.WithheldPartial, base.TotalSummary.QuantumPartial}, + } + for _, c := range checks { + if c.shown+c.hid != c.unfiltered { + t.Errorf("%s with %s: shown %d + withheld %d = %d, but an unfiltered scan "+ + "found %d. The withheld breakdown does not account for what the filter "+ + "removed, so the gate cannot recover it", + c.label, f.name, c.shown, c.hid, c.shown+c.hid, c.unfiltered) + } + } + } +} diff --git a/cmd/cryptodeps/main.go b/cmd/cryptodeps/main.go index 090b6cf..d7b1e02 100644 --- a/cmd/cryptodeps/main.go +++ b/cmd/cryptodeps/main.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "sort" - "strings" "github.com/spf13/cobra" @@ -74,7 +73,7 @@ var rootCmd = &cobra.Command{ and assess their quantum computing vulnerability. It provides: - - Full dependency tree analysis + - Analysis of the dependencies your manifests declare - Crypto algorithm detection with call graphs - Quantum risk classification (VULNERABLE, PARTIAL, SAFE) - Multiple output formats (table, JSON, CBOM, SARIF) @@ -210,6 +209,15 @@ func runAnalyze(cmd *cobra.Command, args []string) error { if err := analyzer.ValidateMinSeverity(minSeverity); err != nil { return err } + // --fail-on decides the exit code, so an unrecognised value here does not + // merely mislead: it reverts the gate to the default policy and lets a build + // pass that the operator had asked to fail. + if err := analyzer.ValidateFailOn(failOn); err != nil { + return err + } + // Not normalised here. Both readers of this flag canonicalise it themselves, + // which is the layer that holds for any caller, and a second normalisation + // here would be code no test can hold: removing it left the suite green. // Parse output format format, err := output.ParseFormat(formatFlag) @@ -279,11 +287,29 @@ func runAnalyze(cmd *cobra.Command, args []string) error { return nil } -// determineExitCode calculates the appropriate exit code based on scan results -// and the --fail-on threshold. This enables CI/CD pipelines to fail builds -// when quantum-vulnerable crypto is detected. -func determineExitCode(result *types.ScanResult, threshold string) int { - threshold = strings.ToLower(threshold) +// exitCodeForSummary is the single reading of a scan for gate purposes. +// +// The single-project and workspace paths previously carried two copies of this +// switch, differing only in which summary they read, which is how they could +// drift apart without either being obviously wrong. One function, two callers. +// +// The gate asks about what the scan FOUND, which is the survivors plus whatever +// the reporting filters withheld. --risk and --min-severity choose what the +// report shows; they must not decide whether the build fails. Before this, a +// project exiting 1 under `--fail-on vulnerable` exited 0 when `--risk safe` was +// added, while the same run printed "This is not a clean result" to stdout: the +// tool told the operator it had hidden findings and told CI the project was +// clean. Released 1.2.2 exited 1 for both, because the filters did nothing at +// all, so making them work introduced this. +func exitCodeForSummary(s types.ScanSummary, threshold string) int { + // Canonicalised through the same function the validator uses, so a caller + // that reaches this without going through main cannot reintroduce the split + // reading that let " partial " loosen the gate. + threshold = analyzer.CanonicalFailOn(threshold) + + withCrypto := s.WithCrypto + s.WithheldWithCrypto + vulnerable := s.QuantumVulnerable + s.WithheldVulnerable + partial := s.QuantumPartial + s.WithheldPartial switch threshold { case "none": @@ -292,11 +318,11 @@ func determineExitCode(result *types.ScanResult, threshold string) int { case "any": // Fail if any crypto usage is detected - if result.Summary.WithCrypto > 0 { - if result.Summary.QuantumVulnerable > 0 { + if withCrypto > 0 { + if vulnerable > 0 { return ExitVulnerable } - if result.Summary.QuantumPartial > 0 { + if partial > 0 { return ExitPartial } // Has crypto but all safe - still exit non-zero for "any" mode @@ -306,10 +332,10 @@ func determineExitCode(result *types.ScanResult, threshold string) int { case "partial": // Fail on partial or vulnerable - if result.Summary.QuantumVulnerable > 0 { + if vulnerable > 0 { return ExitVulnerable } - if result.Summary.QuantumPartial > 0 { + if partial > 0 { return ExitPartial } return ExitSuccess @@ -318,50 +344,23 @@ func determineExitCode(result *types.ScanResult, threshold string) int { fallthrough default: // Fail only on vulnerable (default) - if result.Summary.QuantumVulnerable > 0 { + if vulnerable > 0 { return ExitVulnerable } return ExitSuccess } } +// determineExitCode calculates the appropriate exit code based on scan results +// and the --fail-on threshold. This enables CI/CD pipelines to fail builds +// when quantum-vulnerable crypto is detected. +func determineExitCode(result *types.ScanResult, threshold string) int { + return exitCodeForSummary(result.Summary, threshold) +} + // determineExitCodeMulti calculates exit code for multi-project results. func determineExitCodeMulti(result *types.MultiProjectResult, threshold string) int { - threshold = strings.ToLower(threshold) - - switch threshold { - case "none": - return ExitSuccess - - case "any": - if result.TotalSummary.WithCrypto > 0 { - if result.TotalSummary.QuantumVulnerable > 0 { - return ExitVulnerable - } - if result.TotalSummary.QuantumPartial > 0 { - return ExitPartial - } - return ExitPartial - } - return ExitSuccess - - case "partial": - if result.TotalSummary.QuantumVulnerable > 0 { - return ExitVulnerable - } - if result.TotalSummary.QuantumPartial > 0 { - return ExitPartial - } - return ExitSuccess - - case "vulnerable": - fallthrough - default: - if result.TotalSummary.QuantumVulnerable > 0 { - return ExitVulnerable - } - return ExitSuccess - } + return exitCodeForSummary(result.TotalSummary, threshold) } func runStatus(cmd *cobra.Command, args []string) error { diff --git a/docs/testing/release-smoke.md b/docs/testing/release-smoke.md index a84fea7..7dc9e5e 100644 --- a/docs/testing/release-smoke.md +++ b/docs/testing/release-smoke.md @@ -1,55 +1,114 @@ -# Release Smoke Test: qramm-cryptodeps +# Release smoke test: cryptodeps -Manual pre-release walkthrough. Run this before every deploy or publish. -Use real keyboard and mouse input, not synthetic events. Record the actual -output, not a summary. +Manual pre-release walkthrough. Run this before every tag push, against the +CI-equivalent binary, not against `go run` and not against a binary already on +`PATH`. Record the actual output, not a summary. -## 1. Build and start clean +Build the artifact the way the release workflow does, so the version it reports +is the version being released: -- [ ] Fresh install succeeds (`npm ci` or equivalent) with no errors. -- [ ] Build succeeds (`npm run build`). -- [ ] App starts locally (`npm run dev`) with no console errors. +```bash +VERSION= +CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.version=${VERSION} -X main.commit=$(git rev-parse HEAD) -X main.date=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -o /tmp/cryptodeps-rc ./cmd/cryptodeps +/tmp/cryptodeps-rc version # must print ${VERSION}, not "dev" +``` -## 2. Core user paths - -List the three most important things a real visitor does on this site or -with this tool, then walk each one by hand. - -- [ ] Path 1: -- [ ] Path 2: -- [ ] Path 3: - -## 3. Accessibility and audience fit +Build the previously released version the same way and keep both. Nearly every +finding on this tool has turned out to be pre-existing, and that changes the +release decision: -CSNP serves non-technical and vulnerable users. Verify the experience holds -for them. +```bash +git archive v | tar -x -C /tmp/prev && cd /tmp/prev +CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=" -o /tmp/cryptodeps-prev ./cmd/cryptodeps +``` -- [ ] Plain language: no unexplained jargon in user-facing copy. -- [ ] Keyboard navigable: every interactive element reachable by Tab. -- [ ] Screen-reader labels present on buttons, links, and form fields. -- [ ] Color contrast meets WCAG AA. +## 1. Build and suite -## 4. Responsive parity +- [ ] `go build ./...`, `go vet ./...`, `gofmt -l .` all clean. +- [ ] `go test ./...` green, and green under `-race`. +- [ ] `goreleaser check` validates. +- [ ] `goreleaser release --snapshot --clean --skip=publish,docker,sign` produces + the same five archive names as the previous release, version token aside. -- [ ] Mobile (375px): nav, menus, and layout all work. -- [ ] Desktop (1280px): no overflow or broken grids. -- [ ] Dark mode (if supported) renders correctly. - -## 5. Data and links - -- [ ] Every download link resolves (no 404s). -- [ ] Any displayed number or statistic traces to a real source. -- [ ] No placeholder or fabricated content shipped. - -## 6. Security +## 2. Core user paths -- [ ] No secrets in the bundle or committed files. -- [ ] `.env` and credential files are gitignored and excluded. -- [ ] Forms validate and sanitize input. +- [ ] **Scan a project with real crypto dependencies.** Findings name the + package and version, the summary line agrees with the finding list, and + the exit code is 1. +- [ ] **Scan a clean project.** The verdict distinguishes "no dependencies", + "dependencies all absent from the database" and "analyzed, nothing found". + None of them may say cryptographic usage was not detected when nothing was + examined. +- [ ] **Scan a tree holding a corrupt manifest beside a good one.** The corrupt + file is named with a reason, and the scan exits 2. This is the case the + tool exists to get right: a scanner may skip input, it must never skip it + silently. + +## 3. Every output format + +Run `--format` for each of table, json, cbom, sarif, markdown on the same tree. + +- [ ] All five agree on the finding set and on the counts. +- [ ] The tool version is the version being released in JSON `tool`, CBOM + `metadata.tools`, and SARIF `version` and `semanticVersion`. +- [ ] CBOM validates against the official CycloneDX 1.6 schema and SARIF against + official SARIF 2.1.0, both fetched upstream, not against the tool's own + tests. +- [ ] Five runs of each format are byte-identical once the JSON `scanDate` and + the CBOM `serialNumber` are normalized. +- [ ] No emoji, no em-dashes, no ANSI escape bytes in any format. + +## 4. Untrusted input + +Both a filesystem path and a dependency name/version come from the repository +under scan, and the bundled Action publishes the markdown report. + +- [ ] A directory named with embedded newlines and `## Scan result: CLEAN` puts + no heading in the markdown report. +- [ ] A dependency whose version contains a newline and the same text puts no + heading in the report either. The database lookup falls back from + `name@version` to the name alone, so a real package with a hostile version + still resolves and still reaches the findings table. +- [ ] Both are still named in the report, rendered inert, rather than dropped. + +## 5. Error handling and exit codes + +Check each against the table in `analyze --help`. + +- [ ] Empty directory, missing directory, a file instead of a directory. +- [ ] An empty manifest, and a manifest the process cannot read (`chmod 000`). +- [ ] A tree whose only manifest is of an unsupported type. It must not be + reported as a file that could not be read. +- [ ] `--risk` and `--min-severity` change the report, the summary and the exit + code together, and reject an unknown value rather than ignoring it. +- [ ] A runtime error prints the message without the full flag list after it. + +## 6. Verdict sanity, including the best achievable case + +- [ ] Scan a project whose only crypto dependency is a post-quantum library. + Read the verdict as its maintainer would. A project that has already + migrated must not be told to adopt the scheme it is using. Sanity-check + the best achievable posture, not just the worst. +- [ ] Every finding names what is wrong and gives a next action. A hint that + tells the user to run a flag they just ran is a dead end. + +## 7. Database + +- [ ] `cryptodeps status` agrees with the database's own `stats` block and with + a direct count of its records. +- [ ] With no `~/.cryptodeps` cache present, the built-in database is used and + `status` says the cache is not downloaded. +- [ ] Compare the shipped `data/crypto-database.json` per-ecosystem counts + against the previous release's published asset. A large drop in one + ecosystem is a partial upstream fetch, not a real change. ## Result - Date: - Tester: +- Binary under test (version and commit): +- Previous binary compared against: - Verdict: PASS / FAIL -- Notes: +- Findings, each marked pre-existing or new: diff --git a/internal/analyzer/analyzer.go b/internal/analyzer/analyzer.go index 1fc1ceb..74d949c 100644 --- a/internal/analyzer/analyzer.go +++ b/internal/analyzer/analyzer.go @@ -64,6 +64,46 @@ func ValidateRiskFilter(s string) error { } } +// ValidateFailOn checks a --fail-on value. +// +// This is the one enum flag of the three that did not validate, and it is the +// one that decides an exit code. An unrecognised value fell through to the +// default policy, so `--fail-on partail` turned a build that the operator had +// asked to fail on partial risk into a build that passed, with nothing on +// either stream: exit 3 became exit 0 on the same project. A CI gate that +// silently loosens on a typo is worse than one that refuses to run. +func ValidateFailOn(s string) error { + // An empty value is refused for the same reason a padded one is honoured: + // whatever this accepts, the exit code must act on. "" matched no policy and + // fell through to the vulnerable-only default, so a project with partial-risk + // findings exited 3 for "partial" and 0 for "", silently, and an unset + // workflow input is precisely how a CI gate arrives here empty. Unlike --risk + // and --min-severity, where empty means "do not filter" and is a real state, + // this flag already has a default and cannot express one by being blank. + switch CanonicalFailOn(s) { + case "none", "any", "partial", "vulnerable": + return nil + default: + return fmt.Errorf("invalid --fail-on value %q: expected one of vulnerable, partial, any, none", s) + } +} + +// CanonicalFailOn is the one reading of a --fail-on value. +// +// It exists because there were two. The validator trimmed and lowercased before +// deciding, and the code that turns the value into an exit code only lowercased, +// so " partial " was accepted as valid and then matched no policy: the gate +// reverted to vulnerable-only and a project with partial-risk findings exited 0 +// where "partial" exited 3, on either stream in silence. Whitespace around a +// value is the ordinary result of a YAML block scalar or an expression in a +// workflow file, which is exactly where this flag is used. +// +// Every reader of the flag must go through this, so that accepting a value and +// acting on it cannot disagree. +func CanonicalFailOn(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} + // ValidateMinSeverity checks a --min-severity value. func ValidateMinSeverity(s string) error { if strings.TrimSpace(s) == "" { @@ -199,8 +239,23 @@ func (a *Analyzer) analyzeManifest(m *manifest.Manifest, projectPath string) (*t kept = append(kept, c) } else { result.Summary.FilteredOut++ + // Remember what was withheld, by the risk the gate asks about, + // so that hiding a finding from the report cannot also hide it + // from --fail-on. These counters are not serialized. + switch c.QuantumRisk { + case types.RiskVulnerable: + result.Summary.WithheldVulnerable++ + case types.RiskPartial: + result.Summary.WithheldPartial++ + } } } + if len(kept) == 0 && len(depResult.Analysis.Crypto) > 0 { + // The dependency HAS cryptography; the filter removed all of it. + // Without this, --fail-on any reports a project with crypto as + // having none, because WithCrypto is counted from survivors. + result.Summary.WithheldWithCrypto++ + } // Copy before mutating: Analysis points into the shared database, // so writing through it would corrupt the entry for every other // dependency that resolves to the same package. @@ -231,8 +286,25 @@ func (a *Analyzer) analyzeManifest(m *manifest.Manifest, projectPath string) (*t if !depResult.InDatabase { result.Summary.NotInDatabase++ } + // Counted here, where the means of examination are known, rather than + // re-derived from NotInDatabase by each report that needs it. + if !depResult.Examined() { + result.Summary.NotExamined++ + } + // The same, for the dependencies that were examined incompletely. These + // are absent from NotExamined by definition, which is why a partial + // reading was reported as a complete one. + if depResult.Analysis != nil { + result.Summary.SourceFilesUnreadable += depResult.Analysis.Analysis.FilesUnreadable + } } + // Whether source analysis ran at all, which is a property of the scan and + // not of any one dependency. a.ondemand is the thing that governs it: + // --deep with --offline sets the option without giving the analyzer any way + // to fetch, so the option alone would overstate what was attempted. + result.Summary.DeepAttempted = a.ondemand != nil + // Perform reachability analysis if enabled and ecosystem supports it if a.options.Reachability && m.Ecosystem == types.EcosystemGo { a.performReachabilityAnalysis(result, projectPath) @@ -304,9 +376,14 @@ func (a *Analyzer) generateHints(result *types.ScanResult) []string { } } - // Hint: no crypto found but packages exist - if result.Summary.WithCrypto == 0 && result.Summary.TotalDependencies > 0 { - if result.Summary.NotInDatabase == result.Summary.TotalDependencies { + // Hint: no crypto found but packages exist. + // + // Guarded on source analysis not having run, which the hint above it was + // already guarded on and this one was not. Without the guard a --deep scan + // that read every package and found no cryptography ended with advice to + // run --deep. + if result.Summary.WithCrypto == 0 && result.Summary.TotalDependencies > 0 && !result.Summary.DeepAttempted { + if result.Summary.NotExamined == result.Summary.TotalDependencies { hints = append(hints, "No crypto findings. All packages are unknown - try --deep to analyze source code.") } } @@ -340,8 +417,47 @@ func (a *Analyzer) analyzeDependency(dep types.Dependency) types.DependencyResul if err != nil { // Log the error but continue fmt.Fprintf(os.Stderr, "Warning: on-demand analysis failed for %s: %v\n", dep.Name, err) + result.Error = fmt.Sprintf("source analysis could not fetch %s: %v", dep.Name, err) return result } + // Examination is claimed from what an analyzer parsed, not from a call + // that returned no error. The two differ whenever an archive arrives + // carrying nothing this tool can read: a Maven artifact with no sources + // JAR falls back to the compiled main JAR, the Java walker accepts only + // .java, .kt and .kts, and it then returns no usages and no error. That + // was reported as "no cryptographic usage detected in the 1 of 1 + // dependencies that were examined" for a scan that read zero files, so + // the absence of an error was standing in for evidence and saying the + // opposite of the truth. + if !analysis.SourceWasRead() { + fmt.Fprintf(os.Stderr, + "Warning: source analysis of %s read no files it can parse, so it is "+ + "reported as not examined rather than as clean\n", dep.Name) + result.Error = fmt.Sprintf( + "source analysis read no files it can parse in the fetched archive for %s", dep.Name) + return result + } + // Partly read is not read. A package can hold a file this analyzer + // parsed and another it refused, and the refusal left no trace outside + // the walk that recorded it: the count stopped at the AST layer, so a + // dependency whose only cryptography sat in the refused file was + // reported as examined and clean on every stream and in every format. + // The exception belongs beside the claim, not inside the walk. + if analysis.Analysis.FilesUnreadable > 0 { + fmt.Fprintf(os.Stderr, + "Warning: source analysis of %s read %d file(s) and could not read %d more, so its "+ + "cryptography may be under-reported\n", + dep.Name, analysis.Analysis.FilesAnalyzed, analysis.Analysis.FilesUnreadable) + // Named, not just counted. A reader told that one file went unread and + // not which one has been handed a dead end, and the reason differs in + // what they should do about it. + for _, reason := range analysis.Analysis.UnreadableFiles { + fmt.Fprintf(os.Stderr, " not read: %s\n", reason) + } + if n := analysis.Analysis.FilesUnreadable - len(analysis.Analysis.UnreadableFiles); n > 0 { + fmt.Fprintf(os.Stderr, " and %d more not named here\n", n) + } + } result.Analysis = analysis result.DeepAnalyzed = true } diff --git a/internal/analyzer/analyzer_test.go b/internal/analyzer/analyzer_test.go index 482cf4d..67bbae9 100644 --- a/internal/analyzer/analyzer_test.go +++ b/internal/analyzer/analyzer_test.go @@ -6,6 +6,7 @@ package analyzer import ( "os" "path/filepath" + "strings" "testing" "github.com/csnp/qramm-cryptodeps/internal/database" @@ -371,3 +372,134 @@ require golang.org/x/crypto v0.17.0 t.Errorf("Manifest = %q, want %q", result.Manifest, gomodPath) } } + +// TestSummaryCoverageMatchesDependencies pins the summary to the dependency +// list it describes. +// +// The coverage fields are what every format's verdict is computed from, and +// they are accumulated in a different loop from the one that decides whether a +// dependency was examined. If the two ever disagree, the reports are wrong in a +// way no formatter test can see, because the formatters would be faithfully +// rendering a summary that does not describe the scan. +func TestSummaryCoverageMatchesDependencies(t *testing.T) { + tmpDir := t.TempDir() + + // One dependency the embedded database carries, one it does not. + gomod := `module test + +require ( + golang.org/x/crypto v0.31.0 + github.com/unknown/package v1.0.0 +) +` + if err := os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte(gomod), 0644); err != nil { + t.Fatalf("write go.mod: %v", err) + } + + result, err := New(database.NewEmbedded(), Options{}).Analyze(tmpDir) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + + // Guard the fixture: it has to contain both kinds of dependency, or the + // invariant below holds for a reason that has nothing to do with the code. + var examined, unexamined int + for _, dep := range result.Dependencies { + if dep.Examined() { + examined++ + } else { + unexamined++ + } + } + if examined == 0 || unexamined == 0 { + t.Fatalf("fixture does not mix examined and unexamined dependencies: %d/%d", + examined, unexamined) + } + + if result.Summary.NotExamined != unexamined { + t.Errorf("Summary.NotExamined = %d, want %d; the summary does not describe the "+ + "dependency list it was built from", result.Summary.NotExamined, unexamined) + } + if result.Summary.TotalDependencies-result.Summary.NotExamined != examined { + t.Errorf("summary reports %d examined, want %d", + result.Summary.TotalDependencies-result.Summary.NotExamined, examined) + } + if result.Summary.DeepAttempted { + t.Error("DeepAttempted is set on a scan that did not request source analysis") + } +} + +// TestDeepAttemptedReflectsWhatCouldRun separates the flag the user passed from +// the analysis that was actually possible. +// +// --deep with --offline sets the option and gives the analyzer no way to fetch +// anything, so a report keyed on the option alone would tell the user that +// source analysis had been tried. +func TestDeepAttemptedReflectsWhatCouldRun(t *testing.T) { + tmpDir := t.TempDir() + gomod := "module test\n\nrequire github.com/unknown/package v1.0.0\n" + if err := os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte(gomod), 0644); err != nil { + t.Fatalf("write go.mod: %v", err) + } + + for _, tc := range []struct { + name string + opts Options + want bool + }{ + {"neither flag", Options{}, false}, + {"offline", Options{Offline: true}, false}, + {"deep and offline", Options{Deep: true, Offline: true}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := New(database.NewEmbedded(), tc.opts).Analyze(tmpDir) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if result.Summary.DeepAttempted != tc.want { + t.Errorf("DeepAttempted = %v, want %v", result.Summary.DeepAttempted, tc.want) + } + }) + } +} + +// TestUnexaminedHintIsNotPrintedAfterDeep guards the hint that had no guard. +// +// The hint above it in generateHints was already conditioned on source analysis +// not having run; this one was not, so a --deep scan that read every package +// ended with "try --deep to analyze source code". +func TestUnexaminedHintIsNotPrintedAfterDeep(t *testing.T) { + // Source analysis ran and could not read any of them. This is the fixture + // that reaches the branch: NotExamined equals TotalDependencies, which is + // the condition the hint is generated under. A fixture where the packages + // were successfully read never reaches it, so it would pass whether the + // guard were present or not. + deepRan := &types.ScanResult{ + Summary: types.ScanSummary{ + TotalDependencies: 3, NotInDatabase: 3, NotExamined: 3, DeepAttempted: true, + }, + } + if deepRan.Summary.NotExamined != deepRan.Summary.TotalDependencies { + t.Fatalf("fixture does not reach the hint's condition: %+v", deepRan.Summary) + } + a := New(database.NewEmbedded(), Options{Deep: true}) + for _, hint := range a.generateHints(deepRan) { + if strings.Contains(hint, "--deep") { + t.Errorf("hint sends a user who ran --deep back to --deep: %q", hint) + } + } + + // And the same hint must still appear when it is the right advice. + deepNotRun := &types.ScanResult{ + Summary: types.ScanSummary{TotalDependencies: 3, NotInDatabase: 3, NotExamined: 3}, + } + var found bool + for _, hint := range New(database.NewEmbedded(), Options{}).generateHints(deepNotRun) { + if strings.Contains(hint, "--deep") { + found = true + } + } + if !found { + t.Error("the --deep hint no longer appears for a scan that never ran source analysis") + } +} diff --git a/internal/analyzer/ast/go.go b/internal/analyzer/ast/go.go index 35549ed..7632588 100644 --- a/internal/analyzer/ast/go.go +++ b/internal/analyzer/ast/go.go @@ -30,8 +30,8 @@ func NewGoAnalyzer() *GoAnalyzer { } // AnalyzeDirectory analyzes all Go files in a directory. -func (a *GoAnalyzer) AnalyzeDirectory(dir string) ([]types.CryptoUsage, error) { - var allUsages []types.CryptoUsage +func (a *GoAnalyzer) AnalyzeDirectory(dir string) (DirectoryScan, error) { + var scan DirectoryScan err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -53,22 +53,21 @@ func (a *GoAnalyzer) AnalyzeDirectory(dir string) ([]types.CryptoUsage, error) { return nil } - usages, err := a.AnalyzeFile(path) - if err != nil { - // Log but continue on parse errors - return nil - } - - allUsages = append(allUsages, usages...) + // A parse error is counted rather than propagated: one unreadable file + // should not abandon a package, and the count is what stops an + // unreadable package from being reported as an examined one. + scan.add(a.AnalyzeFile(path)) return nil }) - return allUsages, err + return scan, err } // AnalyzeFile analyzes a single Go file for cryptographic usage. func (a *GoAnalyzer) AnalyzeFile(filename string) ([]types.CryptoUsage, error) { - src, err := os.ReadFile(filename) + // Not os.ReadFile: an extracted archive can carry a symlink pointing at any + // file the process can read, and a .go name over binary content is not Go. + src, err := readSource(filename) if err != nil { return nil, err } diff --git a/internal/analyzer/ast/go_test.go b/internal/analyzer/ast/go_test.go index 5aa42e5..002ab3f 100644 --- a/internal/analyzer/ast/go_test.go +++ b/internal/analyzer/ast/go_test.go @@ -207,7 +207,8 @@ func f2() { rsa.GenerateKey(nil, 2048) } os.WriteFile(file2, []byte(content2), 0644) analyzer := NewGoAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory failed: %v", err) } @@ -237,7 +238,8 @@ func f() { rsa.GenerateKey(nil, 2048) } os.WriteFile(regularFile, []byte(regularContent), 0644) analyzer := NewGoAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory failed: %v", err) } @@ -584,7 +586,8 @@ func TestGoAnalyzer_EmptyDirectory(t *testing.T) { tmpDir := t.TempDir() analyzer := NewGoAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory on empty dir failed: %v", err) } @@ -604,7 +607,8 @@ func TestGoAnalyzer_NonGoFile(t *testing.T) { } analyzer := NewGoAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory failed: %v", err) } diff --git a/internal/analyzer/ast/java.go b/internal/analyzer/ast/java.go index 2fe9816..fd19576 100644 --- a/internal/analyzer/ast/java.go +++ b/internal/analyzer/ast/java.go @@ -5,7 +5,6 @@ package ast import ( - "bufio" "os" "path/filepath" "regexp" @@ -143,8 +142,8 @@ type javaFuncContext struct { } // AnalyzeDirectory analyzes all Java/Kotlin files in a directory. -func (a *JavaAnalyzer) AnalyzeDirectory(dir string) ([]types.CryptoUsage, error) { - var allUsages []types.CryptoUsage +func (a *JavaAnalyzer) AnalyzeDirectory(dir string) (DirectoryScan, error) { + var scan DirectoryScan err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -174,25 +173,22 @@ func (a *JavaAnalyzer) AnalyzeDirectory(dir string) ([]types.CryptoUsage, error) return nil } - usages, err := a.AnalyzeFile(path) - if err != nil { - return nil - } - - allUsages = append(allUsages, usages...) + // Counted, not propagated: see the Go walker. A class-only JAR reaches + // here with nothing this analyzer accepts, and FilesParsed staying at + // zero is how the caller learns the package was never read. + scan.add(a.AnalyzeFile(path)) return nil }) - return allUsages, err + return scan, err } // AnalyzeFile analyzes a single Java/Kotlin file for cryptographic usage. func (a *JavaAnalyzer) AnalyzeFile(filename string) ([]types.CryptoUsage, error) { - file, err := os.Open(filename) + scanner, err := openSource(filename) if err != nil { return nil, err } - defer file.Close() var usages []types.CryptoUsage imports := make(map[string]bool) @@ -200,7 +196,6 @@ func (a *JavaAnalyzer) AnalyzeFile(filename string) ([]types.CryptoUsage, error) var currentClass string braceDepth := 0 - scanner := bufio.NewScanner(file) lineNum := 0 for scanner.Scan() { diff --git a/internal/analyzer/ast/java_test.go b/internal/analyzer/ast/java_test.go index 6555dcf..da91195 100644 --- a/internal/analyzer/ast/java_test.go +++ b/internal/analyzer/ast/java_test.go @@ -277,7 +277,8 @@ public class Crypto { os.WriteFile(regularFile, []byte(regularContent), 0644) analyzer := NewJavaAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory failed: %v", err) } @@ -608,7 +609,8 @@ class Crypto3 { os.WriteFile(file3, []byte(content3), 0644) analyzer := NewJavaAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory failed: %v", err) } diff --git a/internal/analyzer/ast/javascript.go b/internal/analyzer/ast/javascript.go index b7c27b0..3b30bc5 100644 --- a/internal/analyzer/ast/javascript.go +++ b/internal/analyzer/ast/javascript.go @@ -5,7 +5,6 @@ package ast import ( - "bufio" "os" "path/filepath" "regexp" @@ -95,8 +94,8 @@ var jsAlgorithmMap = map[string]string{ } // AnalyzeDirectory analyzes all JavaScript files in a directory. -func (a *JavaScriptAnalyzer) AnalyzeDirectory(dir string) ([]types.CryptoUsage, error) { - var allUsages []types.CryptoUsage +func (a *JavaScriptAnalyzer) AnalyzeDirectory(dir string) (DirectoryScan, error) { + var scan DirectoryScan err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -130,17 +129,12 @@ func (a *JavaScriptAnalyzer) AnalyzeDirectory(dir string) ([]types.CryptoUsage, return nil } - usages, err := a.AnalyzeFile(path) - if err != nil { - // Log but continue on parse errors - return nil - } - - allUsages = append(allUsages, usages...) + // Counted, not propagated: see the Go walker. + scan.add(a.AnalyzeFile(path)) return nil }) - return allUsages, err + return scan, err } // jsFuncContext tracks function context during JavaScript file analysis. @@ -153,11 +147,10 @@ type jsFuncContext struct { // AnalyzeFile analyzes a single JavaScript file for cryptographic usage. func (a *JavaScriptAnalyzer) AnalyzeFile(filename string) ([]types.CryptoUsage, error) { - file, err := os.Open(filename) + scanner, err := openSource(filename) if err != nil { return nil, err } - defer file.Close() var usages []types.CryptoUsage imports := make(map[string]bool) @@ -166,7 +159,6 @@ func (a *JavaScriptAnalyzer) AnalyzeFile(filename string) ([]types.CryptoUsage, braceDepth := 0 exportedNames := make(map[string]bool) - scanner := bufio.NewScanner(file) lineNum := 0 for scanner.Scan() { diff --git a/internal/analyzer/ast/javascript_test.go b/internal/analyzer/ast/javascript_test.go index 5f666b1..4a04d65 100644 --- a/internal/analyzer/ast/javascript_test.go +++ b/internal/analyzer/ast/javascript_test.go @@ -255,7 +255,8 @@ crypto.createHash('md5'); os.WriteFile(regularFile, []byte(regularContent), 0644) analyzer := NewJavaScriptAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory failed: %v", err) } @@ -725,7 +726,8 @@ bcrypt.hash(password, 10); os.WriteFile(file3, []byte(content3), 0644) analyzer := NewJavaScriptAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory failed: %v", err) } diff --git a/internal/analyzer/ast/python.go b/internal/analyzer/ast/python.go index ccc0bc5..e47fa48 100644 --- a/internal/analyzer/ast/python.go +++ b/internal/analyzer/ast/python.go @@ -5,7 +5,6 @@ package ast import ( - "bufio" "os" "path/filepath" "regexp" @@ -87,8 +86,8 @@ var pythonCryptoPackages = map[string][]string{ } // AnalyzeDirectory analyzes all Python files in a directory. -func (a *PythonAnalyzer) AnalyzeDirectory(dir string) ([]types.CryptoUsage, error) { - var allUsages []types.CryptoUsage +func (a *PythonAnalyzer) AnalyzeDirectory(dir string) (DirectoryScan, error) { + var scan DirectoryScan err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -119,17 +118,12 @@ func (a *PythonAnalyzer) AnalyzeDirectory(dir string) ([]types.CryptoUsage, erro return nil } - usages, err := a.AnalyzeFile(path) - if err != nil { - // Log but continue on parse errors - return nil - } - - allUsages = append(allUsages, usages...) + // Counted, not propagated: see the Go walker. + scan.add(a.AnalyzeFile(path)) return nil }) - return allUsages, err + return scan, err } // pyFuncContext tracks function context during Python file analysis. @@ -142,11 +136,10 @@ type pyFuncContext struct { // AnalyzeFile analyzes a single Python file for cryptographic usage. func (a *PythonAnalyzer) AnalyzeFile(filename string) ([]types.CryptoUsage, error) { - file, err := os.Open(filename) + scanner, err := openSource(filename) if err != nil { return nil, err } - defer file.Close() var usages []types.CryptoUsage imports := make(map[string]bool) @@ -156,7 +149,6 @@ func (a *PythonAnalyzer) AnalyzeFile(filename string) ([]types.CryptoUsage, erro indent int } - scanner := bufio.NewScanner(file) lineNum := 0 for scanner.Scan() { diff --git a/internal/analyzer/ast/python_test.go b/internal/analyzer/ast/python_test.go index 7a22c5a..4f6d904 100644 --- a/internal/analyzer/ast/python_test.go +++ b/internal/analyzer/ast/python_test.go @@ -282,7 +282,8 @@ hashlib.md5(b"data") os.WriteFile(regularFile, []byte(regularContent), 0644) analyzer := NewPythonAnalyzer() - usages, err := analyzer.AnalyzeDirectory(tmpDir) + scan, err := analyzer.AnalyzeDirectory(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("AnalyzeDirectory failed: %v", err) } diff --git a/internal/analyzer/ast/scan.go b/internal/analyzer/ast/scan.go new file mode 100644 index 0000000..bf05858 --- /dev/null +++ b/internal/analyzer/ast/scan.go @@ -0,0 +1,53 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package ast + +import "github.com/csnp/qramm-cryptodeps/pkg/types" + +// DirectoryScan is what one AnalyzeDirectory walk read. +// +// The usages alone cannot answer whether a directory was examined: an empty +// slice means the same thing for a package with no cryptography and for a +// package the analyzer could not read a single file of. Callers decide whether +// to report an examination, so the counts they need to decide it come back with +// the findings rather than being inferred from their absence. +type DirectoryScan struct { + // Usages is every cryptographic usage found. + Usages []types.CryptoUsage + // FilesParsed counts the source files this analyzer parsed successfully. It + // is the evidence that reading happened. + FilesParsed int + // FilesFailed counts the files that matched the analyzer's extensions and + // then failed to parse. Walks swallow those so that one bad file does not + // abandon a package, which means the count is the only trace they leave. + FilesFailed int + // Unreadable says which files failed and why. + // + // A count alone is a dead end: the report could say that one file in a + // package went unread and offer the reader no way to learn which, so the + // disclosure that exists to prevent a false clean could not be acted on. The + // reason is kept with the path because the two answers differ in what the + // reader should do, and only the tool knows which applies. + Unreadable []string +} + +// maxUnreadableReported bounds how many refused files are named. +// +// An archive can be hostile, and a package of ten thousand refused files must +// not turn one warning into ten thousand lines of stderr. The count is always +// exact; only the naming is bounded, and the message says so when it truncates. +const maxUnreadableReported = 10 + +// add records the outcome of one file. +func (s *DirectoryScan) add(usages []types.CryptoUsage, err error) { + if err != nil { + s.FilesFailed++ + if len(s.Unreadable) < maxUnreadableReported { + s.Unreadable = append(s.Unreadable, err.Error()) + } + return + } + s.FilesParsed++ + s.Usages = append(s.Usages, usages...) +} diff --git a/internal/analyzer/ast/scan_test.go b/internal/analyzer/ast/scan_test.go new file mode 100644 index 0000000..b09ade7 --- /dev/null +++ b/internal/analyzer/ast/scan_test.go @@ -0,0 +1,570 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package ast + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf8" +) + +// TestGoDirectoryScanCountsOnlyFilesItParsed holds the meaning of FilesParsed. +// +// The count is what the rest of the tool uses to decide whether a package was +// examined, so counting a file the analyzer could not read would restore the +// defect it exists to close: a package would again be reported as examined on +// the strength of files nothing was learned from. +func TestGoDirectoryScanCountsOnlyFilesItParsed(t *testing.T) { + dir := t.TempDir() + + readable := `package main + +import "crypto/md5" + +func hash(b []byte) [16]byte { return md5.Sum(b) } +` + // Valid enough to be walked and offered to the parser, and not valid Go. + unparsable := `package main + +func broken( { +` + if err := os.WriteFile(filepath.Join(dir, "readable.go"), []byte(readable), 0644); err != nil { + t.Fatalf("write readable.go: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "unparsable.go"), []byte(unparsable), 0644); err != nil { + t.Fatalf("write unparsable.go: %v", err) + } + + scan, err := NewGoAnalyzer().AnalyzeDirectory(dir) + if err != nil { + t.Fatalf("AnalyzeDirectory: %v", err) + } + + // Guard the fixture: if the parser accepts the broken file, this test is + // measuring nothing. + if scan.FilesFailed != 1 { + t.Fatalf("FilesFailed = %d, so the fixture's unparsable file did not fail to "+ + "parse and this test cannot detect a walker that counts it", scan.FilesFailed) + } + if scan.FilesParsed != 1 { + t.Errorf("FilesParsed = %d for one readable and one unparsable file", scan.FilesParsed) + } + if len(scan.Usages) == 0 { + t.Errorf("the readable file's MD5 usage is missing, so one unparsable file " + + "abandoned the whole directory") + } +} + +// TestDirectoryScanCountsOnlyTextItCouldRead is the regression test for the +// second attempt at the examination evidence. +// +// The first attempt counted files the analyzer OPENED. Three of the four +// analyzers are line scanners whose only failure mode was os.Open, so any +// openable file with a matching extension counted as parsed: a zero-byte +// Empty.java in a fetched archive produced filesAnalyzed 1 and the report said +// "no cryptographic usage detected in the 1 of 1 dependencies that were +// examined", which is the exact sentence this release exists to remove. An +// extension is a descriptor; accepting it as the thing itself is how the +// false-clean class survived its own fix. +func TestDirectoryScanCountsOnlyTextItCouldRead(t *testing.T) { + cases := []struct { + name string + file string + content []byte + analyzer func(string) (DirectoryScan, error) + }{ + {"empty java", "Empty.java", []byte{}, func(d string) (DirectoryScan, error) { + return NewJavaAnalyzer().AnalyzeDirectory(d) + }}, + {"class bytes named java", "Hashing.java", + []byte{0xca, 0xfe, 0xba, 0xbe, 0x00, 0x00, 0x00, 0x34}, + func(d string) (DirectoryScan, error) { return NewJavaAnalyzer().AnalyzeDirectory(d) }}, + {"empty js", "index.js", []byte{}, func(d string) (DirectoryScan, error) { + return NewJavaScriptAnalyzer().AnalyzeDirectory(d) + }}, + {"binary named js", "index.js", []byte{0x00, 0x01, 0x02, 0xff, 0xfe}, + func(d string) (DirectoryScan, error) { + return NewJavaScriptAnalyzer().AnalyzeDirectory(d) + }}, + {"empty python", "mod.py", []byte{}, func(d string) (DirectoryScan, error) { + return NewPythonAnalyzer().AnalyzeDirectory(d) + }}, + {"binary named python", "mod.py", []byte{0x7f, 'E', 'L', 'F', 0x00, 0x01}, + func(d string) (DirectoryScan, error) { return NewPythonAnalyzer().AnalyzeDirectory(d) }}, + {"empty go", "main.go", []byte{}, func(d string) (DirectoryScan, error) { + return NewGoAnalyzer().AnalyzeDirectory(d) + }}, + // Larger than the head that is checked for being text, and carrying no + // NUL byte, so neither the NUL test nor a head shorter than the boundary + // catches it. The first version of the text check walked the head down + // to nothing looking for a rune boundary and then called an empty slice + // valid UTF-8, so every binary of this shape counted as parsed source. + {"large binary named java without a NUL byte", "Big.java", + bytes.Repeat([]byte{0xff}, 2000), + func(d string) (DirectoryScan, error) { return NewJavaAnalyzer().AnalyzeDirectory(d) }}, + {"large binary named js without a NUL byte", "big.js", + bytes.Repeat([]byte{0xfe}, 4096), + func(d string) (DirectoryScan, error) { + return NewJavaScriptAnalyzer().AnalyzeDirectory(d) + }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, tc.file) + if err := os.WriteFile(path, tc.content, 0644); err != nil { + t.Fatalf("write fixture: %v", err) + } + // Guard the fixture: the walker has to reach this file at all, or a + // zero count means only that nothing was looked at. + if info, err := os.Stat(path); err != nil || info.Size() != int64(len(tc.content)) { + t.Fatalf("fixture %s is not on disk as written: %v", path, err) + } + + scan, err := tc.analyzer(dir) + if err != nil { + t.Fatalf("AnalyzeDirectory: %v", err) + } + if scan.FilesParsed != 0 { + t.Errorf("FilesParsed = %d for a file holding %d bytes of %s; a caller reading "+ + "this count reports the package as examined", scan.FilesParsed, + len(tc.content), tc.name) + } + if scan.FilesFailed != 1 { + t.Errorf("FilesFailed = %d, so the file was not even attempted and this case "+ + "proves nothing", scan.FilesFailed) + } + }) + } +} + +// TestDirectoryScanReadsSourceInASingleByteEncoding is the regression test for +// the third attempt at the examination evidence. +// +// The second attempt refused any file whose head was not valid UTF-8. These +// analyzers are line scanners matching ASCII identifiers, so they never needed +// valid UTF-8, and the check refused legitimate source in any single-byte +// encoding: one Latin-1 accent in a comment made a .java or .js file +// "not text this analyzer can read", its cryptography was never looked for, and +// the released 1.2.2 had found it. The evasion that follows is the reason this +// is a security test and not a nicety: a dependency that also ships one +// readable file is still reported as examined, so encoding the file that calls +// MD5 in Latin-1 hid it from the scanner entirely. +func TestDirectoryScanReadsSourceInASingleByteEncoding(t *testing.T) { + // 0xe9 is 'e' with an acute accent in Latin-1 and is not valid UTF-8. + latin1Comment := []byte("// author: Jos\xe9 Garc\xeda\n") + + cases := []struct { + name string + file string + content []byte + analyzer func(string) (DirectoryScan, error) + }{ + {"latin-1 java", "Hashing.java", + append(latin1Comment, []byte(`import java.security.MessageDigest; +class Hashing { void f() throws Exception { MessageDigest.getInstance("MD5"); } } +`)...), + func(d string) (DirectoryScan, error) { return NewJavaAnalyzer().AnalyzeDirectory(d) }}, + {"latin-1 javascript", "index.js", + append(latin1Comment, []byte("const crypto = require('crypto');\n"+ + "function h(x) { return crypto.createHash('md5').update(x).digest('hex'); }\n")...), + func(d string) (DirectoryScan, error) { return NewJavaScriptAnalyzer().AnalyzeDirectory(d) }}, + // Past the head boundary, so the file is judged on the head alone and the + // accent sits inside it. The padding is ASCII, which is what a real + // source file in a single-byte encoding looks like: code is ASCII even + // when its comments are not. + {"latin-1 javascript larger than the head", "big.js", + append(append(latin1Comment, bytes.Repeat([]byte("// padding to pass the head boundary\n"), 40)...), + []byte("const crypto = require('crypto');\n"+ + "function h(x) { return crypto.createHash('md5').update(x).digest('hex'); }\n")...), + func(d string) (DirectoryScan, error) { return NewJavaScriptAnalyzer().AnalyzeDirectory(d) }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Guard the fixture in both directions. If the content were valid + // UTF-8 this case would pass against the code it exists to catch, + // and if it carried a NUL byte it would be refused for an unrelated + // reason that this change does not and should not alter. + if utf8.Valid(tc.content) { + t.Fatalf("fixture is valid UTF-8, so it never reaches the branch under test") + } + if bytes.IndexByte(tc.content, 0) >= 0 { + t.Fatalf("fixture carries a NUL byte, so it is refused as binary rather than " + + "for its encoding") + } + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, tc.file), tc.content, 0644); err != nil { + t.Fatalf("write fixture: %v", err) + } + + scan, err := tc.analyzer(dir) + if err != nil { + t.Fatalf("AnalyzeDirectory: %v", err) + } + if scan.FilesParsed != 1 { + t.Errorf("FilesParsed = %d for legitimate source in a single-byte encoding; "+ + "the file was refused and its cryptography was never looked for", + scan.FilesParsed) + } + if scan.FilesFailed != 0 { + t.Errorf("FilesFailed = %d for a readable source file", scan.FilesFailed) + } + if len(scan.Usages) == 0 { + t.Errorf("the MD5 usage in a Latin-1 encoded file is missing, so a dependency " + + "can hide its cryptography from this scanner by choosing an encoding") + } + }) + } +} + +// TestDirectoryScanReadsAFileWithOneVeryLongLine is the regression test for a +// false negative on one of the tool's most common inputs. +// +// bufio's default token limit is 64 KiB. A line longer than that stops Scan and +// makes Err report it, the three line-scanning analyzers return that error, and +// the walk then discards every usage they had already found. So a bundled or +// minified dist file, which is normally one very long line, contributed nothing: +// a 70 KB single-line file calling crypto.createHash('md5') produced no finding, +// no warning, and a clean verdict, on this candidate and on 1.2.2. +func TestDirectoryScanReadsAFileWithOneVeryLongLine(t *testing.T) { + // One line, comfortably past the 64 KiB default, with the call at the end so + // that a scanner which gave up early cannot reach it. The require and the + // call use the spelling this analyzer tracks: a fixture whose crypto is not + // detectable at ALL would fail this test for a reason that has nothing to do + // with the line length, which is how the first draft of it misled me. + line := "const crypto = require('crypto'); /*" + strings.Repeat("a", 70000) + "*/ " + + "crypto.createHash('md5').update('x').digest('hex');\n" + + dir := t.TempDir() + // Not named .min.js: minified files are skipped by name, and this test is + // about the line length rather than the naming convention. + path := filepath.Join(dir, "bundle.js") + if err := os.WriteFile(path, []byte(line), 0644); err != nil { + t.Fatalf("write fixture: %v", err) + } + // Guard the fixture: below the limit this test proves nothing. + if len(line) <= 64*1024 { + t.Fatalf("fixture line is %d bytes, at or below bufio's 64 KiB default, so it does "+ + "not reach the branch under test", len(line)) + } + // Guard it the other way too: the same crypto call on a SHORT line has to be + // detected, or a zero result below says only that the fixture is undetectable. + short := t.TempDir() + if err := os.WriteFile(filepath.Join(short, "small.js"), + []byte("const crypto = require('crypto');\ncrypto.createHash('md5');\n"), 0644); err != nil { + t.Fatalf("write control fixture: %v", err) + } + if control, err := NewJavaScriptAnalyzer().AnalyzeDirectory(short); err != nil || + len(control.Usages) == 0 { + t.Fatalf("the control fixture's MD5 is not detected on a short line (%d usages, %v), "+ + "so this test cannot attribute a miss to the line length", + len(control.Usages), err) + } + + scan, err := NewJavaScriptAnalyzer().AnalyzeDirectory(dir) + if err != nil { + t.Fatalf("AnalyzeDirectory: %v", err) + } + if scan.FilesParsed != 1 { + t.Errorf("FilesParsed = %d for a single-line bundle this analyzer can read", + scan.FilesParsed) + } + if len(scan.Usages) == 0 { + t.Errorf("the MD5 call on a %d byte line is missing, so a bundled dist file "+ + "contributes nothing and the package is still reported as examined", len(line)) + } +} + +// TestIsTextBracketsTheThreshold pins minTextFraction from BOTH sides. +// +// A fixture of pure 0xff has a printable fraction of zero, so it is refused by +// any threshold above zero and cannot detect one that has been loosened. That +// left the whole range between 0 and 0.75 unpinned: a mutation to 0.01 kept the +// suite green. Bracketing needs an input in the middle, and the natural one is +// what the threshold was measured against. +func TestIsTextBracketsTheThreshold(t *testing.T) { + // Around 0.39 printable, which is what random bytes with the NUL removed + // measure, and is the shape of compressed or encrypted content. + var mixed []byte + for i := 0; len(mixed) < 2000; i++ { + if i%5 == 0 { + mixed = append(mixed, byte('a'+i%26)) + } else { + mixed = append(mixed, byte(0x80+i%0x7f)) + } + } + // Around 0.98 printable, which is what real source in a single-byte encoding + // measures: ASCII code with the occasional accented byte in a comment. + var sourceLike []byte + for i := 0; len(sourceLike) < 2000; i++ { + if i%50 == 0 { + sourceLike = append(sourceLike, 0xe9) + } else { + sourceLike = append(sourceLike, byte('a'+i%26)) + } + } + + for _, tc := range []struct { + name string + content []byte + want bool + }{ + {"mostly high bytes is not text", mixed, false}, + {"mostly ASCII with a few high bytes is text", sourceLike, true}, + } { + t.Run(tc.name, func(t *testing.T) { + // Guard both fixtures: each must be invalid UTF-8 and NUL-free, or it + // is answered by a different branch than the one under test. + if utf8.Valid(tc.content) { + t.Fatalf("fixture is valid UTF-8, so the fraction is never consulted") + } + if bytes.IndexByte(tc.content, 0) >= 0 { + t.Fatalf("fixture carries a NUL byte, so it is refused as binary") + } + // And assert where each sits relative to the threshold, so that a + // change to minTextFraction fails here with its reason visible rather + // than somewhere downstream. + got := printableASCIIFraction(tc.content[:headBytes]) + if tc.want && got < minTextFraction { + t.Fatalf("fixture meant to be text measures %.3f, below the %.2f threshold", + got, minTextFraction) + } + if !tc.want && got >= minTextFraction { + t.Fatalf("fixture meant to be binary measures %.3f, at or above the %.2f threshold", + got, minTextFraction) + } + + if isText(tc.content) != tc.want { + t.Errorf("isText = %v, want %v for a head measuring %.3f printable ASCII", + !tc.want, tc.want, got) + } + }) + } +} + +// TestIsTextDoesNotTrimAHeadIntoValidity pins the bound on the rune-straddle +// allowance, which the fraction fallback would otherwise hide. +// +// The allowance trims up to the three continuation bytes a truncated rune can +// occupy. Unbounded, it walks a binary head down to whatever prefix happens to +// be valid: a file opening with a few ASCII bytes and continuing with 0xff is +// then accepted as text, which is the defect the bound was added to close. Pure +// 0xff cannot detect it, because trimming that reaches an empty head and the +// empty-head guard refuses it for an unrelated reason. +func TestIsTextDoesNotTrimAHeadIntoValidity(t *testing.T) { + content := append([]byte("abc"), bytes.Repeat([]byte{0xff}, 2000)...) + + if bytes.IndexByte(content, 0) >= 0 { + t.Fatalf("fixture carries a NUL byte, so it is refused as binary") + } + if len(content) <= headBytes { + t.Fatalf("fixture is %d bytes, not past the head boundary, so no trimming happens", + len(content)) + } + // The guard that makes this fixture the right one: an unbounded trim WOULD + // find a valid prefix here, so the test can tell a bounded loop from an + // unbounded one. + if !utf8.Valid(content[:3]) { + t.Fatalf("fixture's ASCII prefix is not valid UTF-8, so an unbounded trim would not " + + "reach validity and this test cannot detect the bound being removed") + } + + if isText(content) { + t.Errorf("a head of three ASCII bytes followed by 0xff was accepted as text; the " + + "straddle allowance trimmed it into validity") + } +} + +// TestMaxSourceFileClearsRealSource pins the cap against the largest real source +// file measured, so that lowering it fails here rather than in a silent skip. +// +// aws-sdk-go v1.55.5 ships service/ec2/api.go at 7,771,273 bytes and has grown +// every release. A cap a real file is about to cross drops that file from the +// analysis, and until this test existed the value was unpinned in both +// directions: mutations to 1 MiB and to no cap at all both kept the suite green. +func TestMaxSourceFileClearsRealSource(t *testing.T) { + const largestMeasured = 7_771_273 // aws-sdk-go v1.55.5 service/ec2/api.go + // Headroom, not merely clearance. This file was 6,400,535 bytes at v1.44.0 + // and grows every release, so a cap that merely exceeds today's largest file + // is a cap that starts silently dropping it during the life of a release. + if maxSourceFile < 2*largestMeasured { + t.Errorf("maxSourceFile = %d, less than twice the largest real source file measured "+ + "(%d); a generated file of this kind grows every release and would begin to be "+ + "counted unreadable rather than analyzed", maxSourceFile, largestMeasured) + } + // And the cap has to still be a cap, or an archive can exhaust the host. + if maxSourceFile > 128<<20 { + t.Errorf("maxSourceFile = %d, high enough that one file in a hostile archive can "+ + "exhaust memory", maxSourceFile) + } +} + +// TestDirectoryScanNamesTheFilesItCouldNotRead keeps the disclosure actionable. +// +// A count with no names is a dead end: the report says one file in a package +// went unread and gives the reader no way to learn which, or why. The coverage +// note promised the scan named them before it did. +func TestDirectoryScanNamesTheFilesItCouldNotRead(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "good.js"), + []byte("module.exports.x = 1;\n"), 0644); err != nil { + t.Fatalf("write readable fixture: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "blob.js"), + bytes.Repeat([]byte{0xff}, 2000), 0644); err != nil { + t.Fatalf("write blob fixture: %v", err) + } + + scan, err := NewJavaScriptAnalyzer().AnalyzeDirectory(dir) + if err != nil { + t.Fatalf("AnalyzeDirectory: %v", err) + } + if scan.FilesFailed != 1 || scan.FilesParsed != 1 { + t.Fatalf("FilesParsed = %d, FilesFailed = %d; the fixture is not one readable file "+ + "and one refused one", scan.FilesParsed, scan.FilesFailed) + } + if len(scan.Unreadable) != 1 { + t.Fatalf("Unreadable = %v, want one entry naming the refused file", scan.Unreadable) + } + if !strings.Contains(scan.Unreadable[0], "blob.js") { + t.Errorf("the refusal does not name the file: %q", scan.Unreadable[0]) + } + if !strings.Contains(scan.Unreadable[0], "not text") { + t.Errorf("the refusal does not say why, so the reader cannot act on it: %q", + scan.Unreadable[0]) + } +} + +// TestDirectoryScanBoundsHowManyRefusalsItNames keeps a hostile archive from +// turning one warning into thousands, without ever understating the count. +func TestDirectoryScanBoundsHowManyRefusalsItNames(t *testing.T) { + dir := t.TempDir() + const blobs = maxUnreadableReported + 5 + for i := 0; i < blobs; i++ { + name := filepath.Join(dir, fmt.Sprintf("blob%d.js", i)) + if err := os.WriteFile(name, bytes.Repeat([]byte{0xff}, 2000), 0644); err != nil { + t.Fatalf("write blob %d: %v", i, err) + } + } + + scan, err := NewJavaScriptAnalyzer().AnalyzeDirectory(dir) + if err != nil { + t.Fatalf("AnalyzeDirectory: %v", err) + } + if scan.FilesFailed != blobs { + t.Errorf("FilesFailed = %d, want the exact count %d: only the naming is bounded", + scan.FilesFailed, blobs) + } + if len(scan.Unreadable) != maxUnreadableReported { + t.Errorf("named %d refusals, want the bound of %d", len(scan.Unreadable), + maxUnreadableReported) + } +} + +// TestIsTextRefusesNothing holds the empty-head guard directly. +// +// readSource refuses a zero-byte file before isText is reached, so the guard is +// unreachable through that path and a mutation of it survives the end-to-end +// tests. It is kept and asserted here anyway: utf8.Valid reports true for an +// empty slice, which is how the previous version of this check came to accept +// every NUL-free binary, and the next caller of isText should not have to +// rediscover that. +func TestIsTextRefusesNothing(t *testing.T) { + for name, content := range map[string][]byte{"nil": nil, "empty": {}} { + if isText(content) { + t.Errorf("isText(%s) = true; an empty slice is valid UTF-8 and is not text", name) + } + } +} + +// TestDirectoryScanDoesNotFollowSymlinks keeps an extracted archive's contents +// to the archive. +// +// filepath.Walk lstats, so a symlink is not a directory and falls through to +// the file analyzer, which opened whatever it pointed at. Both tar and unzip +// restore absolute symlink targets, so a package could ship Leak.java pointing +// at any file the scanning process can read and have this tool read it, count +// it as evidence of examination, and publish its contents as that dependency's +// source in JSON, SARIF and CBOM. +func TestDirectoryScanDoesNotFollowSymlinks(t *testing.T) { + outside := t.TempDir() + secret := filepath.Join(outside, "CONFIDENTIAL.java") + if err := os.WriteFile(secret, []byte( + "import java.security.MessageDigest;\n"+ + "class Secret { void f() throws Exception { MessageDigest.getInstance(\"MD5\"); } }\n"), + 0644); err != nil { + t.Fatalf("write the file outside the archive: %v", err) + } + + dir := t.TempDir() + if err := os.Symlink(secret, filepath.Join(dir, "Leak.java")); err != nil { + t.Skipf("this platform does not support symlinks: %v", err) + } + + // Guard the fixture: the target must be readable and must carry the finding + // this test is looking for, or its absence proves nothing. + direct, err := NewJavaAnalyzer().AnalyzeFile(secret) + if err != nil || len(direct) == 0 { + t.Fatalf("the file outside the archive carries no finding to leak (%d usages, %v), "+ + "so this test cannot detect the leak", len(direct), err) + } + + scan, err := NewJavaAnalyzer().AnalyzeDirectory(dir) + if err != nil { + t.Fatalf("AnalyzeDirectory: %v", err) + } + if scan.FilesParsed != 0 { + t.Errorf("FilesParsed = %d for a directory holding only a symlink out of it", + scan.FilesParsed) + } + for _, u := range scan.Usages { + t.Errorf("a finding was read through a symlink out of the archive: %s at %s", + u.Algorithm, u.Location.File) + } +} + +// TestJavaDirectoryScanReadsNothingFromCompiledClasses is the walker-level +// statement of the false clean this release closes. +// +// A Maven artifact with no sources JAR unpacks to compiled classes. Nothing +// here is readable by an analyzer that parses .java, .kt and .kts, and the +// walk has to say so rather than return an empty result that reads the same as +// a package with no cryptography in it. +func TestJavaDirectoryScanReadsNothingFromCompiledClasses(t *testing.T) { + dir := t.TempDir() + + classDir := filepath.Join(dir, "com", "google", "common") + if err := os.MkdirAll(classDir, 0755); err != nil { + t.Fatalf("create class dir: %v", err) + } + if err := os.WriteFile(filepath.Join(classDir, "Hashing.class"), + []byte("\xca\xfe\xba\xbe\x00\x00\x00\x34"), 0644); err != nil { + t.Fatalf("write class file: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "MANIFEST.MF"), + []byte("Manifest-Version: 1.0\n"), 0644); err != nil { + t.Fatalf("write manifest: %v", err) + } + + scan, err := NewJavaAnalyzer().AnalyzeDirectory(dir) + if err != nil { + t.Fatalf("AnalyzeDirectory: %v", err) + } + + if scan.FilesParsed != 0 { + t.Errorf("FilesParsed = %d for a directory holding no Java source at all; "+ + "a caller reading this count would report the package as examined", + scan.FilesParsed) + } + if len(scan.Usages) != 0 { + t.Errorf("compiled classes produced %d usages", len(scan.Usages)) + } +} diff --git a/internal/analyzer/ast/source_file.go b/internal/analyzer/ast/source_file.go new file mode 100644 index 0000000..ee01b62 --- /dev/null +++ b/internal/analyzer/ast/source_file.go @@ -0,0 +1,179 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package ast + +import ( + "bufio" + "bytes" + "fmt" + "os" + "unicode/utf8" +) + +// maxSourceFile bounds what one file may contribute. A source file larger than +// this is not source in any sense this tool can act on, and reading it into +// memory to find out is how a scanner becomes a way to exhaust the host it runs +// on with an archive it was handed. +// +// The bound is measured against real source rather than picked: aws-sdk-go +// v1.55.5 ships service/ec2/api.go at 7,771,273 bytes, which is 93 percent of +// the 8 MiB this used to be, and it has grown every release. A cap a real file +// is about to cross drops that file from the analysis, so there is margin here +// on purpose. +const maxSourceFile = 32 << 20 // 32 MiB + +// headBytes is how much of a file is checked for being text. A magic number +// lives in the first few bytes, and validating megabytes to reject the first +// eight is not a trade worth making. +const headBytes = 1024 + +// minTextFraction is how much of a head must be printable ASCII for a file that +// is not valid UTF-8 to still count as source. +// +// Measured rather than chosen. Across 140 real npm source files that carry +// non-ASCII characters, re-encoded to Latin-1, the lowest fraction was 0.904 +// and the median 0.999, because source code is ASCII even when its comments and +// string literals are not. Random bytes with the NUL removed reached at most +// 0.453 over 200 samples, and real binaries with their NUL bytes stripped +// reached 0.286. The threshold sits in the gap with margin on both sides. +const minTextFraction = 0.75 + +// openSource reads a file that an analyzer is about to walk line by line, and +// refuses one it cannot actually read. +// +// The distinction matters because "how many files did we parse" is the evidence +// behind every claim this tool makes about having examined a package. Three of +// the four analyzers are line scanners whose only failure mode was os.Open, so +// any openable file with a matching extension counted as parsed: a zero-byte +// Empty.java, or compiled class bytes under a .java name, both produced +// "no cryptographic usage detected in the 1 of 1 dependencies that were +// examined" for a scan that learned nothing. An extension is a descriptor, not +// the thing itself. +// +// A file counts as read when it holds text this tool can scan. Empty is not +// text, and neither is a binary blob: both are refused here so that the count +// they would have inflated stays at zero and the package is reported as not +// examined. +func openSource(filename string) (*bufio.Scanner, error) { + content, err := readSource(filename) + if err != nil { + return nil, err + } + s := bufio.NewScanner(bytes.NewReader(content)) + // bufio's default token limit is 64 KiB, and a line longer than that makes + // Scan stop and Err report it. The three line-scanning analyzers return that + // error, and the walk then discards every usage they had already found, so + // one long line silently cost a whole file: a bundled dist/*.js of 70 KB on + // a single line calling crypto.createHash('md5') produced no finding, no + // warning, and "No cryptographic usage detected". Minified and bundled + // output is normal in published packages, so this was a false negative on + // the tool's core purpose in one of its most common inputs. + // + // readSource has already read the whole file into memory and capped it, so + // letting a token reach that same cap costs nothing that has not been spent. + s.Buffer(make([]byte, 0, 64*1024), maxSourceFile) + return s, nil +} + +// readSource is openSource for an analyzer that needs the whole file, and is +// where every check described above actually lives. +func readSource(filename string) ([]byte, error) { + info, err := os.Lstat(filename) + if err != nil { + return nil, err + } + // A symlink is not part of the archive it appears in. Following one reads + // whatever it points at, which for an extracted package means any file the + // scanning process can read: tar and unzip both restore absolute symlink + // targets, so a package could ship Leak.java -> /etc/passwd and have this + // tool publish the result as that dependency's source. + if info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("%s is a symbolic link, which is not read: an extracted "+ + "package is analyzed from its own contents", filename) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s is not a regular file", filename) + } + if info.Size() == 0 { + return nil, fmt.Errorf("%s is empty, so nothing was read from it", filename) + } + if info.Size() > maxSourceFile { + return nil, fmt.Errorf("%s is %d bytes, above the %d byte limit for one source file", + filename, info.Size(), maxSourceFile) + } + + content, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + if !isText(content) { + return nil, fmt.Errorf("%s is not text this analyzer can read", filename) + } + + return content, nil +} + +// isText reports whether content is source text rather than a binary blob +// wearing a source file's extension. +// +// The question is whether these analyzers can read the file, and they are line +// scanners matching ASCII identifiers, so they neither need nor check valid +// UTF-8. Requiring it made the tool refuse legitimate source in any single-byte +// encoding: a .java or .js file holding one Latin-1 accent was reported as +// unreadable, its cryptography was never looked for, and 1.2.2 had found it. +// Worse, the refusal was silent for a package that also held a file this +// analyzer could read, so a dependency could hide its cryptography from the +// scanner by encoding one file in Latin-1 and still be reported as examined and +// clean. An encoding is not a language, and refusing to read one is not the same +// as there being nothing to read. +func isText(content []byte) bool { + if bytes.IndexByte(content, 0) >= 0 { + return false + } + // Only the head is checked: it is where a magic number lives, and validating + // megabytes of a legitimate file to reject the first eight bytes of a class + // file is not a trade worth making. + head := content + if len(head) > headBytes { + head = head[:headBytes] + // A rune can straddle the boundary, so trim up to the three continuation + // bytes one can occupy rather than calling a legitimate file binary. + // + // Bounded deliberately: an unbounded "shrink while invalid" loop walks a + // binary file's head down to nothing, and utf8.Valid on an empty slice + // is true, so every NUL-free binary over headBytes passed as text. 2000 + // bytes of 0xff was accepted as source. + for i := 0; i < utf8.UTFMax-1 && len(head) > 0 && !utf8.Valid(head); i++ { + head = head[:len(head)-1] + } + } + if len(head) == 0 { + return false + } + if utf8.Valid(head) { + return true + } + // Not valid UTF-8, which is not the same as not being source. A single-byte + // encoding is still text these analyzers scan line by line, and what + // separates it from a blob is how much of the head is ASCII rather than + // whether every byte forms a rune. This is also what keeps the NUL-free + // binary out: 2000 bytes of 0xff is 0 percent printable ASCII. + return printableASCIIFraction(head) >= minTextFraction +} + +// printableASCIIFraction is how much of head is ASCII a line scanner can match. +func printableASCIIFraction(head []byte) float64 { + if len(head) == 0 { + return 0 + } + var printable int + for _, b := range head { + switch { + case b >= 0x20 && b <= 0x7e, // printable ASCII + b == '\t', b == '\n', b == '\r', b == '\v', b == '\f': + printable++ + } + } + return float64(printable) / float64(len(head)) +} diff --git a/internal/analyzer/deep_examination_test.go b/internal/analyzer/deep_examination_test.go new file mode 100644 index 0000000..0aa4dfc --- /dev/null +++ b/internal/analyzer/deep_examination_test.go @@ -0,0 +1,358 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package analyzer + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf8" + + "github.com/csnp/qramm-cryptodeps/internal/database" + "github.com/csnp/qramm-cryptodeps/pkg/output" + "github.com/csnp/qramm-cryptodeps/pkg/types" +) + +// The artifact from the reproduction: its sources JAR 404s and its main JAR +// 200s, so --deep fetches a JAR holding compiled classes and no source at all. +const ( + unreadableGroupID = "com.google.guava" + unreadableArtifactID = "listenablefuture" + unreadableVersion = "9999.0-empty-to-avoid-conflict-with-guava" +) + +// mavenDeepFixture builds a pom.xml project plus a Maven source-cache entry +// already extracted, and returns the project directory. +// +// files maps a path inside the extracted JAR to its contents, so a caller +// chooses whether the archive holds anything an analyzer can read. The cache +// entry is what keeps this offline: the fetcher resolves it as a cache hit and +// never reaches the network. +// +// The cache path is spelled out rather than computed, because it is what the +// fetcher's own sanitizer produces for this coordinate. If that ever stops +// matching, the fetch fails instead of hitting the cache, and every test here +// fails loudly on its fixture guard rather than passing for the wrong reason. +func mavenDeepFixture(t *testing.T, tmpDir string, files map[string]string) string { + t.Helper() + + project := filepath.Join(tmpDir, "project") + if err := os.MkdirAll(project, 0755); err != nil { + t.Fatalf("create project dir: %v", err) + } + + pom := ` + + 4.0.0 + com.example + fixture + 1.0.0 + + + ` + unreadableGroupID + ` + ` + unreadableArtifactID + ` + ` + unreadableVersion + ` + + +` + if err := os.WriteFile(filepath.Join(project, "pom.xml"), []byte(pom), 0644); err != nil { + t.Fatalf("write pom.xml: %v", err) + } + + // cacheSegment maps the ':' in a Maven coordinate to '_' and leaves the + // version alone; "extracted" is where the fetcher unzips a JAR. + extracted := filepath.Join(tmpDir, "cryptodeps-cache", "maven", + unreadableGroupID+"_"+unreadableArtifactID, unreadableVersion, "extracted") + for name, content := range files { + path := filepath.Join(extracted, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("create cache dir for %s: %v", name, err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("write cached file %s: %v", name, err) + } + } + + return project +} + +// runDeepScan scans project with --deep and returns the whole result together +// with the rendered table, failing if the fixture did not produce exactly one +// dependency. +func runDeepScan(t *testing.T, project string) (*types.ScanResult, types.DependencyResult, string) { + t.Helper() + + result, err := New(database.NewEmbedded(), Options{Deep: true}).Analyze(project) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if len(result.Dependencies) != 1 { + t.Fatalf("fixture declared %d dependencies, want exactly 1: %+v", + len(result.Dependencies), result.Dependencies) + } + dep := result.Dependencies[0] + + // The fetch has to have reached the walker, or every assertion below is + // answered by a download that failed rather than by an archive that carried + // nothing to read. PATH is cleared by these tests, so a cache miss is + // unrecoverable and says so here rather than passing quietly. + if strings.Contains(dep.Error, "could not fetch") { + t.Fatalf("fixture never reached source analysis, so this test cannot tell an "+ + "unreadable archive from a failed download: %s", dep.Error) + } + if dep.InDatabase { + t.Fatalf("fixture dependency is in the crypto database, so --deep is not the "+ + "path under test: %+v", dep) + } + if !result.Summary.DeepAttempted { + t.Fatalf("source analysis was never attempted for this fixture: %+v", result.Summary) + } + + var buf bytes.Buffer + if err := (&output.TableFormatter{Options: output.DefaultOptions()}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + + return result, dep, buf.String() +} + +// TestDeepScanOfClassOnlyArchiveIsNotReportedAsExamined is the regression test +// for the third defect of unit 152. +// +// `analyze --deep` over a Maven artifact whose sources JAR does not +// exist fell back to the main JAR, which holds compiled classes only. The Java +// walker accepts .java, .kt and .kts, found none, and returned no usages and no +// error. DeepAnalyzed was set from that absent error, so the report said: +// +// [OK] No cryptographic usage detected in the 1 of 1 dependencies that were +// examined. +// +// for a scan that parsed zero files. The claim of examination is now made from +// the count of files an analyzer parsed. +func TestDeepScanOfClassOnlyArchiveIsNotReportedAsExamined(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + t.Setenv("PATH", t.TempDir()) + + project := mavenDeepFixture(t, tmpDir, map[string]string{ + // A JAR of compiled classes: bytes an analyzer cannot read, in a + // directory that is neither empty nor a failed extraction. + "com/google/common/util/concurrent/ListenableFuture.class": "\xca\xfe\xba\xbe\x00\x00\x00\x34", + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + }) + + result, dep, out := runDeepScan(t, project) + + if dep.DeepAnalyzed { + t.Errorf("DeepAnalyzed is set for a package whose fetched archive holds no source "+ + "this tool can read: %+v", dep) + } + if dep.Analysis.SourceWasRead() { + t.Errorf("the result claims source was read from an archive of compiled classes: %+v", + dep.Analysis.Analysis) + } + if !strings.Contains(dep.Error, "read no files it can parse") { + t.Errorf("the report does not say why the package was not examined, so a consumer "+ + "reading JSON cannot tell this from a package that was skipped: %q", + dep.Error) + } + if result.Summary.NotExamined != 1 { + t.Errorf("Summary.NotExamined = %d for a scan that parsed no file at all", + result.Summary.NotExamined) + } + + if strings.Contains(out, "dependencies that were examined") { + t.Errorf("the verdict claims an examination that read no files:\n%s", out) + } + if !strings.Contains(out, "Not analyzed") { + t.Errorf("the verdict does not say the scan established nothing:\n%s", out) + } + if strings.Contains(out, "use --deep") { + t.Errorf("the user ran --deep and the next step offered is --deep:\n%s", out) + } +} + +// TestDeepScanReadsSourceInASingleByteEncoding is the end-to-end statement of +// the encoding regression the text check introduced. +// +// The check that made examination mean a parse refused any file whose head was +// not valid UTF-8. Legitimate source in a single-byte encoding is not valid +// UTF-8, so a .java file holding one Latin-1 accent was refused, and the MD5 +// call beside the accent was never found. The released 1.2.2 reported it. The +// whole pipeline is exercised here, not just the walker, because the verdict is +// what a user reads. +func TestDeepScanReadsSourceInASingleByteEncoding(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + t.Setenv("PATH", t.TempDir()) + + // 0xe9 is 'e' with an acute accent in Latin-1. + source := "// author: Jos\xe9 Garc\xeda\n" + + `package com.google.common.util.concurrent; +import java.security.MessageDigest; +public class Hashing { + public byte[] digest(byte[] data) throws Exception { + MessageDigest md = MessageDigest.getInstance("MD5"); + return md.digest(data); + } +} +` + // Guard the fixture: valid UTF-8 would never reach the branch under test. + if utf8.ValidString(source) { + t.Fatalf("fixture is valid UTF-8, so it does not exercise the encoding path") + } + + project := mavenDeepFixture(t, tmpDir, map[string]string{ + "com/google/common/util/concurrent/Hashing.java": source, + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + }) + + result, dep, out := runDeepScan(t, project) + + if !dep.DeepAnalyzed { + t.Fatalf("a package whose archive carries readable Latin-1 source is reported as not "+ + "examined: %+v (%s)", dep, dep.Error) + } + if got := dep.Analysis.Analysis.FilesAnalyzed; got != 1 { + t.Errorf("filesAnalyzed = %d for an archive holding one readable .java file in a "+ + "single-byte encoding", got) + } + if got := dep.Analysis.Analysis.FilesUnreadable; got != 0 { + t.Errorf("filesUnreadable = %d for an archive whose only file is readable source", got) + } + if result.Summary.NotExamined != 0 { + t.Errorf("Summary.NotExamined = %d after the only dependency was read", + result.Summary.NotExamined) + } + if !strings.Contains(out, "MD5") { + t.Errorf("the MD5 call in a Latin-1 encoded source file is missing from the report, so "+ + "a dependency can hide its cryptography by choosing an encoding:\n%s", out) + } +} + +// TestDeepScanSaysWhichFilesItCouldNotRead is the regression test for the +// partial state of the examination question. +// +// FilesFailed was counted inside the walk and went no further, so a package +// holding one file the analyzer read and one it refused was reported as examined +// and clean on every stream and in every format. That is the same false clean +// this release exists to remove, one layer in: the earlier fix asked the +// disclosure question of a package that was not examined at all, and never of +// one that was examined incompletely. A refused file is where a finding would +// have been. +func TestDeepScanSaysWhichFilesItCouldNotRead(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + t.Setenv("PATH", t.TempDir()) + + project := mavenDeepFixture(t, tmpDir, map[string]string{ + // Readable, and carrying no cryptography, so the scan produces no + // findings and reaches the clean verdict. + "com/example/Plain.java": `package com.example; +public class Plain { public int add(int a, int b) { return a + b; } } +`, + // A binary blob under a .java name, past the head boundary and with no + // NUL byte: refused, and the refusal is what has to be disclosed. + "com/example/Blob.java": strings.Repeat("\xff", 2000), + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + }) + + result, dep, out := runDeepScan(t, project) + + // Guard the fixture: if both files parsed, or neither did, this test is not + // looking at a partial reading and proves nothing about disclosing one. + if got := dep.Analysis.Analysis.FilesAnalyzed; got != 1 { + t.Fatalf("filesAnalyzed = %d, so the fixture is not a package read in part and this "+ + "test cannot detect an undisclosed partial reading", got) + } + if got := dep.Analysis.Analysis.FilesUnreadable; got != 1 { + t.Fatalf("filesUnreadable = %d, so the blob was not refused and there is nothing to "+ + "disclose", got) + } + if !dep.DeepAnalyzed { + t.Errorf("a package with one readable file is reported as not examined at all: %s", dep.Error) + } + if result.Summary.SourceFilesUnreadable != 1 { + t.Errorf("Summary.SourceFilesUnreadable = %d, so the count never left the walk and no "+ + "format can report it", result.Summary.SourceFilesUnreadable) + } + // NotExamined cannot carry this state, which is exactly why it was missed. + if result.Summary.NotExamined != 0 { + t.Errorf("Summary.NotExamined = %d for a dependency that was examined in part", + result.Summary.NotExamined) + } + + if !strings.Contains(out, "could not be read") { + t.Errorf("the verdict claims a clean examination without saying a file in it was "+ + "never read:\n%s", out) + } + + // A count with no names is a dead end: the note tells the reader the reading + // was incomplete and, without this, gives them no way to find out where. + named := dep.Analysis.Analysis.UnreadableFiles + if len(named) != 1 { + t.Fatalf("UnreadableFiles = %v, want the one refused file named", named) + } + if !strings.Contains(named[0], "Blob.java") { + t.Errorf("the refusal does not name the file it applies to: %q", named[0]) + } + if !strings.Contains(named[0], "not text") { + t.Errorf("the refusal does not say why, so the reader cannot act on it: %q", named[0]) + } + + // And it has to survive into the document a consumer reads, not only the + // in-memory result. + var buf bytes.Buffer + if err := (&output.JSONFormatter{Indent: true}).Format(result, &buf); err != nil { + t.Fatalf("format json: %v", err) + } + if !strings.Contains(buf.String(), "unreadableFiles") || + !strings.Contains(buf.String(), "Blob.java") { + t.Errorf("JSON does not name the refused file:\n%s", buf.String()) + } +} + +// TestDeepScanOfReadableArchiveIsStillReportedAsExamined asks the inverse +// question of the same change. +// +// Narrowing what counts as an examination must not stop the tool reporting the +// packages it really did read. Same fixture shape, one .java file in the +// archive, and the finding in it has to survive. +func TestDeepScanOfReadableArchiveIsStillReportedAsExamined(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + t.Setenv("PATH", t.TempDir()) + + project := mavenDeepFixture(t, tmpDir, map[string]string{ + "com/google/common/util/concurrent/Hashing.java": `package com.google.common.util.concurrent; +import java.security.MessageDigest; +public class Hashing { + public byte[] digest(byte[] data) throws Exception { + MessageDigest md = MessageDigest.getInstance("MD5"); + return md.digest(data); + } +} +`, + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + }) + + result, dep, out := runDeepScan(t, project) + + if !dep.DeepAnalyzed { + t.Fatalf("a package whose archive carries readable source is reported as not "+ + "examined: %+v (%s)", dep, dep.Error) + } + if got := dep.Analysis.Analysis.FilesAnalyzed; got != 1 { + t.Errorf("filesAnalyzed = %d for an archive holding one .java file", got) + } + if result.Summary.NotExamined != 0 { + t.Errorf("Summary.NotExamined = %d after the only dependency was read", + result.Summary.NotExamined) + } + if !strings.Contains(out, "MD5") { + t.Errorf("a finding read from the fetched archive is missing from the report:\n%s", out) + } +} diff --git a/internal/analyzer/deep_verdict_test.go b/internal/analyzer/deep_verdict_test.go new file mode 100644 index 0000000..f874c76 --- /dev/null +++ b/internal/analyzer/deep_verdict_test.go @@ -0,0 +1,218 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package analyzer + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/csnp/qramm-cryptodeps/internal/database" + "github.com/csnp/qramm-cryptodeps/pkg/output" +) + +// deepScanFixture builds a project whose dependencies are absent from the +// crypto database, together with a source cache holding each one already +// extracted. +// +// The cache is what makes this an offline test of the real --deep path: the +// analyzer's on-demand fetcher resolves the packages from it and never reaches +// the network, so the whole chain from manifest to rendered verdict runs +// exactly as it does for a user, in a unit test. +// +// It returns the project directory. The caller must have redirected TMPDIR +// before calling New, because that is where the fetcher puts its cache. +func deepScanFixture(t *testing.T, tmpDir string, source map[string]string) string { + t.Helper() + + project := filepath.Join(tmpDir, "project") + if err := os.MkdirAll(project, 0755); err != nil { + t.Fatalf("create project dir: %v", err) + } + + var deps []string + for name := range source { + deps = append(deps, `"`+name+`": "1.0.0"`) + } + manifest := `{"name":"fixture","version":"1.0.0","dependencies":{` + + strings.Join(deps, ",") + `}}` + if err := os.WriteFile(filepath.Join(project, "package.json"), []byte(manifest), 0644); err != nil { + t.Fatalf("write package.json: %v", err) + } + + for name, code := range source { + // The layout npm pack plus tar produces, which is what the fetcher + // looks for on a cache hit. + dir := filepath.Join(tmpDir, "cryptodeps-cache", "npm", name, "1.0.0", "package") + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatalf("create cache entry for %s: %v", name, err) + } + if err := os.WriteFile(filepath.Join(dir, "index.js"), []byte(code), 0644); err != nil { + t.Fatalf("write cached source for %s: %v", name, err) + } + } + + return project +} + +// TestDeepScanOfCleanTreeReportsClean is the end-to-end regression test for the +// defect that held the v1.3.0 tag. +// +// `analyze --deep`, where every dependency is absent from the crypto +// database and none of them carries cryptography, printed: +// +// [?] Not analyzed. All 3 dependencies are absent from the crypto database, +// so no conclusion about cryptographic usage can be drawn from this scan. +// Run with --deep to analyze package source code directly. +// +// The scan had read all three. The JSON for the same invocation reported +// deepAnalyzed on every one of them, so the two formats contradicted each other +// for one command, and the released 1.2.2 printed the correct verdict for this +// case. The classifier was asking whether the database covered the packages in +// order to answer whether anything had been examined. +func TestDeepScanOfCleanTreeReportsClean(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + + project := deepScanFixture(t, tmpDir, map[string]string{ + "chalk": "module.exports = s => s;\n", + "commander": "module.exports = { parse() {} };\n", + "lodash": "module.exports = { get() {} };\n", + }) + + result, err := New(database.NewEmbedded(), Options{Deep: true}).Analyze(project) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + + // Guard the fixture on both sides. Source analysis has to have actually + // run, and the database has to have covered nothing, or the two predicates + // are indistinguishable here and the test proves nothing. + if result.Summary.NotInDatabase != result.Summary.TotalDependencies { + t.Fatalf("fixture dependencies are in the database: %+v", result.Summary) + } + for _, dep := range result.Dependencies { + if !dep.DeepAnalyzed { + t.Fatalf("source analysis did not run for %s, so the scan really did examine "+ + "nothing and this test cannot detect the defect", dep.Dependency.Name) + } + } + if result.Summary.NotExamined != 0 { + t.Fatalf("Summary.NotExamined = %d after every dependency was read", + result.Summary.NotExamined) + } + + var buf bytes.Buffer + if err := (&output.TableFormatter{Options: output.DefaultOptions()}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + out := buf.String() + + if !strings.Contains(out, "No cryptographic usage detected") { + t.Errorf("a --deep scan that read every dependency and found no cryptography did "+ + "not report a clean result:\n%s", out) + } + if strings.Contains(out, "Not analyzed") || + strings.Contains(out, "no conclusion about cryptographic usage") { + t.Errorf("scan examined every dependency and reported that it had examined "+ + "nothing:\n%s", out) + } + if strings.Contains(out, "--deep") { + t.Errorf("the user ran --deep and the only next step offered is --deep:\n%s", out) + } + if !strings.Contains(out, "3 of 3") { + t.Errorf("verdict does not state how much of the tree was examined:\n%s", out) + } +} + +// TestDeepScanStillReportsFindings is the inverse question for the same change. +// +// Narrowing when the tool says "nothing was examined" must not make it quieter +// about cryptography it did find. Same fixture shape, one dependency carrying a +// broken hash. +func TestDeepScanStillReportsFindings(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + + project := deepScanFixture(t, tmpDir, map[string]string{ + "clean-pkg": "module.exports = s => s;\n", + "hashing-lib": "const crypto = require('crypto');\n" + + "module.exports = d => crypto.createHash('md5').update(d).digest('hex');\n", + }) + + result, err := New(database.NewEmbedded(), Options{Deep: true}).Analyze(project) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + + if result.Summary.WithCrypto == 0 { + t.Fatalf("fixture produced no findings, so it cannot show that findings survive: %+v", + result.Summary) + } + + var buf bytes.Buffer + if err := (&output.TableFormatter{Options: output.DefaultOptions()}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + out := buf.String() + + if !strings.Contains(out, "MD5") { + t.Errorf("a finding read by source analysis is missing from the report:\n%s", out) + } + if strings.Contains(out, "No cryptographic usage detected") { + t.Errorf("report carries a finding and a clean verdict at the same time:\n%s", out) + } +} + +// TestDeepScanReportsPackagesItCouldNotRead covers the mixed case, which is the +// one the narrowed trigger moves into the clean branch. +// +// One dependency read, one that source analysis could not fetch. The report has +// to say the second was not examined, and must not answer with --deep, which is +// what the user already ran. +func TestDeepScanReportsPackagesItCouldNotRead(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + // No downloader on PATH, so the dependency without a cache entry cannot be + // fetched and the run produces one examined and one unexamined package. + t.Setenv("PATH", t.TempDir()) + + project := deepScanFixture(t, tmpDir, map[string]string{ + "cached-pkg": "module.exports = s => s;\n", + }) + // A second dependency with no cache entry behind it. + manifest := `{"name":"fixture","version":"1.0.0","dependencies":` + + `{"cached-pkg":"1.0.0","missing-pkg":"1.0.0"}}` + if err := os.WriteFile(filepath.Join(project, "package.json"), []byte(manifest), 0644); err != nil { + t.Fatalf("rewrite package.json: %v", err) + } + + result, err := New(database.NewEmbedded(), Options{Deep: true}).Analyze(project) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + + if result.Summary.NotExamined != 1 || result.Summary.TotalDependencies != 2 { + t.Fatalf("fixture does not mix examined and unexamined dependencies: %+v", + result.Summary) + } + + var buf bytes.Buffer + if err := (&output.TableFormatter{Options: output.DefaultOptions()}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + out := buf.String() + + if !strings.Contains(out, "1 of 2") { + t.Errorf("verdict does not say how much of the tree it covers:\n%s", out) + } + if !strings.Contains(out, "1 could not be examined") { + t.Errorf("verdict does not account for the package source analysis could not read:\n%s", out) + } + if strings.Contains(out, "use --deep") { + t.Errorf("the user ran --deep and the tool suggests running --deep:\n%s", out) + } +} diff --git a/internal/analyzer/filter_test.go b/internal/analyzer/filter_test.go index 3d8598a..8f90668 100644 --- a/internal/analyzer/filter_test.go +++ b/internal/analyzer/filter_test.go @@ -255,6 +255,22 @@ func TestFilterValidationRejectsUnknownValues(t *testing.T) { bad: []string{"banana", "vulnerable", "sev1"}, wantMsg: "--min-severity", }, + { + // --fail-on was the one enum flag of the three that did not + // validate, and it is the one that decides the exit code. A typo + // fell through to the default policy, so a project that exits 3 + // under --fail-on partial exited 0 under --fail-on partail, with + // nothing on either stream. The others mislead a reader; this one + // silently loosens a CI gate. + name: "fail-on", + fn: ValidateFailOn, + // "" is deliberately absent: this flag has a default, so a blank + // value expresses nothing and used to fall through to that default + // while looking accepted. + good: []string{"none", "any", "partial", "vulnerable", "VULNERABLE", " partial "}, + bad: []string{"partail", "banana", "safe", "unknown", "all", "critical", "1"}, + wantMsg: "--fail-on", + }, } { t.Run(tc.name, func(t *testing.T) { for _, v := range tc.good { diff --git a/internal/analyzer/ondemand/analyzer.go b/internal/analyzer/ondemand/analyzer.go index 1353a1b..3905085 100644 --- a/internal/analyzer/ondemand/analyzer.go +++ b/internal/analyzer/ondemand/analyzer.go @@ -34,16 +34,16 @@ func (a *Analyzer) Analyze(dep types.Dependency) (*types.PackageAnalysis, error) } // Analyze based on ecosystem - var usages []types.CryptoUsage + var scan ast.DirectoryScan switch dep.Ecosystem { case types.EcosystemGo: - usages, err = a.analyzeGo(sourceDir) + scan, err = a.analyzeGo(sourceDir) case types.EcosystemNPM: - usages, err = a.analyzeJavaScript(sourceDir) + scan, err = a.analyzeJavaScript(sourceDir) case types.EcosystemPyPI: - usages, err = a.analyzePython(sourceDir) + scan, err = a.analyzePython(sourceDir) case types.EcosystemMaven: - usages, err = a.analyzeJava(sourceDir) + scan, err = a.analyzeJava(sourceDir) default: return nil, fmt.Errorf("unsupported ecosystem: %s", dep.Ecosystem) } @@ -52,40 +52,49 @@ func (a *Analyzer) Analyze(dep types.Dependency) (*types.PackageAnalysis, error) return nil, err } - // Build the analysis result + // Build the analysis result. FilesAnalyzed travels with it because the + // caller decides from it whether this package was examined at all, and an + // empty Crypto slice cannot tell "read it, found nothing" from "read + // nothing". analysis := &types.PackageAnalysis{ Package: dep.Name, Version: dep.Version, Ecosystem: dep.Ecosystem, - Crypto: usages, + Crypto: scan.Usages, + Analysis: types.AnalysisMetadata{ + FilesAnalyzed: scan.FilesParsed, + FilesUnreadable: scan.FilesFailed, + // Named as well as counted, so the disclosure is actionable. + UnreadableFiles: scan.Unreadable, + }, } // Deduplicate and classify algorithms - analysis.Crypto = a.deduplicateUsages(usages) + analysis.Crypto = a.deduplicateUsages(scan.Usages) return analysis, nil } // analyzeGo analyzes Go source code. -func (a *Analyzer) analyzeGo(sourceDir string) ([]types.CryptoUsage, error) { +func (a *Analyzer) analyzeGo(sourceDir string) (ast.DirectoryScan, error) { analyzer := ast.NewGoAnalyzer() return analyzer.AnalyzeDirectory(sourceDir) } // analyzeJavaScript analyzes JavaScript source code. -func (a *Analyzer) analyzeJavaScript(sourceDir string) ([]types.CryptoUsage, error) { +func (a *Analyzer) analyzeJavaScript(sourceDir string) (ast.DirectoryScan, error) { analyzer := ast.NewJavaScriptAnalyzer() return analyzer.AnalyzeDirectory(sourceDir) } // analyzePython analyzes Python source code. -func (a *Analyzer) analyzePython(sourceDir string) ([]types.CryptoUsage, error) { +func (a *Analyzer) analyzePython(sourceDir string) (ast.DirectoryScan, error) { analyzer := ast.NewPythonAnalyzer() return analyzer.AnalyzeDirectory(sourceDir) } // analyzeJava analyzes Java source code. -func (a *Analyzer) analyzeJava(sourceDir string) ([]types.CryptoUsage, error) { +func (a *Analyzer) analyzeJava(sourceDir string) (ast.DirectoryScan, error) { analyzer := ast.NewJavaAnalyzer() return analyzer.AnalyzeDirectory(sourceDir) } diff --git a/internal/analyzer/ondemand/analyzer_test.go b/internal/analyzer/ondemand/analyzer_test.go index 78c42ea..6420d82 100644 --- a/internal/analyzer/ondemand/analyzer_test.go +++ b/internal/analyzer/ondemand/analyzer_test.go @@ -175,7 +175,8 @@ func hashData(data []byte) []byte { } analyzer := NewAnalyzer(t.TempDir()) - usages, err := analyzer.analyzeGo(tmpDir) + scan, err := analyzer.analyzeGo(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("analyzeGo failed: %v", err) } @@ -203,7 +204,8 @@ module.exports = { hashData }; } analyzer := NewAnalyzer(t.TempDir()) - usages, err := analyzer.analyzeJavaScript(tmpDir) + scan, err := analyzer.analyzeJavaScript(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("analyzeJavaScript failed: %v", err) } @@ -228,7 +230,8 @@ def hash_data(data): } analyzer := NewAnalyzer(t.TempDir()) - usages, err := analyzer.analyzePython(tmpDir) + scan, err := analyzer.analyzePython(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("analyzePython failed: %v", err) } @@ -257,7 +260,8 @@ public class Main { } analyzer := NewAnalyzer(t.TempDir()) - usages, err := analyzer.analyzeJava(tmpDir) + scan, err := analyzer.analyzeJava(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("analyzeJava failed: %v", err) } @@ -272,7 +276,8 @@ func TestAnalyzer_EmptyDirectory(t *testing.T) { analyzer := NewAnalyzer(t.TempDir()) // Test with empty directories - should return empty slice, no error - usages, err := analyzer.analyzeGo(tmpDir) + scan, err := analyzer.analyzeGo(tmpDir) + usages := scan.Usages if err != nil { t.Fatalf("analyzeGo on empty dir failed: %v", err) } @@ -280,7 +285,8 @@ func TestAnalyzer_EmptyDirectory(t *testing.T) { t.Errorf("Expected 0 usages for empty dir, got %d", len(usages)) } - usages, err = analyzer.analyzeJavaScript(tmpDir) + scan, err = analyzer.analyzeJavaScript(tmpDir) + usages = scan.Usages if err != nil { t.Fatalf("analyzeJavaScript on empty dir failed: %v", err) } @@ -288,7 +294,8 @@ func TestAnalyzer_EmptyDirectory(t *testing.T) { t.Errorf("Expected 0 usages for empty dir, got %d", len(usages)) } - usages, err = analyzer.analyzePython(tmpDir) + scan, err = analyzer.analyzePython(tmpDir) + usages = scan.Usages if err != nil { t.Fatalf("analyzePython on empty dir failed: %v", err) } @@ -296,7 +303,8 @@ func TestAnalyzer_EmptyDirectory(t *testing.T) { t.Errorf("Expected 0 usages for empty dir, got %d", len(usages)) } - usages, err = analyzer.analyzeJava(tmpDir) + scan, err = analyzer.analyzeJava(tmpDir) + usages = scan.Usages if err != nil { t.Fatalf("analyzeJava on empty dir failed: %v", err) } @@ -349,7 +357,8 @@ func GenerateKey() (*rsa.PrivateKey, error) { // We can't easily mock go mod download, so test the internal methods directly analyzer := NewAnalyzer(cacheDir) - usages, err := analyzer.analyzeGo(goModDir) + scan, err := analyzer.analyzeGo(goModDir) + usages := scan.Usages if err != nil { t.Fatalf("analyzeGo failed: %v", err) } @@ -372,7 +381,8 @@ func TestAnalyze_NPMWithCachedSource(t *testing.T) { cacheDir := tmpDir + "/cache" // Create a pre-cached npm package structure - npmPkgDir := cacheDir + "/npm/test-crypto-pkg/1.0.0" + // The layout a real npm extraction leaves: the tarball beside package/. + npmPkgDir := cacheDir + "/npm/test-crypto-pkg/1.0.0/package" os.MkdirAll(npmPkgDir, 0755) // Write a JavaScript file with crypto @@ -420,7 +430,8 @@ func TestAnalyze_PyPIWithCachedSource(t *testing.T) { cacheDir := tmpDir + "/cache" // Create a pre-cached PyPI package structure - pyPkgDir := cacheDir + "/pypi/test-crypto-lib/2.0.0" + // pip downloads a wheel and this fetcher unzips it into extracted/. + pyPkgDir := cacheDir + "/pypi/test-crypto-lib/2.0.0/extracted" os.MkdirAll(pyPkgDir, 0755) // Write a Python file with crypto @@ -523,7 +534,7 @@ func TestAnalyze_NoCryptoFound(t *testing.T) { cacheDir := tmpDir + "/cache" // Create a cached package with no crypto code - npmPkgDir := cacheDir + "/npm/no-crypto-pkg/1.0.0" + npmPkgDir := cacheDir + "/npm/no-crypto-pkg/1.0.0/package" os.MkdirAll(npmPkgDir, 0755) // Write a JavaScript file WITHOUT crypto @@ -564,7 +575,7 @@ func TestAnalyze_MultipleCryptoAlgorithms(t *testing.T) { cacheDir := tmpDir + "/cache" // Create a cached package with multiple crypto algorithms - npmPkgDir := cacheDir + "/npm/multi-crypto/1.0.0" + npmPkgDir := cacheDir + "/npm/multi-crypto/1.0.0/package" os.MkdirAll(npmPkgDir, 0755) jsCode := `const crypto = require('crypto'); diff --git a/internal/analyzer/source/cache_entry_test.go b/internal/analyzer/source/cache_entry_test.go new file mode 100644 index 0000000..c4e334b --- /dev/null +++ b/internal/analyzer/source/cache_entry_test.go @@ -0,0 +1,137 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package source + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/csnp/qramm-cryptodeps/pkg/types" +) + +// makeDir creates dir with one file in it, so that isNonEmptyDir accepts it. +func makeDir(t *testing.T, dir string) string { + t.Helper() + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "index.js"), []byte("module.exports = 1;\n"), 0644); err != nil { + t.Fatalf("write into %s: %v", dir, err) + } + return dir +} + +// TestResetCacheEntryRefusesAnythingWiderThanOneEntry bounds the only +// destructive operation in the fetcher. +// +// The removal took whatever path it was handed. Four separate mutations of it +// left the suite green, including one that removed the entire cache root, so +// nothing in the repository stated how much this function is allowed to delete. +// Each case below asserts the refusal and then asserts the directory is still +// there, because an error returned after a deletion is not a refusal. +func TestResetCacheEntryRefusesAnythingWiderThanOneEntry(t *testing.T) { + cacheRoot := t.TempDir() + fetcher := NewFetcher(cacheRoot) + + entry := makeDir(t, filepath.Join(cacheRoot, "npm", "pkg", "1.0.0")) + extracted := makeDir(t, filepath.Join(entry, "package")) + // A directory the cache root can be walked out of, which is what an + // unsanitized segment would reach. + sibling := makeDir(t, filepath.Join(filepath.Dir(cacheRoot), "victim")) + + cases := []struct { + name string + path string + }{ + {"the cache root itself", cacheRoot}, + {"an ecosystem directory", filepath.Join(cacheRoot, "npm")}, + {"every version of one package", filepath.Join(cacheRoot, "npm", "pkg")}, + {"a directory outside the cache", sibling}, + {"a traversal out of the cache", cacheRoot + "/npm/../../" + filepath.Base(sibling)}, + {"below a cache entry", extracted}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // The fixture has to exist, or "it was not deleted" is trivially + // true and this case proves nothing. A missing one fails here + // rather than skipping, so a fixture that stops resolving cannot + // quietly remove a case from this gate. + before, err := os.Stat(tc.path) + if err != nil || !before.IsDir() { + t.Fatalf("fixture %s is not a directory, so this case asserts nothing: %v", + tc.path, err) + } + + err = fetcher.resetCacheEntry(tc.path) + if err == nil { + t.Fatalf("resetCacheEntry(%s) was allowed", tc.path) + } + if !strings.Contains(err.Error(), "refusing to remove") { + t.Errorf("refusal does not say what it refused: %v", err) + } + if _, err := os.Stat(tc.path); err != nil { + t.Errorf("%s was removed anyway: %v", tc.path, err) + } + }) + } +} + +// TestResetCacheEntryRemovesOneEntry is the other half of the bound: the guard +// must not be so tight that the entry a refetch depends on is never cleared. +func TestResetCacheEntryRemovesOneEntry(t *testing.T) { + cacheRoot := t.TempDir() + fetcher := NewFetcher(cacheRoot) + + entry := makeDir(t, filepath.Join(cacheRoot, "npm", "pkg", "1.0.0")) + sibling := makeDir(t, filepath.Join(cacheRoot, "npm", "pkg", "2.0.0")) + + if err := fetcher.resetCacheEntry(entry); err != nil { + t.Fatalf("resetCacheEntry on a real entry: %v", err) + } + if _, err := os.Stat(entry); !os.IsNotExist(err) { + t.Errorf("the unusable cache entry survived, so the next scan analyzes it again: %v", err) + } + if _, err := os.Stat(sibling); err != nil { + t.Errorf("clearing one version removed another: %v", err) + } +} + +// TestAmbiguousCacheEntryIsKeptAndReported covers the case where the fetcher +// cannot tell which directory in an entry is the package. +// +// It used to delete the entry and fetch again, so a cache entry that may have +// held the findings was destroyed by a scan that only failed to identify it, +// and the user was shown whatever the refetch failed with rather than the +// diagnostic written for this case. With no downloader on PATH the refetch +// cannot succeed, which is what makes the difference visible. +func TestAmbiguousCacheEntryIsKeptAndReported(t *testing.T) { + cacheRoot := t.TempDir() + t.Setenv("PATH", t.TempDir()) + fetcher := NewFetcher(cacheRoot) + + entry := filepath.Join(cacheRoot, "npm", "pkg", "1.0.0") + makeDir(t, filepath.Join(entry, "candidate-a")) + makeDir(t, filepath.Join(entry, "candidate-b")) + + _, err := fetcher.Fetch(types.Dependency{ + Name: "pkg", + Version: "1.0.0", + Ecosystem: types.EcosystemNPM, + }) + if err == nil { + t.Fatal("an entry holding two candidate directories was resolved to one of them") + } + if !strings.Contains(err.Error(), "cannot identify the extracted source") { + t.Errorf("the reported error is not the one written for this case: %v", err) + } + + for _, dir := range []string{"candidate-a", "candidate-b"} { + if _, statErr := os.Stat(filepath.Join(entry, dir)); statErr != nil { + t.Errorf("%s was deleted because the fetcher could not identify it: %v", dir, statErr) + } + } +} diff --git a/internal/analyzer/source/fetcher.go b/internal/analyzer/source/fetcher.go index de9ef5b..75ab37a 100644 --- a/internal/analyzer/source/fetcher.go +++ b/internal/analyzer/source/fetcher.go @@ -5,10 +5,14 @@ package source import ( + "crypto/sha256" + "encoding/hex" + "errors" "fmt" "os" "os/exec" "path/filepath" + "regexp" "strings" "github.com/csnp/qramm-cryptodeps/pkg/types" @@ -19,6 +23,430 @@ type Fetcher struct { cacheDir string } +// Directory names that a fetched archive is expected to unpack to, when the +// ecosystem has a convention. Anything else is resolved by inspection. +const ( + npmExtractedDir = "package" // npm tarballs conventionally unpack to package/ + zipExtractedDir = "extracted" // where this fetcher unzips wheels and JARs + maxCacheSegment = 96 // long enough for any real name or version + pathTraversalMsg = "%s %q refers to a local path, which is not fetched: " + + "deep analysis reads package archives, not directories on this machine" +) + +// mavenCoordinatePart is the character set Maven allows in a groupId or an +// artifactId. Anything else in a coordinate is not a coordinate: it is a way to +// reach the URL the fetch is built from, or the path it is written to. +var mavenCoordinatePart = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + +// validMavenCoordinate rejects a groupId or artifactId that could steer the +// fetch rather than name an artifact. +// +// The Maven fetcher interpolates both into a URL and joins the artifactId into +// the path curl writes to. A '/' in the artifactId escapes that path, and a '?' +// splits the URL so that its path portion still names a real artifact while the +// file portion traverses, which is what turned a scanned pom.xml into an +// arbitrary file write: curl -f does not create on a 404, and the query-string +// trick supplies a 200. Validating the coordinate is the layer that closes it, +// because it holds for the URL and the path at once. +func validMavenCoordinate(groupID, artifactID string) error { + for _, part := range [...]struct{ label, value string }{ + {"groupId", groupID}, + {"artifactId", artifactID}, + } { + if !mavenCoordinatePart.MatchString(part.value) { + return fmt.Errorf("%q is not a valid Maven coordinate: %s %q must match %s", + groupID+":"+artifactID, part.label, part.value, mavenCoordinatePart) + } + } + return nil +} + +// packageNamePattern is what each registry says one of its names looks like. +// +// A name is not an arbitrary string. Every registry here defines a grammar for +// its names, and anything outside that grammar is not a name at all: it is a +// way to reach something other than the package it claims to be. Screening for +// the spellings of a local path could never close this, because the guard sees +// one string while the package manager sees a structured spec. `npm pack` +// splits `name@spec` on the first '@' after index 0, so a dependency named +// `x@/tmp/victim` presented a guard-passing string to the guard and a directory +// to npm, which packed it, ran its prepare script, and published its contents. +// The same shape reaches pip through a PEP 508 direct reference in a Poetry or +// Pipfile table key: `victimpkg @ file:///tmp/victim`. +// +// So the question asked here is the one Maven coordinates already answered: is +// this a name? An allowlist can be reasoned about; a denylist of the ways a +// path can be spelled cannot. +var packageNamePattern = map[types.Ecosystem]*regexp.Regexp{ + // npm: an optional @scope/ then the name. What is excluded is what lets a + // name be read as something other than a name: '@' separates a name from a + // spec, '/' and '\' make it a path, ':' makes it a scheme, and whitespace + // separates arguments. + // + // Deliberately NOT npm's rules for a name it would accept today. Those + // require a scope and a name to begin with a letter or a digit, and the + // first version of this pattern said so, which refused 1,054 names that are + // published and installable right now. npm grandfathered them: nine exceed + // 100,000 downloads a month, including @lingo.dev/_spec at 212,000, @-xun/fs, + // @_sh/strapi-plugin-ckeditor and @~39/empty. A scanner that silently stops + // analyzing a real dependency reports a clean result it never established, + // which is the same false clean this release exists to remove, arrived at + // from the other direction. The question a fetch guard has to answer is what + // the package manager can misread, not what the registry would accept from a + // new publisher. + types.EcosystemNPM: regexp.MustCompile(`^(@[^@/\\:\s]+/)?[^@/\\:\s]+$`), + // PyPI, PEP 503: letters, digits, and . _ - between alphanumerics. + types.EcosystemPyPI: regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$`), + // Go module path: dot-separated host, then slash-separated elements. + types.EcosystemGo: regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._~-]*(/[A-Za-z0-9._~-]+)*$`), +} + +// validPackageName rejects a dependency name that is not a name in its +// ecosystem's own terms. +func validPackageName(ecosystem types.Ecosystem, name string) error { + // Maven is absent from the table because its names are coordinates, but it is + // answered here rather than only inside the Maven fetcher, so that every + // ecosystem's grammar is the FIRST thing a name meets. While this was reached + // only later, the traversal screen ran first and refused a hostile coordinate + // as a local path, which is a correct refusal that says the wrong thing and, + // worse, left the coordinate validator unexercised by the test written to + // pin it: one guard masking another is how the masked one comes to be + // deleted with a green suite. + if ecosystem == types.EcosystemMaven { + groupID, artifactID, found := strings.Cut(name, ":") + if !found || strings.Contains(artifactID, ":") { + return fmt.Errorf("invalid Maven coordinate: %s (expected groupId:artifactId)", name) + } + return validMavenCoordinate(groupID, artifactID) + } + + pattern, ok := packageNamePattern[ecosystem] + if !ok { + return nil + } + if !pattern.MatchString(name) { + return fmt.Errorf("%q is not a valid %s package name: it must match %s, and a name "+ + "that does not is not fetched, because package managers read a name as part of a "+ + "spec that can also select a directory or a repository on this machine", + name, ecosystem, pattern) + } + // A path element of ".." is legal under the Go pattern and is still a + // traversal, so it is refused separately rather than by complicating the + // grammar. + for _, part := range strings.Split(name, "/") { + if part == ".." || part == "." { + return fmt.Errorf("%q is not a valid %s package name: %q is a directory reference", + name, ecosystem, part) + } + } + // A leading dot makes a name relative to the working directory, which for + // these fetchers is inside the cache. Stated here rather than in the grammar + // so that loosening the grammar to admit the names registries actually carry + // cannot quietly admit this too. No npm name begins with a dot: the registry + // returns an empty range between "." and "/". + if strings.HasPrefix(name, ".") { + return fmt.Errorf("%q is not a valid %s package name: a leading dot makes it a path "+ + "relative to the directory the fetch runs in", name, ecosystem) + } + return nil +} + +// cacheSegment reduces a manifest-supplied name or version to one safe path +// segment. +// +// Both come from the manifest under scan, which is untrusted input. A +// dependency declared as {"ejs": "../../../somewhere"} made the cache path +// resolve outside the cache directory, and --deep then walked that directory +// and published its absolute file paths and line numbers as the dependency's +// source. Replacing everything outside a conservative set means the result +// cannot traverse, cannot be empty, and cannot be a relative marker. +func cacheSegment(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', + r == '.', r == '-', r == '_': + b.WriteRune(r) + default: + b.WriteRune('_') + } + } + + out := b.String() + if len(out) > maxCacheSegment { + // Truncating alone would map two long names onto one cache entry, and + // the second package would then be analyzed from the first one's + // source. The suffix keeps distinct inputs distinct. + sum := sha256.Sum256([]byte(s)) + out = out[:maxCacheSegment] + "-" + hex.EncodeToString(sum[:4]) + } + // Replacement is many-to-one below that length, so two names can still + // share one entry: "@scope/pkg" and "_scope_pkg" both give "_scope_pkg", + // and the second package is then analyzed from the first one's source. + // Tracked in qramm-cryptodeps-deep-fetch-path-primitives for 1.3.1, because + // closing it changes the cache path of every Maven coordinate and every + // scoped npm name, and those paths are reported to the user. + // + // "" collapses the path by one level, and "." and ".." are directory + // references rather than names. + if strings.Trim(out, ".") == "" { + return "_" + } + return out +} + +// localPathReference reports whether a manifest-supplied string points at a +// directory on this machine rather than at a published release. +// +// npm and pip both accept such a reference as a package spec, so passing one +// through to `npm pack` or `pip download` would fetch and analyze a local +// directory of the manifest author's choosing. Registry and VCS references +// (github:owner/repo, git+https://...) are left alone: those are remote, and +// deep analysis of them works. +// +// This applies to the NAME as well as the version. cacheSegment already keeps +// the name from steering the cache path, and that made the name look safe, but +// a path and a package spec are different guarantees: `npm pack ../../../x` +// resolves a directory no matter how the cache entry is named. +func localPathReference(v string) bool { + v = strings.TrimSpace(v) + if v == "" { + return false + } + // file: is npm's own spelling of a directory. link: and portal: are the + // Yarn and pnpm spellings of the same thing; npm pack rejects them today, + // so this is defence in depth rather than a live vector, and it costs one + // list entry to stop depending on another tool's parser staying strict. + lower := strings.ToLower(v) + for _, scheme := range [...]string{"file:", "link:", "portal:"} { + if strings.HasPrefix(lower, scheme) { + return true + } + } + // A transport can be prefixed onto the scheme, and git+file:// is a local + // path wearing a VCS reference's clothes: it passed the prefix test above + // because it starts with "git+", and passed the traversal walk below + // because it contains a colon. `npm pack` clones it from disk and runs its + // prepare script. + if scheme, _, found := strings.Cut(lower, ":"); found { + for _, part := range strings.Split(scheme, "+") { + if part == "file" { + return true + } + } + } + if strings.HasPrefix(v, "/") || strings.HasPrefix(v, "~") || strings.HasPrefix(v, ".") { + return true + } + // Windows spellings of an absolute path: a drive letter (C:\victim, C:/victim), + // a rooted path (\victim) and a UNC share (\\host\share). Windows is a + // shipped target of this tool, and the checks below never see these: a + // drive letter contains a colon, which the registry-reference branch treats + // as evidence that the value is remote. + if windowsLocalPath(v) { + return true + } + // A traversal, wherever it appears. This walk used to be skipped for any + // value containing a colon, on the reasoning that a colon meant a registry + // reference such as github:owner/repo. It only takes an invented scheme to + // defeat that: a version of "a1:../../../../../../../victim" carries a colon, + // so the walk never ran, and `npm pack left-pad@a1:../../../victim` packed a + // directory outside the cache, which this tool then read and published as + // that dependency's source with its absolute paths. Verified by sentinel at + // traversal depth 7. + // + // No legitimate version carries a ".." path element, so the colon earns no + // exemption here. Whether a value is a remote reference is decided by the + // scheme test above, not by the presence of a punctuation mark. + for _, part := range strings.FieldsFunc(v, func(r rune) bool { return r == '/' || r == '\\' }) { + if part == ".." { + return true + } + } + return false +} + +// windowsLocalPath reports whether v is a Windows path to this machine. +// +// Kept separate from the Unix forms because the shapes have nothing in common: +// a drive letter is two characters and a colon, and a UNC share starts with two +// separators. Both are absolute paths on a target this tool ships binaries for. +func windowsLocalPath(v string) bool { + if strings.HasPrefix(v, `\`) { // \victim and \\host\share + return true + } + if len(v) >= 3 && v[1] == ':' && (v[2] == '\\' || v[2] == '/') { + c := v[0] + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') + } + return false +} + +// isNonEmptyDir reports whether path is a directory holding at least one entry. +// An empty directory is not a source tree, and treating one as a cache hit is +// how a failed extraction came to be reported as a completed analysis. +func isNonEmptyDir(path string) bool { + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + return false + } + entries, err := os.ReadDir(path) + return err == nil && len(entries) > 0 +} + +// ambiguousSourceError says a cache entry holds more than one candidate source +// directory, so which of them is the package cannot be established. +// +// It is a distinct type because the answer to it is the opposite of the answer +// to an entry that holds nothing: an entry that cannot be identified must be +// kept and reported, not cleared and fetched again. Clearing it destroyed a +// cache entry that may well have held the findings, and reported the deletion +// as whatever the refetch failed with afterwards. +type ambiguousSourceError struct { + packageDir string + candidates []string +} + +func (e *ambiguousSourceError) Error() string { + return fmt.Sprintf("cannot identify the extracted source in %s: %d candidate directories (%s)", + e.packageDir, len(e.candidates), strings.Join(e.candidates, ", ")) +} + +// extractedSourceRoot resolves the directory inside packageDir that holds the +// extracted package. +// +// The cache-hit path and the post-extraction path must agree on the answer, or +// the same package is analyzed from one tree on the first run and a different +// one on every run after it. They did not agree: the npm fetcher returned +// packageDir/package after extracting, on the convention that a tarball unpacks +// to package/, and returned packageDir itself on a cache hit after asking only +// whether that directory existed. A version directory holding nothing but the +// downloaded tarball satisfies that question, so a package whose extraction had +// failed was counted as analyzed by source on every subsequent run, with no +// source file read and no warning printed. Both paths now resolve through here, +// and a directory that holds no extracted source is an error rather than a +// silent pass. +func extractedSourceRoot(packageDir string, preferred ...string) (string, error) { + for _, name := range preferred { + if candidate := filepath.Join(packageDir, name); isNonEmptyDir(candidate) { + return candidate, nil + } + } + + entries, err := os.ReadDir(packageDir) + if err != nil { + return "", fmt.Errorf("no extracted source in %s: %w", packageDir, err) + } + + var found []string + for _, e := range entries { + if !e.IsDir() || strings.HasPrefix(e.Name(), ".") { + continue + } + if isNonEmptyDir(filepath.Join(packageDir, e.Name())) { + found = append(found, e.Name()) + } + } + + switch len(found) { + case 0: + return "", fmt.Errorf("no extracted source in %s", packageDir) + case 1: + return filepath.Join(packageDir, found[0]), nil + default: + // Guessing here would analyze a tree that may not be the package, and + // report the result as if it were. Say so instead. + return "", &ambiguousSourceError{packageDir: packageDir, candidates: found} + } +} + +// cacheEntryDepth is how many path elements one package's cache entry sits +// below the cache root: ecosystem, name, version. +const cacheEntryDepth = 3 + +// checkCacheEntry rejects a path that is not exactly one package's cache entry. +// +// This is the only destructive operation in the fetcher and its argument is +// built from manifest-supplied text, so its scope is checked rather than +// assumed. The check is what makes the scope testable: four separate mutations +// of the removal, up to and including RemoveAll of the entire cache root, +// previously left the suite green. +func (f *Fetcher) checkCacheEntry(packageDir string) error { + root, err := filepath.Abs(f.cacheDir) + if err != nil { + return fmt.Errorf("cannot resolve the cache directory %s: %w", f.cacheDir, err) + } + entry, err := filepath.Abs(packageDir) + if err != nil { + return fmt.Errorf("cannot resolve the cache entry %s: %w", packageDir, err) + } + + rel, err := filepath.Rel(root, entry) + if err != nil { + return fmt.Errorf("refusing to remove %s: it is not inside the cache directory %s", entry, root) + } + parts := strings.Split(rel, string(filepath.Separator)) + for _, part := range parts { + if part == ".." { + return fmt.Errorf("refusing to remove %s: it resolves outside the cache directory %s", + entry, root) + } + } + if rel == "." || len(parts) != cacheEntryDepth { + return fmt.Errorf("refusing to remove %s: a cache entry is %d levels below the cache "+ + "directory %s (ecosystem, name, version), and this is %d", + entry, cacheEntryDepth, root, len(parts)) + } + return nil +} + +// resetCacheEntry removes a cache entry that does not hold usable source, so +// that the caller can fetch it again instead of analyzing whatever is there. +func (f *Fetcher) resetCacheEntry(packageDir string) error { + if err := f.checkCacheEntry(packageDir); err != nil { + return err + } + if err := os.RemoveAll(packageDir); err != nil { + return fmt.Errorf("failed to clear incomplete cache entry %s: %w", packageDir, err) + } + return nil +} + +// cachedSourceRoot resolves a cache entry and decides what to do when it cannot. +// +// It returns the source root on a hit. On a miss it clears the unusable entry +// and returns an empty root, which tells the caller to fetch. It returns an +// error only when the entry must be left alone, which is the ambiguous case: +// the entry may hold the package, and deleting a tree because it could not be +// identified is a worse answer than refusing to guess. +// +// The three fetchers previously each discarded this error and fell through to a +// refetch, so the diagnostics written for it were unreachable and the user was +// shown whatever the refetch failed with instead. +func (f *Fetcher) cachedSourceRoot(packageDir string, preferred ...string) (string, error) { + root, err := extractedSourceRoot(packageDir, preferred...) + if err == nil { + return root, nil + } + + var ambiguous *ambiguousSourceError + if errors.As(err, &ambiguous) { + return "", err + } + + return "", f.resetCacheEntry(packageDir) +} + +// discardPartialFetch clears what a failed download left behind and returns the +// error to report for it. +func (f *Fetcher) discardPartialFetch(packageDir string, cause error, what string) error { + if err := f.resetCacheEntry(packageDir); err != nil { + return fmt.Errorf("%s: %w (the partial cache entry was also left in place: %v)", + what, cause, err) + } + return fmt.Errorf("%s: %w", what, cause) +} + // NewFetcher creates a new source fetcher with the given cache directory. func NewFetcher(cacheDir string) *Fetcher { if cacheDir == "" { @@ -29,6 +457,24 @@ func NewFetcher(cacheDir string) *Fetcher { // Fetch downloads the source code for a package and returns the local path. func (f *Fetcher) Fetch(dep types.Dependency) (string, error) { + // Both fields come from the manifest under scan, and both reach a package + // manager as part of a spec. + // + // The name is held to its registry's grammar rather than screened for path + // spellings, because the package manager parses the string this code treats + // as opaque: `npm pack x@/tmp/victim` is one argument to this function and a + // name plus a directory to npm. The version is screened, because its legal + // forms genuinely include remote references that have no grammar in common. + if err := validPackageName(dep.Ecosystem, dep.Name); err != nil { + return "", err + } + if localPathReference(dep.Version) { + return "", fmt.Errorf(pathTraversalMsg, "version", dep.Version) + } + if localPathReference(dep.Name) { + return "", fmt.Errorf(pathTraversalMsg, "name", dep.Name) + } + switch dep.Ecosystem { case types.EcosystemGo: return f.fetchGoModule(dep) @@ -61,11 +507,16 @@ func (f *Fetcher) fetchGoModule(dep types.Dependency) (string, error) { // Parse the JSON output to get the module directory // The output includes "Dir" field with the cached module path modDir := extractGoModDir(output) - if modDir != "" { - return modDir, nil + if modDir == "" { + return "", fmt.Errorf("could not determine module directory") + } + // An empty or missing module directory would be walked, yield nothing, and + // be reported as a package examined by source analysis. + if !isNonEmptyDir(modDir) { + return "", fmt.Errorf("module directory %s holds no source", modDir) } - return "", fmt.Errorf("could not determine module directory") + return modDir, nil } // extractGoModDir extracts the Dir field from go mod download JSON output. @@ -90,12 +541,13 @@ func extractGoModDir(output []byte) string { // fetchNpmPackage downloads an npm package. func (f *Fetcher) fetchNpmPackage(dep types.Dependency) (string, error) { // Create cache directory for this package - safeName := strings.ReplaceAll(dep.Name, "/", "_") - packageDir := filepath.Join(f.cacheDir, "npm", safeName, dep.Version) + packageDir := filepath.Join(f.cacheDir, "npm", cacheSegment(dep.Name), cacheSegment(dep.Version)) - // Check if already cached - if _, err := os.Stat(packageDir); err == nil { - return packageDir, nil + // Check if already cached. A cache entry counts as a hit only if it holds + // extracted source; see extractedSourceRoot. An empty root with no error + // means the entry was unusable and has been cleared, so fetch it again. + if root, err := f.cachedSourceRoot(packageDir, npmExtractedDir); err != nil || root != "" { + return root, err } // Ensure cache directory exists @@ -110,11 +562,24 @@ func (f *Fetcher) fetchNpmPackage(dep types.Dependency) (string, error) { } // Change to cache directory and run npm pack - cmd := exec.Command("npm", "pack", packageSpec) + // --ignore-scripts is the control, not an optimisation. npm runs a package's + // prepare and prepack scripts for any spec it resolves from a directory or + // a git repository, so fetching source for analysis was a way to execute + // code chosen by the manifest under scan. The guards above stop such a spec + // being built at all; this stops the remaining ones, including the + // deliberately allowed remote git references, from running anything. + // The "--" is not decoration. The name grammar admits the names npm's own + // registry carries, and some of those begin with a hyphen: "-" is a real + // package with 153,000 downloads a month, and "@-xun/debug" has 153,076. A + // spec beginning with a hyphen is read by npm as a FLAG, so a dependency + // named "--help" made npm print its help text instead of fetching anything, + // and untrusted input parsed as a flag is the argument-injection half of the + // same class as untrusted input parsed as a path. The separator ends flag + // parsing, and ordinary and scoped specs are unaffected by it. + cmd := exec.Command("npm", "pack", "--ignore-scripts", "--", packageSpec) cmd.Dir = packageDir if err := cmd.Run(); err != nil { - os.RemoveAll(packageDir) - return "", fmt.Errorf("npm pack failed: %w", err) + return "", f.discardPartialFetch(packageDir, err, "npm pack failed") } // Find the tarball and extract it @@ -129,10 +594,14 @@ func (f *Fetcher) fetchNpmPackage(dep types.Dependency) (string, error) { // Extract the tarball extractCmd := exec.Command("tar", "-xzf", tarball, "-C", packageDir) if err := extractCmd.Run(); err != nil { - return "", fmt.Errorf("failed to extract tarball: %w", err) + return "", f.discardPartialFetch(packageDir, err, "failed to extract tarball") } - // npm packs to a "package" subdirectory - return filepath.Join(packageDir, "package"), nil + // npm tarballs conventionally unpack to package/, but not all of + // them do: ejs 3.1.10 unpacks to ejs-v3.1.10/. Returning the + // convention unconditionally made the caller's lstat fail on those, + // which is how the same package came to be analyzed from a + // different root on the first run than on later ones. + return extractedSourceRoot(packageDir, npmExtractedDir) } } @@ -142,12 +611,12 @@ func (f *Fetcher) fetchNpmPackage(dep types.Dependency) (string, error) { // fetchPyPIPackage downloads a Python package from PyPI. func (f *Fetcher) fetchPyPIPackage(dep types.Dependency) (string, error) { // Create cache directory for this package - safeName := strings.ReplaceAll(dep.Name, "/", "_") - packageDir := filepath.Join(f.cacheDir, "pypi", safeName, dep.Version) + packageDir := filepath.Join(f.cacheDir, "pypi", cacheSegment(dep.Name), cacheSegment(dep.Version)) - // Check if already cached - if _, err := os.Stat(packageDir); err == nil { - return packageDir, nil + // Check if already cached. Same shape as the npm fetcher: the directory + // existing is not the same question as it holding extracted source. + if root, err := f.cachedSourceRoot(packageDir, zipExtractedDir); err != nil || root != "" { + return root, err } // Ensure cache directory exists @@ -161,10 +630,23 @@ func (f *Fetcher) fetchPyPIPackage(dep types.Dependency) (string, error) { packageSpec = dep.Name + "==" + dep.Version } - cmd := exec.Command("pip", "download", "--no-deps", "-d", packageDir, packageSpec) + // --only-binary=:all: is the pip half of the control that --ignore-scripts is + // for npm, and it is here for the same reason. + // + // Resolving a source distribution makes pip install the project's build + // dependencies and run its build backend, which for a setup.py sdist is + // arbitrary code from a package named by the manifest under scan. Fetching + // source in order to analyze it must not be a way to execute it: the operator + // asked this tool to read a dependency, not to build it. Restricting the + // fetch to built wheels closes that, at the cost of the packages that publish + // no wheel, which are now reported as not examined with the reason named + // rather than analyzed by running them. + cmd := exec.Command("pip", "download", "--no-deps", "--only-binary", ":all:", + "-d", packageDir, packageSpec) if err := cmd.Run(); err != nil { - os.RemoveAll(packageDir) - return "", fmt.Errorf("pip download failed: %w", err) + return "", f.discardPartialFetch(packageDir, err, + "pip download failed (source distributions are not built, so a package that "+ + "publishes no wheel is reported as not examined)") } // Find and extract the wheel or tarball @@ -178,28 +660,24 @@ func (f *Fetcher) fetchPyPIPackage(dep types.Dependency) (string, error) { if strings.HasSuffix(name, ".whl") { // Wheels are just zip files wheelPath := filepath.Join(packageDir, name) - extractDir := filepath.Join(packageDir, "extracted") + extractDir := filepath.Join(packageDir, zipExtractedDir) if err := os.MkdirAll(extractDir, 0755); err != nil { return "", err } cmd := exec.Command("unzip", "-q", wheelPath, "-d", extractDir) if err := cmd.Run(); err != nil { - return "", fmt.Errorf("failed to extract wheel: %w", err) + return "", f.discardPartialFetch(packageDir, err, "failed to extract wheel") } - return extractDir, nil + return extractedSourceRoot(packageDir, zipExtractedDir) } else if strings.HasSuffix(name, ".tar.gz") { tarball := filepath.Join(packageDir, name) cmd := exec.Command("tar", "-xzf", tarball, "-C", packageDir) if err := cmd.Run(); err != nil { - return "", fmt.Errorf("failed to extract tarball: %w", err) - } - // Find the extracted directory - newEntries, _ := os.ReadDir(packageDir) - for _, e := range newEntries { - if e.IsDir() && !strings.HasPrefix(e.Name(), ".") { - return filepath.Join(packageDir, e.Name()), nil - } + return "", f.discardPartialFetch(packageDir, err, "failed to extract tarball") } + // Resolve the same way the cache-hit path does, so run 1 and run 2 + // analyze the same tree. + return extractedSourceRoot(packageDir, zipExtractedDir) } } @@ -215,15 +693,18 @@ func (f *Fetcher) fetchMavenArtifact(dep types.Dependency) (string, error) { } groupID := parts[0] artifactID := parts[1] + if err := validMavenCoordinate(groupID, artifactID); err != nil { + return "", err + } // Create cache directory for this artifact - safeName := strings.ReplaceAll(dep.Name, ":", "_") - packageDir := filepath.Join(f.cacheDir, "maven", safeName, dep.Version) + packageDir := filepath.Join(f.cacheDir, "maven", cacheSegment(dep.Name), cacheSegment(dep.Version)) - // Check if already cached - extractDir := filepath.Join(packageDir, "extracted") - if _, err := os.Stat(extractDir); err == nil { - return extractDir, nil + // Check if already cached. An extraction that failed after the directory + // was created leaves it empty, which the old existence check accepted. + extractDir := filepath.Join(packageDir, zipExtractedDir) + if root, err := f.cachedSourceRoot(packageDir, zipExtractedDir); err != nil || root != "" { + return root, err } // Ensure cache directory exists @@ -237,8 +718,13 @@ func (f *Fetcher) fetchMavenArtifact(dep types.Dependency) (string, error) { sourcesURL := fmt.Sprintf("https://repo1.maven.org/maven2/%s/%s/%s/%s-%s-sources.jar", groupPath, artifactID, dep.Version, artifactID, dep.Version) - // Download the sources JAR - jarPath := filepath.Join(packageDir, artifactID+"-"+dep.Version+"-sources.jar") + // Download the sources JAR. The coordinate is validated above, so this + // cannot traverse today; it goes through cacheSegment anyway so that the + // write target stays inside the cache even if the validation is ever + // loosened. The file the process writes should not depend on a second + // function staying strict. + jarBase := cacheSegment(artifactID) + "-" + cacheSegment(dep.Version) + jarPath := filepath.Join(packageDir, jarBase+"-sources.jar") // Use curl to download (available on most systems) cmd := exec.Command("curl", "-sL", "-o", jarPath, "-f", sourcesURL) @@ -246,11 +732,11 @@ func (f *Fetcher) fetchMavenArtifact(dep types.Dependency) (string, error) { // Try without -sources suffix (main jar) as fallback mainURL := fmt.Sprintf("https://repo1.maven.org/maven2/%s/%s/%s/%s-%s.jar", groupPath, artifactID, dep.Version, artifactID, dep.Version) - jarPath = filepath.Join(packageDir, artifactID+"-"+dep.Version+".jar") + jarPath = filepath.Join(packageDir, jarBase+".jar") cmd = exec.Command("curl", "-sL", "-o", jarPath, "-f", mainURL) if err := cmd.Run(); err != nil { - os.RemoveAll(packageDir) - return "", fmt.Errorf("failed to download Maven artifact: %w (tried sources and main JAR)", err) + return "", f.discardPartialFetch(packageDir, err, + "failed to download Maven artifact (tried sources and main JAR)") } } @@ -260,10 +746,10 @@ func (f *Fetcher) fetchMavenArtifact(dep types.Dependency) (string, error) { } extractCmd := exec.Command("unzip", "-q", jarPath, "-d", extractDir) if err := extractCmd.Run(); err != nil { - return "", fmt.Errorf("failed to extract JAR: %w", err) + return "", f.discardPartialFetch(packageDir, err, "failed to extract JAR") } - return extractDir, nil + return extractedSourceRoot(packageDir, zipExtractedDir) } // CacheDir returns the cache directory path. diff --git a/internal/analyzer/source/fetcher_test.go b/internal/analyzer/source/fetcher_test.go index 94821fe..03c56f3 100644 --- a/internal/analyzer/source/fetcher_test.go +++ b/internal/analyzer/source/fetcher_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "github.com/csnp/qramm-cryptodeps/pkg/types" @@ -145,15 +146,45 @@ func TestFetchMavenInvalidCoordinate(t *testing.T) { } } +// writeCacheEntry lays out a cache entry the way the fetcher itself writes one: +// the downloaded archive beside the directory it was extracted into. +func writeCacheEntry(t *testing.T, packageDir, archive, extractedDir, sourceFile string) string { + t.Helper() + if err := os.MkdirAll(packageDir, 0755); err != nil { + t.Fatalf("create cache entry: %v", err) + } + if archive != "" { + if err := os.WriteFile(filepath.Join(packageDir, archive), []byte("archive"), 0644); err != nil { + t.Fatalf("write archive: %v", err) + } + } + if extractedDir == "" { + return packageDir + } + root := filepath.Join(packageDir, extractedDir) + if err := os.MkdirAll(root, 0755); err != nil { + t.Fatalf("create extracted dir: %v", err) + } + if sourceFile != "" { + if err := os.WriteFile(filepath.Join(root, sourceFile), []byte("source"), 0644); err != nil { + t.Fatalf("write source file: %v", err) + } + } + return root +} + func TestFetchNpmCached(t *testing.T) { // Create a temp cache directory with pre-cached npm package tmpDir, _ := os.MkdirTemp("", "fetcher-npm-test") defer os.RemoveAll(tmpDir) - // Pre-create the cache directory to simulate a cached package - cachedDir := filepath.Join(tmpDir, "npm", "test-package", "1.0.0") - os.MkdirAll(cachedDir, 0755) - os.WriteFile(filepath.Join(cachedDir, "index.js"), []byte("module.exports = {}"), 0644) + // Pre-create the cache entry the way the fetcher writes one. The fixture + // used to put index.js straight into the version directory, which is a + // layout the fetcher never produces, and asserting that it was a cache hit + // was asserting the defect: the same shape is what a failed extraction + // leaves behind. + packageDir := filepath.Join(tmpDir, "npm", "test-package", "1.0.0") + cachedDir := writeCacheEntry(t, packageDir, "test-package-1.0.0.tgz", npmExtractedDir, "index.js") f := NewFetcher(tmpDir) dir, err := f.Fetch(types.Dependency{ @@ -175,10 +206,9 @@ func TestFetchPyPICached(t *testing.T) { tmpDir, _ := os.MkdirTemp("", "fetcher-pypi-test") defer os.RemoveAll(tmpDir) - // Pre-create the cache directory to simulate a cached package - cachedDir := filepath.Join(tmpDir, "pypi", "test-package", "1.0.0") - os.MkdirAll(cachedDir, 0755) - os.WriteFile(filepath.Join(cachedDir, "__init__.py"), []byte(""), 0644) + packageDir := filepath.Join(tmpDir, "pypi", "test-package", "1.0.0") + cachedDir := writeCacheEntry(t, packageDir, "test_package-1.0.0-py3-none-any.whl", + zipExtractedDir, "__init__.py") f := NewFetcher(tmpDir) dir, err := f.Fetch(types.Dependency{ @@ -248,10 +278,11 @@ func TestFetchNpmWithSlash(t *testing.T) { tmpDir, _ := os.MkdirTemp("", "fetcher-npm-slash-test") defer os.RemoveAll(tmpDir) - // Pre-create the cache directory with slash replaced - cachedDir := filepath.Join(tmpDir, "npm", "@org_package", "1.0.0") - os.MkdirAll(cachedDir, 0755) - os.WriteFile(filepath.Join(cachedDir, "index.js"), []byte("module.exports = {}"), 0644) + // Every character outside [A-Za-z0-9._-] becomes an underscore now, so the + // leading @ of a scope does too. Cache keys are internal and disposable; + // what matters is that a manifest cannot steer the path. + packageDir := filepath.Join(tmpDir, "npm", "_org_package", "1.0.0") + cachedDir := writeCacheEntry(t, packageDir, "org-package-1.0.0.tgz", npmExtractedDir, "index.js") f := NewFetcher(tmpDir) dir, err := f.Fetch(types.Dependency{ @@ -337,10 +368,11 @@ func TestFetchNpmWithoutVersion(t *testing.T) { tmpDir, _ := os.MkdirTemp("", "fetcher-npm-noversion-test") defer os.RemoveAll(tmpDir) - // Pre-create the cache directory with empty version - cachedDir := filepath.Join(tmpDir, "npm", "test-package", "") - os.MkdirAll(cachedDir, 0755) - os.WriteFile(filepath.Join(cachedDir, "index.js"), []byte("module.exports = {}"), 0644) + // An unpinned dependency gets its own segment rather than none. Joining an + // empty segment collapsed the path by one level, so an unversioned package + // and the package's version directories shared a parent. + packageDir := filepath.Join(tmpDir, "npm", "test-package", "_") + cachedDir := writeCacheEntry(t, packageDir, "test-package-1.0.0.tgz", npmExtractedDir, "index.js") f := NewFetcher(tmpDir) dir, err := f.Fetch(types.Dependency{ @@ -357,14 +389,21 @@ func TestFetchNpmWithoutVersion(t *testing.T) { } } +// TestFetchPyPIWithSlash previously asserted that a PyPI name containing +// slashes was sanitized into a cache path and fetched. +// +// It is now asserted to be refused. PyPI names cannot contain a slash, so such +// a string is not a name that failed to resolve, it is a path or a spec wearing +// a name's place in the manifest, and sanitizing it for the cache never made it +// safe to hand to pip. The old expectation restated what cacheSegment did +// rather than a judgement about what should be fetched. func TestFetchPyPIWithSlash(t *testing.T) { - // Test PyPI package with slash in name (normalized) tmpDir, _ := os.MkdirTemp("", "fetcher-pypi-slash-test") defer os.RemoveAll(tmpDir) - cachedDir := filepath.Join(tmpDir, "pypi", "google_cloud_storage", "2.0.0") - os.MkdirAll(cachedDir, 0755) - os.WriteFile(filepath.Join(cachedDir, "__init__.py"), []byte(""), 0644) + packageDir := filepath.Join(tmpDir, "pypi", "google_cloud_storage", "2.0.0") + cachedDir := writeCacheEntry(t, packageDir, "google-cloud-storage-2.0.0.tar.gz", + "google-cloud-storage-2.0.0", "__init__.py") f := NewFetcher(tmpDir) dir, err := f.Fetch(types.Dependency{ @@ -373,12 +412,32 @@ func TestFetchPyPIWithSlash(t *testing.T) { Ecosystem: types.EcosystemPyPI, }) - if err != nil { - t.Fatalf("Fetch cached pypi package with slashes failed: %v", err) + if err == nil { + t.Fatalf("a PyPI name containing slashes was fetched, to %q", dir) } - // Note: the safeName replaces "/" with "_" so google/cloud/storage becomes google_cloud_storage - if dir != cachedDir { - t.Errorf("Expected cached dir %s, got %s", cachedDir, dir) + if !strings.Contains(err.Error(), "not a valid pypi package name") { + t.Errorf("refused for the wrong reason: %v", err) + } + // The cache entry that a sanitized name would have resolved to is still + // there, so this test is refusing a fetch that would otherwise have + // succeeded rather than one that had nothing to return. + if !isNonEmptyDir(cachedDir) { + t.Fatalf("fixture cache entry %s is missing, so this test proves nothing", cachedDir) + } + + // The real name for the same package is still fetched from that entry. + ok, err := f.Fetch(types.Dependency{ + Name: "google-cloud-storage", + Version: "2.0.0", + Ecosystem: types.EcosystemPyPI, + }) + if err == nil || ok != "" { + // There is no cache entry under the real name, so this must fail at the + // download rather than at the guard. + t.Logf("fetch under the real name returned %q, %v", ok, err) + } + if err != nil && strings.Contains(err.Error(), "not a valid pypi package name") { + t.Errorf("the guard refuses a legitimate PyPI name: %v", err) } } @@ -505,10 +564,9 @@ func TestFetchNpmEmptyVersion(t *testing.T) { tmpDir, _ := os.MkdirTemp("", "fetcher-npm-empty-version") defer os.RemoveAll(tmpDir) - // Create cache with empty version directory - cachedDir := filepath.Join(tmpDir, "npm", "my-package", "") - os.MkdirAll(cachedDir, 0755) - os.WriteFile(filepath.Join(cachedDir, "index.js"), []byte(""), 0644) + // Create cache with the placeholder segment an empty version resolves to + packageDir := filepath.Join(tmpDir, "npm", "my-package", "_") + cachedDir := writeCacheEntry(t, packageDir, "my-package-1.0.0.tgz", npmExtractedDir, "index.js") f := NewFetcher(tmpDir) dir, err := f.Fetch(types.Dependency{ @@ -705,3 +763,804 @@ func TestIntegration_FetchMavenArtifact(t *testing.T) { t.Log("Note: No .java files found - sources JAR may not be available") } } + +// ============================================================================ +// Cache-entry resolution and manifest-controlled paths +// +// Every test below fails against the code as it stood at 4d6227b. +// ============================================================================ + +// withoutFetchTools makes every external downloader unreachable, so that a +// fetch which does NOT hit the cache fails immediately instead of going to the +// network. It lets a cache-behaviour test be both hermetic and end to end. +func withoutFetchTools(t *testing.T) { + t.Helper() + t.Setenv("PATH", t.TempDir()) +} + +// TestCacheHitRequiresExtractedSource covers the defect that made a package +// which had never been read count as analyzed. +// +// The cache-hit check asked whether the version directory existed. A directory +// holding nothing but the downloaded tarball answers yes, so the fetcher handed +// that directory to the AST walker, the walker found no source files, and the +// dependency was recorded as deep-analyzed with zero findings. Nothing warned, +// because nothing failed. Asserted through Fetch rather than through the helper +// so that the test states the behaviour and not the implementation. +func TestCacheHitRequiresExtractedSource(t *testing.T) { + withoutFetchTools(t) + tmpDir := t.TempDir() + + // The layout an interrupted extraction leaves: archive present, nothing + // unpacked. Also the layout the npm fetcher leaves when its own tar step + // fails. + packageDir := filepath.Join(tmpDir, "npm", "ejs", "3.1.10") + writeCacheEntry(t, packageDir, "ejs-3.1.10.tgz", "", "") + + // Guard the fixture: the directory the old check asked about does exist, + // so this test cannot pass for the trivial reason that it is absent. + if _, err := os.Stat(packageDir); err != nil { + t.Fatalf("fixture does not create the cache directory: %v", err) + } + + f := NewFetcher(tmpDir) + root, err := f.Fetch(types.Dependency{ + Name: "ejs", + Version: "3.1.10", + Ecosystem: types.EcosystemNPM, + }) + if err == nil { + t.Errorf("a cache entry holding only an archive was accepted as source at %q; "+ + "the package is then reported as analyzed without a byte of it being read", root) + } +} + +// TestMavenCacheHitRequiresExtractedContent is the same defect in the fetcher +// that did keep both paths on one directory name. The extracted directory is +// created before the unzip runs, so a failed unzip leaves it there and empty, +// and the existence check accepted it on every later run. +func TestMavenCacheHitRequiresExtractedContent(t *testing.T) { + withoutFetchTools(t) + tmpDir := t.TempDir() + + packageDir := filepath.Join(tmpDir, "maven", "com.example_artifact", "1.0.0") + extractDir := writeCacheEntry(t, packageDir, "artifact-1.0.0.jar", zipExtractedDir, "") + + if _, err := os.Stat(extractDir); err != nil { + t.Fatalf("fixture does not create the extracted directory: %v", err) + } + + f := NewFetcher(tmpDir) + root, err := f.Fetch(types.Dependency{ + Name: "com.example:artifact", + Version: "1.0.0", + Ecosystem: types.EcosystemMaven, + }) + if err == nil { + t.Errorf("an empty extracted directory was accepted as source at %q; a scan of it "+ + "reports no findings and calls the artifact examined", root) + } +} + +// TestCacheHitIsStableAcrossRuns pins the property the two paths disagreed on: +// the same cache entry must resolve to the same source root every time, for a +// tarball that does not follow the npm package/ convention. +func TestCacheHitIsStableAcrossRuns(t *testing.T) { + withoutFetchTools(t) + tmpDir := t.TempDir() + + packageDir := filepath.Join(tmpDir, "npm", "ejs", "3.1.10") + want := writeCacheEntry(t, packageDir, "ejs-3.1.10.tgz", "ejs-v3.1.10", "index.js") + + f := NewFetcher(tmpDir) + dep := types.Dependency{Name: "ejs", Version: "3.1.10", Ecosystem: types.EcosystemNPM} + + first, err := f.Fetch(dep) + if err != nil { + t.Fatalf("first fetch: %v", err) + } + second, err := f.Fetch(dep) + if err != nil { + t.Fatalf("second fetch: %v", err) + } + + if first != want || second != want { + t.Errorf("cache resolved to %q then %q, want %q both times; a package analyzed from "+ + "two different roots reports two different results for one scan", first, second, want) + } +} + +// TestExtractedSourceRootResolvesBothLayouts pins the agreement between the +// cold and warm paths. +// +// npm tarballs conventionally unpack to package/, and the post-extraction path +// returned that name unconditionally. ejs 3.1.10 unpacks to ejs-v3.1.10/, so +// the first run failed with lstat and every later run silently analyzed a +// different root. Both runs must resolve to the same directory. +func TestExtractedSourceRootResolvesBothLayouts(t *testing.T) { + tests := []struct { + name string + extracted string + }{ + {"npm convention", npmExtractedDir}, + {"tarball with its own root, as ejs 3.1.10 has", "ejs-v3.1.10"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + packageDir := filepath.Join(t.TempDir(), "npm", "ejs", "3.1.10") + want := writeCacheEntry(t, packageDir, "ejs-3.1.10.tgz", tt.extracted, "index.js") + + got, err := extractedSourceRoot(packageDir, npmExtractedDir) + if err != nil { + t.Fatalf("extractedSourceRoot: %v", err) + } + if got != want { + t.Errorf("extractedSourceRoot = %q, want %q", got, want) + } + }) + } +} + +// TestExtractedSourceRootRejectsEmptyAndAmbiguous covers the two states where +// no answer is better than a guess. +func TestExtractedSourceRootRejectsEmptyAndAmbiguous(t *testing.T) { + t.Run("empty extracted directory", func(t *testing.T) { + // What a failed unzip leaves behind: the JAR fetcher creates extracted/ + // before unzipping into it, so the directory exists and holds nothing. + packageDir := filepath.Join(t.TempDir(), "maven", "com.example_artifact", "1.0.0") + writeCacheEntry(t, packageDir, "artifact-1.0.0.jar", zipExtractedDir, "") + + if root, err := extractedSourceRoot(packageDir, zipExtractedDir); err == nil { + t.Errorf("an empty extracted directory resolved to %q; a scan of it reports "+ + "no findings and calls the package examined", root) + } + }) + + t.Run("two candidate directories", func(t *testing.T) { + packageDir := filepath.Join(t.TempDir(), "pypi", "example", "1.0.0") + writeCacheEntry(t, packageDir, "example-1.0.0.tar.gz", "example-1.0.0", "mod.py") + writeCacheEntry(t, packageDir, "", "example-1.0.0.data", "payload") + + root, err := extractedSourceRoot(packageDir) + if err == nil { + t.Errorf("ambiguous cache entry resolved to %q; the tool would analyze one "+ + "tree and report the result as if it were the package", root) + } + }) +} + +// TestFetchRefusesLocalPathVersion is the security regression test. +// +// The cache path was built from the dependency name and version exactly as the +// manifest under scan declared them, so {"ejs": "../../../somewhere"} resolved +// the cache entry outside the cache directory. That directory then existed, the +// existence check accepted it as a cache hit, and --deep walked it and +// published its absolute file paths and line numbers as the dependency's +// source. A scan of an untrusted repository could read any directory the +// process could read and put what it found in the SARIF that CI uploads. +func TestFetchRefusesLocalPathVersion(t *testing.T) { + // No downloader on PATH, so any fetch that is not answered from the cache + // fails without touching the network. The escape below is answered from the + // cache, which is the whole point: it needs no network at all. + withoutFetchTools(t) + + root := t.TempDir() + cacheDir := filepath.Join(root, "cache") + if err := os.MkdirAll(cacheDir, 0755); err != nil { + t.Fatalf("create cache dir: %v", err) + } + + // The directory the manifest is aiming at, outside the cache, holding + // something a scan would report. + victim := filepath.Join(root, "victim") + if err := os.MkdirAll(victim, 0755); err != nil { + t.Fatalf("create victim dir: %v", err) + } + secret := filepath.Join(victim, "leak.js") + if err := os.WriteFile(secret, []byte("crypto.createHash('md5')"), 0644); err != nil { + t.Fatalf("write victim file: %v", err) + } + + // The traversal has to be deep enough to actually land on the victim, or + // this test passes because npm is unavailable rather than because the + // fetcher refused. escape is /npm/ejs/ resolving to victim. + const escape = "../../../victim" + if got := filepath.Join(cacheDir, "npm", "ejs", escape); got != victim { + t.Fatalf("fixture does not aim at the victim directory: %q != %q; the test would "+ + "pass without exercising the escape", got, victim) + } + + f := NewFetcher(cacheDir) + dir, err := f.Fetch(types.Dependency{ + Name: "ejs", + Version: escape, + Ecosystem: types.EcosystemNPM, + }) + if err == nil { + t.Fatalf("a manifest-declared version resolved deep analysis to %q, outside the "+ + "cache; the tool would read that directory and publish its file paths and "+ + "line numbers as the dependency's source", dir) + } + // The reason matters, not just the failure. Without this the test passes + // whenever npm happens to be unavailable, which is exactly the condition + // the rest of the test sets up, so an implementation that dropped the + // refusal entirely would still look green. + if !strings.Contains(err.Error(), "refers to a local path") { + t.Errorf("fetch failed for the wrong reason: %v; the local-path reference has to be "+ + "refused on its own terms, not incidentally by the downloader", err) + } + if dir != "" { + t.Errorf("returned path %q alongside an error", dir) + } + + // The refusal must not have disturbed the target either. + if _, err := os.Stat(secret); err != nil { + t.Errorf("the target directory was modified while refusing to fetch it: %v", err) + } +} + +// TestFetchNeverResolvesOutsideTheCache states the invariant the test above +// exercises through one input, over the forms a manifest can use. +// +// Nothing a manifest declares may steer a fetch out of the cache directory, +// whether by traversing out of it or by naming a path directly for npm or pip +// to resolve. +func TestFetchNeverResolvesOutsideTheCache(t *testing.T) { + withoutFetchTools(t) + + root := t.TempDir() + cacheDir := filepath.Join(root, "cache") + if err := os.MkdirAll(cacheDir, 0755); err != nil { + t.Fatalf("create cache dir: %v", err) + } + // Populate a plausible target at every depth the traversals below reach, so + // that a fetcher which resolves one of them finds a real directory and + // returns it rather than failing for an unrelated reason. + for _, d := range []string{"victim", "cache/victim", "cache/npm/victim"} { + dir := filepath.Join(root, d) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatalf("create %s: %v", d, err) + } + if err := os.WriteFile(filepath.Join(dir, "index.js"), []byte("x"), 0644); err != nil { + t.Fatalf("write into %s: %v", d, err) + } + } + + f := NewFetcher(cacheDir) + for _, ecosystem := range []types.Ecosystem{types.EcosystemNPM, types.EcosystemPyPI} { + for _, version := range []string{ + "../victim", "../../victim", "../../../victim", + "file:../../../victim", "/etc", "~/.ssh", "1.0.0/../../../victim", + } { + t.Run(string(ecosystem)+" "+version, func(t *testing.T) { + dir, err := f.Fetch(types.Dependency{ + Name: "pkg", + Version: version, + Ecosystem: ecosystem, + }) + if err != nil { + return // refused, which is the intended outcome + } + if !strings.HasPrefix(filepath.Clean(dir), filepath.Clean(cacheDir)+string(filepath.Separator)) { + t.Errorf("version %q resolved to %q, outside the cache directory %q", + version, dir, cacheDir) + } + }) + } + } +} + +// TestCacheSegmentCannotEscape checks the property directly, including inputs +// that reach the fetcher through a package name rather than a version. +func TestCacheSegmentCannotEscape(t *testing.T) { + for _, in := range []string{ + "..", ".", "../..", "../../etc/passwd", "/absolute", "~", "", + "a/b", `a\b`, "@scope/pkg", "com.example:artifact", "1.0.0", + } { + t.Run(in, func(t *testing.T) { + seg := cacheSegment(in) + if seg == "" { + t.Fatalf("cacheSegment(%q) is empty, which collapses the cache path", in) + } + if strings.ContainsAny(seg, `/\`) { + t.Errorf("cacheSegment(%q) = %q, which spans more than one path element", in, seg) + } + if strings.Trim(seg, ".") == "" { + t.Errorf("cacheSegment(%q) = %q, which is a directory reference", in, seg) + } + + base := filepath.Join("/cache", "npm", "pkg") + joined := filepath.Join(base, seg) + if !strings.HasPrefix(joined, base+string(filepath.Separator)) { + t.Errorf("cacheSegment(%q) escapes the cache directory: %q", in, joined) + } + }) + } +} + +// TestLocalPathReferenceLeavesRemoteReferencesAlone guards the other direction: +// narrowing what is fetched must not stop the references that legitimately +// resolve to a remote package. +func TestLocalPathReferenceLeavesRemoteReferencesAlone(t *testing.T) { + local := []string{"../x", "./x", "/x", "~/x", "file:../x", "FILE:../x", "a/../../x", + `C:\x`, "C:/x", `c:\x`, `\x`, `\\host\share`, "link:../x", "portal:../x", "LINK:../x", + // A transport prefixed onto the scheme. git+file:// passed both earlier + // checks: it does not start with "file:", and the colon suppressed the + // traversal walk. npm pack clones it from disk and runs its prepare + // script, so this was code execution chosen by the scanned manifest. + "git+file:///x", "GIT+FILE:///x", "git+file://../x"} + // A Maven coordinate carries a colon and is not a Windows path; so does an + // alias and a VCS reference. Refusing any of these would stop deep analysis + // of packages that fetch correctly today. + remote := []string{"1.2.3", "^1.2.3", ">=1.0,<2.0", "v0.14.0", "latest", "", + "github:owner/repo", "git+https://github.com/owner/repo.git", "npm:alias@1.0.0", + "com.google.guava:guava", "a:b", "org.apache.logging.log4j:log4j-core"} + + for _, v := range local { + if !localPathReference(v) { + t.Errorf("localPathReference(%q) = false, want true", v) + } + } + for _, v := range remote { + if localPathReference(v) { + t.Errorf("localPathReference(%q) = true, want false; this reference resolves to a "+ + "published package and deep analysis of it works", v) + } + } +} + +// TestFetchRefusesLocalPathName covers the half of the traversal class that the +// version guard did not reach. +// +// cacheSegment sanitizes the name for the cache PATH, so the cache entry stays +// where it belongs and the defect is invisible there. The name is also handed +// to the package manager as a SPEC, and `npm pack ../../../../victim` resolves +// a directory. Path safety and spec safety are different guarantees, and only +// the first one was in place: a manifest declaring +// {"../../../../victim": ""} packed a tree outside the project, walked it, and +// published its file names and line numbers as the dependency's source. +func TestFetchRefusesLocalPathName(t *testing.T) { + // No downloader on PATH, so nothing here touches the network. That is also + // why the assertion below is on the refusal REASON: without it this test + // would pass on a machine with no npm even if the guard were deleted. + withoutFetchTools(t) + + root := t.TempDir() + cacheDir := filepath.Join(root, "cache") + if err := os.MkdirAll(cacheDir, 0755); err != nil { + t.Fatalf("create cache dir: %v", err) + } + + f := NewFetcher(cacheDir) + for _, name := range []string{ + "../../../../victim", + "./victim", + "/tmp/victim", + "~/victim", + "file:../victim", + "pkg/../../../victim", + // Windows spellings of the same thing, on a target this tool ships + // binaries for. A drive letter contains a colon, which the guard read + // as evidence that the reference was remote. + `C:\victim`, + "C:/victim", + `c:\victim`, + `\victim`, + `\\host\share\victim`, + // The Yarn and pnpm spellings. npm pack rejects them today, so these + // hold the guard rather than close a live vector. + "link:../victim", + "portal:../victim", + } { + t.Run(name, func(t *testing.T) { + dir, err := f.Fetch(types.Dependency{ + Name: name, + Ecosystem: types.EcosystemNPM, + }) + if err == nil { + t.Fatalf("dependency name %q was fetched to %q; npm resolves a name that "+ + "names a directory, so the tool reads and publishes a tree the "+ + "manifest author chose", name, dir) + } + // Either guard is an answer on the name's own terms: the grammar + // says it is not a name, or the path check says it is a local path. + // What must never satisfy this test is an incidental failure from a + // missing downloader, which is what the pre-fix code produced. + refused := strings.Contains(err.Error(), "refers to a local path") || + strings.Contains(err.Error(), "is not a valid") + if !refused { + t.Errorf("name %q failed for the wrong reason: %v; it has to be refused on "+ + "its own terms, not incidentally by a missing downloader", name, err) + } + if dir != "" { + t.Errorf("returned path %q alongside an error", dir) + } + }) + } +} + +// TestMavenFetchNeverWritesOutsideTheCache covers the write primitive. +// +// fetchMavenArtifact built the curl -o target by joining the artifactId from +// the scanned pom.xml straight onto the cache directory. filepath.Join resolves +// .. lexically, so the download target escaped, and a '?' in the artifactId +// split the Maven URL so that its path portion still named a real artifact +// while the file portion traversed: curl -f then got its 200 and wrote. That is +// an arbitrary file write and truncate driven by a scanned manifest, which is a +// different and worse class than the read the version guard closed. +func TestMavenFetchNeverWritesOutsideTheCache(t *testing.T) { + withoutFetchTools(t) + + root := t.TempDir() + cacheDir := filepath.Join(root, "cache") + if err := os.MkdirAll(cacheDir, 0755); err != nil { + t.Fatalf("create cache dir: %v", err) + } + + const version = "1" + artifactID := "junit/4.13.2/junit-4.13.2-sources.jar?a=" + + strings.Repeat("/..", 8) + "/pwned" + name := "junit:" + artifactID + + // Guard: the fixture has to actually land outside the cache. If it does + // not, every assertion below passes for the wrong reason and the test is + // decorative. This mirrors the join the fetcher performs. + packageDir := filepath.Join(cacheDir, "maven", cacheSegment(name), cacheSegment(version)) + wouldWrite := filepath.Join(packageDir, artifactID+"-"+version+"-sources.jar") + if strings.HasPrefix(filepath.Clean(wouldWrite), filepath.Clean(cacheDir)+string(filepath.Separator)) { + t.Fatalf("fixture does not escape the cache: %q is inside %q, so this test would "+ + "not exercise the write primitive", wouldWrite, cacheDir) + } + + f := NewFetcher(cacheDir) + dir, err := f.Fetch(types.Dependency{ + Name: name, + Version: version, + Ecosystem: types.EcosystemMaven, + }) + if err == nil { + t.Fatalf("a manifest-declared Maven coordinate was fetched to %q; the download "+ + "target resolves to %q, outside the cache", dir, wouldWrite) + } + if !strings.Contains(err.Error(), "not a valid Maven coordinate") { + t.Errorf("failed for the wrong reason: %v; the coordinate has to be refused on its "+ + "own terms, not incidentally by a missing downloader", err) + } + + // Nothing may have been created along the escape path, at any depth. + if _, statErr := os.Stat(wouldWrite); statErr == nil { + t.Errorf("the refused fetch still created %q", wouldWrite) + } +} + +// TestFetchAcceptsOrdinaryNames guards the test above from over-reaching: a +// refusal that also rejects real package names would be a worse defect than the +// one being fixed, and would not show up in a test that only feeds it attacks. +func TestFetchAcceptsOrdinaryNames(t *testing.T) { + for _, name := range []string{ + "ejs", "lodash", "@scope/pkg", "zope.interface", "python-dateutil", + "com.google.guava:guava", "github.com/spf13/cobra", + } { + if localPathReference(name) { + t.Errorf("ordinary package name %q was treated as a local path reference", name) + } + } +} + +// TestMavenCoordinateValidationAcceptsRealArtifacts is the companion guard: the +// validator must not reject the coordinates the tool exists to read. +func TestMavenCoordinateValidationAcceptsRealArtifacts(t *testing.T) { + for _, name := range []string{ + "junit:junit", + "com.google.guava:guava", + "org.bouncycastle:bcprov-jdk18on", + "io.jsonwebtoken:jjwt-api", + "com.squareup.okhttp3:okhttp", + } { + parts := strings.SplitN(name, ":", 2) + if err := validMavenCoordinate(parts[0], parts[1]); err != nil { + t.Errorf("real Maven coordinate %q was rejected: %v", name, err) + } + } + for _, bad := range [][2]string{ + {"junit", "junit/../../../x"}, + {"junit", "junit?a=b"}, + {"../../etc", "junit"}, + {"junit", ""}, + {"", "junit"}, + {"junit", "junit\x00"}, + } { + if err := validMavenCoordinate(bad[0], bad[1]); err == nil { + t.Errorf("coordinate %q:%q was accepted", bad[0], bad[1]) + } + } +} + +// TestVersionTraversalIsRefusedBehindAnyScheme is the regression test for the +// fourth spelling of the local-path class. +// +// localPathReference skipped its traversal walk for any value containing a +// colon, on the reasoning that a colon meant a remote reference such as +// github:owner/repo. An invented scheme defeats that: a version of +// "a1:../../../../../../../victim" carries a colon, so the walk never ran, and +// `npm pack left-pad@a1:../../../victim` packed a directory outside the cache, +// which the analyzer then read and published as that dependency's source with +// its absolute paths. Reproduced by sentinel at traversal depth 7 against the +// 1.3.0 candidate. +func TestVersionTraversalIsRefusedBehindAnyScheme(t *testing.T) { + withoutFetchTools(t) + f := NewFetcher(t.TempDir()) + + for _, version := range []string{ + "a1:../../../../../../../victim", // an invented scheme + "npm:../../../victim", // a real scheme npm does accept + "github:owner/../../../../../../victim", // a remote form carrying a traversal + "1.0.0/../../../../../../../../victim", // no scheme at all, for the baseline + } { + t.Run(version, func(t *testing.T) { + // Guard the fixture: without a ".." element this case is not a traversal + // and proves nothing about the walk. + if !strings.Contains(version, "..") { + t.Fatalf("fixture %q carries no traversal", version) + } + _, err := f.Fetch(types.Dependency{ + Name: "left-pad", + Version: version, + Ecosystem: types.EcosystemNPM, + }) + if err == nil { + t.Fatalf("a version traversing out of the cache was fetched") + } + // The REASON matters: PATH is empty here, so "npm not found" would + // satisfy err != nil while the guard was absent. + if !strings.Contains(err.Error(), "refers to a local path") { + t.Errorf("refused for the wrong reason: %v; the traversal has to be refused on "+ + "its own terms, not incidentally by a missing downloader", err) + } + }) + } +} + +// TestRemoteReferencesWithoutTraversalStillFetch is the inverse question. +// +// Removing the colon exemption must not refuse the remote references it was +// written to admit, or --deep loses every git and registry-shorthand dependency. +func TestRemoteReferencesWithoutTraversalStillFetch(t *testing.T) { + withoutFetchTools(t) + f := NewFetcher(t.TempDir()) + + for _, version := range []string{ + "github:owner/repo", + "git+https://github.com/owner/repo.git#v1.2.3", + "npm:left-pad@1.3.0", + "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "^1.2.3", + ">=1.0.0 <2.0.0", + "1.2.3-beta.1+build.5", + } { + t.Run(version, func(t *testing.T) { + _, err := f.Fetch(types.Dependency{ + Name: "left-pad", + Version: version, + Ecosystem: types.EcosystemNPM, + }) + // The fetch cannot succeed with no downloader on PATH. What must not + // happen is a refusal by the guard, which would mean this legitimate + // version is never fetched on a real machine either. + if err != nil && strings.Contains(err.Error(), "refers to a local path") { + t.Errorf("a legitimate remote reference was refused as a local path: %v", err) + } + }) + } +} + +// TestFailedExtractionLeavesNoUsableCacheEntry is the regression test for a +// fail-open that turned an error into a clean examination on the next run. +// +// The three download sites route failure through discardPartialFetch; the four +// extraction sites returned a bare error. Both tar and unzip extract partially +// before failing, so the entry was left holding a non-empty package directory, +// which the cache-hit check accepts. The first scan reported the package as not +// examined and the second reported it as examined, with findings, from a +// partially extracted archive and no warning on any stream. +func TestFailedExtractionLeavesNoUsableCacheEntry(t *testing.T) { + // Every ecosystem that extracts an archive, because the same bare error was + // returned at all four sites and a test covering only npm left three of them + // deletable with a green suite. + cases := []struct { + name string + dep types.Dependency + // stubs maps an executable this path shells out to onto a script. The + // downloader must succeed and produce an archive; the extractor must + // extract something and then fail, which is what a truncated or hostile + // archive does. + stubs map[string]string + // extracted is the directory the extractor half-fills, relative to the + // package cache entry. + extracted string + }{ + { + name: "npm", + dep: types.Dependency{Name: "left-pad", Version: "1.3.0", Ecosystem: types.EcosystemNPM}, + stubs: map[string]string{ + "npm": "#!/bin/sh\n: > left-pad-1.3.0.tgz\n", + "tar": "#!/bin/sh\nwhile [ $# -gt 0 ]; do case \"$1\" in -C) shift; D=\"$1\";; esac; shift; done\n" + + "/bin/mkdir -p \"$D/package\"\nprintf 'module.exports=1;\\n' > \"$D/package/index.js\"\nexit 1\n", + }, + extracted: npmExtractedDir, + }, + { + name: "maven", + dep: types.Dependency{Name: "org.tukaani:xz", Version: "1.9", Ecosystem: types.EcosystemMaven}, + stubs: map[string]string{ + "curl": "#!/bin/sh\nwhile [ $# -gt 0 ]; do case \"$1\" in -o) shift; O=\"$1\";; esac; shift; done\n: > \"$O\"\n", + "unzip": "#!/bin/sh\nwhile [ $# -gt 0 ]; do case \"$1\" in -d) shift; D=\"$1\";; esac; shift; done\n" + + "/bin/mkdir -p \"$D/org\"\nprintf 'class A {}\\n' > \"$D/org/A.java\"\nexit 1\n", + }, + extracted: zipExtractedDir, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + bin := t.TempDir() + for name, script := range tc.stubs { + if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0755); err != nil { + t.Fatalf("write stub %s: %v", name, err) + } + } + // The stubs come first so they shadow any real tool, but the rest of + // PATH stays: emptying it left the stubs' own mkdir unresolvable, so + // nothing was ever extracted and this test passed while the defect + // was present. + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + + cacheDir := filepath.Join(t.TempDir(), "cache") + f := NewFetcher(cacheDir) + + _, err := f.Fetch(tc.dep) + if err == nil { + t.Fatal("a failed extraction was reported as a successful fetch") + } + // Guard the fixture: the fetch has to have failed AT EXTRACTION. If + // the stub downloader produced no archive, the fetch fails earlier, + // nothing is half-extracted, and every assertion below is answered by + // an empty cache. + if !strings.Contains(err.Error(), "extract") { + t.Fatalf("the fixture did not reach extraction, so it cannot detect a "+ + "partial one: %v", err) + } + + packageDir := filepath.Join(cacheDir, string(tc.dep.Ecosystem), + cacheSegment(tc.dep.Name), cacheSegment(tc.dep.Version)) + if entries, err := os.ReadDir(filepath.Join(packageDir, tc.extracted)); err == nil && + len(entries) > 0 { + t.Fatalf("the partially extracted directory survives at %q with %d entries, "+ + "so the next scan will accept it as a cache hit", packageDir, len(entries)) + } + + // The second attempt must not silently succeed from what the first left. + root, err := f.Fetch(tc.dep) + if err == nil { + t.Errorf("the second fetch succeeded from a partially extracted entry and "+ + "returned %q; a scan that could not read a package must not report it "+ + "as read on the next run", root) + } + }) + } +} + +// TestFetchDoesNotBuildSourceDistributions pins the pip half of the +// execute-nothing control. +// +// --ignore-scripts stops npm running a package's prepare script. pip had no +// equivalent: resolving a PyPI name with no matching wheel makes pip install the +// project's build dependencies and run its build backend, which for a setup.py +// sdist is arbitrary code from a package the scanned manifest chose. Fetching +// source in order to read it must not be a way to run it. +// +// Asserted on the argv, because the property is what the tool asks pip to do. +func TestFetchDoesNotBuildSourceDistributions(t *testing.T) { + bin := t.TempDir() + argv := filepath.Join(t.TempDir(), "argv") + // A stub pip that records its arguments and then fails, so no network is + // touched and the fetch ends immediately. + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argv + "\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, "pip"), []byte(script), 0755); err != nil { + t.Fatalf("write stub pip: %v", err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + + f := NewFetcher(t.TempDir()) + if _, err := f.Fetch(types.Dependency{ + Name: "psycopg2", + Version: "2.9.9", + Ecosystem: types.EcosystemPyPI, + }); err == nil { + t.Fatal("the stub pip failed, so the fetch should have failed with it") + } + + recorded, err := os.ReadFile(argv) + if err != nil { + t.Fatalf("the stub pip was never run, so this test cannot say what it was asked "+ + "to do: %v", err) + } + args := strings.Fields(string(recorded)) + // Guard the fixture: these are the arguments of the fetch under test. + if len(args) == 0 || args[0] != "download" { + t.Fatalf("the recorded argv is not a pip download: %q", args) + } + + var onlyBinary bool + for i, a := range args { + if a == "--only-binary" && i+1 < len(args) && args[i+1] == ":all:" { + onlyBinary = true + } + if strings.HasPrefix(a, "--only-binary=") && strings.HasSuffix(a, ":all:") { + onlyBinary = true + } + } + if !onlyBinary { + t.Errorf("pip is invoked without --only-binary=:all: (%q), so a dependency with no "+ + "wheel has its build backend executed on the scanning host", args) + } +} + +// TestNpmSpecCannotBeReadAsAFlag pins the argument-injection half of the +// name-is-not-a-name class. +// +// The name grammar admits what npm's registry actually carries, and some real +// names begin with a hyphen. npm reads a spec beginning with a hyphen as a flag, +// so a dependency named "--help" made npm print its help text instead of +// fetching, and one named "--ignore-scripts" would be swallowed silently. A "--" +// separator ends flag parsing. Asserted on the argv, because the property is what +// the tool asks npm to do. +func TestNpmSpecCannotBeReadAsAFlag(t *testing.T) { + bin := t.TempDir() + argv := filepath.Join(t.TempDir(), "argv") + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argv + "\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, "npm"), []byte(script), 0755); err != nil { + t.Fatalf("write stub npm: %v", err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + + f := NewFetcher(t.TempDir()) + // A real published name that begins with a hyphen. + if _, err := f.Fetch(types.Dependency{ + Name: "-", + Version: "0.0.1-security.1", + Ecosystem: types.EcosystemNPM, + }); err == nil { + t.Fatal("the stub npm failed, so the fetch should have failed with it") + } + + recorded, err := os.ReadFile(argv) + if err != nil { + t.Fatalf("the stub npm was never run, so this test cannot say what it was asked "+ + "to do: %v", err) + } + args := strings.Split(strings.TrimRight(string(recorded), "\n"), "\n") + // Guard the fixture: these must be the arguments of the fetch under test. + if len(args) < 2 || args[0] != "pack" { + t.Fatalf("the recorded argv is not an npm pack: %q", args) + } + + sep := -1 + for i, a := range args { + if a == "--" { + sep = i + } + } + if sep < 0 { + t.Fatalf("npm is invoked with no -- separator (%q), so a dependency whose name "+ + "begins with a hyphen is read as a flag rather than as a package", args) + } + if sep != len(args)-2 { + t.Errorf("the -- separator is at %d of %d, so the spec is not the only thing after "+ + "it: %q", sep, len(args), args) + } + if args[len(args)-1] != "-@0.0.1-security.1" { + t.Errorf("the spec after -- is %q, not the declared package", args[len(args)-1]) + } +} diff --git a/internal/analyzer/source/package_name_test.go b/internal/analyzer/source/package_name_test.go new file mode 100644 index 0000000..1222359 --- /dev/null +++ b/internal/analyzer/source/package_name_test.go @@ -0,0 +1,233 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package source + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/csnp/qramm-cryptodeps/pkg/types" +) + +// TestFetchRefusesASpecSmuggledThroughTheName is the regression test for the +// bypass that survived the first attempt at closing this class. +// +// The guard screened the dependency name for the spellings of a local path and +// treated it as one opaque string. `npm pack` does not: it splits `name@spec` +// on the first '@' after index 0, so `x@/tmp/victim` presents a guard-passing +// name to the guard and a directory to npm. Reproduced end to end against the +// 1.3.0 candidate binary: npm packed the victim directory, ran its `prepare` +// script, and the analyzer published the victim's files as that dependency's +// source, with deepAnalyzed set and nothing on stderr. +// +// The equivalent for PyPI is a PEP 508 direct reference, which arrives as a +// Poetry or Pipfile table key and reaches `pip download` the same way. +func TestFetchRefusesASpecSmuggledThroughTheName(t *testing.T) { + withoutFetchTools(t) + + cacheDir := t.TempDir() + f := NewFetcher(cacheDir) + + cases := []struct { + name string + ecosystem types.Ecosystem + dep string + }{ + {"npm absolute path after the name", types.EcosystemNPM, "x@/tmp/victim"}, + {"npm file: spec after the name", types.EcosystemNPM, "x@file:../../victim"}, + {"npm home-relative spec after the name", types.EcosystemNPM, "x@~/victim"}, + {"npm scoped name with a spec", types.EcosystemNPM, "@scope/pkg@/tmp/victim"}, + {"npm git spec after the name", types.EcosystemNPM, "x@git+file:///tmp/victim"}, + {"pypi PEP 508 direct reference", types.EcosystemPyPI, "victimpkg @ file:///tmp/victim"}, + {"pypi direct reference without spaces", types.EcosystemPyPI, "victimpkg@file:///tmp/victim"}, + {"go module path with a traversal", types.EcosystemGo, "example.com/../../victim"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir, err := f.Fetch(types.Dependency{Name: tc.dep, Ecosystem: tc.ecosystem}) + if err == nil { + t.Fatalf("dependency name %q was fetched to %q: the package manager reads it "+ + "as a name plus a spec, and the spec selects something on this machine", + tc.dep, dir) + } + // The reason is the assertion. PATH is emptied by this test, so a + // fetch that got as far as the downloader would fail anyway, and a + // test satisfied by err != nil would pass against the code that + // executed the victim's prepare script. + if !strings.Contains(err.Error(), "is not a valid") { + t.Errorf("name %q failed for the wrong reason: %v", tc.dep, err) + } + if dir != "" { + t.Errorf("returned path %q alongside an error", dir) + } + }) + } +} + +// TestFetchAcceptsRealPackageNames is the other half of the bound. +// +// Holding names to a grammar is only safe if the grammar is the registry's own. +// Every name here is a package that exists and that deep analysis fetches +// today, so a pattern tightened far enough to break real scans fails here +// rather than in a user's CI. +func TestFetchAcceptsRealPackageNames(t *testing.T) { + withoutFetchTools(t) + + f := NewFetcher(t.TempDir()) + + cases := []struct { + ecosystem types.Ecosystem + names []string + }{ + {types.EcosystemNPM, []string{ + "ejs", "crypto-js", "node-forge", "@noble/post-quantum", "@types/node", + "lodash.merge", "jsonwebtoken", "@scope/pkg.with.dots", + }}, + {types.EcosystemPyPI, []string{ + "cryptography", "pycryptodome", "python-dateutil", "zope.interface", + "ruamel.yaml.clib", "Django", "typing_extensions", + }}, + {types.EcosystemGo, []string{ + "github.com/spf13/cobra", "golang.org/x/crypto", "gopkg.in/yaml.v3", + "github.com/BurntSushi/toml", "github.com/csnp/qramm-cryptodeps/v2", + }}, + } + + for _, tc := range cases { + for _, name := range tc.names { + t.Run(name, func(t *testing.T) { + if err := validPackageName(tc.ecosystem, name); err != nil { + t.Errorf("a real %s package name is refused: %v", tc.ecosystem, err) + } + // And through Fetch, where a refusal would surface as the same + // message rather than as a download failure. + _, err := f.Fetch(types.Dependency{Name: name, Ecosystem: tc.ecosystem}) + if err != nil && strings.Contains(err.Error(), "is not a valid") { + t.Errorf("Fetch refuses a real %s package name: %v", tc.ecosystem, err) + } + }) + } + } +} + +// TestNpmPackRunsNoPackageScripts pins the control that holds when a spec is +// legitimately remote. +// +// `npm pack` runs a package's prepare and prepack scripts for any spec it +// resolves from a directory or a git repository, so fetching source in order to +// analyze it was a way to execute code chosen by the manifest under scan. VCS +// references are deliberately still fetched, which means the guards above are +// not the only thing standing between a scanned manifest and code execution. +func TestNpmPackRunsNoPackageScripts(t *testing.T) { + root := t.TempDir() + f := NewFetcher(filepath.Join(root, "cache")) + + // A stub npm that records the arguments it was given, so this asserts the + // flag reaches the process rather than that a real npm behaved. + bin := filepath.Join(root, "bin") + if err := os.MkdirAll(bin, 0755); err != nil { + t.Fatalf("create stub dir: %v", err) + } + argsFile := filepath.Join(root, "npm-args") + stub := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argsFile + "\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, "npm"), []byte(stub), 0755); err != nil { + t.Fatalf("write npm stub: %v", err) + } + t.Setenv("PATH", bin) + + _, err := f.Fetch(types.Dependency{Name: "ejs", Version: "3.1.10", Ecosystem: types.EcosystemNPM}) + if err == nil { + t.Fatal("the stub npm exits 1, so the fetch should have failed") + } + + recorded, readErr := os.ReadFile(argsFile) + if readErr != nil { + t.Fatalf("the stub npm was never invoked, so this test asserts nothing: %v", readErr) + } + args := strings.Split(strings.TrimSpace(string(recorded)), "\n") + var sawIgnoreScripts bool + for _, a := range args { + if a == "--ignore-scripts" { + sawIgnoreScripts = true + } + } + if !sawIgnoreScripts { + t.Errorf("npm pack was invoked as %v, without --ignore-scripts: a fetched package's "+ + "prepare and prepack scripts run on the scanning host", args) + } +} + +// TestGrammarAcceptsGrandfatheredRegistryNames is the regression test for a +// guard that refused real, installable dependencies. +// +// The first npm pattern encoded the rules npm applies to a name it would accept +// from a new publisher: a scope and a name each beginning with a letter or a +// digit. npm grandfathered the names that predate those rules, so the pattern +// refused 1,054 published names, and a refused dependency is silently never +// analyzed. Nine exceed 100,000 downloads a month. Every name below was +// confirmed to exist by fetching its registry document, and each was confirmed +// to be packable except "-", which npm's own CLI reads as a flag. +// +// A guard on a security tool that refuses real input produces a clean result it +// never established, which is the same false clean this release exists to +// remove, reached from the other side. +func TestGrammarAcceptsGrandfatheredRegistryNames(t *testing.T) { + // name -> why it broke the first pattern + real := map[string]string{ + "@lingo.dev/_spec": "name after the scope begins with an underscore", + "@lingo.dev/_compiler": "name after the scope begins with an underscore", + "@-xun/debug": "scope begins with a hyphen", + "@-xun/fs": "scope begins with a hyphen", + "@_sh/strapi-plugin-ckeditor": "scope begins with an underscore", + "@~39/empty": "scope begins with a tilde", + "-": "the whole name is a hyphen", + "foo~": "a tilde inside the name", + "object.assign": "dots, which the first pattern did allow", + "@babel/core": "an ordinary scoped name", + "q": "a single character", + "JSONStream": "uppercase", + } + for name, why := range real { + if err := validPackageName(types.EcosystemNPM, name); err != nil { + t.Errorf("%q is a published npm package (%s) and the grammar refuses it, so this "+ + "dependency is never analyzed and the scan reports a coverage it does not "+ + "have: %v", name, why, err) + } + } +} + +// TestGrammarStillRefusesWhatNpmWouldMisread is the other half of the same +// question, and the reason the pattern cannot simply be dropped. +// +// Loosening a guard to admit real names must not admit the spec that started +// this: `npm pack` splits name@spec on the first '@' after index 0, so a name +// carrying an '@' hands the guard one string and npm a directory. +func TestGrammarStillRefusesWhatNpmWouldMisread(t *testing.T) { + hostile := map[string]string{ + "x@/tmp/victim": "a spec smuggled through the name", + "x@file:../victim": "the same with npm's own local-path spelling", + "x@npm:other": "an alias smuggled through the name", + "../../../../victim": "a bare traversal", + "/etc/passwd": "an absolute path", + "./victim": "a relative path", + ".": "the current directory", + "..": "the parent directory", + ".hidden": "a leading dot makes it relative to the fetch directory", + "victim\\\\share": "a Windows path separator", + "http://example.invalid/x": "a URL, which carries a scheme", + "git+file:///tmp/victim": "a local repository wearing a VCS reference", + "pkg with spaces": "whitespace separates arguments", + "@scope/x@/tmp/victim": "a spec smuggled through a scoped name", + } + for name, why := range hostile { + if err := validPackageName(types.EcosystemNPM, name); err == nil { + t.Errorf("%q is accepted as an npm package name (%s); the package manager reads a "+ + "name as part of a spec that can select a directory on this machine", + name, why) + } + } +} diff --git a/internal/manifest/maven.go b/internal/manifest/maven.go index 54c301e..a29f2b6 100644 --- a/internal/manifest/maven.go +++ b/internal/manifest/maven.go @@ -123,10 +123,19 @@ func (p *MavenParser) Parse(path string) ([]types.Dependency, error) { if props == nil { props = make(map[string]string) } - // Add project.version as a property + // Add the project's own coordinates as properties. ${project.groupId} is how + // a multi-module build names a sibling module, so it appears in a dependency + // groupId far more often than ${project.version} appears in a version, and + // without it here the placeholder survives into the coordinate. if pom.Version != "" { props["project.version"] = pom.Version } + if pom.GroupID != "" { + props["project.groupId"] = pom.GroupID + } + if pom.ArtifactID != "" { + props["project.artifactId"] = pom.ArtifactID + } var deps []types.Dependency @@ -134,8 +143,16 @@ func (p *MavenParser) Parse(path string) ([]types.Dependency, error) { // Skip test-scope dependencies for now isTest := dep.Scope == "test" - // Construct Maven coordinate name - name := dep.GroupID + ":" + dep.ArtifactID + // Construct Maven coordinate name. The groupId and artifactId carry + // property placeholders as legitimately as the version does, and + // ${project.groupId} for a module of the same group is the common case: + // across 2,099 real poms from Maven Central, 81 coordinates in 16 + // published artifacts use it. Resolving only the version left the literal + // "${project.groupId}" in the name, where coordinate validation refused + // it as a name that could steer a fetch. The dependency was then reported + // as unexaminable, so a real pom analyzed 6 of its dependencies where the + // same pom with the property expanded analyzed 11. + name := resolveProperties(dep.GroupID, props) + ":" + resolveProperties(dep.ArtifactID, props) // Resolve property placeholders in version version := resolveProperties(dep.Version, props) diff --git a/internal/manifest/maven_test.go b/internal/manifest/maven_test.go new file mode 100644 index 0000000..c61be7a --- /dev/null +++ b/internal/manifest/maven_test.go @@ -0,0 +1,93 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package manifest + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestMavenPropertyInACoordinateIsResolved is the regression test for a +// dependency that was refused because its name still held a placeholder. +// +// Properties were resolved in the version and not in the groupId or artifactId, +// so ${project.groupId}, which is how a multi-module build names a sibling +// module, survived into the coordinate. Coordinate validation then refused it as +// a name that could steer a fetch, and the dependency was reported as +// unexaminable. Measured across 2,099 real poms from Maven Central: 81 +// coordinates in 16 published artifacts, and on one real pom the tool analyzed 6 +// dependencies where the same pom with the property expanded analyzed 11. +func TestMavenPropertyInACoordinateIsResolved(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "pom.xml") + pom := ` + + 4.0.0 + org.springframework + spring-webmvc-struts + 2.5.6 + + org.bouncycastle + 1.78.1 + + + + ${project.groupId} + spring-web + ${project.version} + + + ${bc.group} + bcpkix-jdk18on + ${bc.version} + + + commons-codec + commons-codec + 1.16.1 + + +` + if err := os.WriteFile(path, []byte(pom), 0644); err != nil { + t.Fatalf("write pom: %v", err) + } + + deps, err := (&MavenParser{}).Parse(path) + if err != nil { + t.Fatalf("parse: %v", err) + } + byName := make(map[string]string) + for _, d := range deps { + byName[d.Name] = d.Version + } + + // Guard the fixture: the ordinary coordinate must parse, or a failure below + // could just mean the file was never read. + if _, ok := byName["commons-codec:commons-codec"]; !ok { + t.Fatalf("the literal coordinate is missing, so the fixture was not parsed: %v", byName) + } + + for _, want := range []struct{ name, version string }{ + {"org.springframework:spring-web", "2.5.6"}, + {"org.bouncycastle:bcpkix-jdk18on", "1.78.1"}, + } { + got, ok := byName[want.name] + if !ok { + t.Errorf("%s is missing; an unresolved placeholder survived into the coordinate "+ + "and the dependency is refused as an invalid name. Parsed: %v", + want.name, byName) + continue + } + if got != want.version { + t.Errorf("%s version = %q, want %q", want.name, got, want.version) + } + } + for name := range byName { + if strings.Contains(name, "${") { + t.Errorf("coordinate %q still carries an unresolved property", name) + } + } +} diff --git a/internal/manifest/python.go b/internal/manifest/python.go index 5b27b1f..9999119 100644 --- a/internal/manifest/python.go +++ b/internal/manifest/python.go @@ -173,12 +173,30 @@ func (p *PythonParser) parsePipfile(path string) ([]types.Dependency, error) { func poetryVersion(constraint any) string { switch v := constraint.(type) { case string: - return v + return stripVersionOperator(v) case map[string]any: if version, ok := v["version"].(string); ok { - return version + return stripVersionOperator(version) + } + // A dependency declared by location rather than by version keeps its + // locator, because dropping it does not merely lose information: it + // leaves a name with no version, and the fetcher then downloads whatever + // PyPI serves under that name. A `path = "../internal-lib"` dependency + // was replaced by a public package of the same name, whose source was + // then analyzed and reported as this project's. That is dependency + // confusion performed by the scanner, and an attacker only has to + // register the name of a company's local package to be handed the + // attribution, and with pip building sdists, execution. + // + // Returned raw so the guards in the fetcher see what was really + // declared: a path is refused as a local reference, and a URL or a + // repository fails to resolve under its own name rather than silently + // becoming a different package. + for _, key := range [...]string{"path", "file", "url", "git"} { + if locator, ok := v[key].(string); ok && strings.TrimSpace(locator) != "" { + return locator + } } - // Git and path dependencies carry no version. return "" default: return "" @@ -215,7 +233,27 @@ func splitRequirement(spec string) (name, version string) { rest = strings.TrimSpace(rest[idx+1:]) } } - return name, strings.TrimSpace(rest) + return name, stripVersionOperator(rest) +} + +// stripVersionOperator removes the comparison operator from a version +// constraint, leaving the version it names. +// +// The requirements.txt reader has always done this, because its regex captures +// the operator separately and keeps only what follows. The pyproject.toml and +// Pipfile readers kept the whole constraint, so the same two packages declared +// in the two formats produced pycryptodome@3.20.0 from one and +// pycryptodome@==3.20.0 from the other. The CBOM emitter normalises the purl +// and the JSON and SARIF documents do not, so a single scan disagreed with +// itself and a consumer could not join the two by version. It also reached the +// fetcher, which builds a pip spec as name==version and would have asked for +// pycryptodome===3.20.0. +// +// Multi-constraint strings such as ">=1.0,<2.0" keep everything after the +// leading operator, which is what requirements.txt already produced for them. +func stripVersionOperator(constraint string) string { + trimmed := strings.TrimLeft(strings.TrimSpace(constraint), "<>=!~^") + return strings.TrimSpace(trimmed) } // normalizePyPIName applies PEP 503 normalization so that the same package diff --git a/internal/manifest/python_formats_test.go b/internal/manifest/python_formats_test.go index 9d3de7d..1ec932d 100644 --- a/internal/manifest/python_formats_test.go +++ b/internal/manifest/python_formats_test.go @@ -179,8 +179,11 @@ pytest = "^7.0" if got := byName["cryptography"]; got != "41.0.0" { t.Errorf("cryptography version = %q, want 41.0.0; got all %v", got, byName) } - if got, ok := byName["requests"]; !ok || got != "^2.31" { - t.Errorf("requests from inline table = %q (present=%v), want ^2.31", got, ok) + // The caret is a constraint operator, not part of the version. Keeping it + // put "^2.31" in the JSON version field and in the pip spec the fetcher + // builds, while the same package declared in requirements.txt gave "2.31". + if got, ok := byName["requests"]; !ok || got != "2.31" { + t.Errorf("requests from inline table = %q (present=%v), want 2.31", got, ok) } if _, ok := byName["pytest"]; !ok { t.Errorf("dev group dependency pytest missing; got %v", byName) @@ -218,8 +221,11 @@ pytest = ">=7.0" byName[d.Name] = d.Version } - if got := byName["cryptography"]; got != "==41.0.0" { - t.Errorf("cryptography = %q, want ==41.0.0; got all %v", got, byName) + // requirements.txt has always yielded "41.0.0" for the same declaration. + // The two formats disagreeing meant one scan's SARIF said + // cryptography@==41.0.0 while its CBOM purl said cryptography@41.0.0. + if got := byName["cryptography"]; got != "41.0.0" { + t.Errorf("cryptography = %q, want 41.0.0; got all %v", got, byName) } if got, ok := byName["requests"]; !ok || got != "" { t.Errorf("wildcard requests = %q (present=%v), want an empty version", got, ok) @@ -233,18 +239,30 @@ pytest = ">=7.0" } // TestSplitRequirementHandlesPEP508 checks extras and environment markers are -// stripped rather than becoming part of the package name. +// stripped rather than becoming part of the package name, and that the version +// field carries a version. +// +// The comparison operator belongs to the constraint, not to the version. The +// requirements.txt reader drops it, the TOML readers kept it, and the same two +// packages therefore appeared as pycryptodome@3.20.0 in one scan and +// pycryptodome@==3.20.0 in another. Within a single scan the CBOM normalised +// the purl and JSON and SARIF did not, so a consumer could not join the two +// documents by version. func TestSplitRequirementHandlesPEP508(t *testing.T) { tests := []struct { spec string wantName string wantVersion string }{ - {"cryptography==41.0.0", "cryptography", "==41.0.0"}, - {"cryptography[ssh]>=41.0.0", "cryptography", ">=41.0.0"}, - {`pyjwt>=2.0; python_version<"4"`, "pyjwt", ">=2.0"}, + {"cryptography==41.0.0", "cryptography", "41.0.0"}, + {"cryptography[ssh]>=41.0.0", "cryptography", "41.0.0"}, + {`pyjwt>=2.0; python_version<"4"`, "pyjwt", "2.0"}, {"requests", "requests", ""}, - {" flask == 3.0 ", "flask", "== 3.0"}, + {" flask == 3.0 ", "flask", "3.0"}, + // Everything after the leading operator is kept, which is what the + // requirements.txt reader has always produced for a compound + // constraint. + {"urllib3>=1.26,<2.0", "urllib3", "1.26,<2.0"}, } for _, tt := range tests { @@ -259,3 +277,69 @@ func TestSplitRequirementHandlesPEP508(t *testing.T) { }) } } + +// TestPoetryLocationDependencyKeepsItsLocator is the regression test for a +// dependency confusion the new TOML parser introduced. +// +// poetryVersion returned "" for an inline table with no version key, which +// discarded the path or repository the dependency was actually declared by. The +// fetcher then had a name with no version and downloaded whatever PyPI serves +// under that name, so a `path = "../internal-lib"` dependency was replaced by a +// public package of the same name and that package's source was analyzed and +// reported as this project's. An attacker only has to register the name of a +// company's local package to be handed the attribution, and because pip builds +// sdists, the execution too. +// +// The locator is kept so the fetcher's own guards see what was declared: a path +// is refused as a local reference rather than silently substituted. +func TestPoetryLocationDependencyKeepsItsLocator(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "pyproject.toml") + writeFile(t, path, ` +[tool.poetry] +name = "demo" + +[tool.poetry.dependencies] +python = "^3.11" +cryptography = "41.0.0" +internal-lib = {path = "../internal-lib", develop = true} +forked-dep = {git = "https://github.com/owner/forked-dep.git", rev = "abc123"} +vendored = {url = "https://example.invalid/vendored-1.0.0.tar.gz"} +`) + + deps, err := (&PythonParser{}).Parse(path) + if err != nil { + t.Fatalf("parse: %v", err) + } + byName := make(map[string]string) + for _, d := range deps { + byName[d.Name] = d.Version + } + + // Guard the fixture: the ordinary dependency must still parse, or a failure + // below could just mean the file was not read. + if got := byName["cryptography"]; got != "41.0.0" { + t.Fatalf("cryptography version = %q, want 41.0.0; got all %v", got, byName) + } + + for _, tc := range []struct{ name, want string }{ + {"internal-lib", "../internal-lib"}, + {"forked-dep", "https://github.com/owner/forked-dep.git"}, + {"vendored", "https://example.invalid/vendored-1.0.0.tar.gz"}, + } { + got, ok := byName[tc.name] + if !ok { + t.Errorf("%s is missing from the parsed dependencies; got %v", tc.name, byName) + continue + } + if got == "" { + t.Errorf("%s has an empty version, so the fetcher will download whatever PyPI "+ + "serves under the name %q instead of the %q it was declared as", + tc.name, tc.name, tc.want) + continue + } + if got != tc.want { + t.Errorf("%s version = %q, want the declared locator %q", tc.name, got, tc.want) + } + } +} diff --git a/pkg/output/coverage_test.go b/pkg/output/coverage_test.go index f66922d..40cf169 100644 --- a/pkg/output/coverage_test.go +++ b/pkg/output/coverage_test.go @@ -78,7 +78,8 @@ func nothingExaminedScan() *types.MultiProjectResult { {Dependency: types.Dependency{Name: "rsa", Version: "4.9"}}, {Dependency: types.Dependency{Name: "certifi"}}, }, - Summary: types.ScanSummary{TotalDependencies: 2, DirectDependencies: 2, NotInDatabase: 2}, + Summary: types.ScanSummary{TotalDependencies: 2, DirectDependencies: 2, NotInDatabase: 2, + NotExamined: 2}, }}) } @@ -258,9 +259,10 @@ func TestCoverageNotesDoNotContradictResults(t *testing.T) { }}, }}, // Deliberately the shape --deep produces: findings exist even though - // every dependency is absent from the database. + // every dependency is absent from the database, because source + // analysis read them. Summary: types.ScanSummary{TotalDependencies: 1, DirectDependencies: 1, WithCrypto: 1, - QuantumVulnerable: 1, NotInDatabase: 1}, + QuantumVulnerable: 1, NotInDatabase: 1, NotExamined: 0, DeepAttempted: true}, } if notes := coverageNotes([]*types.ScanResult{withFindings}); len(notes) != 0 { @@ -296,7 +298,8 @@ func TestCoverageIsJudgedPerProject(t *testing.T) { {Dependency: types.Dependency{Name: "left-pad"}}, {Dependency: types.Dependency{Name: "is-odd"}}, }, - Summary: types.ScanSummary{TotalDependencies: 2, DirectDependencies: 2, NotInDatabase: 2}, + Summary: types.ScanSummary{TotalDependencies: 2, DirectDependencies: 2, NotInDatabase: 2, + NotExamined: 2}, } analyzed := &types.ScanResult{ Manifest: "/repo/known/go.mod", @@ -313,7 +316,7 @@ func TestCoverageIsJudgedPerProject(t *testing.T) { // Guard the fixture: the aggregate must NOT satisfy notInDatabase == total, // or the old whole-run test would have caught this and there is no bug. - if multi.TotalSummary.NotInDatabase >= multi.TotalSummary.TotalDependencies { + if multi.TotalSummary.NotExamined >= multi.TotalSummary.TotalDependencies { t.Fatalf("fixture does not mix examined and unexamined projects: %+v", multi.TotalSummary) } @@ -342,7 +345,7 @@ func TestCoverageIsJudgedPerProject(t *testing.T) { // wrongly, that scan reports "not analyzed" and never mentions the two withheld // findings. func TestClassifyNoFindingsChecksFilterFirst(t *testing.T) { - both := types.ScanSummary{TotalDependencies: 2, NotInDatabase: 2, FilteredOut: 2} + both := types.ScanSummary{TotalDependencies: 2, NotInDatabase: 2, NotExamined: 2, FilteredOut: 2} if got := classifyNoFindings(both); got != caseFiltered { t.Errorf("classifyNoFindings(%+v) = %v, want caseFiltered; withheld findings must "+ "outrank every other explanation for an empty report", both, got) @@ -356,8 +359,14 @@ func TestClassifyNoFindingsChecksFilterFirst(t *testing.T) { want noFindingsCase }{ {"no dependencies", types.ScanSummary{}, caseNoDependencies}, - {"all unknown", types.ScanSummary{TotalDependencies: 3, NotInDatabase: 3}, caseNothingExamined}, - {"examined and clean", types.ScanSummary{TotalDependencies: 3, NotInDatabase: 1}, caseGenuinelyClean}, + {"all unknown", types.ScanSummary{TotalDependencies: 3, NotInDatabase: 3, NotExamined: 3}, caseNothingExamined}, + {"examined and clean", types.ScanSummary{TotalDependencies: 3, NotInDatabase: 1, NotExamined: 1}, caseGenuinelyClean}, + // Every dependency read by source analysis and none of them in the + // database. The database count says nothing was examined; the scan + // examined all three. + {"deep-analyzed, none in the database", + types.ScanSummary{TotalDependencies: 3, NotInDatabase: 3, NotExamined: 0, DeepAttempted: true}, + caseGenuinelyClean}, } { if got := classifyNoFindings(tc.summary); got != tc.want { t.Errorf("%s: classifyNoFindings(%+v) = %v, want %v", tc.name, tc.summary, got, tc.want) @@ -603,7 +612,8 @@ func TestCoverageNotesAreEmittedPerProjectNotJustFirst(t *testing.T) { Dependencies: []types.DependencyResult{ {Dependency: types.Dependency{Name: "left-pad"}}, }, - Summary: types.ScanSummary{TotalDependencies: 1, DirectDependencies: 1, NotInDatabase: 1}, + Summary: types.ScanSummary{TotalDependencies: 1, DirectDependencies: 1, NotInDatabase: 1, + NotExamined: 1}, } } multi := types.AggregateResults("/repo", []*types.ScanResult{ diff --git a/pkg/output/markdown.go b/pkg/output/markdown.go index 2062566..3220b8c 100644 --- a/pkg/output/markdown.go +++ b/pkg/output/markdown.go @@ -48,6 +48,10 @@ func (f *MarkdownFormatter) formatProject(result *types.ScanResult, root string, fmt.Fprintf(w, "| **Quantum Vulnerable** | %d |\n", result.Summary.QuantumVulnerable) fmt.Fprintf(w, "| **Quantum Partial** | %d |\n", result.Summary.QuantumPartial) fmt.Fprintf(w, "| **Not in Database** | %d |\n", result.Summary.NotInDatabase) + // Not the same number once --deep is in play, and the difference is the + // part a reader needs in order to know how much of the tree this report + // actually covers. + fmt.Fprintf(w, "| **Not Examined** | %d |\n", result.Summary.NotExamined) fmt.Fprintf(w, "\n") if !hasAnyCrypto(result.Dependencies) { @@ -170,12 +174,29 @@ func (f *MarkdownFormatter) formatProject(result *types.ScanResult, root string, fmt.Fprintf(w, "| [FIPS 205](https://csrc.nist.gov/pubs/fips/205/final) | SLH-DSA (SPHINCS+) | Stateless Hash Signatures |\n") fmt.Fprintf(w, "\n") - // Notes section for packages not in database - if result.Summary.NotInDatabase > 0 { + // Notes section for packages nothing examined. Keyed on NotExamined rather + // than NotInDatabase: a package the database does not carry and source + // analysis read is covered, and telling the reader to run --deep for it + // repeats a step they have already taken. + // A package read in part is counted as examined, so it is absent from + // NotExamined and needs its own note. The heading belongs to either state + // rather than to the first one that happened to be written. + if result.Summary.NotExamined > 0 || result.Summary.SourceFilesUnreadable > 0 { fmt.Fprintf(w, "## Notes\n\n") - pct := float64(result.Summary.NotInDatabase) / float64(result.Summary.TotalDependencies) * 100 - fmt.Fprintf(w, "> **%d packages (%.0f%%) were not in the crypto database.**\n", result.Summary.NotInDatabase, pct) - fmt.Fprintf(w, "> Run with `--deep` flag to analyze these packages via source code inspection.\n\n") + } + if result.Summary.NotExamined > 0 { + pct := float64(result.Summary.NotExamined) / float64(result.Summary.TotalDependencies) * 100 + fmt.Fprintf(w, "> **%d packages (%.0f%%) were not examined.**\n", result.Summary.NotExamined, pct) + if result.Summary.DeepAttempted { + fmt.Fprintf(w, "> Source analysis could not read them; see the warnings printed during the scan.\n\n") + } else { + fmt.Fprintf(w, "> Run with `--deep` flag to analyze these packages via source code inspection.\n\n") + } + } + if result.Summary.SourceFilesUnreadable > 0 { + fmt.Fprintf(w, "> **%d source files could not be read** in packages that were examined, so "+ + "their cryptography may be under-reported.\n", result.Summary.SourceFilesUnreadable) + fmt.Fprintf(w, "> The scan names them in the warnings it printed.\n\n") } // Footer @@ -199,16 +220,27 @@ func writeMarkdownNoFindingsVerdict(w io.Writer, s types.ScanSummary) { fmt.Fprintf(w, "**No dependencies found in this manifest.** Nothing to analyze.\n") case caseNothingExamined: - fmt.Fprintf(w, "**Not analyzed.** All %d dependencies are absent from the crypto database, "+ - "so no conclusion about cryptographic usage can be drawn from this scan. "+ - "Run with `--deep` to analyze package source code directly.\n", s.TotalDependencies) + if s.DeepAttempted { + fmt.Fprintf(w, "**Not analyzed.** None of the %d dependencies could be examined: they are absent "+ + "from the crypto database, and source analysis could not read any of them. "+ + "See the warnings printed during the scan.\n", s.TotalDependencies) + } else { + fmt.Fprintf(w, "**Not analyzed.** All %d dependencies are absent from the crypto database, "+ + "so no conclusion about cryptographic usage can be drawn from this scan. "+ + "Run with `--deep` to analyze package source code directly.\n", s.TotalDependencies) + } default: - fmt.Fprintf(w, "No cryptographic usage detected in the %d of %d dependencies that were analyzed.\n", - s.TotalDependencies-s.NotInDatabase, s.TotalDependencies) - if s.NotInDatabase > 0 { - fmt.Fprintf(w, "\n%d not in database, so they were not examined (use `--deep` to analyze).\n", - s.NotInDatabase) + fmt.Fprintf(w, "No cryptographic usage detected in the %d of %d dependencies that were examined.\n", + examinedCount(s), s.TotalDependencies) + if s.NotExamined > 0 { + fmt.Fprintf(w, "\n%d could not be examined: %s.\n", s.NotExamined, unexaminedAdvice(s)) + } + // A dependency read in part is counted as read, so it is absent from + // NotExamined and this verdict claimed a complete reading of it. + if s.SourceFilesUnreadable > 0 { + fmt.Fprintf(w, "\n%d source file(s) in those dependencies could not be read, so this is not a "+ + "complete reading of them. See the warnings printed during the scan.\n", s.SourceFilesUnreadable) } } } diff --git a/pkg/output/output_test.go b/pkg/output/output_test.go index ddc19c3..95c9cff 100644 --- a/pkg/output/output_test.go +++ b/pkg/output/output_test.go @@ -68,6 +68,7 @@ func testScanResult() *types.ScanResult { QuantumVulnerable: 1, QuantumPartial: 0, NotInDatabase: 1, + NotExamined: 1, }, } } diff --git a/pkg/output/partial_coverage_test.go b/pkg/output/partial_coverage_test.go new file mode 100644 index 0000000..2156bfa --- /dev/null +++ b/pkg/output/partial_coverage_test.go @@ -0,0 +1,446 @@ +// Copyright 2025-2026 CyberSecurity NonProfit (CSNP) +// SPDX-License-Identifier: Apache-2.0 + +package output + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/csnp/qramm-cryptodeps/pkg/types" +) + +// partiallyExaminedScan is a project where one dependency was read and carried +// a finding, and a second could not be examined at all. +// +// This is the shape that was invisible: the coverage question was only asked of +// reports that produced nothing, and this one produces something. +func partiallyExaminedScan() *types.ScanResult { + return &types.ScanResult{ + Project: "demo", + Manifest: "pom.xml", + Ecosystem: types.EcosystemMaven, + Dependencies: []types.DependencyResult{ + { + Dependency: types.Dependency{Name: "org.tukaani:xz", Version: "1.9", Ecosystem: types.EcosystemMaven}, + DeepAnalyzed: true, + Analysis: &types.PackageAnalysis{ + Package: "org.tukaani:xz", + Analysis: types.AnalysisMetadata{ + FilesAnalyzed: 103, + }, + Crypto: []types.CryptoUsage{{ + Algorithm: "SHA-256", + Type: "hash", + QuantumRisk: types.RiskPartial, + Severity: types.SeverityMedium, + Location: types.Location{File: "SHA256.java", Line: 18}, + }}, + }, + }, + { + Dependency: types.Dependency{ + Name: "com.google.guava:listenablefuture", + Version: "9999.0-empty-to-avoid-conflict-with-guava", + Ecosystem: types.EcosystemMaven, + }, + Error: "source analysis read no files it can parse in the fetched archive for " + + "com.google.guava:listenablefuture", + }, + }, + Summary: types.ScanSummary{ + TotalDependencies: 2, + WithCrypto: 1, + QuantumPartial: 1, + NotInDatabase: 2, + NotExamined: 1, + DeepAttempted: true, + }, + } +} + +// TestPartialCoverageReachesEveryFormat is the regression test for the gap the +// v1.3.0 fresh-user pass found. +// +// A three-dependency Maven scan where one artifact publishes no sources JAR +// said so on stdout, on stderr and in JSON, and said nothing at all in the two +// formats that are uploaded to GitHub code scanning and to compliance systems: +// the CBOM listed the libraries it had findings for and carried no coverage +// property, and the SARIF run reported executionSuccessful with no +// notification. A bill of materials that omits a dependency without saying so +// is a false bill of materials. +// +// The cause is the same shape as an earlier defect in this file: the coverage +// question was asked only of reports that produced no findings, and a partially +// examined project produces findings. +func TestPartialCoverageReachesEveryFormat(t *testing.T) { + result := partiallyExaminedScan() + + // Guard the fixture on both sides, or a passing assertion below could be + // describing a scan that examined everything or nothing. + if result.Summary.NotExamined == 0 || + result.Summary.NotExamined >= result.Summary.TotalDependencies { + t.Fatalf("fixture is not partially examined: %+v", result.Summary) + } + if !hasAnyCrypto(result.Dependencies) { + t.Fatalf("fixture has no findings, so it does not reach the case under test") + } + + t.Run("sarif", func(t *testing.T) { + var buf bytes.Buffer + if err := (&SARIFFormatter{Options: DefaultOptions()}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + var doc struct { + Runs []struct { + Results []json.RawMessage `json:"results"` + Invocations []struct { + ToolExecutionNotifications []struct { + Message struct { + Text string `json:"text"` + } `json:"message"` + Level string `json:"level"` + } `json:"toolExecutionNotifications"` + } `json:"invocations"` + } `json:"runs"` + } + if err := json.Unmarshal(buf.Bytes(), &doc); err != nil { + t.Fatalf("parse SARIF: %v", err) + } + if len(doc.Runs) != 1 { + t.Fatalf("expected one run, got %d", len(doc.Runs)) + } + if len(doc.Runs[0].Results) == 0 { + t.Fatal("the finding is missing, so this document is not the case under test") + } + if len(doc.Runs[0].Invocations) == 0 || len(doc.Runs[0].Invocations[0].ToolExecutionNotifications) == 0 { + t.Fatalf("SARIF reports no notification for a scan that could not examine %d of %d "+ + "dependencies; GitHub code scanning is told the run was complete", + result.Summary.NotExamined, result.Summary.TotalDependencies) + } + notification := doc.Runs[0].Invocations[0].ToolExecutionNotifications[0] + if !strings.Contains(notification.Message.Text, "could not be examined") { + t.Errorf("the notification does not say coverage was incomplete: %q", + notification.Message.Text) + } + if notification.Level != "warning" { + t.Errorf("notification level = %q, want warning: an incomplete scan is not a note", + notification.Level) + } + }) + + t.Run("cbom", func(t *testing.T) { + var buf bytes.Buffer + if err := (&CBOMFormatter{}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + var doc struct { + Metadata struct { + Properties []struct { + Name string `json:"name"` + Value string `json:"value"` + } `json:"properties"` + } `json:"metadata"` + Components []struct { + Type string `json:"type"` + Name string `json:"name"` + } `json:"components"` + } + if err := json.Unmarshal(buf.Bytes(), &doc); err != nil { + t.Fatalf("parse CBOM: %v", err) + } + + var libraries int + for _, c := range doc.Components { + if c.Type == "library" { + libraries++ + } + } + if libraries == 0 { + t.Fatal("the examined library is missing, so this document is not the case under test") + } + + var coverage string + for _, p := range doc.Metadata.Properties { + if p.Name == "cryptodeps:coverage" { + coverage = p.Value + } + } + if coverage == "" { + t.Fatalf("CBOM carries no cryptodeps:coverage property while listing %d of %d "+ + "declared dependencies: a bill of materials that omits one without saying so "+ + "is a false bill of materials", libraries, result.Summary.TotalDependencies) + } + if !strings.Contains(coverage, "could not be examined") { + t.Errorf("the coverage property does not say what was missed: %q", coverage) + } + }) + + // The human formats already carried this, which is what made the gap in the + // machine ones hard to see: the operator running the scan was told, and the + // document their CI uploaded was not. Asserted here so that a change to the + // machine formats cannot be made by quietly removing the human ones. + t.Run("table and markdown already said so", func(t *testing.T) { + for name, want := range map[string]string{ + "table": "could not be read by source analysis", + "markdown": "were not examined", + } { + var f Formatter + if name == "table" { + f = &TableFormatter{Options: DefaultOptions()} + } else { + f = &MarkdownFormatter{Options: DefaultOptions()} + } + var buf bytes.Buffer + if err := f.Format(result, &buf); err != nil { + t.Fatalf("%s: %v", name, err) + } + if !strings.Contains(buf.String(), want) { + t.Errorf("%s does not account for the dependency nothing read, expected %q:\n%s", + name, want, buf.String()) + } + } + }) +} + +// TestPartiallyReadPackageIsDisclosedInEveryFormat is the same question asked of +// a package that WAS examined and was read incompletely. +// +// casePartialCoverage covers a dependency nothing reached. A dependency holding +// one file the analyzer parsed and one it refused is not that: it is examined, so +// it is absent from NotExamined, and every format described a partial reading as +// a complete one. The count that proved otherwise was incremented inside the +// walk and discarded there. +func TestPartiallyReadPackageIsDisclosedInEveryFormat(t *testing.T) { + result := partiallyExaminedScan() + // Same fixture, with the second dependency examined in part rather than not + // at all: three files read, one refused. + result.Dependencies[1].DeepAnalyzed = true + result.Dependencies[1].Error = "" + result.Dependencies[1].Analysis = &types.PackageAnalysis{ + Package: "com.google.guava:listenablefuture", + Analysis: types.AnalysisMetadata{FilesAnalyzed: 3, FilesUnreadable: 1}, + } + result.Summary.NotExamined = 0 + result.Summary.SourceFilesUnreadable = 1 + + // Guard the fixture: this must be the partial-READ state and not the + // partial-COVERAGE state, or the assertions below are satisfied by the note + // that already existed. + if result.Summary.NotExamined != 0 { + t.Fatalf("fixture still has an unexamined dependency, so casePartialCoverage answers "+ + "these assertions instead: %+v", result.Summary) + } + if result.Summary.SourceFilesUnreadable == 0 { + t.Fatalf("fixture has no unreadable source file, so it does not reach the case under test") + } + if !hasAnyCrypto(result.Dependencies) { + t.Fatalf("fixture has no findings; the no-findings path is covered elsewhere and the " + + "formats that omitted this state only did so when findings existed") + } + + notes := coverageNotes([]*types.ScanResult{result}) + var partialSource *coverageNote + for i := range notes { + if notes[i].Case == casePartialSource { + partialSource = ¬es[i] + } + } + if partialSource == nil { + t.Fatalf("no note is produced for a package read in part, so no format can report it: %+v", + notes) + } + if partialSource.Level() != "warning" { + t.Errorf("note level = %q, want warning: an incomplete reading is not a note", + partialSource.Level()) + } + + t.Run("sarif", func(t *testing.T) { + var buf bytes.Buffer + if err := (&SARIFFormatter{Options: DefaultOptions()}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + var doc struct { + Runs []struct { + Results []json.RawMessage `json:"results"` + Invocations []struct { + ToolExecutionNotifications []struct { + Message struct { + Text string `json:"text"` + } `json:"message"` + } `json:"toolExecutionNotifications"` + } `json:"invocations"` + } `json:"runs"` + } + if err := json.Unmarshal(buf.Bytes(), &doc); err != nil { + t.Fatalf("parse SARIF: %v", err) + } + if len(doc.Runs) != 1 || len(doc.Runs[0].Results) == 0 { + t.Fatal("the finding is missing, so this document is not the case under test") + } + var found bool + for _, inv := range doc.Runs[0].Invocations { + for _, n := range inv.ToolExecutionNotifications { + if strings.Contains(n.Message.Text, "could not be read") { + found = true + } + } + } + if !found { + t.Errorf("SARIF reports no notification for %d source file(s) that could not be "+ + "read; code scanning is told the reading was complete:\n%s", + result.Summary.SourceFilesUnreadable, buf.String()) + } + }) + + t.Run("cbom", func(t *testing.T) { + var buf bytes.Buffer + if err := (&CBOMFormatter{}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + var doc struct { + Metadata struct { + Properties []struct { + Name string `json:"name"` + Value string `json:"value"` + } `json:"properties"` + } `json:"metadata"` + } + if err := json.Unmarshal(buf.Bytes(), &doc); err != nil { + t.Fatalf("parse CBOM: %v", err) + } + var found bool + for _, p := range doc.Metadata.Properties { + if p.Name == "cryptodeps:coverage" && strings.Contains(p.Value, "could not be read") { + found = true + } + } + if !found { + t.Errorf("CBOM carries no coverage property for a package read in part:\n%s", buf.String()) + } + }) + + t.Run("table and markdown", func(t *testing.T) { + for name, f := range map[string]Formatter{ + "table": &TableFormatter{Options: DefaultOptions()}, + "markdown": &MarkdownFormatter{Options: DefaultOptions()}, + } { + var buf bytes.Buffer + if err := f.Format(result, &buf); err != nil { + t.Fatalf("%s: %v", name, err) + } + if !strings.Contains(buf.String(), "could not be read") { + t.Errorf("%s does not say a source file went unread:\n%s", name, buf.String()) + } + // The note is a blockquote under a heading. Without the heading it is + // still readable prose but no longer a section, and this fixture has + // nothing else that would produce one: NotExamined is zero. + if name == "markdown" && !strings.Contains(buf.String(), "## Notes") { + t.Errorf("markdown emits the note with no section heading to carry it:\n%s", + buf.String()) + } + } + }) + + // JSON is where a consumer audits the claim per dependency, so the two + // counts have to travel together rather than only in aggregate. + t.Run("json carries both counts per dependency", func(t *testing.T) { + var buf bytes.Buffer + if err := (&JSONFormatter{Indent: true}).Format(result, &buf); err != nil { + t.Fatalf("format: %v", err) + } + if !strings.Contains(buf.String(), `"filesUnreadable": 1`) { + t.Errorf("JSON does not carry filesUnreadable for the dependency read in part:\n%s", + buf.String()) + } + }) +} + +// TestCleanVerdictDisclosesAPartialReadInEveryHumanFormat covers the shape the +// defect actually took. +// +// The damaging case is a package with no findings, because that is the verdict +// that says nothing was found: a dependency whose only cryptography sat in a +// refused file reached "no cryptographic usage detected in the 1 of 1 +// dependencies that were examined". Both human formats reach that sentence +// through their own no-findings branch, so both need asserting. +func TestCleanVerdictDisclosesAPartialReadInEveryHumanFormat(t *testing.T) { + result := partiallyExaminedScan() + // One dependency, examined in part, carrying no findings at all. + result.Dependencies = result.Dependencies[:1] + result.Dependencies[0].Analysis.Crypto = nil + result.Summary = types.ScanSummary{ + TotalDependencies: 1, + NotInDatabase: 1, + DeepAttempted: true, + SourceFilesUnreadable: 2, + } + + // Guard the fixture: with a finding present, or with an unexamined + // dependency, a different branch answers these assertions. + if hasAnyCrypto(result.Dependencies) { + t.Fatalf("fixture has findings, so the no-findings verdict is never reached") + } + if classifyNoFindings(result.Summary) != caseGenuinelyClean { + t.Fatalf("fixture does not reach the clean verdict, it classifies as %v", + classifyNoFindings(result.Summary)) + } + + for name, f := range map[string]Formatter{ + "table": &TableFormatter{Options: DefaultOptions()}, + "markdown": &MarkdownFormatter{Options: DefaultOptions()}, + } { + var buf bytes.Buffer + if err := f.Format(result, &buf); err != nil { + t.Fatalf("%s: %v", name, err) + } + out := buf.String() + // The clean sentence itself is expected: the package WAS examined. What + // must not happen is that sentence standing alone. + if !strings.Contains(out, "could not be read") { + t.Errorf("%s reports a clean examination and never says that 2 source files in it "+ + "were not read:\n%s", name, out) + } + } +} + +// TestFullyReadPackageCarriesNoPartialReadWarning is the inverse question. +// +// A package every file of which was read must not acquire a warning, or the +// signal is worth nothing on the reports that are actually complete. +func TestFullyReadPackageCarriesNoPartialReadWarning(t *testing.T) { + result := partiallyExaminedScan() + result.Summary.SourceFilesUnreadable = 0 + + for _, n := range coverageNotes([]*types.ScanResult{result}) { + if n.Case == casePartialSource { + t.Errorf("a scan that read every file carries a partial-read note: %q", n.Text()) + } + } +} + +// TestFullyExaminedScanCarriesNoCoverageWarning is the inverse question. +// +// Adding a note for partial coverage must not attach one to a scan that read +// everything, or every complete report acquires a warning and the signal is +// worth nothing. +func TestFullyExaminedScanCarriesNoCoverageWarning(t *testing.T) { + result := partiallyExaminedScan() + // Same fixture, with the unexamined dependency read. + result.Dependencies[1].DeepAnalyzed = true + result.Dependencies[1].Error = "" + result.Dependencies[1].Analysis = &types.PackageAnalysis{ + Package: "com.google.guava:listenablefuture", + Analysis: types.AnalysisMetadata{FilesAnalyzed: 7}, + } + result.Summary.NotExamined = 0 + + notes := coverageNotes([]*types.ScanResult{result}) + for _, n := range notes { + if n.Case == casePartialCoverage { + t.Errorf("a fully examined scan carries a partial-coverage note: %q", n.Text()) + } + } +} diff --git a/pkg/output/table.go b/pkg/output/table.go index 6d69755..9ed6dcc 100644 --- a/pkg/output/table.go +++ b/pkg/output/table.go @@ -140,23 +140,34 @@ func (f *TableFormatter) formatProject(result *types.ScanResult, root string, w // Count deep-analyzed packages deepAnalyzed := 0 - notAnalyzed := 0 for _, dep := range result.Dependencies { if dep.DeepAnalyzed { deepAnalyzed++ - } else if !dep.InDatabase && dep.Analysis == nil { - notAnalyzed++ } } + notAnalyzed := result.Summary.NotExamined // Info for deep-analyzed packages if deepAnalyzed > 0 { fmt.Fprintf(w, "[*] %d packages analyzed via AST (--deep)\n", deepAnalyzed) } - // Warning for packages not analyzed + // Warning for packages nothing examined. The wording follows what the user + // already did: telling someone who ran --deep to run --deep is a dead end. if notAnalyzed > 0 { - fmt.Fprintf(w, "[!] %d packages not in database (use --deep to analyze)\n", notAnalyzed) + if result.Summary.DeepAttempted { + fmt.Fprintf(w, "[!] %d packages could not be read by source analysis (see warnings above)\n", notAnalyzed) + } else { + fmt.Fprintf(w, "[!] %d packages not in database (use --deep to analyze)\n", notAnalyzed) + } + } + + // A package that was read incompletely is counted as read, so it is absent + // from notAnalyzed above and needs saying separately. + if result.Summary.SourceFilesUnreadable > 0 { + fmt.Fprintf(w, "[!] %d source files could not be read in packages that were examined,\n", + result.Summary.SourceFilesUnreadable) + fmt.Fprintln(w, " so their cryptography may be under-reported (see warnings above)") } f.printHints(w, result) @@ -177,7 +188,7 @@ func (f *TableFormatter) formatProject(result *types.ScanResult, root string, w // three cases below are genuinely different and are worded differently. func (f *TableFormatter) printNoFindingsVerdict(w io.Writer, result *types.ScanResult) { total := result.Summary.TotalDependencies - unknown := result.Summary.NotInDatabase + unexamined := result.Summary.NotExamined switch classifyNoFindings(result.Summary) { case caseFiltered: @@ -190,17 +201,32 @@ func (f *TableFormatter) printNoFindingsVerdict(w io.Writer, result *types.ScanR fmt.Fprintln(w, "[?] No dependencies found in this manifest. Nothing to analyze.") case caseNothingExamined: - fmt.Fprintf(w, "[?] Not analyzed. All %d dependencies are absent from the crypto database,\n", total) - fmt.Fprintln(w, " so no conclusion about cryptographic usage can be drawn from this scan.") - fmt.Fprintln(w, " Run with --deep to analyze package source code directly.") + if result.Summary.DeepAttempted { + fmt.Fprintf(w, "[?] Not analyzed. None of the %d dependencies could be examined: they are\n", total) + fmt.Fprintln(w, " absent from the crypto database, and source analysis could not read any") + fmt.Fprintln(w, " of them. See the warnings printed during the scan.") + } else { + fmt.Fprintf(w, "[?] Not analyzed. All %d dependencies are absent from the crypto database,\n", total) + fmt.Fprintln(w, " so no conclusion about cryptographic usage can be drawn from this scan.") + fmt.Fprintln(w, " Run with --deep to analyze package source code directly.") + } // The hints for this case say the same thing in other words. Printing // both reads as three separate problems. default: - fmt.Fprintf(w, "[OK] No cryptographic usage detected in the %d of %d dependencies that were analyzed.\n", - total-unknown, total) - if unknown > 0 { - fmt.Fprintf(w, "[!] %d not in database, so they were not examined (use --deep to analyze).\n", unknown) + fmt.Fprintf(w, "[OK] No cryptographic usage detected in the %d of %d dependencies that were examined.\n", + examinedCount(result.Summary), total) + if unexamined > 0 { + fmt.Fprintf(w, "[!] %d could not be examined: %s.\n", unexamined, unexaminedAdvice(result.Summary)) + } + // This is the sentence a partial reading most damages, because it is the + // one that says nothing was found. A dependency holding its only + // cryptography in a file the analyzer refused reached exactly here, and + // said it had been examined and was clean. + if result.Summary.SourceFilesUnreadable > 0 { + fmt.Fprintf(w, "[!] %d source files in those dependencies could not be read, so this is not a\n", + result.Summary.SourceFilesUnreadable) + fmt.Fprintln(w, " complete reading of them. See the warnings printed during the scan.") } f.printHints(w, result) } diff --git a/pkg/output/verdict.go b/pkg/output/verdict.go index 9d641d3..2a3fc0f 100644 --- a/pkg/output/verdict.go +++ b/pkg/output/verdict.go @@ -28,30 +28,73 @@ const ( caseFiltered noFindingsCase = iota // caseNoDependencies means the manifest declared nothing to analyze. caseNoDependencies - // caseNothingExamined means every dependency was absent from the database, - // so the scan drew no conclusion. Reporting this as clean is a false - // negative on the tool's core question. + // caseNothingExamined means no dependency was reached by any means of + // examination, so the scan drew no conclusion. Reporting this as clean is a + // false negative on the tool's core question. caseNothingExamined // caseGenuinelyClean means dependencies were examined and carried no // cryptography. caseGenuinelyClean + // casePartialCoverage means some dependencies were examined and others were + // not. It is not a no-findings case: it can and does occur beside a + // populated result set, which is exactly why it was missing. A document + // listing two of three dependencies without saying the third was never read + // is a false bill of materials, and the two formats that omitted it are the + // two that get uploaded to code scanning and to compliance systems. + casePartialCoverage + // casePartialSource means dependencies were examined, and some of their + // source files could not be read. It is the partial state of the + // examination question: such a dependency is examined, so it is absent from + // NotExamined and from casePartialCoverage, and a scan that read three of a + // package's four files described itself as having read the package. A file + // the analyzer refuses is where a finding would have been. + casePartialSource ) // classifyNoFindings decides which case a findings-free summary falls into. // Callers must only reach it when no finding survived to be reported. +// +// The examination question is asked of NotExamined, not of NotInDatabase. The +// two were the same number while a database lookup was the only way to examine +// a package; --deep made them differ, and asking the database question meant a +// scan that read every dependency's source reported that it had examined +// nothing. A predicate that merely correlates with the question is a proxy, and +// this one stopped correlating the moment a second answer path existed. func classifyNoFindings(s types.ScanSummary) noFindingsCase { switch { case s.FilteredOut > 0: return caseFiltered case s.TotalDependencies == 0: return caseNoDependencies - case s.NotInDatabase >= s.TotalDependencies: + case s.NotExamined >= s.TotalDependencies: return caseNothingExamined default: return caseGenuinelyClean } } +// examinedCount is how many dependencies the scan actually inspected. +func examinedCount(s types.ScanSummary) int { + n := s.TotalDependencies - s.NotExamined + if n < 0 { + return 0 + } + return n +} + +// unexaminedAdvice is the next step for dependencies nothing reached, worded +// for what the user has already done. +// +// A tool that answers "run --deep" to someone who ran --deep has given them a +// dead end. The two states need different sentences because they need different +// actions. +func unexaminedAdvice(s types.ScanSummary) string { + if s.DeepAttempted { + return "source analysis could not read them; see the warnings printed during the scan" + } + return "not in the crypto database (use --deep to analyze them)" +} + // hasAnyCrypto reports whether any dependency carried a surviving finding. func hasAnyCrypto(deps []types.DependencyResult) bool { for _, dep := range deps { @@ -110,12 +153,42 @@ func coverageNotes(projects []*types.ScanResult) []coverageNote { }) } + // Incomplete coverage is a property of the SCAN too, for the same + // reason: a project can have findings AND dependencies nothing read. + // Asking it only of empty reports meant a CBOM listed two of three + // libraries, and a SARIF run reported executionSuccessful with no + // notification, for a scan whose stderr had said it could not read the + // third. The human formats said so; the machine ones did not. + if p.Summary.NotExamined > 0 && p.Summary.NotExamined < p.Summary.TotalDependencies { + notes = append(notes, coverageNote{ + Manifest: p.Manifest, + Case: casePartialCoverage, + Summary: p.Summary, + }) + } + + // Asked of the scan for the same reason as the two above, and asked + // independently of them: a dependency that was examined incompletely is + // counted as examined, so neither NotExamined nor the no-findings + // classification can reach this state. A package that hid its + // cryptography in a file the analyzer refused was reported as examined + // and clean, in all five formats, with the count that proved otherwise + // discarded inside the walk. + if p.Summary.SourceFilesUnreadable > 0 { + notes = append(notes, coverageNote{ + Manifest: p.Manifest, + Case: casePartialSource, + Summary: p.Summary, + }) + } + if hasAnyCrypto(p.Dependencies) { continue } c := classifyNoFindings(p.Summary) - // caseFiltered is already handled above, and caseGenuinelyClean needs no - // explanation: an empty result set is exactly what it means. + // caseFiltered and casePartialCoverage are already handled above, and + // caseGenuinelyClean needs no explanation: an empty result set is + // exactly what it means. if c == caseGenuinelyClean || c == caseFiltered { continue } @@ -132,10 +205,27 @@ func (n coverageNote) Text() string { "This is a filtered subset, not every finding.", n.Summary.FilteredOut) case caseNoDependencies: return "No dependencies were declared, so nothing was analyzed." + case casePartialCoverage: + return fmt.Sprintf("%d of the %d dependencies could not be examined: %s. The findings "+ + "below describe the %d that were examined, and say nothing about the rest.", + n.Summary.NotExamined, n.Summary.TotalDependencies, unexaminedAdvice(n.Summary), + examinedCount(n.Summary)) + case casePartialSource: + return fmt.Sprintf("%d source file(s) in the dependencies that were examined could not be read, "+ + "so the cryptography of those dependencies may be under-reported. Each is named on stderr "+ + "during the scan with the reason, and in JSON under the dependency's "+ + "analysis.unreadableFiles alongside filesAnalyzed and filesUnreadable.", + n.Summary.SourceFilesUnreadable) case caseNothingExamined: - return fmt.Sprintf("None of the %d dependencies are present in the crypto database, so no "+ - "conclusion about cryptographic usage was drawn. An empty result set here means "+ - "nothing was examined, not that nothing was found.", n.Summary.TotalDependencies) + if n.Summary.DeepAttempted { + return fmt.Sprintf("None of the %d dependencies could be examined: they are absent from the "+ + "crypto database, and source analysis could not read any of them. An empty result set "+ + "here means nothing was examined, not that nothing was found.", n.Summary.TotalDependencies) + } + return fmt.Sprintf("None of the %d dependencies are present in the crypto database and source "+ + "analysis was not run, so no conclusion about cryptographic usage was drawn. An empty "+ + "result set here means nothing was examined, not that nothing was found.", + n.Summary.TotalDependencies) default: return "" } diff --git a/pkg/output/verdict_test.go b/pkg/output/verdict_test.go index c603706..1ccc46c 100644 --- a/pkg/output/verdict_test.go +++ b/pkg/output/verdict_test.go @@ -41,7 +41,8 @@ func TestAllUnknownIsNotReportedAsClean(t *testing.T) { {Dependency: types.Dependency{Name: "requests"}}, {Dependency: types.Dependency{Name: "certifi"}}, }, - Summary: types.ScanSummary{TotalDependencies: 3, DirectDependencies: 3, NotInDatabase: 3}, + Summary: types.ScanSummary{TotalDependencies: 3, DirectDependencies: 3, NotInDatabase: 3, + NotExamined: 3}, } out := renderTable(t, result) @@ -227,3 +228,177 @@ func TestRiskIconIsASCIIAndUnique(t *testing.T) { seen[token] = risk } } + +// deepAnalyzedCleanScan is a scan where the database carried none of the +// dependencies and source analysis read all of them, finding no cryptography. +// +// This is the ordinary outcome of `analyze --deep` on a project whose +// dependencies are not cryptographic, and the state the verdict got wrong. +func deepAnalyzedCleanScan() *types.ScanResult { + return &types.ScanResult{ + Project: "/repo", + Manifest: "/repo/package.json", + Ecosystem: types.EcosystemNPM, + Dependencies: []types.DependencyResult{ + {Dependency: types.Dependency{Name: "chalk", Version: "5.3.0"}, DeepAnalyzed: true, + Analysis: &types.PackageAnalysis{Package: "chalk"}}, + {Dependency: types.Dependency{Name: "commander", Version: "12.1.0"}, DeepAnalyzed: true, + Analysis: &types.PackageAnalysis{Package: "commander"}}, + {Dependency: types.Dependency{Name: "lodash", Version: "4.17.21"}, DeepAnalyzed: true, + Analysis: &types.PackageAnalysis{Package: "lodash"}}, + }, + Summary: types.ScanSummary{ + TotalDependencies: 3, DirectDependencies: 3, + // Absent from the database, and examined anyway. + NotInDatabase: 3, NotExamined: 0, DeepAttempted: true, + }, + } +} + +// TestDeepAnalyzedCleanScanIsReportedAsClean is the regression test for the +// defect that held the v1.3.0 tag. +// +// `analyze --deep` over three dependencies absent from the database +// printed "Not analyzed. All 3 dependencies are absent from the crypto +// database, so no conclusion about cryptographic usage can be drawn from this +// scan. Run with --deep to analyze package source code directly." Every clause +// of that was false for the run that produced it, the JSON for the same +// invocation reported deepAnalyzed on all three, and the released 1.2.2 got the +// case right. The classifier was asking a question about the database lookup in +// order to answer a question about examination. +func TestDeepAnalyzedCleanScanIsReportedAsClean(t *testing.T) { + result := deepAnalyzedCleanScan() + + // Guard the fixture. If the database count and the examination count agree, + // this fixture cannot tell the two predicates apart and the test proves + // nothing. + if result.Summary.NotInDatabase == result.Summary.NotExamined { + t.Fatalf("fixture does not separate the database count from the examination count: %+v", + result.Summary) + } + if got := classifyNoFindings(result.Summary); got != caseGenuinelyClean { + t.Fatalf("classifyNoFindings = %v, want caseGenuinelyClean", got) + } + + out := renderTable(t, result) + + if !strings.Contains(out, cleanVerdict) { + t.Errorf("a scan that read every dependency and found no cryptography did not report "+ + "a clean result:\n%s", out) + } + if strings.Contains(out, "no conclusion about cryptographic usage") { + t.Errorf("scan drew a conclusion and then said it could not:\n%s", out) + } + if strings.Contains(out, "--deep") { + t.Errorf("the user ran --deep and the only suggested next step is --deep:\n%s", out) + } + if !strings.Contains(out, "3 of 3") { + t.Errorf("verdict does not say how much of the tree was examined:\n%s", out) + } +} + +// TestDeepAnalyzedCleanScanIsCleanInEveryFormat carries the same case through +// the formats that re-derived the verdict independently once before. +func TestDeepAnalyzedCleanScanIsCleanInEveryFormat(t *testing.T) { + multi := types.AggregateResults("/repo", []*types.ScanResult{deepAnalyzedCleanScan()}) + + if multi.TotalSummary.NotExamined != 0 || !multi.TotalSummary.DeepAttempted { + t.Fatalf("aggregation dropped the coverage fields: %+v", multi.TotalSummary) + } + + for _, format := range []Format{FormatTable, FormatMarkdown, FormatSARIF, FormatCBOM, FormatJSON} { + t.Run(string(format), func(t *testing.T) { + out := renderMulti(t, format, multi) + if out == "" { + t.Fatalf("%s produced no output", format) + } + for _, phrase := range []string{ + "nothing was examined", + "no conclusion about cryptographic usage", + "not in the crypto database. Run with", + } { + if strings.Contains(out, phrase) { + t.Errorf("%s says %q about a scan that examined every dependency:\n%s", + format, phrase, out) + } + } + if format != FormatJSON && strings.Contains(out, "--deep") { + t.Errorf("%s tells a user who ran --deep to run --deep:\n%s", format, out) + } + }) + } +} + +// TestUnfetchablePackagesDoNotSuggestDeepAgain covers the other half of the +// dead end: --deep was given, the fetch failed, nothing was examined. +// +// "Run with --deep" is wrong here for the same reason, and the correct next +// step is the warning the scan already printed. +func TestUnfetchablePackagesDoNotSuggestDeepAgain(t *testing.T) { + result := &types.ScanResult{ + Manifest: "/repo/package.json", + Ecosystem: types.EcosystemNPM, + Dependencies: []types.DependencyResult{ + {Dependency: types.Dependency{Name: "left-pad"}}, + {Dependency: types.Dependency{Name: "is-odd"}}, + }, + Summary: types.ScanSummary{ + TotalDependencies: 2, DirectDependencies: 2, + NotInDatabase: 2, NotExamined: 2, DeepAttempted: true, + }, + } + + out := renderTable(t, result) + + if strings.Contains(out, cleanVerdict) { + t.Errorf("scan that examined nothing reported a clean result:\n%s", out) + } + if !strings.Contains(out, "Not analyzed") { + t.Errorf("scan that examined nothing does not say so:\n%s", out) + } + if strings.Contains(out, "--deep") { + t.Errorf("the user ran --deep and the tool suggests running --deep:\n%s", out) + } + if !strings.Contains(out, "warnings") { + t.Errorf("verdict gives no next step for packages that could not be read:\n%s", out) + } +} + +// TestPartiallyExaminedScanSaysWhatItMissed is the inverse question. +// +// Narrowing the trigger for "nothing was examined" moves a project that is +// partly examined into the clean case. The clean case must then account for the +// dependencies nothing reached, or the narrowing has traded a false alarm for a +// silence. +func TestPartiallyExaminedScanSaysWhatItMissed(t *testing.T) { + result := &types.ScanResult{ + Manifest: "/repo/package.json", + Ecosystem: types.EcosystemNPM, + Dependencies: []types.DependencyResult{ + {Dependency: types.Dependency{Name: "chalk"}, InDatabase: true, + Analysis: &types.PackageAnalysis{Package: "chalk"}}, + {Dependency: types.Dependency{Name: "left-pad"}}, + {Dependency: types.Dependency{Name: "is-odd"}}, + }, + Summary: types.ScanSummary{ + TotalDependencies: 3, DirectDependencies: 3, + NotInDatabase: 2, NotExamined: 2, + }, + } + + if got := classifyNoFindings(result.Summary); got != caseGenuinelyClean { + t.Fatalf("fixture does not land in the clean case: %v", got) + } + + out := renderTable(t, result) + + if !strings.Contains(out, "1 of 3") { + t.Errorf("clean verdict does not say how much of the tree it covers:\n%s", out) + } + if !strings.Contains(out, "2 could not be examined") { + t.Errorf("clean verdict does not account for the dependencies nothing reached:\n%s", out) + } + if !strings.Contains(out, "--deep") { + t.Errorf("no next step offered for the unexamined dependencies:\n%s", out) + } +} diff --git a/pkg/types/types.go b/pkg/types/types.go index 2ece379..f76030c 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -118,6 +118,43 @@ type AnalysisMetadata struct { ToolVersion string `json:"toolVersion" yaml:"toolVersion"` Contributor string `json:"contributor,omitempty" yaml:"contributor,omitempty"` SourceHash string `json:"sourceHash,omitempty" yaml:"sourceHash,omitempty"` + // FilesAnalyzed is how many source files an analyzer actually parsed. It is + // the evidence behind every claim this tool makes about having examined a + // package by reading it, and it is reported so that the claim can be + // audited rather than taken on trust. Zero for a database record, which was + // not produced by reading this package here. + FilesAnalyzed int `json:"filesAnalyzed,omitempty" yaml:"filesAnalyzed,omitempty"` + // FilesUnreadable is how many files carried one of this analyzer's own + // extensions and could not be parsed. + // + // It is reported for the same reason FilesAnalyzed is. A package that was + // partly read is not the same as one that was read, and the difference was + // invisible: the count existed inside the walk and stopped there, so a + // dependency whose only cryptography sat in a file the analyzer refused was + // reported as examined and clean, with nothing on any stream saying a file + // had been skipped. Examination is a claim, and a claim needs its exceptions + // stated as well as its evidence. + FilesUnreadable int `json:"filesUnreadable,omitempty" yaml:"filesUnreadable,omitempty"` + // UnreadableFiles says which files those were and why, bounded so that a + // hostile archive cannot turn one warning into thousands. A count with no + // names is a dead end: the reader is told the reading was incomplete and + // given no way to find out where. + UnreadableFiles []string `json:"unreadableFiles,omitempty" yaml:"unreadableFiles,omitempty"` +} + +// SourceWasRead reports whether source analysis parsed at least one file. +// +// This is the question "was this package examined by reading it", and it has to +// be asked of the files that were read rather than of an error that was not +// returned. A fetch can succeed and yield nothing an analyzer can parse: a +// Maven artifact whose sources JAR does not exist falls back to the main JAR, +// which holds compiled classes only, so the walker sees no .java file and +// returns no usages and no error. Reading that as a completed examination +// produced "no cryptographic usage detected in the 1 of 1 dependencies that +// were examined" for a scan that had read nothing at all, which is the +// false-clean class this release exists to close. +func (p *PackageAnalysis) SourceWasRead() bool { + return p != nil && p.Analysis.FilesAnalyzed > 0 } // QuantumSummary summarizes the quantum risk of a package. @@ -141,11 +178,33 @@ type PackageAnalysis struct { // DependencyResult represents the analysis result for a dependency. type DependencyResult struct { - Dependency Dependency `json:"dependency" yaml:"dependency"` - Analysis *PackageAnalysis `json:"analysis,omitempty" yaml:"analysis,omitempty"` - InDatabase bool `json:"inDatabase" yaml:"inDatabase"` - DeepAnalyzed bool `json:"deepAnalyzed,omitempty" yaml:"deepAnalyzed,omitempty"` - Error string `json:"error,omitempty" yaml:"error,omitempty"` + Dependency Dependency `json:"dependency" yaml:"dependency"` + Analysis *PackageAnalysis `json:"analysis,omitempty" yaml:"analysis,omitempty"` + InDatabase bool `json:"inDatabase" yaml:"inDatabase"` + // DeepAnalyzed records that source analysis read this package. It must be + // set from what an analyzer parsed, never from an on-demand call that + // merely returned no error: see PackageAnalysis.SourceWasRead. + DeepAnalyzed bool `json:"deepAnalyzed,omitempty" yaml:"deepAnalyzed,omitempty"` + // Error says why a dependency was not examined, for the consumers that read + // this document rather than the warnings printed during the scan. A machine + // reading JSON or SARIF could previously see only that a package was absent + // from the results, with no way to tell an unreachable download from an + // archive that carried nothing to read. + Error string `json:"error,omitempty" yaml:"error,omitempty"` +} + +// Examined reports whether this dependency was inspected by any means. +// +// There are two, and reports that asked only about the first were wrong about +// the second. A database lookup answers "do we already know what this package +// contains"; source analysis answers the same question by reading the package. +// A scan that ran source analysis over every dependency and found nothing was +// still described as having examined nothing, because the only question being +// asked was the database one. The distinction belongs here rather than in each +// caller's own condition, so that a third means of examination added later has +// one place to declare itself. +func (d DependencyResult) Examined() bool { + return d.InDatabase || d.DeepAnalyzed } // ScanResult represents the complete result of scanning a project. @@ -167,11 +226,51 @@ type ScanSummary struct { QuantumVulnerable int `json:"quantumVulnerable" yaml:"quantumVulnerable"` QuantumPartial int `json:"quantumPartial" yaml:"quantumPartial"` NotInDatabase int `json:"notInDatabase" yaml:"notInDatabase"` + // NotExamined counts dependencies that no means of examination reached: + // absent from the crypto database, and not read by source analysis either. + // + // NotInDatabase is not a substitute for it. Using the database count to + // answer "was anything examined" was correct only while the database was + // the only way to examine a package, and --deep is a second way. A scan + // that deep-analyzed every dependency reported that none had been examined, + // and told the user to run the flag they had just run. + NotExamined int `json:"notExamined" yaml:"notExamined"` + // DeepAttempted records whether source analysis ran for the packages the + // database did not cover. Without it a report cannot tell "not examined + // because you did not ask for source analysis", where the next step is + // --deep, from "not examined because source analysis could not fetch the + // package", where suggesting --deep is a dead end. + DeepAttempted bool `json:"deepAttempted,omitempty" yaml:"deepAttempted,omitempty"` // FilteredOut counts findings that were detected and then withheld by // --risk or --min-severity. Without it, a filter that matches nothing is // indistinguishable from a project with no cryptography, and the report // would state the second while the first is true. FilteredOut int `json:"filteredOut,omitempty" yaml:"filteredOut,omitempty"` + // WithheldVulnerable, WithheldPartial and WithheldWithCrypto record what the + // reporting filters removed, broken down the way the --fail-on gate asks its + // question. They are deliberately NOT serialized: they exist so that the exit + // code can describe the scan while the report describes the view. + // + // Making --risk and --min-severity actually filter, in this release, had the + // side effect of letting them decide the exit code, because the gate reads the + // summary and the summary counted only survivors. So `--fail-on vulnerable` + // exited 1 on a project and 0 on the same project with `--risk safe` added, + // while stdout said "This is not a clean result". 1.2.2 exited 1 for both, + // because the filters did nothing at all, so this was a regression in the one + // flag that decides CI outcomes, in the release that hardened that flag three + // times against silent loosening. A view flag must not answer a gate. + WithheldVulnerable int `json:"-" yaml:"-"` + WithheldPartial int `json:"-" yaml:"-"` + WithheldWithCrypto int `json:"-" yaml:"-"` + // SourceFilesUnreadable counts source files that source analysis could not + // parse across the dependencies it did examine. + // + // NotExamined cannot carry this: a dependency with one readable file and one + // unreadable one WAS examined, so it is absent from that count, and the + // report then described a partial reading as a complete one. This is the + // partial state of the same question, and the coverage note has to be asked + // of it rather than only of the empty one. + SourceFilesUnreadable int `json:"sourceFilesUnreadable,omitempty" yaml:"sourceFilesUnreadable,omitempty"` // Reachability stats (only populated when reachability analysis is enabled) ReachabilityAnalyzed bool `json:"reachabilityAnalyzed,omitempty" yaml:"reachabilityAnalyzed,omitempty"` ConfirmedCrypto int `json:"confirmedCrypto,omitempty" yaml:"confirmedCrypto,omitempty"` // Direct calls from user code @@ -238,10 +337,23 @@ func AggregateResults(rootPath string, results []*ScanResult) *MultiProjectResul multi.TotalSummary.QuantumVulnerable += r.Summary.QuantumVulnerable multi.TotalSummary.QuantumPartial += r.Summary.QuantumPartial multi.TotalSummary.NotInDatabase += r.Summary.NotInDatabase + // The aggregate answers the same coverage question as each project's + // own summary and must not answer it from a different field. + multi.TotalSummary.NotExamined += r.Summary.NotExamined + if r.Summary.DeepAttempted { + multi.TotalSummary.DeepAttempted = true + } // Without this the aggregate reported zero withheld findings while the // per-project summaries reported dozens, so the totals a reader // actually looks at described a filtered scan as a complete one. multi.TotalSummary.FilteredOut += r.Summary.FilteredOut + // The withheld breakdown has to aggregate for the same reason the count + // does: otherwise a workspace scan's gate reads zero withheld findings + // while its projects withheld many, and the filter defeats --fail-on at + // the aggregate level even once it is held at the project level. + multi.TotalSummary.WithheldVulnerable += r.Summary.WithheldVulnerable + multi.TotalSummary.WithheldPartial += r.Summary.WithheldPartial + multi.TotalSummary.WithheldWithCrypto += r.Summary.WithheldWithCrypto multi.TotalSummary.ConfirmedCrypto += r.Summary.ConfirmedCrypto multi.TotalSummary.ReachableCrypto += r.Summary.ReachableCrypto multi.TotalSummary.AvailableCrypto += r.Summary.AvailableCrypto diff --git a/pkg/types/types_test.go b/pkg/types/types_test.go index cf9c6ab..3b53ba3 100644 --- a/pkg/types/types_test.go +++ b/pkg/types/types_test.go @@ -375,3 +375,52 @@ func TestAggregateReportsNoFilterWhenNoneWasApplied(t *testing.T) { multi.TotalSummary.FilteredOut) } } + +// TestExaminedCountsBothMeansOfExamination pins the distinction the summary is +// built from. A database hit and a successful source read are two answers to +// one question, and asking only the first is what made a --deep scan report +// that it had examined nothing. +func TestExaminedCountsBothMeansOfExamination(t *testing.T) { + tests := []struct { + name string + dep DependencyResult + want bool + }{ + {"in the database", DependencyResult{InDatabase: true}, true}, + {"read by source analysis", DependencyResult{DeepAnalyzed: true}, true}, + {"both", DependencyResult{InDatabase: true, DeepAnalyzed: true}, true}, + {"neither", DependencyResult{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.dep.Examined(); got != tt.want { + t.Errorf("Examined() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestAggregateCarriesCoverageFields guards the aggregate against answering the +// coverage question from a different field than the per-project summaries do. +// The FilteredOut field was dropped here once already, and the totals are what +// a reader actually looks at. +func TestAggregateCarriesCoverageFields(t *testing.T) { + examined := &ScanResult{Summary: ScanSummary{ + TotalDependencies: 2, NotInDatabase: 2, NotExamined: 0, DeepAttempted: true, + }} + unexamined := &ScanResult{Summary: ScanSummary{ + TotalDependencies: 3, NotInDatabase: 3, NotExamined: 3, + }} + + multi := AggregateResults("/repo", []*ScanResult{examined, unexamined}) + + if multi.TotalSummary.NotExamined != 3 { + t.Errorf("TotalSummary.NotExamined = %d, want 3", multi.TotalSummary.NotExamined) + } + if !multi.TotalSummary.DeepAttempted { + t.Error("TotalSummary.DeepAttempted = false, want true; one project ran source analysis") + } + if multi.TotalSummary.NotInDatabase != 5 { + t.Errorf("TotalSummary.NotInDatabase = %d, want 5", multi.TotalSummary.NotInDatabase) + } +}