From 5019bd1099f73443d682cb60ebcc25b0445b9382 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 26 Oct 2026 02:18:40 +0530 Subject: [PATCH 1/2] feat(imports): support OpenGrep reports locally and in Cloud --- docs/scanner-imports/opengrep.md | 146 ++++++ .../skills/publish-findings/SKILL.md | 6 +- sdk/typescript/README.md | 6 + sdk/typescript/docs/cli.md | 10 + .../scripts/fixtures/package-consumer.ts | 5 +- sdk/typescript/scripts/package-dist-files.mjs | 2 + sdk/typescript/scripts/smoke-package.mjs | 2 +- sdk/typescript/src/cli.ts | 28 +- sdk/typescript/src/external-findings-local.ts | 8 +- .../src/external-findings-publish.ts | 50 +- sdk/typescript/src/external-findings-read.ts | 14 + sdk/typescript/src/external-findings.ts | 2 + .../src/external-import-contract.ts | 57 +- sdk/typescript/src/index.ts | 5 + sdk/typescript/src/native-json-findings.ts | 163 ++++++ sdk/typescript/src/opengrep-findings.ts | 249 +++++++++ sdk/typescript/src/record.ts | 16 + sdk/typescript/src/semgrep-findings.ts | 126 +---- .../tests-ts/cli-local-import.test.ts | 66 +++ .../tests-ts/external-findings-local.test.ts | 20 + .../external-findings-publish.test.ts | 246 ++++++++- .../tests-ts/opengrep-findings-local.test.ts | 149 ++++++ .../tests-ts/opengrep-findings.test.ts | 494 ++++++++++++++++++ .../support/opengrep-import-fixtures.ts | 72 +++ 24 files changed, 1792 insertions(+), 150 deletions(-) create mode 100644 docs/scanner-imports/opengrep.md create mode 100644 sdk/typescript/src/native-json-findings.ts create mode 100644 sdk/typescript/src/opengrep-findings.ts create mode 100644 sdk/typescript/tests-ts/opengrep-findings-local.test.ts create mode 100644 sdk/typescript/tests-ts/opengrep-findings.test.ts create mode 100644 sdk/typescript/tests-ts/support/opengrep-import-fixtures.ts diff --git a/docs/scanner-imports/opengrep.md b/docs/scanner-imports/opengrep.md new file mode 100644 index 0000000000..4270c17525 --- /dev/null +++ b/docs/scanner-imports/opengrep.md @@ -0,0 +1,146 @@ +# Import OpenGrep findings + +Set `CODEX_SECURITY_ENABLE_EXTERNAL_IMPORTS=1` in the command environment to enable external imports. + +Import native OpenGrep JSON, including gzip-compressed reports, into the local +workbench or Cloud. This reader accepts scanner output, not SARIF, normalized +JSONL, or an API summary. It preserves findings as unassessed scanner evidence. + +From the source repository root, use a fresh report directory outside the +checkout. The following commands use the shell's temporary directory: + +```sh +report_dir=$(mktemp -d) && +opengrep scan --config /path/to/rules.yaml --json --output "$report_dir/opengrep.json" . && +codex-security publish findings "$report_dir/opengrep.json" --to local \ + --provider opengrep --source-key opengrep/ORGANIZATION/PROJECT/BRANCH \ + --source-repository-url https://github.com/example/project \ + --repository . --dry-run --format json +``` + +Native JSON includes dataflow traces when the scanner produces them. +`--dataflow-traces` controls text and SARIF rendering; it is not required for JSON. +See OpenGrep's [output options](https://github.com/opengrep/opengrep/blob/main/src/osemgrep/cli_scan/Scan_CLI.ml) +and [JSON formatter](https://github.com/opengrep/opengrep/blob/main/src/osemgrep/reporting/Cli_json_output.ml). + +Review the source repository, findings, exclusions, diagnostics, and destination. +Repeat the import without `--dry-run` to confirm interactively, or add `--yes` +after reviewing the preview. Local imports require no Cloud login or model call. +Read the stored evidence with: + +```sh +codex-security findings imported . --format json +``` + +For Cloud, use the same report and source namespace with a Cloud destination: + +```sh +codex-security publish findings "$report_dir/opengrep.json" --to cloud \ + --provider opengrep --source-key opengrep/ORGANIZATION/PROJECT/BRANCH \ + --source-repository-url https://github.com/example/project \ + --repository https://github.com/example/project --dry-run --format json +``` + +`--source-repository-url` is required and declares where the report came from. +The importer checks it against configured local Git remotes or the selected +Cloud repository. It does not infer the scanner's revision from the current +checkout. Cloud preview requires the existing ChatGPT login and an authorized +repository and environment; review the preview before approving the upload. + +## Provider identity and evidence + +`--provider opengrep` selects the OpenGrep reader. Local storage and Cloud use the +existing `semgrep` source-provider contract for compatibility, while typed +`scanner_origin` identifies **OpenGrep**. The preview shows this mapping. This +does not add a native OpenGrep provider to the Cloud service. Cloud publication +requires that deployment to support the existing Semgrep import contract; +successful local imports do not verify hosted availability. + +Keep an OpenGrep-specific source key, such as +`opengrep/ORGANIZATION/PROJECT/BRANCH`, stable across runs. Occurrence IDs use the +rule, path, and source range. The importer normalizes redundant slash-separated +components in repository-relative paths and preserves literal backslashes in +filenames. An unmapped path keeps its original spelling in the occurrence ID. +Keep Windows path spellings consistent across runs because changing them can +create a second record. Multiple dataflow traces for the same occurrence are +grouped into one finding. For Cloud imports, check `excluded` for traces that +could not fit. If independently valid traces exceed the 256 KiB limit together, +the whole occurrence is excluded. Moving a match creates a new record and leaves +its earlier record intact. The raw scanner +fingerprints remain evidence and do not override this location-based identity. +Omitting a finding from a later report does not close or delete it. + +The adapter retains rule metadata, source ranges, snippets, traces, ignored state, +and original severity. Normalized severity is `unknown`: OpenGrep rule levels +such as `ERROR` or `WARNING` are not a verified vulnerability severity. Ignored +matches remain source evidence, separate from a Codex assessment. Importing a +report does not assess findings, apply fixes, or create a completed Codex scan. + +## Diagnostics and coverage + +OpenGrep can return matches alongside parse failures, partial parsing, timeouts, +and other diagnostics. Warning diagnostics can accompany exit code 0 with the +default `--no-strict`; `--strict` changes warning handling, while `--error` +separately makes findings produce exit code 1. Inspect the report's diagnostics +and the scanner's exit status. Neither zero findings nor a successful exit +establishes complete coverage. See OpenGrep's +[error classification](https://github.com/opengrep/opengrep/blob/main/src/core/Core_error.ml). + +The JSON preview and result retain the native report envelope, without `results`, +in `report`, including diagnostics and path metadata for an empty local report. +Individual stored findings retain the original observations in +`source_data.results` and a `source_data.scan_error_count`. Their compact +`source_data.report` retains only the scanner version, keeping scan-wide metadata +such as diagnostics and matching explanations outside each finding's evidence +budget. Full report metadata is +returned by the SDK and CLI; it is not saved as a separate durable scan record. +Keep the original JSON if you need the complete run metadata later. +Add OpenGrep's `--verbose` when collecting +skipped-target details; its [output handling](https://github.com/opengrep/opengrep/blob/main/src/osemgrep/reporting/Output.ml) +can omit those details otherwise. + +Publication status describes import processing and acknowledged records, not +scanner coverage. A `complete` publication does not resolve scan diagnostics or +establish a clean security result. Cloud publication requires at least one +supported finding; an empty report can still be previewed locally with its +metadata. Review exclusions before approving either destination. + +## Assess an imported finding locally + +Copy an imported record's `id` from `findings imported --format json`. Use an +authorized checkout corresponding to the report's revision, then pass its +evidence to the existing SDK validator: + +```ts +import { + CodexSecurity, + listLocalExternalFindings, +} from "@openai/codex-security"; + +const repositoryPath = "/path/to/repository"; +const [record] = await listLocalExternalFindings({ + repository: repositoryPath, + id: "IMPORTED_FINDING_ID", +}); +if (!record) throw new Error("Imported finding not found."); + +const security = new CodexSecurity(); +try { + const assessment = await security.validate({ + repositoryPath, + finding: record.evidence, + }); + console.log(record.id, assessment.disposition); + console.log(assessment.report); +} finally { + await security.close(); +} +``` + +Validation sends the finding evidence, including snippets and traces, to the +configured Codex model provider. A hosted provider requires network connectivity. +The validator reads the checkout in a read-only tool sandbox with network access +disabled. It returns an assessment and leaves the stored import `not_assessed`. See +[standalone validation](../../sdk/typescript/README.md#validate-an-existing-finding) +for dispositions and report handling. Hosts that already provide model execution +can use [validateRecords](../../sdk/typescript/docs/validate-records.md) instead. diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index 12b476a4d2..2d8d094dfe 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -1,6 +1,6 @@ --- name: publish-findings -description: Disabled by default; requires user opt-in with CODEX_SECURITY_ENABLE_EXTERNAL_IMPORTS=1. Import selected Snyk Open Source findings, Semgrep Code CLI/CI reports, Endor Labs findings, GitHub code-scanning alerts or GHSA advisories, or Wiz package vulnerabilities, SAST, repository secrets, and IaC findings locally or to Codex Security Cloud after previewing the destination and evidence. Prefers native JSON; also supports Snyk plugin SCA bundles, Snyk Open Source and Semgrep Code SARIF, endorctl SARIF, saved GitHub JSON/JSONL, Wiz normalized JSONL and repository package-vulnerability CSV, and gzip; does not import workload secrets or cloud configuration, run a scan, assess findings, or synchronize triage decisions. +description: Requires explicit opt-in with CODEX_SECURITY_ENABLE_EXTERNAL_IMPORTS=1. Preview and import supported findings from Snyk Open Source, Semgrep Code, OpenGrep, Endor Labs, GitHub, or Wiz into the local workbench or Codex Security Cloud. Use each provider's documented native JSON, SARIF, plugin bundle, JSONL or CSV format, including gzip-compressed reports. --- # Publish external findings @@ -80,6 +80,10 @@ Import the native report with `--provider semgrep`, a stable organization/projec Keep supplied fingerprints. Reimporting the same fingerprint after code moves updates the existing source record with the new range. If the fingerprint is absent or `requires login`, the fallback uses rule, literal path and range: movement creates a second retained record, and the earlier record remains stored. Do not infer a merge or deletion. Read the persisted evidence with `codex-security findings imported /path/to/project --format json` before assessment. Source rule, CWE, snippets, traces and status are vendor evidence, not Codex assessments. Do not infer a report revision from the current checkout. Resolve scan errors and review all exclusions before uploading; do not remove error metadata to force acceptance. +## OpenGrep + +Import native OpenGrep JSON with `--provider opengrep`, `--source-repository-url`, and a stable OpenGrep-specific `--source-key`. Preview the destination and scan diagnostics before importing. Local and Cloud imports use the Semgrep provider and retain OpenGrep as the scanner origin. + ## GitHub alerts and advisories Acquire code-scanning findings with `codex-security import github OWNER/REPO --format json` and save its complete JSON output for local import or Cloud publication. This existing collector completes pagination, fetches alert details, and retains the selected ref's instance. In Codex, use the official [GitHub Codex plugin](https://github.com/openai/plugins/tree/main/plugins/github) for discovery and selecting alerts, then pass that selection to the collector. Its GitHub MCP server paginates each list call separately. Do not ask the user to download JSON from a website or reconstruct an import envelope from MCP summaries; raw MCP/API responses are not accepted by `--provider github`. diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index a0d6b59aab..f61e611cd4 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -667,6 +667,12 @@ options where supported, and changing formats can change record identities or replace richer evidence with less detail. See the [import guide](docs/cli.md#import-vendor-findings-locally) for format selection. +For OpenGrep, use `--provider opengrep` with native JSON or gzip and an explicit +`--source-repository-url`. Local and Cloud imports retain unassessed OpenGrep +evidence through the existing Semgrep source-provider contract. See the +[OpenGrep guide](https://github.com/openai/codex-security/blob/main/docs/scanner-imports/opengrep.md) for preview commands, +coverage diagnostics, stable source identities, and local SDK assessment. + For Wiz, prefer supported native JSON acquired through an authorized MCP or API connection. The [CSV compatibility reader](docs/cli.md#import-wiz-findings) covers repository package vulnerabilities (SCA) only; it does not add SAST, CSPM or diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 5bf692c18e..f764085d53 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -1303,6 +1303,7 @@ repository path instead of a Cloud repository ID or URL. Supported inputs includ Wiz findings, [Snyk Open Source JSON and SARIF](#import-snyk-open-source-findings), [Endor Labs Finding JSON and SARIF](#import-endor-labs-findings), [Semgrep Code JSON and SARIF](https://github.com/openai/codex-security/blob/main/docs/scanner-imports/semgrep.md), +[OpenGrep native JSON](https://github.com/openai/codex-security/blob/main/docs/scanner-imports/opengrep.md), and [saved GitHub alerts and GHSA advisories](#import-github-findings-locally-or-to-cloud): ```bash @@ -1338,6 +1339,15 @@ remain stored; omission does not resolve or delete a finding. Excluded records are reported, and an import with exclusions exits with status 1. The same supported vendor formats and evidence requirements apply locally and in Cloud. +For OpenGrep, use `--provider opengrep` and supply `--source-repository-url`. +The reader supports native JSON and gzip for both destinations. It uses the +existing `semgrep` source-provider contract while retaining `OpenGrep` as the +scanner origin; keep a stable OpenGrep-specific source key. Review report +diagnostics as well as findings: import completion does not establish scan +coverage. The [OpenGrep guide](https://github.com/openai/codex-security/blob/main/docs/scanner-imports/opengrep.md) +covers evidence identity, empty reports, and assessing an imported finding with +the existing SDK validator. + `scan import --json` and `scan import --csv` remain separate commands for importing Codex Security's findings schema as a saved scan; they do not accept vendor exports. diff --git a/sdk/typescript/scripts/fixtures/package-consumer.ts b/sdk/typescript/scripts/fixtures/package-consumer.ts index 01257f0baa..dd0f46ddaa 100644 --- a/sdk/typescript/scripts/fixtures/package-consumer.ts +++ b/sdk/typescript/scripts/fixtures/package-consumer.ts @@ -306,7 +306,10 @@ export async function externalPublication( const fromFile = await prepareExternalPublication(file, options); fromFile.preview.requests[0]?.source.provider satisfies "wiz" | "endor" | "snyk" | "github" | "semgrep" | undefined; - const prepared = await prepareNormalizedExternalPublication(input, options); + const prepared = await prepareNormalizedExternalPublication(input, { + ...options, + provider: fromFile.preview.source.provider, + }); return prepared.publish(); } diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs index 02a7682e33..847d4010f7 100644 --- a/sdk/typescript/scripts/package-dist-files.mjs +++ b/sdk/typescript/scripts/package-dist-files.mjs @@ -30,6 +30,8 @@ const distModules = [ "scanner-import", "snyk-findings", "semgrep-findings", + "native-json-findings", + "opengrep-findings", "external-findings-publish", "external-import-contract", "external-import-models", diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjs index 9ffd9dd226..421480b87a 100644 --- a/sdk/typescript/scripts/smoke-package.mjs +++ b/sdk/typescript/scripts/smoke-package.mjs @@ -396,7 +396,7 @@ try { "--input-type=module", "--eval", `const sdk = await import(${JSON.stringify(packageManifest.name)}); - for (const name of ["CodexSecurity", "publishScan", "publishScanToCustom", "checkScanPublication", "deduplicateScan", "deduplicateRecords", "classifySeverity", "classifyScanSeverity", "classifyScanDirectorySeverity", "matchScanFindings", "securityPolicyDiff", "loadProjectConfig", "resolveProjectConfig", "prepareExternalPublication", "prepareNormalizedExternalPublication"]) { + for (const name of ["CodexSecurity", "publishScan", "publishScanToCustom", "checkScanPublication", "deduplicateScan", "deduplicateRecords", "classifySeverity", "classifyScanSeverity", "classifyScanDirectorySeverity", "matchScanFindings", "securityPolicyDiff", "loadProjectConfig", "resolveProjectConfig", "prepareExternalPublication", "prepareNormalizedExternalPublication", "normalizeOpenGrepFindings", "readOpenGrepFindings"]) { if (typeof sdk[name] !== "function") { throw new Error("The installed package does not export " + name + "."); } diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index dcc891b306..12ff041e4c 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -3544,15 +3544,15 @@ export async function main( function registerFindingPublication() { publication.command("findings", { description: - "Preview and import selected Wiz, Snyk, Semgrep Code, Endor, or GitHub findings locally or to Cloud.", - hint: "For Snyk Open Source SCA vulnerabilities only, prefer native snyk test JSON or an official plugin snyk-findings-export/v1 bundle; Open Source SARIF 2.1.0 is also supported. Snyk licenses, Code, Container, IaC and platform backlogs are unsupported. For Semgrep Code, supply native JSON or SARIF and --source-repository-url. For Endor, supply native Finding JSON or endorctl SARIF 2.1.0. For GitHub, supply complete JSON/JSONL saved by import github or import github-advisories. GHSA advisories do not establish an installed dependency or applicability. For Wiz, prefer a native package vulnerability JSON report or named SAST/repository-secret/IaC JSON collections with repository metadata. Normalized JSONL and repository package-vulnerability CSV (SCA only, GitHub repositories) are also supported. Gzip is supported for each format; Wiz CSV does not cover SAST or cloud posture. Other Wiz CSV profiles, scan events, workload secrets, and cloud configuration exports are not supported. --to local stores unassessed evidence for a local repository path without Cloud login. --full-output requires --format json or --format jsonl.\nExample: codex-security publish findings selected-wiz.json --to local --repository ./project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", + "Preview and import selected Wiz, Snyk, Semgrep Code, OpenGrep, Endor, or GitHub findings locally or to Cloud.", + hint: "For Snyk Open Source SCA vulnerabilities only, prefer native snyk test JSON or an official plugin snyk-findings-export/v1 bundle; Open Source SARIF 2.1.0 is also supported. Snyk licenses, Code, Container, IaC and platform backlogs are unsupported. For Semgrep Code, supply native JSON or SARIF and --source-repository-url. For OpenGrep, supply native JSON and --source-repository-url; imports use the semgrep storage provider and preserve OpenGrep as the scanner origin. For Endor, supply native Finding JSON or endorctl SARIF 2.1.0. For GitHub, supply complete JSON/JSONL saved by import github or import github-advisories. GHSA advisories do not establish an installed dependency or applicability. For Wiz, prefer a native package vulnerability JSON report or named SAST/repository-secret/IaC JSON collections with repository metadata. Normalized JSONL and repository package-vulnerability CSV (SCA only, GitHub repositories) are also supported. Gzip is supported for each format; Wiz CSV does not cover SAST or cloud posture. Other Wiz CSV profiles, scan events, workload secrets, and cloud configuration exports are not supported. --to local stores unassessed evidence for a local repository path without Cloud login. --full-output requires --format json or --format jsonl.\nExample: codex-security publish findings selected-wiz.json --to local --repository ./project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", destructive: true, mcp: false, args: z.object({ file: z .string() .describe( - "Snyk native test JSON, plugin SCA bundle or Open Source SARIF; Semgrep Code JSON/SARIF; Endor Finding JSON/endorctl SARIF; saved GitHub import output; native Wiz JSON, normalized JSONL, or repository package-vulnerability CSV.", + "Snyk native test JSON, plugin SCA bundle or Open Source SARIF; Semgrep Code JSON/SARIF; OpenGrep native JSON; Endor Finding JSON/endorctl SARIF; saved GitHub import output; native Wiz JSON, normalized JSONL, or repository package-vulnerability CSV.", ), }), options: z.object({ @@ -3564,11 +3564,13 @@ export async function main( ), provider: z .enum(EXTERNAL_FINDINGS_READERS) - .describe("Provider that produced these findings."), + .describe( + "Report reader; OpenGrep uses the semgrep storage provider.", + ), sourceRepositoryUrl: optionValue("--source-repository-url") .optional() .describe( - "Required for Semgrep: source repository URL, checked against the Cloud destination or configured local Git remotes.", + "Required for Semgrep and OpenGrep: source repository URL, checked against the Cloud destination or configured local Git remotes.", ), sourceKey: optionValue("--source-key").describe( "Stable vendor tenant and finding namespace; not a project selection filter.", @@ -3632,6 +3634,20 @@ export async function main( controller.signal, ); }; + const showScannerReport = (preview: ExternalFindings) => { + if (options.provider !== "opengrep") return; + writeDiagnostic("Reader: OpenGrep (stored provider: semgrep)\n"); + const diagnostics = preview.report?.["errors"]; + if (Array.isArray(diagnostics) && diagnostics.length) { + writeDiagnostic( + `Scanner diagnostics: ${diagnostics.length}. Scan coverage may be incomplete.\n`, + ); + for (const diagnostic of diagnostics) + writeDiagnostic( + `${diagnosticValue(JSON.stringify(diagnostic))}\n`, + ); + } + }; let lastProgress = ""; const onProgress = (event: ExternalPublicationProgress) => { if (errorOutput.isTTY !== true) return; @@ -3676,6 +3692,7 @@ export async function main( ); const { preview } = prepared; const showPreview = () => { + showScannerReport(preview); writeDiagnostic( `Local repository: ${diagnosticValue(preview.repository)}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}\n`, ); @@ -3720,6 +3737,7 @@ export async function main( ); const { preview } = prepared; const showPreview = () => { + showScannerReport(preview); writeDiagnostic( `Cloud API: ${diagnosticValue(preview.cloudApiUrl)}\nAccount: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${[...new Set(preview.requests.map((request) => request.repository.environment_id))].map(diagnosticValue).join(", ")}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, ); diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.ts index 8fefd5fc88..d56c7e8a23 100644 --- a/sdk/typescript/src/external-findings-local.ts +++ b/sdk/typescript/src/external-findings-local.ts @@ -14,6 +14,7 @@ import type { import { cloudRepositoryUrlKey, repositoryUrlKey, + validateExternalReport, } from "./external-import-contract.js"; import type { ExternalFindingEvidence, @@ -78,6 +79,7 @@ export interface LocalExternalImportPreview extends ExternalFindings { export interface LocalExternalImportResult { repository: string; read: number; + report?: ExternalFindings["report"]; ready: number; excluded: ExternalFindings["excluded"]; findings: StoredExternalFinding[]; @@ -489,6 +491,7 @@ export async function prepareNormalizedLocalImport( requireExternalImports(dependencies.environment); requireIdentity(options.sourceKey, "Source key"); const current = structuredClone(input); + validateExternalReport(current.report); const identities = new Set(); for (const finding of current.findings) { requireIdentity(finding.source_finding_id, "Source finding id"); @@ -584,8 +587,11 @@ export async function prepareNormalizedLocalImport( return { repository, read: current.read, + ...(current.report === undefined + ? {} + : { report: structuredClone(current.report) }), ready: current.findings.length, - excluded: current.excluded, + excluded: structuredClone(current.excluded), findings: result["findings"] as unknown as StoredExternalFinding[], counts: result["counts"] as LocalExternalImportResult["counts"], }; diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 37cc62ae86..8b36f25c41 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -30,6 +30,7 @@ import { readExternalFindings, type ExternalFindingsReaderOptions, } from "./external-findings-read.js"; +import { canonicalJson } from "./record.js"; import { wizCsvRepositoryUrl } from "./wiz-csv.js"; import { snykRepositoryUrls } from "./snyk-findings.js"; import { endorRepositoryUrls } from "./endor-findings.js"; @@ -41,6 +42,7 @@ import { cloudRepositoryUrlKey, validateImportRequest, validateExternalEvidence, + validateExternalReport, validateImportReceipt, validateRepositories, validateSourceReports, @@ -98,6 +100,7 @@ export interface ExternalPublicationResult { cloudUrl?: string; status: "complete" | "partial" | "interrupted"; read: number; + report?: ExternalFindings["report"]; ready: number; excluded: ExternalFindings["excluded"]; receipts: FindingImportReceipt[]; @@ -161,23 +164,6 @@ async function readInBatches( return output; } -function canonicalJson( - value: unknown, - compareKeys?: (left: string, right: string) => number, -): string { - return JSON.stringify(value, (_key, child: unknown) => { - if (child !== null && typeof child === "object" && !Array.isArray(child)) { - const record = child as Record; - return Object.fromEntries( - Object.keys(record) - .sort(compareKeys) - .map((key) => [key, record[key]]), - ); - } - return child; - }); -} - function sameSubmission( left: SavedSubmission, right: SavedSubmission, @@ -219,6 +205,15 @@ async function readOptionalFile(path: string): Promise { }); } +function reportForJson(report: ExternalFindings["report"]) { + const errors = report?.["errors"]; + if (!Array.isArray(errors)) return report; + const json = JSON.stringify(errors, (_key, entry: unknown) => + entry instanceof Error ? { message: entry.message } : entry, + ); + return { ...report, errors: JSON.parse(json) as unknown[] }; +} + async function writeAtomicJson(path: string, value: unknown): Promise { const temporary = `${path}.${randomUUID()}.tmp`; try { @@ -378,7 +373,12 @@ export async function prepareNormalizedExternalPublication( ): Promise { requireExternalImports(dependencies.environment); // Validation adds Cloud defaults; leave the caller's objects unchanged. - const current = structuredClone(input); + const { report, ...parsed } = input; + const current: ExternalFindings = structuredClone(parsed); + if (report !== undefined) { + current.report = structuredClone(report); + validateExternalReport(current.report); + } // structuredClone preserves aliases. Detach every raw record before adding // typed defaults, including aliases shared with another finding's evidence. for (const finding of current.findings) @@ -441,8 +441,9 @@ async function preparePublicationInput( legacyIdentity, } = input; if (parsed.findings.length === 0) { + const diagnostics = reportForJson(parsed.report)?.["errors"]; throw new CodexSecurityError( - `No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}`, + `No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}${Array.isArray(diagnostics) && diagnostics.length ? ` Scanner diagnostics: ${JSON.stringify(diagnostics)}` : ""}`, ); } progress({ phase: "discovering", completed: 0 }); @@ -842,6 +843,9 @@ async function preparePublicationInput( }; const preview: ExternalPublicationPreview = { ...parsed, + ...(parsed.report === undefined + ? {} + : { report: structuredClone(parsed.report) }), findings: submissionFindings(requests), cloudApiUrl, cloudUrl: cloudFindingsUrl(environment, destination.id), @@ -883,6 +887,9 @@ async function preparePublicationInput( status: parsed.excluded.length || counts.error ? "partial" : "complete", read: parsed.read, + ...(parsed.report === undefined + ? {} + : { report: structuredClone(parsed.report) }), ready: parsed.findings.length, excluded: parsed.excluded, receipts: acknowledged, @@ -1001,7 +1008,10 @@ async function preparePublicationInput( }); }); const completed = result(acknowledged, verified); - await writeAtomicJson(join(state, `${key}.result.json`), completed); + await writeAtomicJson(join(state, `${key}.result.json`), { + ...completed, + report: reportForJson(completed.report), + }); await rm(pendingPath, { force: true }); return completed; } catch (error) { diff --git a/sdk/typescript/src/external-findings-read.ts b/sdk/typescript/src/external-findings-read.ts index 1115783d5c..d9b43d2fe8 100644 --- a/sdk/typescript/src/external-findings-read.ts +++ b/sdk/typescript/src/external-findings-read.ts @@ -8,6 +8,7 @@ import type { ExternalImportTarget } from "./external-import-contract.js"; import { wizCsvRepositoryUrl } from "./wiz-csv.js"; import { readSnykFindings, snykRepositoryUrls } from "./snyk-findings.js"; import { readSemgrepFindings } from "./semgrep-findings.js"; +import { readOpenGrepFindings } from "./opengrep-findings.js"; import { readEndorFindings, endorRepositoryUrls } from "./endor-findings.js"; import { readGitHubFindings } from "./github-findings.js"; import { isGitHubCloudHost } from "./git-remote.js"; @@ -19,6 +20,8 @@ export const EXTERNAL_FINDINGS_PROVIDERS = { wiz: "wiz", snyk: "snyk", semgrep: "semgrep", + // OpenGrep uses the existing SAST contract; evidence retains its scanner origin. + opengrep: "semgrep", endor: "endor", github: "github", } as const satisfies Record; @@ -49,6 +52,17 @@ export async function readExternalFindings( options: ExternalFindingsReaderOptions, target: ExternalImportTarget = "cloud", ): Promise { + if (options.provider === "opengrep") { + const sourceRepositoryUrl = options.sourceRepositoryUrl ?? ""; + return { + findings: await readOpenGrepFindings( + path, + { sourceRepositoryUrl }, + target, + ), + sourceRepositories: [{ url: sourceRepositoryUrl }], + }; + } if (options.provider === "snyk") { const findings = await readSnykFindings(path, target); return { diff --git a/sdk/typescript/src/external-findings.ts b/sdk/typescript/src/external-findings.ts index 763260e8ca..113d3e9079 100644 --- a/sdk/typescript/src/external-findings.ts +++ b/sdk/typescript/src/external-findings.ts @@ -14,6 +14,8 @@ export interface ExternalSourceRepository { export interface ExternalFindings { read: number; + /** Reader-retained run metadata, including diagnostics for empty reports. */ + report?: Record; findings: ExternalFinding[]; excluded: { position: number; diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index e384fdeb6f..293430efe4 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -1,4 +1,5 @@ import { isIP } from "node:net"; +import { isNumberObject } from "node:util/types"; import Ajv2020 from "ajv/dist/2020.js"; import schema from "../schemas/external-findings.schema.json" with { type: "json" }; import type { @@ -225,6 +226,27 @@ export const validateSourceReports = validator("SourceReportPage"); export const validateSourceReport = validator("SourceReport"); +function requireJsonNumber(value: number, label: string): void { + if (!Number.isFinite(value)) + throw new Error(`${label} must contain finite JSON numbers.`); + if (Number.isInteger(value) && !Number.isSafeInteger(value)) + throw new Error( + `${label} integers must be within JavaScript's safe integer range; export larger values as strings to preserve them exactly.`, + ); +} + +/** Report metadata is retained in previews and receipts without Cloud evidence size limits. */ +export function validateExternalReport( + report: Record | undefined, +): void { + if (report === undefined) return; + JSON.stringify(report, (_key, value: unknown) => { + if (isNumberObject(value)) value = +value; + if (typeof value === "number") requireJsonNumber(value, "Scanner report"); + return value; + }); +} + export function validateExternalEvidence( input: unknown, target: ExternalImportTarget = "cloud", @@ -303,10 +325,20 @@ export function validateExternalEvidence( "Evidence URLs must be HTTP(S) links without credentials.", ); } + validateExternalEvidenceJson(result, target); + return result; +} + +/** Return the validated Cloud wire size, or zero for local evidence. */ +export function validateExternalEvidenceJson( + input: unknown, + target: ExternalImportTarget, + additionalCloudBytes = 0, +): number { // Local evidence still has to be serializable JSON, including acyclic source data. - if (target === "local") JSON.stringify(result); + if (target === "local") JSON.stringify(input); let numericByteAdjustment = 0; - const pending: [unknown, number][] = [[result, 0]]; + const pending: [unknown, number][] = [[input, 0]]; while (pending.length > 0) { const [value, depth] = pending.pop()!; if (target === "cloud" && depth > 20) @@ -315,12 +347,7 @@ export function validateExternalEvidence( for (const child of Object.values(value)) pending.push([child, depth + 1]); } else if (typeof value === "number") { - if (!Number.isFinite(value)) - throw new Error("Evidence must contain finite JSON numbers."); - if (Number.isInteger(value) && !Number.isSafeInteger(value)) - throw new Error( - "Evidence integers must be within JavaScript's safe integer range; export larger values as strings to preserve them exactly.", - ); + requireJsonNumber(value, "Evidence"); const wire = JSON.stringify(value); // Cloud pads short exponents and formats small fractions scientifically. const cloud = ( @@ -331,13 +358,15 @@ export function validateExternalEvidence( numericByteAdjustment += cloud.length - wire.length; } } - if ( - target === "cloud" && - Buffer.byteLength(JSON.stringify(result)) + numericByteAdjustment > - 256 * 1024 - ) + const bytes = + target === "cloud" + ? Buffer.byteLength(JSON.stringify(input)) + + numericByteAdjustment + + additionalCloudBytes + : 0; + if (bytes > 256 * 1024) throw new Error("Evidence exceeds the Cloud limit of 256 KiB."); - return result; + return bytes; } export function validateImportRequest(input: unknown): FindingImportRequest { diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 7c9cc96e36..fdf5901456 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -325,6 +325,11 @@ export type { } from "./validation-records.js"; export { prepareLocalExternalImport } from "./external-findings-import.js"; +export { + normalizeOpenGrepFindings, + readOpenGrepFindings, + type OpenGrepImportOptions, +} from "./opengrep-findings.js"; export { normalizeSnykFindingsExport, normalizeSnykJson, diff --git a/sdk/typescript/src/native-json-findings.ts b/sdk/typescript/src/native-json-findings.ts new file mode 100644 index 0000000000..8db1e9f39c --- /dev/null +++ b/sdk/typescript/src/native-json-findings.ts @@ -0,0 +1,163 @@ +import { + type ExternalImportTarget, + validateExternalEvidence, +} from "./external-import-contract.js"; +import type { ExternalFindingEvidence } from "./external-import-models.js"; +import { isRecord } from "./record.js"; +import { + record, + relativePath, + requiredText, + strings, + text, +} from "./scanner-import.js"; + +export function sourceRepositoryUrl(value: unknown, origin: string): string { + const supplied = requiredText( + value, + `${origin} sourceRepositoryUrl (--source-repository-url)`, + ); + const url = new URL(supplied); + if ( + !["http:", "https:"].includes(url.protocol) || + url.username || + url.password + ) + throw new Error( + "Use an HTTP(S) source repository URL without credentials.", + ); + return supplied; +} + +export function ungatedText(value: unknown): string | null { + const supplied = text(value); + // Semgrep CE emits this placeholder for gated fields. + return supplied === "requires login" ? null : supplied; +} + +export function messageText(value: unknown, origin: string): string | null { + if (typeof value !== "string") + throw new Error(`Expected a ${origin} message string.`); + return text(value); +} + +export function weaknessIds(value: unknown): string[] { + const labels = typeof value === "string" ? [value] : strings(value); + return [ + ...new Set( + labels.flatMap((label) => { + const match = /^(CWE-\d+)(?::|$)/u.exec(label); + return match ? [match[1]!] : []; + }), + ), + ]; +} + +export function position(value: unknown, label: string): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1) + throw new Error(`${label} must be a positive integer.`); + return value; +} + +/** Parse the native code-result fields shared by Semgrep and OpenGrep. */ +export function parseNativeJsonResult( + value: unknown, + origin: "Semgrep" | "OpenGrep", +) { + const result = record(value, `${origin} result`); + const extra = record(result["extra"], `${origin} extra`); + const rawMetadata = extra["metadata"]; + const metadata = + origin === "OpenGrep" && !isRecord(rawMetadata) + ? {} + : record(rawMetadata ?? {}, `${origin} metadata`); + const product = metadata["product"]; + const validation = extra["validation_state"]; + if ( + (origin === "OpenGrep" + ? product === "secrets" + : product != null && product !== "sast") || + extra["sca_info"] != null || + extra["historical_info"] != null || + (validation != null && validation !== "NO_VALIDATOR") + ) + throw new Error( + `This adapter supports ${origin} SAST; Supply Chain and Secrets reports need separate qualification.`, + ); + const rule = requiredText(result["check_id"], `${origin} check_id`); + const rawPath = requiredText(result["path"], `${origin} path`); + const start = record(result["start"], `${origin} start`); + const end = record(result["end"], `${origin} end`); + const line = position(start["line"], "Start line"); + const column = position(start["col"], "Start column"); + const endLine = position(end["line"], "End line"); + const endColumn = position(end["col"], "End column"); + if (endLine < line || (endLine === line && endColumn < column)) + throw new Error(`${origin} source range ends before it starts.`); + return { + result, + extra, + metadata, + rule, + rawPath, + path: relativePath(rawPath), + line, + column, + endLine, + endColumn, + }; +} + +export function projectNativeJsonResult( + parsed: ReturnType, + options: { + origin: string; + repositoryUrl: string; + severity: (value: unknown) => string; + reportContext: Record; + target: ExternalImportTarget; + scanErrorCount?: number; + observations?: Record[]; + }, +): ExternalFindingEvidence { + const { result, extra, metadata, rule, path, line, endLine } = parsed; + const lines = extra["lines"]; + return validateExternalEvidence( + { + title: rule, + description: messageText(extra["message"], options.origin), + severity: options.severity(extra["severity"]), + locations: path ? [{ path, line }] : [], + details: { + kind: "sast", + repository: { url: options.repositoryUrl }, + scanner_origin: options.origin, + rule: { id: rule }, + weakness_ids: weaknessIds(metadata["cwe"]), + code: { + end_line: path ? endLine : null, + snippet: + options.origin === "OpenGrep" && typeof lines === "string" + ? lines + : ungatedText(lines), + }, + source_status: + typeof extra["is_ignored"] === "boolean" + ? extra["is_ignored"] + ? "ignored" + : "open" + : null, + }, + source_data: structuredClone({ + report: options.reportContext, + ...(options.scanErrorCount === undefined + ? {} + : { scan_error_count: options.scanErrorCount }), + ...(options.observations + ? { results: options.observations } + : { result }), + }), + }, + options.target, + ); +} diff --git a/sdk/typescript/src/opengrep-findings.ts b/sdk/typescript/src/opengrep-findings.ts new file mode 100644 index 0000000000..60ee33f061 --- /dev/null +++ b/sdk/typescript/src/opengrep-findings.ts @@ -0,0 +1,249 @@ +import { posix } from "node:path"; +import { + validateExternalEvidenceJson, + validateExternalReport, + type ExternalImportTarget, +} from "./external-import-contract.js"; +import { canonicalJson } from "./record.js"; +import { errorMessage } from "./errors.js"; +import type { ExternalFindings } from "./external-findings.js"; +import { + messageText, + parseNativeJsonResult, + projectNativeJsonResult, + sourceRepositoryUrl, +} from "./native-json-findings.js"; +import { + array, + readScannerJson, + record, + relativePath, + requiredText, + stableId, +} from "./scanner-import.js"; + +export interface OpenGrepImportOptions { + /** Source repository declared by the caller, independent of the destination. */ + sourceRepositoryUrl: string; +} + +/** Import native scanner evidence without assigning a Codex assessment. */ +export function normalizeOpenGrepFindings( + input: unknown, + options: OpenGrepImportOptions, + target: ExternalImportTarget = "cloud", +): ExternalFindings { + const document = record(input, "OpenGrep report"); + const repositoryUrl = sourceRepositoryUrl( + options?.sourceRepositoryUrl, + "OpenGrep", + ); + const version = requiredText(document["version"], "OpenGrep version"); + const { results, ...report } = document; + const values = array(results, "OpenGrep results"); + const errors = array(report["errors"], "OpenGrep errors"); + validateExternalReport(report); + const projection = { + origin: "OpenGrep", + repositoryUrl, + severity: () => "unknown", + reportContext: { version }, + target, + scanErrorCount: errors.length, + }; + const output: ExternalFindings = { + read: values.length, + findings: [], + excluded: [], + report: structuredClone(report), + }; + type Observation = { + position: number; + parsed: ReturnType; + }; + function exclude(position: number, id: string | null, error: unknown) { + output.excluded.push({ + position, + source_finding_id: id, + reason: errorMessage(error), + }); + } + function projectGroup(observations: Observation[]) { + const results = observations.map(({ parsed }) => parsed.result); + if (results.length === 1) + return projectNativeJsonResult(observations[0]!.parsed, { + ...projection, + observations: results, + }); + const sourceData = { + report: { version }, + scan_error_count: errors.length, + results, + }; + // Replacing [] with the same raw results adds the same wire bytes to every candidate. + const rawBytes = + validateExternalEvidenceJson({ source_data: sourceData }, target) - + validateExternalEvidenceJson( + { source_data: { ...sourceData, results: [] } }, + target, + ); + let projectionError: unknown; + for (const { parsed } of observations) { + try { + const evidence = projectNativeJsonResult(parsed, { + ...projection, + observations: [], + }); + validateExternalEvidenceJson(evidence, target, rawBytes); + evidence.source_data!["results"] = results; + return evidence; + } catch (error) { + projectionError ??= error; + } + } + throw projectionError; + } + function retainPublishableTraces(observations: Observation[], id: string) { + const emptyWrapperBytes = validateExternalEvidenceJson( + { source_data: { results: [] } }, + target, + ); + function sizeWithTrace(observation: Observation, evidenceBytes: number) { + return validateExternalEvidenceJson( + { source_data: { results: [observation.parsed.result] } }, + target, + evidenceBytes - emptyWrapperBytes, + ); + } + const singletonErrors = new Map(); + let smallestProjectionBytes = Infinity; + let coreRawBytes = 0; + for (const observation of observations) { + try { + const evidence = projectNativeJsonResult(observation.parsed, { + ...projection, + observations: [], + }); + const projectedBytes = validateExternalEvidenceJson(evidence, target); + const singletonBytes = sizeWithTrace(observation, projectedBytes); + coreRawBytes += singletonBytes - projectedBytes; + smallestProjectionBytes = Math.min( + smallestProjectionBytes, + projectedBytes, + ); + } catch (error) { + singletonErrors.set(observation, errorMessage(error)); + } + } + if (!singletonErrors.size) return observations; + // Reserve every independently publishable trace, including the commas in + // their results array, before trying raw-only traces in canonical order. + const coreCount = observations.length - singletonErrors.size; + let groupBytes = smallestProjectionBytes + coreRawBytes + coreCount - 1; + return observations.filter((observation) => { + let reason = singletonErrors.get(observation); + if (reason === undefined) return true; + if (coreCount) { + try { + groupBytes = sizeWithTrace(observation, groupBytes + 1); + return true; + } catch (error) { + const groupReason = errorMessage(error); + if (groupReason !== reason) + reason = `${groupReason} This trace also fails standalone validation. ${reason}`; + } + } + exclude(observation.position, id, reason); + return false; + }); + } + const groups = new Map(); + for (const [index, value] of values.entries()) { + let id: string | null = null; + try { + const parsed = parseNativeJsonResult(value, "OpenGrep"); + if (parsed.path !== null) { + // Validate the emitted spelling across platforms, keeping literal POSIX backslashes. + const path = posix.normalize(parsed.rawPath); + parsed.path = + relativePath(posix.normalize(path.replaceAll("\\", "/"))) === null + ? null + : path; + } + // Native fingerprint counters change when another analysis pass adds matches. + // A source move creates a new occurrence; this is not cross-revision matching. + id = stableId("opengrep-location", [ + parsed.rule, + parsed.path ?? parsed.rawPath, + parsed.line, + parsed.column, + parsed.endLine, + parsed.endColumn, + ]); + messageText(parsed.extra["message"], "OpenGrep"); + const group = groups.get(id) ?? []; + group.push({ position: index + 1, parsed }); + groups.set(id, group); + } catch (error) { + exclude(index + 1, id, error); + } + } + for (const [id, observations] of [...groups].sort(([left], [right]) => + left < right ? -1 : left > right ? 1 : 0, + )) { + const ordered = + observations.length === 1 + ? observations + : observations + .flatMap((observation) => { + try { + const result = structuredClone(observation.parsed.result); + validateExternalEvidenceJson( + { source_data: { results: [result] } }, + target, + ); + observation.parsed.result = result; + return [{ observation, key: canonicalJson(result) }]; + } catch (error) { + exclude(observation.position, id, error); + return []; + } + }) + .sort((left, right) => + left.key < right.key ? -1 : left.key > right.key ? 1 : 0, + ) + .map(({ observation }) => observation); + if (!ordered.length) continue; + let retained = ordered; + try { + let evidence; + try { + evidence = projectGroup(retained); + } catch (error) { + if (retained.length === 1) throw error; + retained = retainPublishableTraces(retained, id); + if (!retained.length) continue; + if (retained.length === ordered.length) throw error; + evidence = projectGroup(retained); + } + output.findings.push({ source_finding_id: id, evidence }); + } catch (error) { + for (const observation of retained) + exclude(observation.position, id, error); + } + } + output.excluded.sort((left, right) => left.position - right.position); + return output; +} + +export async function readOpenGrepFindings( + path: string, + options: OpenGrepImportOptions, + target: ExternalImportTarget = "cloud", +): Promise { + return normalizeOpenGrepFindings( + await readScannerJson(path), + options, + target, + ); +} diff --git a/sdk/typescript/src/record.ts b/sdk/typescript/src/record.ts index 35f1d60d9a..3850998531 100644 --- a/sdk/typescript/src/record.ts +++ b/sdk/typescript/src/record.ts @@ -1,3 +1,19 @@ export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } + +export function canonicalJson( + value: unknown, + compareKeys?: (left: string, right: string) => number, +): string { + return JSON.stringify(value, (_key, child: unknown) => { + if (isRecord(child)) { + return Object.fromEntries( + Object.keys(child) + .sort(compareKeys) + .map((key) => [key, child[key]]), + ); + } + return child; + }); +} diff --git a/sdk/typescript/src/semgrep-findings.ts b/sdk/typescript/src/semgrep-findings.ts index 9002d9f762..e97a630aef 100644 --- a/sdk/typescript/src/semgrep-findings.ts +++ b/sdk/typescript/src/semgrep-findings.ts @@ -4,6 +4,15 @@ import { validateExternalEvidence, } from "./external-import-contract.js"; import type { ExternalFindings } from "./external-findings.js"; +import { + messageText, + parseNativeJsonResult, + position, + projectNativeJsonResult, + sourceRepositoryUrl, + ungatedText, + weaknessIds, +} from "./native-json-findings.js"; import { array, hasSarifExternalPropertyReferences, @@ -14,7 +23,6 @@ import { requiredText, sarifRunContextForResult, stableId, - strings, text, } from "./scanner-import.js"; @@ -24,32 +32,7 @@ export interface SemgrepImportOptions { } function sourceRepository(options: SemgrepImportOptions): string { - const value = requiredText( - options?.sourceRepositoryUrl, - "Semgrep sourceRepositoryUrl (--source-repository-url)", - ); - const url = new URL(value); - if ( - !["http:", "https:"].includes(url.protocol) || - url.username || - url.password - ) - throw new Error( - "Use an HTTP(S) source repository URL without credentials.", - ); - return value; -} - -function ungatedText(value: unknown): string | null { - const supplied = text(value); - // The current CLI emits this placeholder for gated Community Edition fields. - return supplied === "requires login" ? null : supplied; -} - -function messageText(value: unknown): string | null { - if (typeof value !== "string") - throw new Error("Expected a Semgrep message string."); - return text(value); + return sourceRepositoryUrl(options?.sourceRepositoryUrl, "Semgrep"); } function severity(value: unknown): string { @@ -65,24 +48,6 @@ function severity(value: unknown): string { return typeof value === "string" ? (levels[value] ?? "unknown") : "unknown"; } -function weaknessIds(value: unknown): string[] { - const labels = typeof value === "string" ? [value] : strings(value); - return [ - ...new Set( - labels.flatMap((label) => { - const match = /^(CWE-\d+)(?::|$)/u.exec(label); - return match ? [match[1]!] : []; - }), - ), - ]; -} - -function position(value: unknown, label: string): number { - if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1) - throw new Error(`${label} must be a positive integer.`); - return value; -} - function identity( rule: string, path: string, @@ -195,63 +160,19 @@ export function normalizeSemgrepFindings( const { results, paths: _paths, time: _time, ...context } = document; return normalizeRecords( array(results, "Semgrep results"), - (value, sourceFindingId) => { - const result = record(value, "Semgrep result"); - const extra = record(result["extra"], "Semgrep extra"); - const metadata = record(extra["metadata"] ?? {}, "Semgrep metadata"); - const product = metadata["product"]; - const validation = extra["validation_state"]; - if ( - (product != null && product !== "sast") || - extra["sca_info"] != null || - extra["historical_info"] != null || - (validation != null && validation !== "NO_VALIDATOR") - ) - throw new Error( - "This adapter supports Semgrep SAST; Supply Chain and Secrets reports need separate qualification.", - ); - const rule = requiredText(result["check_id"], "Semgrep check_id"); - const rawPath = requiredText(result["path"], "Semgrep path"); - const start = record(result["start"], "Semgrep start"); - const end = record(result["end"], "Semgrep end"); - const line = position(start["line"], "Start line"); - const column = position(start["col"], "Start column"); - const endLine = position(end["line"], "End line"); - const endColumn = position(end["col"], "End column"); - if (endLine < line || (endLine === line && endColumn < column)) - throw new Error("Semgrep source range ends before it starts."); - const path = relativePath(rawPath); - return { - source_finding_id: sourceFindingId, - evidence: validateExternalEvidence( - { - title: rule, - description: messageText(extra["message"]), - severity: severity(extra["severity"]), - locations: path ? [{ path, line }] : [], - details: { - kind: "sast", - repository: { url: repositoryUrl }, - scanner_origin: "Semgrep", - rule: { id: rule }, - weakness_ids: weaknessIds(metadata["cwe"]), - code: { - end_line: path ? endLine : null, - snippet: ungatedText(extra["lines"]), - }, - source_status: - typeof extra["is_ignored"] === "boolean" - ? extra["is_ignored"] - ? "ignored" - : "open" - : null, - }, - source_data: structuredClone({ report: context, result }), - }, + (value, sourceFindingId) => ({ + source_finding_id: sourceFindingId, + evidence: projectNativeJsonResult( + parseNativeJsonResult(value, "Semgrep"), + { + origin: "Semgrep", + repositoryUrl, + severity, + reportContext: context, target, - ), - }; - }, + }, + ), + }), jsonIdentity, target, ); @@ -420,7 +341,8 @@ export function normalizeSemgrepSarif( { title: ruleId, description: - text(message["markdown"]) ?? messageText(message["text"]), + text(message["markdown"]) ?? + messageText(message["text"], "Semgrep"), severity: mappedSeverity, locations: path ? [{ path, line }] : [], details: { diff --git a/sdk/typescript/tests-ts/cli-local-import.test.ts b/sdk/typescript/tests-ts/cli-local-import.test.ts index c9e6f4322f..2f3ba4e133 100644 --- a/sdk/typescript/tests-ts/cli-local-import.test.ts +++ b/sdk/typescript/tests-ts/cli-local-import.test.ts @@ -16,6 +16,7 @@ import { FakeSignals, } from "./cli-fixtures.js"; import { createCliTest } from "./support/cli-run.js"; +import { opengrepJson } from "./support/opengrep-import-fixtures.js"; const dependencies: typeof cliDependencies = (options = {}) => { const result = cliDependencies(options); @@ -202,6 +203,71 @@ test("CLI imports a vendor report, reads local evidence, and updates the same so expect(result.findings[0].evidence.confidence).toBeUndefined(); }); +test("CLI imports OpenGrep evidence locally without assessing it", async () => { + const f = await fixture(); + const report = opengrepJson(); + await writeFile(f.file, JSON.stringify(report)); + const sourceRepositoryUrl = "https://github.com/example/project"; + const imported = createCliTest(main); + expect( + await imported.runCli( + [ + "publish", + "findings", + "selected.json", + "--to", + "local", + "--repository", + "example project", + "--provider", + "opengrep", + "--source-repository-url", + sourceRepositoryUrl, + "--source-key", + "example/opengrep", + "--yes", + "--format", + "json", + ], + f.deps, + ), + ).toBe(0); + const result = JSON.parse(imported.stdout.text()); + expect(result.report.errors).toEqual(report.errors); + expect(result.counts).toEqual({ + created: 1, + updated: 0, + unchanged: 0, + }); + const readback = createCliTest(main); + expect( + await readback.runCli( + ["findings", "imported", "example project", "--format", "json"], + f.deps, + ), + ).toBe(0); + const findings = JSON.parse(readback.stdout.text()).findings; + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ + provider: "semgrep", + assessment: { state: "not_assessed" }, + evidence: { + severity: "unknown", + details: { + scanner_origin: "OpenGrep", + repository: { url: sourceRepositoryUrl }, + }, + source_data: { + report: { + version: report.version, + }, + scan_error_count: report.errors.length, + results: report.results, + }, + }, + }); +}); + test("local import confirms exclusions and returns partial status after storing ready records", async () => { const f = await fixture(); await writeFile( diff --git a/sdk/typescript/tests-ts/external-findings-local.test.ts b/sdk/typescript/tests-ts/external-findings-local.test.ts index fcf1a57ecb..6cf86f9c74 100644 --- a/sdk/typescript/tests-ts/external-findings-local.test.ts +++ b/sdk/typescript/tests-ts/external-findings-local.test.ts @@ -197,9 +197,22 @@ describe("local normalized scanner findings", () => { expect(existsSync(disabledState)).toBe(false); }); + test("rejects unserializable report metadata before storing findings", async () => { + const { repository, stateDirectory, environment } = await fixture(); + await expect( + prepareNormalizedLocalImport( + { ...input(), report: { elapsedNs: 1n } }, + { repository, ...source }, + { environment }, + ), + ).rejects.toThrow(/bigint/i); + expect(existsSync(stateDirectory)).toBe(false); + }); + test("persists sparse evidence through the native helper and reads it in a separate process", async () => { const { repository, stateDirectory, environment } = await fixture(); const normalized = input(); + normalized.report = { errors: normalized.excluded }; const prepared = await prepareNormalizedLocalImport( normalized, { repository, ...source }, @@ -221,6 +234,13 @@ describe("local normalized scanner findings", () => { excluded: input().excluded, counts: { created: 2, updated: 0, unchanged: 0 }, }); + (result.report!["errors"] as typeof normalized.excluded).pop(); + Object.assign(result.excluded[0]!, { onSelect: () => undefined }); + expect(await prepared.import()).toMatchObject({ + excluded: normalized.excluded, + report: normalized.report, + counts: { created: 0, updated: 0, unchanged: 2 }, + }); expect(existsSync(join(stateDirectory, "workbench.sqlite3"))).toBe(true); expect(result.findings.map((row) => row.evidence)).toEqual( input().findings.map((row) => row.evidence), diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index b7d0604cb6..1f1f01d386 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -2364,22 +2364,39 @@ test("interrupted CLI output preserves acknowledged batches and resumes only unf expect(f.posts[2]).toBe(f.posts[1]); }); -test("readback interruption distinguishes acknowledged findings from verified findings", async () => { - const f = await fixture(); +test("readback retry preserves acknowledged findings and scanner diagnostics", async () => { + const { opengrepJson } = + await import("./support/opengrep-import-fixtures.js"); + const input = opengrepJson(); + const f = await fixture(input); f.state.brokenReadback = true; - const prepared = await prepareExternalPublication(f.file, options, f.deps); + const prepared = await prepareExternalPublication( + f.file, + { + ...options, + provider: "opengrep", + sourceRepositoryUrl: f.destination().url, + }, + f.deps, + ); try { await prepared.publish(); throw new Error("Expected readback interruption"); } catch (error) { - expect(error).toBeInstanceOf(ExternalPublicationError); - expect((error as ExternalPublicationError).result).toMatchObject({ + if (!(error instanceof ExternalPublicationError)) throw error; + expect(error.result).toMatchObject({ status: "interrupted", counts: { created: 1 }, unacknowledged: 0, verified: 0, }); + (error.result.report!["errors"] as unknown[]).pop(); } + f.state.brokenReadback = false; + const retried = await prepared.publish(); + expect(retried.report?.["errors"]).toEqual(input.errors); + expect(retried.verified).toBe(1); + expect(f.posts).toHaveLength(1); }); test("repository selection is sent on every discovery page", async () => { @@ -2914,6 +2931,7 @@ test("normalized SDK publication detaches raw evidence aliases before validating const sharedPackage = { name: "example-package" }; const input = { read: 2, + report: { package: sharedPackage }, findings: [ { source_finding_id: "shared-first", @@ -2942,6 +2960,9 @@ test("normalized SDK publication detaches raw evidence aliases before validating f.deps, ); expect(JSON.stringify(input)).toBe(original); + expect(prepared.preview.report).toEqual({ + package: { name: "example-package" }, + }); expect(prepared.preview.findings[0]!.evidence.source_data).toEqual({ package: { name: "example-package" }, }); @@ -3012,6 +3033,95 @@ test("normalized SDK publication rejects nonfinite raw evidence without JSON coe expect(f.posts).toHaveLength(0); }); +test("normalized SDK publication rejects unserializable report metadata before upload", async () => { + const f = await fixture(); + const input = { + read: 1, + report: { elapsedNs: 1n }, + findings: [ + { + source_finding_id: "synthetic-report", + evidence: { title: "Example", severity: "unknown" as const }, + }, + ], + excluded: [], + }; + await expect( + prepareNormalizedExternalPublication(input, options, f.deps), + ).rejects.toThrow(/bigint/i); + await expect( + prepareNormalizedExternalPublication( + { ...input, report: { elapsed: new Number(Infinity) } }, + options, + f.deps, + ), + ).rejects.toThrow("finite JSON numbers"); + expect(f.posts).toHaveLength(0); +}); + +test("publication JSON retains native Error messages and scanner diagnostics", async () => { + const { opengrepJson } = + await import("./support/opengrep-import-fixtures.js"); + const scannerErrors = opengrepJson().errors; + const error = new Error("Scanner rule timed out."); + const nested = { + path: "src/example.ts", + cause: new Error("Parser stopped."), + }; + const report = { errors: [error, nested, ...scannerErrors] }; + const jsonErrors = [ + { message: error.message }, + { path: nested.path, cause: { message: nested.cause.message } }, + ...scannerErrors, + ]; + const f = await fixture(); + await expect( + prepareNormalizedExternalPublication( + { read: 0, findings: [], excluded: [], report }, + options, + f.deps, + ), + ).rejects.toThrow(`Scanner diagnostics: ${JSON.stringify(jsonErrors)}`); + expect(f.posts).toHaveLength(0); + + const prepared = await prepareNormalizedExternalPublication( + { + read: 1, + findings: [ + { + source_finding_id: "synthetic-report", + evidence: { title: "Example finding", severity: "high" }, + }, + ], + excluded: [], + report, + }, + options, + f.deps, + ); + const original = structuredClone(report); + nested.cause.message = "Caller changed the diagnostic."; + const completed = await prepared.publish(); + expect(completed.verified).toBe(1); + for (const value of [prepared.preview.report, completed.report]) { + expect(value).toEqual(original); + expect(value?.["errors"]).toEqual([ + expect.any(Error), + { path: nested.path, cause: expect.any(Error) }, + ...scannerErrors, + ]); + } + const dir = join(f.root, "state", "external-finding-publications"); + const receipt = (await readdir(dir)).find((name) => + name.endsWith(".result.json"), + )!; + expect(JSON.parse(await readFile(join(dir, receipt), "utf8")).report).toEqual( + { + errors: jsonErrors, + }, + ); +}); + test("invalid source namespaces fail before authentication or Cloud requests", async () => { const f = await fixture(); const contacted: string[] = []; @@ -3602,6 +3712,132 @@ test("Semgrep publication preserves identity across JSON and SARIF with explicit }); }); +test("OpenGrep publication retains grouped evidence through Cloud readback and replay", async () => { + const { opengrepTraceJson } = + await import("./support/opengrep-import-fixtures.js"); + const input = opengrepTraceJson(); + const { results, ...report } = input; + const f = await fixture(input); + const settings = { + ...options, + provider: "opengrep" as const, + sourceKey: "organization/project/main/opengrep", + sourceRepositoryUrl: "https://github.com/example/project", + }; + const prepared = await prepareExternalPublication(f.file, settings, f.deps); + expect(prepared.preview.source).toEqual({ + provider: "semgrep", + source_key: settings.sourceKey, + }); + expect(prepared.preview.report).toEqual(report); + expect(prepared.preview.findings).toHaveLength(1); + expect(prepared.preview.excluded).toEqual([]); + prepared.preview.report!["errors"] = []; + const first = await prepared.publish(); + expect(first.counts.created).toBe(1); + expect(first.verified).toBe(1); + expect(first.report).toEqual(report); + const original = structuredClone([...f.reports.values()][0]!); + expect(original).toMatchObject({ + origin: "imported", + source: { provider: "semgrep" }, + assessment: { state: "not_assessed" }, + evidence: { + details: { kind: "sast", scanner_origin: "OpenGrep" }, + source_data: { scan_error_count: input.errors.length }, + }, + }); + expect(original.evidence.source_data!["results"]).toHaveLength(2); + expect(original.evidence.source_data!["results"]).toEqual( + expect.arrayContaining(results), + ); + expect(original.evidence).not.toHaveProperty("confidence"); + expect(JSON.parse(f.posts[0]!).source.provider).toBe("semgrep"); + const repeated = await ( + await prepareExternalPublication(f.file, settings, f.deps) + ).publish(); + expect(repeated.counts.unchanged).toBe(1); + expect(repeated.report).toEqual(report); + + input.results[0]!.extra.message = "Updated OpenGrep evidence"; + input.results[0]!.extra.fingerprint = "new-scanner-fingerprint_9"; + await writeFile(f.file, JSON.stringify(input)); + const changed = await ( + await prepareExternalPublication(f.file, settings, f.deps) + ).publish(); + expect(changed.counts.updated).toBe(1); + expect(changed.verified).toBe(1); + expect(changed.report).toEqual(report); + expect([...f.reports.values()]).toHaveLength(1); + const updated = [...f.reports.values()][0]!; + expect(updated).toMatchObject({ + id: original.id, + canonical_finding_id: original.canonical_finding_id, + source_finding_id: original.source_finding_id, + version: 2, + origin: "imported", + assessment: { state: "not_assessed" }, + }); + expect(updated.evidence.source_data!["results"]).toHaveLength(2); + expect(updated.evidence.source_data!["results"]).toEqual( + expect.arrayContaining(input.results), + ); +}); + +test("empty OpenGrep Cloud imports retain scanner diagnostics without uploading", async () => { + const { opengrepJson } = + await import("./support/opengrep-import-fixtures.js"); + const input = opengrepJson(); + input.results = []; + const f = await fixture(input); + await expect( + prepareExternalPublication( + f.file, + { + ...options, + provider: "opengrep", + sourceRepositoryUrl: "https://github.com/example/project", + }, + f.deps, + ), + ).rejects.toThrow(input.errors[0]!.message); + expect(f.posts).toHaveLength(0); +}); + +test("OpenGrep CLI Cloud preview forwards source provenance without uploading", async () => { + const { opengrepJson } = + await import("./support/opengrep-import-fixtures.js"); + const input = opengrepJson(); + const f = await fixture(input); + const command = [ + ...f.command, + "--dry-run", + "--source-repository-url", + "https://github.com/example/project", + ]; + command[command.indexOf("--provider") + 1] = "opengrep"; + const cli = createCliTest(main); + expect(await cli.runCli(command, f.cliDeps)).toBe(0); + expect(JSON.parse(cli.stdout.text())).toMatchObject({ + source: { provider: "semgrep", source_key: options.sourceKey }, + report: { errors: input.errors }, + findings: [ + { + evidence: { + details: { scanner_origin: "OpenGrep" }, + source_data: { results: input.results }, + }, + }, + ], + }); + const humanCommand = [...command]; + humanCommand.splice(humanCommand.indexOf("--format"), 2); + const human = createCliTest(main); + expect(await human.runCli(humanCommand, f.cliDeps)).toBe(0); + expect(human.stderr.text()).toContain(input.errors[0]!.message); + expect(f.posts).toHaveLength(0); +}); + test("Semgrep native percent paths retain CE records through JSON-to-SARIF publication", async () => { const { semgrepJson, semgrepSarif } = await import("./support/semgrep-import-fixtures.js"); diff --git a/sdk/typescript/tests-ts/opengrep-findings-local.test.ts b/sdk/typescript/tests-ts/opengrep-findings-local.test.ts new file mode 100644 index 0000000000..6034a55039 --- /dev/null +++ b/sdk/typescript/tests-ts/opengrep-findings-local.test.ts @@ -0,0 +1,149 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, test } from "bun:test"; +import { prepareLocalExternalImport } from "../src/external-findings-import.js"; +import { listLocalExternalFindings } from "../src/external-findings-local.js"; +import { localFindingsFixture } from "./external-findings-local-helpers.js"; +import { + opengrepJson, + opengrepTraceJson, +} from "./support/opengrep-import-fixtures.js"; + +const repositoryUrl = "https://github.com/example/project"; + +test("OpenGrep grouped traces survive SQLite import, replay, and evidence updates", async () => { + const fixture = await localFindingsFixture({ remote: repositoryUrl }); + try { + const input = opengrepTraceJson(); + // Grouped observations can exceed Cloud's per-finding source data limit. + for (const result of input.results) + result.extra.metavars.$VALUE.abstract_content = "x".repeat(140_000); + const { results, ...report } = input; + const file = join(fixture.root, "opengrep.json"); + await writeFile(file, JSON.stringify(input)); + const options = { + repository: fixture.repository, + provider: "opengrep" as const, + sourceKey: "example-project:main:opengrep", + sourceRepositoryUrl: repositoryUrl, + }; + const dependencies = { environment: fixture.environment }; + const prepared = await prepareLocalExternalImport( + file, + options, + dependencies, + ); + expect(prepared.preview).toMatchObject({ + read: 2, + excluded: [], + report, + source: { provider: "semgrep", source_key: options.sourceKey }, + }); + expect(prepared.preview.findings).toHaveLength(1); + const imported = await prepared.import(); + expect(imported.counts).toEqual({ created: 1, updated: 0, unchanged: 0 }); + expect(imported.report).toEqual(report); + + const [stored] = await listLocalExternalFindings(options, dependencies); + expect(stored).toMatchObject({ + provider: "semgrep", + sourceKey: options.sourceKey, + sourceFindingId: prepared.preview.findings[0]!.source_finding_id, + assessment: { state: "not_assessed" }, + version: 1, + evidence: { + locations: [{ path: "src/example.ts", line: 3 }], + details: { kind: "sast", scanner_origin: "OpenGrep" }, + source_data: { scan_error_count: input.errors.length }, + }, + }); + expect(stored!.evidence.source_data!["results"]).toHaveLength(2); + expect(stored!.evidence.source_data!["results"]).toEqual( + expect.arrayContaining(results), + ); + expect(stored!.evidence).not.toHaveProperty("confidence"); + expect( + ( + await ( + await prepareLocalExternalImport(file, options, dependencies) + ).import() + ).counts, + ).toEqual({ created: 0, updated: 0, unchanged: 1 }); + expect(await listLocalExternalFindings(options, dependencies)).toEqual([ + stored!, + ]); + + input.results[0]!.extra.message = "Updated scanner evidence"; + input.results[0]!.extra.fingerprint = "new-scanner-fingerprint_9"; + await writeFile(file, JSON.stringify(input)); + const updated = await ( + await prepareLocalExternalImport(file, options, dependencies) + ).import(); + expect(updated.counts).toEqual({ created: 0, updated: 1, unchanged: 0 }); + expect(updated.report).toEqual(report); + const records = await listLocalExternalFindings(options, dependencies); + expect(records).toHaveLength(1); + expect(records[0]).toMatchObject({ + id: stored!.id, + sourceFindingId: stored!.sourceFindingId, + createdAt: stored!.createdAt, + version: 2, + assessment: { state: "not_assessed" }, + evidence: { + details: { scanner_origin: "OpenGrep" }, + }, + }); + expect(records[0]!.evidence.source_data!["results"]).toHaveLength(2); + expect(records[0]!.evidence.source_data!["results"]).toEqual( + expect.arrayContaining(input.results), + ); + } finally { + await fixture.dispose(); + } +}); + +test("empty OpenGrep local imports retain scanner diagnostics without creating findings", async () => { + const fixture = await localFindingsFixture({ remote: repositoryUrl }); + try { + const input = opengrepJson(); + input.results = []; + const { results: _results, ...report } = input; + const file = join(fixture.root, "opengrep.json"); + await writeFile(file, JSON.stringify(input)); + const options = { + repository: fixture.repository, + provider: "opengrep" as const, + sourceKey: "example-project:main:opengrep", + sourceRepositoryUrl: repositoryUrl, + }; + const dependencies = { environment: fixture.environment }; + await expect( + prepareLocalExternalImport( + file, + { ...options, sourceRepositoryUrl: "https://github.com/example/other" }, + dependencies, + ), + ).rejects.toThrow("does not match"); + const prepared = await prepareLocalExternalImport( + file, + options, + dependencies, + ); + expect(prepared.preview).toMatchObject({ + read: 0, + findings: [], + excluded: [], + report, + }); + expect(await prepared.import()).toMatchObject({ + read: 0, + ready: 0, + findings: [], + report, + counts: { created: 0, updated: 0, unchanged: 0 }, + }); + expect(await listLocalExternalFindings(options, dependencies)).toEqual([]); + } finally { + await fixture.dispose(); + } +}); diff --git a/sdk/typescript/tests-ts/opengrep-findings.test.ts b/sdk/typescript/tests-ts/opengrep-findings.test.ts new file mode 100644 index 0000000000..dd94e24e44 --- /dev/null +++ b/sdk/typescript/tests-ts/opengrep-findings.test.ts @@ -0,0 +1,494 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { + normalizeOpenGrepFindings as normalize, + readOpenGrepFindings, +} from "../src/opengrep-findings.js"; +import { + opengrepJson, + opengrepTraceJson, +} from "./support/opengrep-import-fixtures.js"; + +const options = { sourceRepositoryUrl: "https://github.com/example/project" }; + +test("OpenGrep keeps native evidence and incomplete-run metadata without an assessment", () => { + const input = opengrepJson(); + const original = structuredClone(input); + const output = normalize(input, options); + expect(output.read).toBe(1); + expect(output.excluded).toEqual([]); + expect(output.findings).toHaveLength(1); + expect(output.findings[0]!.evidence).toMatchObject({ + title: "example.synthetic-rule", + severity: "unknown", + locations: [{ path: "src/example.ts", line: 3 }], + details: { + kind: "sast", + scanner_origin: "OpenGrep", + source_status: "open", + repository: { url: options.sourceRepositoryUrl }, + code: { end_line: 3, snippet: "example()" }, + weakness_ids: ["CWE-000"], + }, + source_data: { + report: { + version: input.version, + }, + scan_error_count: input.errors.length, + results: input.results, + }, + }); + expect(output.findings[0]!.evidence).not.toHaveProperty("assessment"); + expect(output.findings[0]!.evidence.source_data!["report"]).toEqual({ + version: input.version, + }); + const { results: _results, ...report } = original; + expect(output.report).toEqual(report); + expect(input).toEqual(original); +}); + +test.each(["INFO", "WARNING", "ERROR", "MEDIUM", "FUTURE_LEVEL"])( + "scanner severity %s remains raw evidence rather than assessed risk", + (severity) => { + const input = opengrepJson(); + input.results[0]!.extra.severity = severity; + const evidence = normalize(input, options).findings[0]!.evidence; + expect(evidence.severity).toBe("unknown"); + expect(evidence.source_data).toMatchObject({ results: input.results }); + }, +); + +test("location identities survive fingerprint renumbering and result reordering", () => { + const input = opengrepJson(); + input.results[0]!.start.line = 179; + input.results[0]!.end.line = 179; + const originalId = normalize(input, options).findings[0]!.source_finding_id; + const earlier = structuredClone(input.results[0]!); + earlier.start.line = 92; + earlier.end.line = 92; + input.results[0]!.extra.fingerprint = "synthetic-fingerprint_2"; + input.results[0]!.extra.message = "Updated rule message"; + input.results[0]!.start.offset = 500; + input.results[0]!.end.offset = 508; + input.results.unshift(earlier); + const output = normalize(input, options); + expect( + new Set(output.findings.map((item) => item.source_finding_id)).size, + ).toBe(2); + expect( + output.findings.find((item) => item.evidence.locations![0]!.line === 179)! + .source_finding_id, + ).toBe(originalId); + input.results.reverse(); + expect(normalize(input, options)).toEqual(output); +}); + +test("redundant slash components share an identity while source moves and rules stay distinct", () => { + const input = opengrepJson(); + const first = normalize(input, options).findings[0]!; + input.results[0]!.path = "./src/./example.ts"; + const equivalent = normalize(input, options).findings[0]!; + expect(equivalent.source_finding_id).toBe(first.source_finding_id); + expect(equivalent.evidence.locations).toEqual(first.evidence.locations); + input.results[0]!.start.line++; + input.results[0]!.end.line++; + expect(normalize(input, options).findings[0]!.source_finding_id).not.toBe( + first.source_finding_id, + ); + input.results[0]!.start.line--; + input.results[0]!.end.line--; + input.results[0]!.check_id = "example.other-rule"; + expect(normalize(input, options).findings[0]!.source_finding_id).not.toBe( + first.source_finding_id, + ); +}); + +test("literal backslashes keep a distinct occurrence when both targets or either target is imported", () => { + const input = opengrepJson(); + input.results.push({ ...input.results[0]!, path: "src\\example.ts" }); + const together = normalize(input, options); + const separate = input.results.map( + (result) => + normalize({ ...input, results: [result] }, options).findings[0]!, + ); + expect(together.excluded).toEqual([]); + expect(together.findings).toHaveLength(2); + expect(together.findings).toEqual(expect.arrayContaining(separate)); + expect( + separate.map((finding) => finding.evidence.locations![0]!.path), + ).toEqual(input.results.map((result) => result.path)); +}); + +test("one source occurrence retains every trace observation in deterministic order", () => { + const input = opengrepTraceJson(); + const original = structuredClone(input); + const output = normalize(input, options); + expect(output.read).toBe(2); + expect(output.findings).toHaveLength(1); + expect(output.excluded).toEqual([]); + expect(output.findings[0]!.evidence.source_data!["results"]).toHaveLength(2); + expect(output.findings[0]!.evidence.source_data!["results"]).toEqual( + expect.arrayContaining(input.results), + ); + input.results.reverse(); + expect(normalize(input, options)).toEqual(output); + expect(input.results.reverse()).toEqual(original.results); +}); + +test("an invalid observation does not hide a valid sibling at the same source range", () => { + const input: any = opengrepJson(); + const valid = structuredClone(input.results[0]); + const invalid = structuredClone(valid); + invalid.extra.message = { text: "Invalid native message" }; + const backwards = structuredClone(valid); + backwards.end.line = 1; + input.results = [invalid, valid, null, backwards]; + const output = normalize(input, options); + expect(output.read).toBe(4); + expect(output.findings).toHaveLength(1); + expect(output.findings[0]!.evidence.source_data!["results"]).toEqual([valid]); + expect(output.excluded.map((item) => item.position)).toEqual([1, 3, 4]); + expect(output.excluded[0]!.source_finding_id).toBe( + output.findings[0]!.source_finding_id, + ); + expect(output.excluded[0]!.reason).toContain("message string"); + expect(output.excluded[2]!.reason).toContain("ends before"); +}); + +test("grouped observations still honor the existing Cloud evidence size contract", () => { + const input: any = opengrepTraceJson(); + for (const result of input.results) { + result.extra.context = "x".repeat(140_000); + expect( + normalize({ ...input, results: [result] }, options).excluded, + ).toEqual([]); + } + const output = normalize(input, options); + expect(output.findings).toEqual([]); + expect(output.excluded.map((item) => item.position)).toEqual([1, 2]); + expect(output.excluded.every((item) => item.reason.includes("256 KiB"))).toBe( + true, + ); +}); + +test("an oversized trace leaves the publishable traces at its source occurrence", () => { + const input = opengrepJson(); + const valid = input.results[0]!; + const oversized = structuredClone(valid); + Object.assign(oversized.extra.metadata, { padding: "" }); + const singleton = normalize({ ...input, results: [oversized] }, options); + const bytes = Buffer.byteLength( + JSON.stringify(singleton.findings[0]!.evidence), + ); + Object.assign(oversized.extra.metadata, { + padding: "x".repeat(256 * 1024 - bytes + 1), + }); + const rawOnly = structuredClone(valid); + rawOnly.extra.message = "x".repeat(66_000); + for (const publishable of [[valid], [rawOnly, valid]]) { + const expected = normalize({ ...input, results: publishable }, options); + const results = [oversized, ...publishable]; + for (const position of [1, results.length]) { + const output = normalize({ ...input, results }, options); + expect(output.findings).toEqual(expected.findings); + expect(output.excluded).toEqual([ + { + position, + source_finding_id: expected.findings[0]!.source_finding_id, + reason: expect.stringContaining("256 KiB"), + }, + ]); + results.reverse(); + } + const local = normalize({ ...input, results }, options, "local"); + expect(local.excluded).toEqual([]); + expect(local.findings[0]!.evidence.source_data!["results"]).toHaveLength( + results.length, + ); + } +}); + +test("raw-only traces consume the space left by all independently publishable siblings", () => { + const input = opengrepJson(); + const result = input.results[0]!; + const padded = (tag: string) => { + const trace = structuredClone(result); + Object.assign(trace.extra.metadata, { padding: tag.repeat(70_000) }); + return trace; + }; + const first = padded("a"); + const second = padded("b"); + const fits = structuredClone(result); + fits.extra.message = "x".repeat(66_000); + const tooLarge = structuredClone(result); + tooLarge.extra.message = "y".repeat(130_000); + const expected = normalize( + { ...input, results: [first, second, fits] }, + options, + ); + expect(expected.excluded).toEqual([]); + expect(expected.findings[0]!.evidence.source_data!["results"]).toHaveLength( + 3, + ); + + const results = [tooLarge, first, fits, second]; + for (const position of [1, results.length]) { + const output = normalize({ ...input, results }, options); + expect(output.findings).toEqual(expected.findings); + expect(output.excluded).toEqual([ + { + position, + source_finding_id: expected.findings[0]!.source_finding_id, + reason: expect.stringContaining("256 KiB"), + }, + ]); + expect(output.excluded[0]!.reason).toContain("description"); + results.reverse(); + } +}); + +test.each([ + ["cloud", "x".repeat(66_000)], + ["local", "a\0b"], +] as const)( + "a %s group without a representative reports each trace's own rejected field", + (target, invalidField) => { + const input = opengrepJson(); + const message = structuredClone(input.results[0]!); + message.extra.message = invalidField; + const snippet = structuredClone(input.results[0]!); + snippet.extra.lines = invalidField; + const results = [message, snippet]; + const reasons = results.map( + (result) => + normalize({ ...input, results: [result] }, options, target).excluded[0]! + .reason, + ); + const output = normalize({ ...input, results }, options, target); + expect(output.findings).toEqual([]); + expect(reasons[0]).toContain("description"); + expect(reasons[1]).toContain("snippet"); + expect(output.excluded.map((item) => item.reason)).toEqual(reasons); + }, +); + +test("raw trace errors exclude only their own positions and keep the source occurrence ID", () => { + const input = opengrepJson(); + const valid = input.results[0]!; + const id = normalize(input, options).findings[0]!.source_finding_id; + const deep = structuredClone(valid); + let nested: unknown = "leaf"; + for (let level = 0; level < 20; level++) nested = { child: nested }; + Object.assign(deep.extra.metadata, { trace: nested }); + const unsafe = structuredClone(valid); + Object.assign(unsafe.extra.metadata, { count: Number.MAX_SAFE_INTEGER + 1 }); + input.results = [deep, valid, unsafe]; + const cloud = normalize(input, options); + expect(cloud.excluded).toEqual([ + { + position: 1, + source_finding_id: id, + reason: expect.stringContaining("depth limit"), + }, + { + position: 3, + source_finding_id: id, + reason: expect.stringContaining("safe integer range"), + }, + ]); + expect(cloud.findings[0]!.evidence.source_data!["results"]).toEqual([valid]); + const local = normalize(input, options, "local"); + expect(local.excluded).toEqual([cloud.excluded[1]!]); + expect(local.findings[0]!.evidence.source_data!["results"]).toHaveLength(2); + expect(local.findings[0]!.evidence.source_data!["results"]).toEqual( + expect.arrayContaining([deep, valid]), + ); +}); + +test.each([ + ["typed description", 66_000, 0, 1], + ["combined evidence", 60_000, 80_000, 0], +] as const)( + "a representative that fits the Cloud %s limit retains every raw trace", + (_, size, padding, singletonExclusions) => { + const input = opengrepJson(); + const valid = input.results[0]!; + Object.assign(valid.extra.metadata, { padding: "x".repeat(padding) }); + const trace = structuredClone(valid); + trace.extra.message = "A".repeat(size); + expect( + normalize({ ...input, results: [trace] }, options).excluded, + ).toHaveLength(singletonExclusions); + input.results.push(trace); + const output = normalize(input, options); + expect(output.excluded).toEqual([]); + expect(output.findings).toHaveLength(1); + expect(output.findings[0]!.evidence.description).toBe(valid.extra.message); + expect(output.findings[0]!.evidence.source_data!["results"]).toEqual([ + trace, + valid, + ]); + input.results.reverse(); + expect(normalize(input, options)).toEqual(output); + }, +); + +test("large run diagnostics do not consume each finding's evidence budget", () => { + const input = opengrepJson(); + input.errors = Array.from({ length: 2000 }, (_, index) => ({ + ...input.errors[0]!, + path: `src/file-${index}.ts`, + })); + const output = normalize(input, options); + expect(output.excluded).toEqual([]); + expect(output.findings).toHaveLength(1); + expect(output.report!["errors"]).toEqual(input.errors); + const sourceData = output.findings[0]!.evidence.source_data!; + expect(sourceData["report"]).not.toHaveProperty("errors"); + expect(sourceData["scan_error_count"]).toBe(2000); +}); + +test("scan-wide matching explanations stay outside each finding's evidence budget", () => { + const input = { + ...opengrepJson(), + explanations: [ + { + op: "pattern", + children: [], + matches: [], + pattern: "x".repeat(300_000), + }, + ], + }; + const output = normalize(input, options); + expect(output.excluded).toEqual([]); + expect(output.findings).toHaveLength(1); + expect(output.report!["explanations"]).toEqual(input.explanations); + expect(output.findings[0]!.evidence.source_data!["report"]).toEqual({ + version: input.version, + }); +}); + +test("ignored matches remain visible as unassessed source evidence", () => { + const input = opengrepJson(); + input.results[0]!.extra.is_ignored = true; + const output = normalize(input, options); + expect(output.excluded).toEqual([]); + expect(output.findings[0]!.evidence.details!.source_status).toBe("ignored"); + expect(output.findings[0]!.evidence.source_data!["results"]).toEqual( + input.results, + ); + expect(output.findings[0]!.evidence).not.toHaveProperty("assessment"); +}); + +test.each([ + "/tmp/example.ts", + "C:\\repo\\example.ts", + "C:file.ts", + "./C:file.ts", + "./D:/other/file.ts", + ".\\D:\\other\\file.ts", + ".\\C:file.ts", + "./\\example.ts", + "../example.ts", +])( + "unattributed path %s stays raw and cannot become a repository location", + (path) => { + const input = opengrepJson(); + input.results[0]!.path = path; + const output = normalize(input, options); + expect(output.excluded).toEqual([]); + expect(output.findings[0]!.evidence.locations).toEqual([]); + expect(output.findings[0]!.evidence.details!.code!.end_line).toBeNull(); + expect(output.findings[0]!.evidence.source_data!["results"]).toEqual( + input.results, + ); + }, +); + +test("SAST rules retain custom product metadata and literal source lines", () => { + const input = opengrepJson(); + Object.assign(input.results[0]!.extra.metadata, { product: "website" }); + input.results[0]!.extra.lines = "requires login"; + const output = normalize(input, options); + expect(output.excluded).toEqual([]); + expect(output.findings[0]!.evidence.details?.code?.snippet).toBe( + "requires login", + ); + expect(output.findings[0]!.evidence.source_data?.["results"]).toEqual( + input.results, + ); + Object.assign(input.results[0]!.extra.metadata, { product: "secrets" }); + expect(normalize(input, options).excluded[0]?.reason).toContain("SAST"); + + Object.assign(input.results[0]!.extra, { metadata: ["audit"], lines: " " }); + const rule = normalize(input, options); + expect(rule.excluded).toEqual([]); + expect(rule.findings[0]!.evidence.details?.code?.snippet).toBe(" "); + expect(rule.findings[0]!.evidence.source_data?.["results"]).toEqual( + input.results, + ); +}); + +test("zero-result reports retain partial parsing, skipped paths and unknown run metadata", () => { + const input: any = opengrepJson(); + input.results = []; + input.errors.push({ + type: ["PartialParsing", [{ path: "src/skipped.ts", start: { line: 3 } }]], + message: "Synthetic unsupported syntax", + }); + const output = normalize(input, options); + expect(output).toMatchObject({ read: 0, findings: [], excluded: [] }); + const { results: _results, ...report } = input; + expect(output.report).toEqual(report); +}); + +test("missing result arrays and SARIF are not treated as empty native reports", () => { + for (const errors of [undefined, null]) { + expect(() => + normalize({ version: "1.32.0", results: [], errors }, options), + ).toThrow("OpenGrep errors must be an array"); + } + expect(() => normalize({ version: "1.32.0" }, options)).toThrow( + "OpenGrep results must be an array", + ); + expect(() => normalize({ version: "2.1.0", runs: [] }, options)).toThrow( + "OpenGrep results must be an array", + ); + expect(() => normalize(opengrepJson(), { sourceRepositoryUrl: "" })).toThrow( + "OpenGrep sourceRepositoryUrl", + ); +}); + +test("run metadata rejects integer tokens beyond JavaScript precision", async () => { + const directory = await mkdtemp(join(tmpdir(), "opengrep-report-")); + try { + const path = join(directory, "report.json"); + await writeFile( + path, + '{"version":"1.32.0","results":[],"errors":[],"ruleMetadata":{"id":9007199254740993}}', + ); + await expect(readOpenGrepFindings(path, options)).rejects.toThrow( + "safe integer range", + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("reads native gzip JSON through the shared scanner reader", async () => { + const directory = await mkdtemp(join(tmpdir(), "opengrep-import-")); + try { + const path = join(directory, "scanner-output.json.gz"); + const input = opengrepJson(); + await writeFile(path, gzipSync(`\uFEFF${JSON.stringify(input)}`)); + expect(await readOpenGrepFindings(path, options)).toEqual( + normalize(input, options), + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/sdk/typescript/tests-ts/support/opengrep-import-fixtures.ts b/sdk/typescript/tests-ts/support/opengrep-import-fixtures.ts new file mode 100644 index 0000000000..4d8f46f5e3 --- /dev/null +++ b/sdk/typescript/tests-ts/support/opengrep-import-fixtures.ts @@ -0,0 +1,72 @@ +// Synthetic native scanner output; no rule implementations or real findings. +export function opengrepJson() { + const location = { + path: "src/example.ts", + start: { line: 3, col: 1, offset: 20 }, + end: { line: 3, col: 9, offset: 28 }, + }; + return { + version: "1.32.0", + errors: [ + { + code: 2, + level: "warn", + type: "Timeout", + rule_id: "example.other-rule", + path: "src/skipped.ts", + message: "Synthetic rule timeout", + }, + ], + paths: { + scanned: ["src/example.ts", "src/skipped.ts"], + skipped: [{ path: "src/skipped.ts", reason: "analysis_failed" }], + }, + time: { total: 0.5 }, + configuration: { digest: "synthetic-config-digest" }, + results: [ + { + check_id: "example.synthetic-rule", + ...location, + extra: { + message: "Synthetic finding", + severity: "WARNING", + fingerprint: "synthetic-fingerprint_0", + is_ignored: false, + lines: "example()", + validation_state: "NO_VALIDATOR", + engine_kind: "OSS", + metadata: { + category: "security", + confidence: "HIGH", + cwe: ["CWE-000"], + references: ["https://example.invalid/rule"], + }, + metavars: { $VALUE: { abstract_content: "input" } }, + dataflow_trace: { + taint_source: ["CliLoc", [structuredClone(location), "input"]], + intermediate_vars: [], + taint_sink: ["CliLoc", [structuredClone(location), "example()"]], + }, + }, + }, + ], + }; +} + +export function opengrepTraceJson() { + const input = opengrepJson(); + const observation = structuredClone(input.results[0]!); + observation.extra.dataflow_trace.taint_source = [ + "CliLoc", + [ + { + path: "src/input.ts", + start: { line: 1, col: 1, offset: 0 }, + end: { line: 1, col: 6, offset: 5 }, + }, + "other", + ], + ]; + input.results.push(observation); + return input; +} From 02a2137b5a9197a0352aaf3f7e4994749bd62eed Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 26 Oct 2026 02:29:42 +0530 Subject: [PATCH 2/2] fix(imports): bind scanner diagnostics to publication retries --- .../src/external-findings-publish.ts | 8 ++++-- .../external-findings-publish.test.ts | 26 ++++++++++++------- .../tests-ts/opengrep-findings-local.test.ts | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 8b36f25c41..0574e7f883 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -400,12 +400,16 @@ export async function prepareNormalizedExternalPublication( identities.add(id); finding.evidence = validateExternalEvidence(finding.evidence); } + const identity = + current.report === undefined + ? current.findings + : { findings: current.findings, report: reportForJson(current.report) }; return preparePublicationInput( { current, legacy: current, - currentIdentity: current.findings, - legacyIdentity: current.findings, + currentIdentity: identity, + legacyIdentity: identity, }, options, dependencies, diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 1f1f01d386..32adc0be63 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -2370,15 +2370,12 @@ test("readback retry preserves acknowledged findings and scanner diagnostics", a const input = opengrepJson(); const f = await fixture(input); f.state.brokenReadback = true; - const prepared = await prepareExternalPublication( - f.file, - { - ...options, - provider: "opengrep", - sourceRepositoryUrl: f.destination().url, - }, - f.deps, - ); + const settings = { + ...options, + provider: "opengrep" as const, + sourceRepositoryUrl: f.destination().url, + }; + const prepared = await prepareExternalPublication(f.file, settings, f.deps); try { await prepared.publish(); throw new Error("Expected readback interruption"); @@ -2392,8 +2389,17 @@ test("readback retry preserves acknowledged findings and scanner diagnostics", a }); (error.result.report!["errors"] as unknown[]).pop(); } + const changed = structuredClone(input); + changed.errors[0]!.message = "A different scanner diagnostic"; + await writeFile(f.file, JSON.stringify(changed)); + const different = await prepareExternalPublication(f.file, settings, f.deps); + expect(different.preview.resumed).toBe(false); + expect(different.preview.report?.["errors"]).toEqual(changed.errors); + await writeFile(f.file, JSON.stringify(input)); + const original = await prepareExternalPublication(f.file, settings, f.deps); + expect(original.preview.resumed).toBe(true); f.state.brokenReadback = false; - const retried = await prepared.publish(); + const retried = await original.publish(); expect(retried.report?.["errors"]).toEqual(input.errors); expect(retried.verified).toBe(1); expect(f.posts).toHaveLength(1); diff --git a/sdk/typescript/tests-ts/opengrep-findings-local.test.ts b/sdk/typescript/tests-ts/opengrep-findings-local.test.ts index 6034a55039..8d4aefafad 100644 --- a/sdk/typescript/tests-ts/opengrep-findings-local.test.ts +++ b/sdk/typescript/tests-ts/opengrep-findings-local.test.ts @@ -114,7 +114,7 @@ test("empty OpenGrep local imports retain scanner diagnostics without creating f repository: fixture.repository, provider: "opengrep" as const, sourceKey: "example-project:main:opengrep", - sourceRepositoryUrl: repositoryUrl, + sourceRepositoryUrl: "https://github.com/Example/PROJECT", }; const dependencies = { environment: fixture.environment }; await expect(