fix(exaforce): let LLM verdict override static severity floor

conftest.py

@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Fork-owned pytest hook: xfail the upstream tests the exaforce patches invert.
+
+``exaforce/_schema_patches`` prunes keys from the LLM structured-output schema,
+so the upstream tests that assert the un-pruned shape fail.
+
+Fork policy is to keep upstream test files byte-identical — no deselect
+markers, no ``xfail`` decorators, no edits — so an upstream sync never
+conflicts in them. This file satisfies that: it lives at the repo root, is not
+upstream-tracked, and attaches the marker at collection time.
+
+``strict=True`` is the point. A doc that merely lists the expected failures
+cannot tell a genuine regression from the expected noise, and cannot notice
+when an upstream sync makes one of these pass again. Under strict xfail both
+show up: an unexpected failure elsewhere in the file still fails the run, and
+an XPASS here fails too — meaning the fork patch no longer changes this
+behavior, so re-check the patch and drop the entry.
+
+``exaforce/_filter_patches`` needs no entry in its default ``semantic`` mode.
+It lifts the CRITICAL/HIGH floor only for a finding that both carries a
+``category`` (i.e. came from a static rule) and was actually adjudicated by the
+meta-analyzer; every fixture in upstream's ``TestApplyFilterSeverityFloor``
+fails one of those two conditions, so those tests still pass. Under the opt-in
+``SKILLSPECTOR_META_SEVERITY_FLOOR=none`` the floor is empty for everything,
+which does invert one of them — marked conditionally below, so the run stays
+green and strictly checked in that mode too. Fork-side coverage lives in
+``tests/exaforce/test_patches.py``.
+
+Keep this list in sync with ``docs/superpowers/EXPECTED_TEST_FAILURES.md``.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+_SCHEMA_PRUNING_REASON = (
+    "exaforce _schema_patches prunes explanation/remediation/intent from the "
+    "LLM structured-output schema; upstream asserts the un-pruned shape"
+)
+
+_NO_FLOOR_REASON = (
+    "SKILLSPECTOR_META_SEVERITY_FLOOR=none empties the floor for every finding, "
+    "including LLM-backed ones; upstream asserts the floor"
+)
+
+_SCHEMA_PRUNING = (
+    "tests/nodes/test_llm_analyzer_base.py::TestLLMAnalysisResult::test_to_finding",
+    "tests/nodes/test_llm_analyzer_base.py::TestLLMAnalysisResult::test_model_dump",
+    "tests/nodes/test_llm_analyzer_base.py::TestMetaAnalyzerResult::test_intent_validation",
+    "tests/nodes/test_semantic_quality_policy.py::TestFixtureMaliciousSkill"
+    "::test_malicious_skill_findings_preserve_metadata",
+)
+
+# Only inverted by the fully-empty floor; passes under "semantic" and "upstream".
+_NO_FLOOR_ONLY = (
+    "tests/nodes/test_llm_analyzer_base.py::TestApplyFilterSeverityFloor"
+    "::test_critical_unconfirmed_kept_with_llm_unconfirmed_tag",
+)
+
+
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    """Attach ``xfail(strict=True)`` to the upstream tests the fork patches invert."""
+    from skillspector.exaforce._filter_patches import resolve_mode
+
+    expected = dict.fromkeys(_SCHEMA_PRUNING, _SCHEMA_PRUNING_REASON)
+    if resolve_mode() == "none":
+        expected.update(dict.fromkeys(_NO_FLOOR_ONLY, _NO_FLOOR_REASON))
+    for item in items:
+        reason = expected.get(item.nodeid)
+        if reason is not None:
+            item.add_marker(pytest.mark.xfail(reason=reason, strict=True))

docs/superpowers/EXPECTED_TEST_FAILURES.md

@@ -1,22 +1,65 @@
-# Expected test failures (fork: exaforce schema pruning)
+# Expected test failures (fork: exaforce runtime patches)
 
-These upstream tests are kept at upstream parity on purpose and therefore
-assert the *un-pruned* schema, which the exaforce runtime patch removes. They
-are expected to FAIL. A failure here is only a problem if the failure is NOT an
-assertion about a pruned key (e.g. an import/collection error).
+`exaforce/_schema_patches` prunes keys from the LLM structured-output schema, so
+the four upstream tests that assert the un-pruned shape fail. Those tests are
+kept at upstream parity on purpose — no deselect markers, no `xfail`
+decorators, no edits — so an upstream sync never conflicts in them.
 
-Captured from:
-`uv run pytest tests/nodes/test_llm_analyzer_base.py tests/nodes/test_semantic_quality_policy.py -q`
+They are marked `xfail(strict=True)` at collection time by the fork-owned
+`conftest.py` at the repo root, which is not upstream-tracked. CI
+(`.github/workflows/ci.yml` → `make test-ci`) therefore stays green, and the
+expectation is machine-checked in both directions:
 
-- tests/nodes/test_llm_analyzer_base.py::TestLLMAnalysisResult::test_to_finding
-- tests/nodes/test_llm_analyzer_base.py::TestLLMAnalysisResult::test_model_dump
-- tests/nodes/test_llm_analyzer_base.py::TestMetaAnalyzerResult::test_intent_validation
-- tests/nodes/test_semantic_quality_policy.py::TestFixtureMaliciousSkill::test_malicious_skill_findings_preserve_metadata
+- a genuine new failure in one of these files still fails the run, instead of
+  hiding inside a documented block of expected noise;
+- if an upstream sync ever makes one of these pass again, `strict` turns the
+  XPASS into a failure — so the stale entry gets noticed rather than quietly
+  masking the fact that the fork patch no longer changes anything.
 
-All four fail with an `AssertionError` (or `KeyError`) about a pruned key
-(`explanation`, `intent`) being absent — not an import/collection error.
-Confirmed bounded to these two files via `uv run pytest -q -rf`:
+Keep this list in sync with `conftest.py`.
+
+## Schema pruning (`exaforce/_schema_patches.py`)
+
+Each fails with an `AssertionError` (or `KeyError`) about a pruned key
+(`explanation`, `intent`) being absent — never an import/collection error.
+
+- `tests/nodes/test_llm_analyzer_base.py::TestLLMAnalysisResult::test_to_finding`
+- `tests/nodes/test_llm_analyzer_base.py::TestLLMAnalysisResult::test_model_dump`
+- `tests/nodes/test_llm_analyzer_base.py::TestMetaAnalyzerResult::test_intent_validation`
+- `tests/nodes/test_semantic_quality_policy.py::TestFixtureMaliciousSkill::test_malicious_skill_findings_preserve_metadata`
+
+## Severity floor (`exaforce/_filter_patches.py`) — none in the default mode
+
+`_filter_patches` lifts the CRITICAL/HIGH floor only for a finding that both
+carries a `category` — i.e. came from a static rule, since `LLMFinding.to_finding`
+sets none — and was actually adjudicated by the meta-analyzer. Every fixture in
+upstream's `TestApplyFilterSeverityFloor` fails one of those two conditions:
+the fixtures build a bare `Finding` with no `category`, and the omission cases
+pass an empty verdict list, which keeps the upstream floor by design (see the
+module docstring). So under the default `SKILLSPECTOR_META_SEVERITY_FLOOR=semantic`,
+and under `=upstream`, upstream's assertions all still hold — and they still
+exercise the floored path rather than passing vacuously.
+
+The opt-in `=none` empties the floor for every finding regardless of source,
+which inverts exactly one of them:
+
+- `tests/nodes/test_llm_analyzer_base.py::TestApplyFilterSeverityFloor::test_critical_unconfirmed_kept_with_llm_unconfirmed_tag`
+
+`conftest.py` marks that one only when the resolved mode is `none`, so the run
+is green and strictly checked in all three modes.
+
+Fork-side coverage of the three modes, the empty-verdict case, and the
+`category` invariant that makes this work lives in
+`tests/exaforce/test_patches.py`.
+
+## Captured
+
+`uv run pytest -m "not integration and not provider" tests/ -q` (2026-09-08):
 
 ```
-4 failed, 1261 passed, 13 skipped, 34 deselected, 6 xfailed
+1924 passed, 13 skipped, 38 deselected, 8 xfailed
 ```
+
+The 8 xfailed are the 4 schema-pruning entries above plus 4 pre-existing
+upstream xfails. With `SKILLSPECTOR_META_SEVERITY_FLOOR=none` it is
+1923 passed / 9 xfailed. Nothing fails in any mode.

src/skillspector/__init__.py

@@ -38,4 +38,5 @@
 
 # ExaForce fork: apply runtime schema/prompt patches (kept out of upstream files).
 from skillspector import exaforce as _exaforce  # noqa: E402
+
 _exaforce.apply_patches()

src/skillspector/exaforce/__init__.py

@@ -2,14 +2,15 @@
 """ExaForce fork-local runtime patches.
 
 Keeps fork behavior — pruning unused LLM structured-output keys and prompt text
-to shrink requests and reduce LLM timeouts — out of upstream-tracked source
-files. All mutations are guarded: an upstream rename/rewrite raises
-``PatchDriftError`` at import time rather than silently going stale.
+to shrink requests and reduce LLM timeouts, and trusting the meta-analyzer LLM
+verdict over static severity — out of upstream-tracked source files. All
+mutations are guarded: an upstream rename/rewrite raises ``PatchDriftError``
+at import time rather than silently going stale.
 """
 
 from __future__ import annotations
 
-from . import _prompt_patches, _sampling_patches, _schema_patches
+from . import _filter_patches, _prompt_patches, _sampling_patches, _schema_patches
 
 _PATCHED = False
 
@@ -22,4 +23,5 @@ def apply_patches() -> None:
     _schema_patches.apply()
     _prompt_patches.apply()
     _sampling_patches.apply()
+    _filter_patches.apply()
     _PATCHED = True