Exit 5 for content never obtained, one paid solve per page, readable outputs, canary without excuses

.github/ci_checks.py

@@ -98,8 +98,8 @@
 def _is_in_virtualenv(path):
     """True if any ancestor of `path` under REPO is a virtualenv root.
 
-    A directory holding `pyvenv.cfg` IS a virtualenv, whatever it is called
-    (family template §23). The name set in SKIP_DIRS missed CI's own
+    A directory holding `pyvenv.cfg` IS a virtualenv, whatever it is called.
+    The name set in SKIP_DIRS missed CI's own
     `.venv-<engine>`, so the engine jobs scanned pip's and selenium's vendored
     code and failed on it — hidden until pipefail made that job's exit status
     real (2026-09-29).
@@ -119,7 +119,10 @@ def scanned_files():
     for path in sorted(REPO.rglob("*")):
         if not path.is_file() or path.suffix not in SCANNED_SUFFIXES:
             continue
-        if any(part in SKIP_DIRS for part in path.parts):
+        # RELATIVE parts only. Matching the absolute path made a clone under
+        # any ancestor named like a skipped directory (…/legacy/flippa-scraper,
+        # …/venv/…) skip every file and print "ok, 0 files scanned".
+        if any(part in SKIP_DIRS for part in path.relative_to(REPO).parts):
             continue
         if _is_in_virtualenv(path):
             continue
@@ -212,6 +215,9 @@ def secret_check():
                 line_no = text[:match.start()].count("\n") + 1
                 failed.append(f"{rel}:{line_no} contains {what}")
 
+    if scanned == 0:
+        failed.append("the credential scan read 0 files — it is looking in the "
+                      "wrong place, and 'nothing found' would mean nothing")
     if not failed:
         print(f"ok       {scanned} files scanned, nothing credential-shaped, "
               f"no session material, no personal data")

.github/workflows/canary.yml

@@ -20,10 +20,8 @@ on:
 jobs:
   live-run:
     runs-on: ubuntu-latest
-    # `secrets` is not available in a job-level `if`, so presence is tested
-    # through job env and read in each step's `if`.
-    env:
-      HAS_PROXY: ${{ secrets.FLIPPA_PROXY != '' }}
+    # Needs no secret: the run goes direct from the runner. FLIPPA_PROXY is
+    # optional and only changes the route when it is set.
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
@@ -63,21 +61,21 @@ jobs:
           code="${{ steps.run.outputs.exit_code }}"
           case "$code" in
             0) echo "OK — listings parsed from a live run." ;;
-            3) if [ "${{ env.HAS_PROXY }}" = "true" ]; then
-                 echo "::error::Blocked before parsing (exit 3) even through the configured proxy — this looks like a site-side change."
-                 exit 1
-               else
-                 echo "::notice::Blocked before parsing (exit 3) from a bare GitHub runner, which is a datacentre address. flippa.com served this URL from an ordinary connection on 2026-09-19, so this is more likely the runner's IP than a site change. Set the FLIPPA_PROXY secret to check it properly."
-                 exit 0
-               fi ;;
+            # flippa.com serves this URL to a bare GitHub runner: every
+            # scheduled run from 2026-09-29 to 2026-10-14 reached the page and
+            # parsed it (exit 4 there was the parser, not a block). So a block
+            # is a real finding here and is never excused as "the runner's IP".
+            3) echo "::error::Blocked before parsing (exit 3). flippa.com has served this URL to GitHub's runners with no proxy, so this is a change on the site's side (or a new block on runner addresses) — read the dumped page in the artefacts." ; exit 1 ;;
             4) echo "::error::Ran fine but parsed 0 listings (exit 4) on a URL known to have hundreds — this is the shape a real site-side break takes." ; exit 1 ;;
+            5) echo "::error::The content was never obtained (exit 5): navigation timed out, the page never painted, or it held listings the parser could not read. See the log and the dumped page." ; exit 1 ;;
             6) echo "::error::Partial run (exit 6) — see the sidecar's stop_reason. 'pagination_not_addressable' means the ?page[number]= convention has stopped working." ; exit 1 ;;
             *) echo "::error::Unexpected exit code $code (see the step above)." ; exit 1 ;;
           esac
 
       - name: Sanity-check the scraped data
         # Exit 0 only means some listings were written; it says nothing about
-        # whether the columns are populated.
+        # whether the columns are populated. (Any other exit already failed
+        # the job in the step above.)
         if: steps.run.outputs.exit_code == '0'
         run: |
           python3 - <<'EOF'

.github/workflows/tests.yml

@@ -123,7 +123,7 @@ jobs:
             selenium)   module=selenium_scraper ;;
           esac
           # The import itself, first: it fails BY NAME whatever wording the
-          # suite uses for a skip (family template §26).
+          # suite uses for a skip.
           python -c "import $module"
           python smoke_test.py | tee smoke.log
           if grep -q "SKIPPED.*$module" smoke.log; then

.gitignore

@@ -9,12 +9,12 @@ flippa_listings.*
 *_debug.html
 *_debug.png
 # --dump-html X over several pages writes X.page2, X.page3 …, which no
-# suffix rule above covers (family template §26).
+# suffix rule above covers.
 *.page[0-9]*
 
 # ANY working directory a live run writes into, not one chosen name: a
 # sibling committed a 1.5 MB raw dump through `live/` because only
-# `live_results/` was listed (family template §24).
+# `live_results/` was listed.
 live/
 live_results/
 out/
@@ -53,6 +53,6 @@ legacy/
 
 # Every spelling of a working .env, not just `.env`: a `.env.bak` or
 # `.env.local` is exactly how a live key gets one `git add -A` from a commit
-# (family template §22). The negation keeps the documented example tracked.
+# The negation keeps the documented example tracked.
 .env*
 !.env.example

CHANGELOG.md

@@ -19,6 +19,21 @@ a bill.
 > `verified_listing` and `image_url` are null on every row, because the card
 > no longer shows them. The columns stay in the schema.
 
+> **Behaviour changes.**
+> - A run that ends with **zero listings because it never obtained the
+>   content** — a navigation timeout, a page that never painted, a page of
+>   listings the parser could not read, a lost worker, a Scraper API error —
+>   now exits **5**, not **4**. Exit 4 is kept for a listing the run can show
+>   is empty (the site says it matches nothing, or the listing ran out). A
+>   pipeline that treated 4 as "nothing listed today" was being told that
+>   about runs that never saw the listing.
+> - Output files and the `.meta.json` sidecar are created with the mode a
+>   plain `open()` would give them (`0o666` minus the umask) instead of
+>   `0600`, and an existing file keeps its mode.
+> - `--fingerprint` is now refused (exit 2) by the pyppeteer and Selenium
+>   engines, and by the Playwright engine together with `--cdp-endpoint`,
+>   where it used to be accepted with a warning and ignored.
+
 ### Fixed
 
 - **The parser reads the rebuilt search page.** Measured 2026-10-14 on the
@@ -56,6 +71,40 @@ when a column is planted null. `sample_output.*` is cut from that run (broker
 names redacted). The pyppeteer, Selenium and Scraper API engines share the
 parser but were not re-run live.
 
+- **One paid captcha solve per page.** Every engine now routes its solve
+  through a per-page allowance (`page_flow.SolveBudget`, limit
+  `SOLVES_PER_PAGE = 1`) checked before the solver is called and not reset by
+  a retry or an exit rotation. Measured with a stubbed solver and a page that
+  is always a Cloudflare interstitial: a three-exit pool bought 3 solves for
+  one page before, 1 after.
+- **The HTTP status reaches the page classifier** in the Playwright and
+  pyppeteer engines (from what `goto()` returns), so a refusal page with no
+  vendor marker is classified by its status. Selenium exposes no status and
+  passes none, explicitly. The Scraper API client reads the target's status
+  from `http_code`; `status` is the API's own verdict string.
+- **Scraper API `waitFor` is sent as an object.** The JSON-string form is
+  answered with HTTP 422 and still billed. `waitFor.state` is checked against
+  `load` / `domcontentloaded`.
+- **The Scraper API's `x-debug` header is redacted** before it is logged.
+- **CSV formula guard**: a cell starting with a line feed is escaped too, and
+  the sidecar records `csv_cells_escaped` — how many CSV cells differ from
+  the JSON because of it.
+- **The credential scan** matches its skipped directories against paths
+  RELATIVE to the repository, so a clone under an ancestor called `legacy`,
+  `captures` or `venv` is no longer skipped whole, and a scan that reads 0
+  files now fails instead of printing "ok".
+- **A suite check needed a browser binary and a clean `~/.cache`**: the
+  fingerprint-refusal check now skips when Playwright has no Chromium, and
+  runs with a private HOME so a cached fingerprint cannot answer it.
+- Public files no longer cite private instruction files by section number;
+  the rules are stated in this repository's own words, and a suite check
+  keeps it that way.
+- The weekly `pip-audit` job had been red since 2026-10-05 on two new urllib3
+  1.x advisories (PYSEC-2026-4175, PYSEC-2026-4177) in the pyppeteer lock.
+  pyppeteer 2.0.0 still requires `urllib3<2` and the fix is only in 2.8.0, so
+  they are ignored by ID beside the five already listed, and the README says
+  so.
+
 ## [0.2.1] — 2026-09-29
 
 ### Fixed
@@ -84,7 +133,7 @@ success.
   step's exit status real, all three engine jobs went red on the secret
   scan: it walked into CI's own `.venv-<engine>` and read pip's and
   selenium's vendored code as credentials. The scan now skips any directory
-  holding `pyvenv.cfg`, whatever it is called (template §23), and still
+  holding `pyvenv.cfg`, whatever it is called, and still
   reads ordinary untracked files — both planted in a check.
 - **The Playwright engine job never had a browser**, so the suite's
   fingerprint-refusal check — which starts Chromium before it fetches —
@@ -111,7 +160,7 @@ success.
   2026-09-29.
 
 - **Connecting to the Scraping Browser API** follows one policy in every
-  engine (family template §26): three attempts 3 s apart for a locked
+  engine: three attempts 3 s apart for a locked
   profile or an outage, none for an expired login (401), and a message that
   names the status's own cause. pyppeteer's per-attempt timeout is 10 s (it
   reports a refused handshake only as a timeout), and the orphaned-task

CONTRIBUTING.md

@@ -106,7 +106,8 @@ silently regress:
 - **A run that finds nothing writes nothing.** It must not replace a good output
   file with `[]`. `--allow-empty` is the opt-out.
 - **Exit codes are a contract**, not decoration: `0` ok, `1` crash, `2` bad
-  usage, `3` blocked by a challenge, `4` zero listings, `5` remote API error,
+  usage, `3` blocked by a challenge, `4` zero listings from a listing shown
+  to be empty, `5` zero listings because the content was never obtained,
   `6` partial run. A pipeline branches on these, and all four engines produce
   the same code for the same situation because they share `finish_run`.
 - **A sku already written by an earlier page of the same run is dropped, not

README.md

@@ -130,6 +130,16 @@ Install exactly one engine per environment: playwright and pyppeteer pin
 incompatible `pyee` versions, and pyppeteer and selenium collide on
 `urllib3`. Use a virtualenv per engine if you want more than one.
 
+**The pyppeteer engine carries a known-vulnerable urllib3.** pyppeteer
+requires `urllib3<2`, and seven published urllib3 advisories (as of
+2026-10-15) are fixed only in 2.x — among them one where a cross-origin
+redirect can forward `Proxy-Authorization` to the new host
+(GHSA-qccp-gfcp-xxvc), and one where the TLS settings meant for an HTTPS
+proxy and for the target server can be mixed up (PYSEC-2026-4175). In that
+engine's virtualenv `requests` uses the same urllib3, so the proxy preflight
+and the captcha API calls do too. The weekly `pip-audit` job ignores exactly
+those seven, by ID; prefer Playwright if you run with proxy credentials.
+
 Credentials live in `.env`, never on a command line — a secret in `argv` is
 readable by anything that can run `ps` and lands in shell history:
 
@@ -200,8 +210,8 @@ same situation.
 | 1 | crash |
 | 2 | bad usage, including an exit that fails its preflight — one small request through the proxy before any browser starts, so a rotated password costs a second rather than three minutes of timeouts |
 | 3 | blocked before parsing — a challenge, a refusal status, or a page not built out of Flippa's own assets |
-| 4 | ran fine, zero listings (including "the site says this query matches nothing") |
-| 5 | the remote API failed (`scraper_api_client.py`) |
+| 4 | zero listings, and the run can show the listing is empty: the site says the query matches nothing, or the listing ran out |
+| 5 | zero listings because the content was never obtained: a remote API error, a navigation timeout, a dead proxy, a page that never painted, or a page of listings the parser could not read |
 | 6 | partial: some pages were gathered, then the run stopped early |
 
 Every run that writes output also writes `<out>.meta.json`. Two fields there

SECURITY.md

@@ -97,13 +97,16 @@ it is recoverable. In order:
    anything pushed to a public repository should be treated as public
    permanently, whether or not the commit still exists.
 
-Three places leak credentials that people do not expect, because unlike our own
+Two places leak credentials that people do not expect, because unlike our own
 log lines they are **not** masked:
 
 - **raw HTML dumps** (`--dump-html`, and the automatic dump on a zero-product
   run) — these can contain session cookies
-- **the Scraper API's `x-debug` response header**
 - **your shell history**, if you passed a key on the command line
 
+The Scraper API's `x-debug` response header is logged, and since the release
+after 0.2.1 it goes through the same redaction as every other log line; it was
+logged verbatim before that.
+
 Use `.env` for credentials. It is in `.gitignore`, and `python3 env_config.py`
 reports what is configured without printing any values.

fingerprint_client.py

@@ -237,8 +237,8 @@ def playwright_context_kwargs(fp: dict) -> dict:
     if found["locale"]["value"]:
         kwargs["locale"] = str(found["locale"]["value"])
     else:
-        # Deliberately NOT en-{country}: that is the exact bug §16 names, and
-        # it produced "en-DE" for a German fingerprint.
+        # Deliberately NOT en-{country}: an earlier version did exactly that,
+        # and it produced "en-DE" for a German fingerprint.
         logger.warning("This fingerprint carries no locale — leaving the "
                        "browser's own rather than inventing one from the "
                        "country.")