From e0ec0e720b2e7e913ad8983dc39b224b97451f3d Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:56:02 +0000 Subject: [PATCH 1/2] Exit 5 when content was never obtained, one paid solve per page, readable output files - A zero-row run exits 4 only when the listing is shown empty; a timeout, an unpainted or unparsable page, a lost worker or an API error is 5. - One captcha-solve allowance per page in every engine, checked before the purchase and not reset by retries or exit rotation (3 solves -> 1). - Playwright/pyppeteer pass the goto() status to the classifier; the Scraper API reads http_code, sends waitFor as an object, redacts x-debug. - Outputs and sidecar get 0o666 & ~umask (existing files keep their mode). - CSV guard escapes a leading LF; csv_cells_escaped in the sidecar. - Credential scan uses paths relative to the repo and fails on 0 files. - --fingerprint refused where it is not applied. - Canary: exit 3 and 5 are errors; the site serves GitHub runners. - No citations of private instruction files; a suite check guards it. Co-Authored-By: Claude Opus 5.5 --- .github/ci_checks.py | 12 +- .github/workflows/canary.yml | 22 ++- .github/workflows/tests.yml | 2 +- .gitignore | 6 +- CHANGELOG.md | 48 +++++- CONTRIBUTING.md | 3 +- README.md | 4 +- SECURITY.md | 7 +- fingerprint_client.py | 4 +- output_writer.py | 94 +++++++++-- page_flow.py | 24 +++ playwright_scraper.py | 58 +++++-- proxy_pool.py | 2 +- puppeteer_scraper.py | 52 ++++-- scraper_api_client.py | 44 +++-- selenium_scraper.py | 44 +++-- smoke_test.py | 318 ++++++++++++++++++++++++++++++++++- 17 files changed, 628 insertions(+), 116 deletions(-) diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 22fee7c..ae03658 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -98,8 +98,8 @@ def _is_in_virtualenv(path): """True if any ancestor of `path` under REPO is a virtualenv root. - A directory holding `pyvenv.cfg` IS a virtualenv, whatever it is called - (family template §23). The name set in SKIP_DIRS missed CI's own + A directory holding `pyvenv.cfg` IS a virtualenv, whatever it is called. + The name set in SKIP_DIRS missed CI's own `.venv-`, so the engine jobs scanned pip's and selenium's vendored code and failed on it — hidden until pipefail made that job's exit status real (2026-09-29). @@ -119,7 +119,10 @@ def scanned_files(): for path in sorted(REPO.rglob("*")): if not path.is_file() or path.suffix not in SCANNED_SUFFIXES: continue - if any(part in SKIP_DIRS for part in path.parts): + # RELATIVE parts only. Matching the absolute path made a clone under + # any ancestor named like a skipped directory (…/legacy/flippa-scraper, + # …/venv/…) skip every file and print "ok, 0 files scanned". + if any(part in SKIP_DIRS for part in path.relative_to(REPO).parts): continue if _is_in_virtualenv(path): continue @@ -212,6 +215,9 @@ def secret_check(): line_no = text[:match.start()].count("\n") + 1 failed.append(f"{rel}:{line_no} contains {what}") + if scanned == 0: + failed.append("the credential scan read 0 files — it is looking in the " + "wrong place, and 'nothing found' would mean nothing") if not failed: print(f"ok {scanned} files scanned, nothing credential-shaped, " f"no session material, no personal data") diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 294d8bb..b1870b1 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -20,10 +20,8 @@ on: jobs: live-run: runs-on: ubuntu-latest - # `secrets` is not available in a job-level `if`, so presence is tested - # through job env and read in each step's `if`. - env: - HAS_PROXY: ${{ secrets.FLIPPA_PROXY != '' }} + # Needs no secret: the run goes direct from the runner. FLIPPA_PROXY is + # optional and only changes the route when it is set. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -63,21 +61,21 @@ jobs: code="${{ steps.run.outputs.exit_code }}" case "$code" in 0) echo "OK — listings parsed from a live run." ;; - 3) if [ "${{ env.HAS_PROXY }}" = "true" ]; then - echo "::error::Blocked before parsing (exit 3) even through the configured proxy — this looks like a site-side change." - exit 1 - else - echo "::notice::Blocked before parsing (exit 3) from a bare GitHub runner, which is a datacentre address. flippa.com served this URL from an ordinary connection on 2026-09-19, so this is more likely the runner's IP than a site change. Set the FLIPPA_PROXY secret to check it properly." - exit 0 - fi ;; + # flippa.com serves this URL to a bare GitHub runner: every + # scheduled run from 2026-09-29 to 2026-10-14 reached the page and + # parsed it (exit 4 there was the parser, not a block). So a block + # is a real finding here and is never excused as "the runner's IP". + 3) echo "::error::Blocked before parsing (exit 3). flippa.com has served this URL to GitHub's runners with no proxy, so this is a change on the site's side (or a new block on runner addresses) — read the dumped page in the artefacts." ; exit 1 ;; 4) echo "::error::Ran fine but parsed 0 listings (exit 4) on a URL known to have hundreds — this is the shape a real site-side break takes." ; exit 1 ;; + 5) echo "::error::The content was never obtained (exit 5): navigation timed out, the page never painted, or it held listings the parser could not read. See the log and the dumped page." ; exit 1 ;; 6) echo "::error::Partial run (exit 6) — see the sidecar's stop_reason. 'pagination_not_addressable' means the ?page[number]= convention has stopped working." ; exit 1 ;; *) echo "::error::Unexpected exit code $code (see the step above)." ; exit 1 ;; esac - name: Sanity-check the scraped data # Exit 0 only means some listings were written; it says nothing about - # whether the columns are populated. + # whether the columns are populated. (Any other exit already failed + # the job in the step above.) if: steps.run.outputs.exit_code == '0' run: | python3 - <<'EOF' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4fef695..6d0fdc9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -123,7 +123,7 @@ jobs: selenium) module=selenium_scraper ;; esac # The import itself, first: it fails BY NAME whatever wording the - # suite uses for a skip (family template §26). + # suite uses for a skip. python -c "import $module" python smoke_test.py | tee smoke.log if grep -q "SKIPPED.*$module" smoke.log; then diff --git a/.gitignore b/.gitignore index 05470d0..3868dff 100644 --- a/.gitignore +++ b/.gitignore @@ -9,12 +9,12 @@ flippa_listings.* *_debug.html *_debug.png # --dump-html X over several pages writes X.page2, X.page3 …, which no -# suffix rule above covers (family template §26). +# suffix rule above covers. *.page[0-9]* # ANY working directory a live run writes into, not one chosen name: a # sibling committed a 1.5 MB raw dump through `live/` because only -# `live_results/` was listed (family template §24). +# `live_results/` was listed. live/ live_results/ out/ @@ -53,6 +53,6 @@ legacy/ # Every spelling of a working .env, not just `.env`: a `.env.bak` or # `.env.local` is exactly how a live key gets one `git add -A` from a commit -# (family template §22). The negation keeps the documented example tracked. +# The negation keeps the documented example tracked. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 834bb93..6be541a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,21 @@ a bill. > `verified_listing` and `image_url` are null on every row, because the card > no longer shows them. The columns stay in the schema. +> **Behaviour changes.** +> - A run that ends with **zero listings because it never obtained the +> content** — a navigation timeout, a page that never painted, a page of +> listings the parser could not read, a lost worker, a Scraper API error — +> now exits **5**, not **4**. Exit 4 is kept for a listing the run can show +> is empty (the site says it matches nothing, or the listing ran out). A +> pipeline that treated 4 as "nothing listed today" was being told that +> about runs that never saw the listing. +> - Output files and the `.meta.json` sidecar are created with the mode a +> plain `open()` would give them (`0o666` minus the umask) instead of +> `0600`, and an existing file keeps its mode. +> - `--fingerprint` is now refused (exit 2) by the pyppeteer and Selenium +> engines, and by the Playwright engine together with `--cdp-endpoint`, +> where it used to be accepted with a warning and ignored. + ### Fixed - **The parser reads the rebuilt search page.** Measured 2026-10-14 on the @@ -56,6 +71,35 @@ when a column is planted null. `sample_output.*` is cut from that run (broker names redacted). The pyppeteer, Selenium and Scraper API engines share the parser but were not re-run live. +- **One paid captcha solve per page.** Every engine now routes its solve + through a per-page allowance (`page_flow.SolveBudget`, limit + `SOLVES_PER_PAGE = 1`) checked before the solver is called and not reset by + a retry or an exit rotation. Measured with a stubbed solver and a page that + is always a Cloudflare interstitial: a three-exit pool bought 3 solves for + one page before, 1 after. +- **The HTTP status reaches the page classifier** in the Playwright and + pyppeteer engines (from what `goto()` returns), so a refusal page with no + vendor marker is classified by its status. Selenium exposes no status and + passes none, explicitly. The Scraper API client reads the target's status + from `http_code`; `status` is the API's own verdict string. +- **Scraper API `waitFor` is sent as an object.** The JSON-string form is + answered with HTTP 422 and still billed. `waitFor.state` is checked against + `load` / `domcontentloaded`. +- **The Scraper API's `x-debug` header is redacted** before it is logged. +- **CSV formula guard**: a cell starting with a line feed is escaped too, and + the sidecar records `csv_cells_escaped` — how many CSV cells differ from + the JSON because of it. +- **The credential scan** matches its skipped directories against paths + RELATIVE to the repository, so a clone under an ancestor called `legacy`, + `captures` or `venv` is no longer skipped whole, and a scan that reads 0 + files now fails instead of printing "ok". +- **A suite check needed a browser binary and a clean `~/.cache`**: the + fingerprint-refusal check now skips when Playwright has no Chromium, and + runs with a private HOME so a cached fingerprint cannot answer it. +- Public files no longer cite private instruction files by section number; + the rules are stated in this repository's own words, and a suite check + keeps it that way. + ## [0.2.1] — 2026-09-29 ### Fixed @@ -84,7 +128,7 @@ success. step's exit status real, all three engine jobs went red on the secret scan: it walked into CI's own `.venv-` and read pip's and selenium's vendored code as credentials. The scan now skips any directory - holding `pyvenv.cfg`, whatever it is called (template §23), and still + holding `pyvenv.cfg`, whatever it is called, and still reads ordinary untracked files — both planted in a check. - **The Playwright engine job never had a browser**, so the suite's fingerprint-refusal check — which starts Chromium before it fetches — @@ -111,7 +155,7 @@ success. 2026-09-29. - **Connecting to the Scraping Browser API** follows one policy in every - engine (family template §26): three attempts 3 s apart for a locked + engine: three attempts 3 s apart for a locked profile or an outage, none for an expired login (401), and a message that names the status's own cause. pyppeteer's per-attempt timeout is 10 s (it reports a refused handshake only as a timeout), and the orphaned-task diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc296cc..980de82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,8 @@ silently regress: - **A run that finds nothing writes nothing.** It must not replace a good output file with `[]`. `--allow-empty` is the opt-out. - **Exit codes are a contract**, not decoration: `0` ok, `1` crash, `2` bad - usage, `3` blocked by a challenge, `4` zero listings, `5` remote API error, + usage, `3` blocked by a challenge, `4` zero listings from a listing shown + to be empty, `5` zero listings because the content was never obtained, `6` partial run. A pipeline branches on these, and all four engines produce the same code for the same situation because they share `finish_run`. - **A sku already written by an earlier page of the same run is dropped, not diff --git a/README.md b/README.md index 0848b06..b04c577 100644 --- a/README.md +++ b/README.md @@ -200,8 +200,8 @@ same situation. | 1 | crash | | 2 | bad usage, including an exit that fails its preflight — one small request through the proxy before any browser starts, so a rotated password costs a second rather than three minutes of timeouts | | 3 | blocked before parsing — a challenge, a refusal status, or a page not built out of Flippa's own assets | -| 4 | ran fine, zero listings (including "the site says this query matches nothing") | -| 5 | the remote API failed (`scraper_api_client.py`) | +| 4 | zero listings, and the run can show the listing is empty: the site says the query matches nothing, or the listing ran out | +| 5 | zero listings because the content was never obtained: a remote API error, a navigation timeout, a dead proxy, a page that never painted, or a page of listings the parser could not read | | 6 | partial: some pages were gathered, then the run stopped early | Every run that writes output also writes `.meta.json`. Two fields there diff --git a/SECURITY.md b/SECURITY.md index 76495f0..5bfbbf6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -97,13 +97,16 @@ it is recoverable. In order: anything pushed to a public repository should be treated as public permanently, whether or not the commit still exists. -Three places leak credentials that people do not expect, because unlike our own +Two places leak credentials that people do not expect, because unlike our own log lines they are **not** masked: - **raw HTML dumps** (`--dump-html`, and the automatic dump on a zero-product run) — these can contain session cookies -- **the Scraper API's `x-debug` response header** - **your shell history**, if you passed a key on the command line +The Scraper API's `x-debug` response header is logged, and since the release +after 0.2.1 it goes through the same redaction as every other log line; it was +logged verbatim before that. + Use `.env` for credentials. It is in `.gitignore`, and `python3 env_config.py` reports what is configured without printing any values. diff --git a/fingerprint_client.py b/fingerprint_client.py index 8536349..4d80b3e 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -237,8 +237,8 @@ def playwright_context_kwargs(fp: dict) -> dict: if found["locale"]["value"]: kwargs["locale"] = str(found["locale"]["value"]) else: - # Deliberately NOT en-{country}: that is the exact bug §16 names, and - # it produced "en-DE" for a German fingerprint. + # Deliberately NOT en-{country}: an earlier version did exactly that, + # and it produced "en-DE" for a German fingerprint. logger.warning("This fingerprint carries no locale — leaving the " "browser's own rather than inventing one from the " "country.") diff --git a/output_writer.py b/output_writer.py index d3b9460..423991e 100644 --- a/output_writer.py +++ b/output_writer.py @@ -113,6 +113,14 @@ def dedupe_by_sku(products: List[Product], seen: Set[str]) -> List[Product]: return fresh +def _current_umask() -> int: + """The process umask. os.umask can only be read by setting it, so it is + set and immediately restored.""" + mask = os.umask(0o022) + os.umask(mask) + return mask + + def _atomic_write(path: str, write: Callable) -> None: """Write `path` through a temporary file in the same directory, then rename. @@ -121,9 +129,20 @@ def _atomic_write(path: str, write: Callable) -> None: consumer can tell a truncated JSON from a short run. os.replace is atomic within a filesystem, so a reader sees either the previous complete file or the new complete one and never a mixture of the two. + + The file's MODE is the one a plain open() would have given it: mkstemp + creates its temporary at 0600, and os.replace keeps that, so without the + chmod every output and sidecar came out readable by its owner only. A new + file gets 0o666 minus the umask; an existing file keeps the mode it had, + so a file someone made group-readable stays that way. Never a flat 0644, + which would narrow 0664 under umask 002. """ directory = os.path.dirname(os.path.abspath(path)) os.makedirs(directory, exist_ok=True) + try: + mode = os.stat(path).st_mode & 0o7777 + except FileNotFoundError: + mode = 0o666 & ~_current_umask() fd, tmp = tempfile.mkstemp(dir=directory, prefix=".tmp-", suffix=f"-{os.path.basename(path)}") try: @@ -132,6 +151,7 @@ def _atomic_write(path: str, write: Callable) -> None: write(f) f.flush() os.fsync(f.fileno()) + os.chmod(tmp, mode) os.replace(tmp, path) except BaseException: try: @@ -142,9 +162,9 @@ def _atomic_write(path: str, write: Callable) -> None: # A leading one of these makes a spreadsheet treat the cell as a formula -# rather than as text. \t and \r are here because Excel strips them and then -# reads what follows. -_FORMULA_LEAD = ("=", "+", "-", "@", "\t", "\r") +# rather than as text. \t, \r and \n are here because spreadsheet apps strip +# leading whitespace of that kind and then read what follows. +_FORMULA_LEAD = ("=", "+", "-", "@", "\t", "\r", "\n") def csv_safe(value): @@ -214,29 +234,52 @@ def write_json(products: List[Product], path: str) -> None: [asdict(p) for p in products], f, ensure_ascii=False, indent=2)) -def write_csv(products: List[Product], path: str) -> None: +def write_csv(products: List[Product], path: str) -> int: + """Write the CSV; return how many cells csv_safe() had to escape. + + The count goes into the run's sidecar as `csv_cells_escaped`, so that the + CSV and the JSON (which carries the site's bytes unchanged) differing is + declared rather than discovered. + """ # An empty result still gets the header row. A zero-byte file makes a # consumer fail on read (no columns to parse) instead of reading a valid # table with zero rows. if not products: _atomic_write(path, lambda f: csv.DictWriter( f, fieldnames=list(asdict(Product()).keys())).writeheader()) - return + return 0 fieldnames = list(asdict(products[0]).keys()) + escaped = 0 def _write(f): + nonlocal escaped + escaped = 0 writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for p in products: - writer.writerow({k: csv_safe(v) for k, v in asdict(p).items()}) + row = {} + for k, v in asdict(p).items(): + safe = csv_safe(v) + if safe is not v: + escaped += 1 + row[k] = safe + writer.writerow(row) _atomic_write(path, _write) + return escaped # Exit code used when a run completes but produced nothing. Distinct from 1 # (crash) so a caller can tell "ran, found nothing" from "blew up". EXIT_NO_PRODUCTS = 4 +# Exit code for a run that holds nothing because the CONTENT WAS NEVER +# OBTAINED: a remote API error, a navigation timeout, a dead proxy, or a page +# full of listings this parser could not read. Distinct from 4, which claims +# the listing itself is empty — a claim nothing in such a run supports. +EXIT_CONTENT_NOT_OBTAINED = 5 +EXIT_API_ERROR = EXIT_CONTENT_NOT_OBTAINED + # Exit code for a run blocked by a bot-check/challenge page before parsing # even started — distinct from EXIT_NO_PRODUCTS so a caller can tell "the # search genuinely matched nothing" from "something stood between us and the @@ -382,7 +425,7 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, def save(products: List[Product], out_prefix: str, fmt: str, - allow_empty: bool = False) -> int: + allow_empty: bool = False, stats: Optional[dict] = None) -> int: """Write JSON/CSV and return a process exit code. On zero products, nothing is written at all unless `allow_empty`. A @@ -405,7 +448,9 @@ def save(products: List[Product], out_prefix: str, fmt: str, write_json(products, f"{out_prefix}.json") print(f"[+] Saved {len(products)} listings -> {out_prefix}.json") if fmt in ("csv", "both"): - write_csv(products, f"{out_prefix}.csv") + escaped = write_csv(products, f"{out_prefix}.csv") + if stats is not None: + stats["csv_cells_escaped"] = escaped print(f"[+] Saved {len(products)} listings -> {out_prefix}.csv") return 0 if products else EXIT_NO_PRODUCTS @@ -427,6 +472,14 @@ def save(products: List[Product], out_prefix: str, fmt: str, # file hold the whole listing". EXHAUSTIVE_STOP_REASONS = ("listing_exhausted", "no_new_listings", "no_results") +# Stop reasons under which a run that holds NO rows may honestly say "the +# listing is empty" (exit 4). Every other reason, including one added later +# and not listed anywhere, means the content was never obtained (exit 5): an +# allowlist of the reasons that prove emptiness, rather than a list of the +# failures, so a new failure cannot fall through to "empty". +EMPTY_STOP_REASONS = ("no_results", "listing_exhausted", "no_new_listings", + "completed") + def finish_run(products: List[Product], out_prefix: str, fmt: str, allow_empty: bool, *, blocked: bool, stop_reason: str, @@ -481,7 +534,8 @@ def finish_run(products: List[Product], out_prefix: str, fmt: str, f"{total_results_last} while this run was collecting: the catalogue " f"was edited underneath it, so a listing may have been stepped over " f"between two pages. Recorded as catalog_mutated in the metadata.") - rc = save(products, out_prefix, fmt, allow_empty=allow_empty) + save_stats: dict = {} + rc = save(products, out_prefix, fmt, allow_empty=allow_empty, stats=save_stats) wrote_output = bool(products) or allow_empty if wrote_output: @@ -491,7 +545,7 @@ def finish_run(products: List[Product], out_prefix: str, fmt: str, # An explicitly empty result that the site itself reported as # empty is a complete answer, not a failure. status = "complete" - write_run_meta(out_prefix, run_meta( + meta = run_meta( status=status, stop_reason=stop_reason, pages_requested=pages_requested, pages_completed=pages_completed, pages_failed=pages_failed, total_results=total_results, @@ -504,12 +558,24 @@ def finish_run(products: List[Product], out_prefix: str, fmt: str, total_results_first=total_results_first, total_results_last=total_results_last, catalog_mutated=catalog_mutated, - start_url=start_url, final_url=final_url, products=len(products))) + start_url=start_url, final_url=final_url, products=len(products)) + # Housekeeping goes UNDER the run's own fields: a name collision must + # never let a write statistic overwrite something the run reported. + write_run_meta(out_prefix, { + "csv_cells_escaped": save_stats.get("csv_cells_escaped"), **meta}) if not products: - # Nothing gathered at all: a challenge outranks "empty listing", - # because it says something stood between the run and the content. - return EXIT_BLOCKED if blocked else rc + # Nothing gathered at all: a challenge outranks everything, because + # it says something stood between the run and the content. After + # that, only a reason that PROVES the listing empty may say so. + if blocked: + return EXIT_BLOCKED + if stop_reason in EMPTY_STOP_REASONS: + return rc + print(f"[!] 0 listings, and the run never obtained the content " + f"({stop_reason}) — exit {EXIT_CONTENT_NOT_OBTAINED}, not " + f"{EXIT_NO_PRODUCTS}: nothing here says the listing is empty.") + return EXIT_CONTENT_NOT_OBTAINED if not complete: print(f"[!] Partial run: stopped after {pages_completed} of " f"{pages_requested} page(s) ({stop_reason}). The output holds " diff --git a/page_flow.py b/page_flow.py index 79b7ad5..3f60791 100644 --- a/page_flow.py +++ b/page_flow.py @@ -122,6 +122,30 @@ class PagePolicy: } +# How many captcha solves ONE page may buy, across every attempt, retry and +# exit rotation spent on it. Checked BEFORE a purchase, never reset by a retry +# or a rotation: a fresh exit is a reason to fetch again, not a fresh +# allowance. Before this existed, a page behind a challenge bought one solve +# per exit — 3 with a three-exit pool. +SOLVES_PER_PAGE = 1 + + +class SolveBudget: + """The per-page solve allowance. One instance per page, shared by every + call site that might pay, in every engine.""" + + def __init__(self, limit: int = SOLVES_PER_PAGE): + self.limit = limit + self.spent = 0 + + def spend(self) -> bool: + """Claim one solve; False (and nothing claimed) once the limit is reached.""" + if self.spent >= self.limit: + return False + self.spent += 1 + return True + + @dataclass class PageState: """The classification of one response, with the evidence behind it.""" diff --git a/playwright_scraper.py b/playwright_scraper.py index 9c2b521..45bc9b6 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -64,7 +64,8 @@ from captcha_solver import (INJECT_TOKEN_FN, detect_in_html, detect_in_page, reconcile_detections, solve) from fingerprint_client import FingerprintError -from output_writer import failure_stop_reason, finish_run, merge_pages +from output_writer import (EXIT_API_ERROR, failure_stop_reason, finish_run, + merge_pages) from product_parser import (SELECTORS, category_from_url, locale_from_url, page_url, parse_products) from proxy_pool import (CDP_CONNECT_ATTEMPTS, CDP_CONNECT_PAUSE_S, CDP_CONNECT_TIMEOUT_S, @@ -82,8 +83,6 @@ # match resolves on the first thing that looks like a card long before the # grid paints. Flippa serves 50 per page, so 5 is comfortably below a full # page and safely above an accident. -# The family's remote-API exit code; scraper_api_client.py defines the same. -EXIT_API_ERROR = 5 MIN_CARD_MATCHES = 5 # How long to wait for the grid to paint. Not a precondition for data (see the # module docstring) — it buys the DOM cross-check. @@ -216,7 +215,7 @@ def _connect_remote(pw, args): # message is a log. # # Retried per proxy_pool's CDP policy: a profile stays locked for ~2 s - # after a clean disconnect (family template §26), so a run started right + # after a clean disconnect (measured 2026-09-24), so a run started right # after another met HTTP 500. A 401 (expired credentials) is not retried. for attempt in range(1, CDP_CONNECT_ATTEMPTS + 1): try: @@ -343,33 +342,38 @@ def _wait_for_cards(page, timeout_ms: int = RENDER_WAIT_MS) -> int: return count -def handle_captcha_if_present(page, args) -> None: +def handle_captcha_if_present(page, args, budget: page_flow.SolveBudget, + status_code: Optional[int] = None) -> bool: """Runs after EVERY navigation, for ANY page — not scoped to one URL. + Every purchase goes through `budget`, the page's one allowance, checked + before the solver is called. Returns True when the page was reloaded, so + the caller knows the status it holds belongs to the previous document. + Both detectors run and are reconciled; neither short-circuits the other. A challenge configured in JavaScript never appears in the served markup, and a challenge in the markup can be one the live loader contradicts. """ html = _content_when_settled(page) if html is None: - return + return False challenge = reconcile_detections( detect_in_html(html, page.url), detect_in_page(lambda js: page.evaluate(js), page_url=page.url)) if not challenge: - return + return False # Detected is not blocking. Flippa renders Turnstile on its account forms, # not on listings, so a detection on a page whose listings are already # here guards nothing. Counting what is already on the page is instant — # no readiness wait — which is why this check can sit here rather than # forcing a 15s wait on a page a captcha genuinely gates. - state = page_flow.classify(html, url=page.url) + state = page_flow.classify(html, status_code=status_code, url=page.url) if args.solve_captcha == "when-blocked" and not state.policy.may_solve: logger.info("%s detected via %s, but the page is %s — not paying to " "solve it. Pass --solve-captcha always to solve anyway.", challenge.kind, challenge.source, state) - return + return False logger.warning("%s detected via %s (sitekey=%s) — attempting to solve.", challenge.kind, challenge.source, challenge.sitekey) @@ -381,14 +385,19 @@ def handle_captcha_if_present(page, args) -> None: logger.warning("No 2captcha API key, so this challenge cannot be solved " "— continuing with whatever the page already holds. The " "run reports exit 3 if it really was blocking.") - return + return False + if not budget.spend(): + logger.warning("%s still present, but this page has already used its " + "%d paid solve(s) — not buying another.", + challenge.kind, budget.limit) + return False try: token = solve(challenge, args.twocaptcha_key, api_version=args.captcha_api, min_score=args.min_score) except Exception as e: # noqa: BLE001 — a solver failure is not a crash logger.error("Solving the challenge failed (%s) — continuing with " "whatever the page holds.", redact_secret_patterns(str(e))) - return + return False page.evaluate(INJECT_TOKEN_FN, token) if state.policy.blocked: @@ -398,6 +407,7 @@ def handle_captcha_if_present(page, args) -> None: logger.info("Token injected. Reloading page to continue.") page.wait_for_timeout(1500) page.reload(wait_until="domcontentloaded", timeout=60000) + return True else: # Measured on 2026-09-19 against flippa.com/signup, which carries a # real Turnstile: reloading here THROWS THE TOKEN AWAY, because a form @@ -409,6 +419,7 @@ def handle_captcha_if_present(page, args) -> None: "This page was not blocked, so it is NOT reloaded — a " "reload would discard the token, and nothing here submits " "a form.") + return False def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome: @@ -428,13 +439,20 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # same address just burns it further. block_retries = args.proxy_block_retries if (pool and len(pool) > 1) else 0 html, state, load_failed = None, None, False + # One allowance for the whole page: not per attempt, not per exit. + budget = page_flow.SolveBudget() for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) - load_failed, exit_failed = False, None + load_failed, exit_failed, status = False, None, None for attempt in range(1, args.retries + 1): try: - session.page.goto(url, wait_until="domcontentloaded", timeout=60000) + response = session.page.goto(url, wait_until="domcontentloaded", + timeout=60000) + # The HTTP status is the one signal a refusal page with no + # vendor marker gives; page_flow reads it only when the page + # carries no listings. + status = response.status if response is not None else None load_failed = False break except (PWTimeout, PWError) as e: @@ -469,10 +487,11 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if load_failed: break - handle_captcha_if_present(session.page, args) + if handle_captcha_if_present(session.page, args, budget, status): + status = None # the reload replaced the document it described cards = _wait_for_cards(session.page) html = _content_when_settled(session.page) or "" - state = page_flow.classify(html, url=session.page.url) + state = page_flow.classify(html, status_code=status, url=session.page.url) logger.info("Page %d is %s (%d card(s) rendered).", page_num, state, cards) if state.state == page_flow.UNPAINTED and state.policy.wait_first: @@ -481,7 +500,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # before deciding. session.page.wait_for_timeout(RENDER_WAIT_MS) html = _content_when_settled(session.page) or html - state = page_flow.classify(html, url=session.page.url) + state = page_flow.classify(html, status_code=status, url=session.page.url) logger.info("After waiting, page %d is %s.", page_num, state) if not state.policy.rotate_exit: @@ -990,8 +1009,11 @@ def parse_args(): "uses the same key, though it is a separate subscription).") sys.exit(2) if args.fingerprint and args.cdp_endpoint: - logger.warning("--fingerprint is ignored with --cdp-endpoint: the Scraping " - "Browser supplies its own.") + logger.error("--fingerprint cannot be used with --cdp-endpoint: the " + "Scraping Browser supplies its own device identity, and " + "stacking a second one contradicts it rather than " + "covering it.") + sys.exit(2) try: sys.exit(scrape(args)) except ProxyError as e: diff --git a/proxy_pool.py b/proxy_pool.py index dec60f9..d77d2e1 100644 --- a/proxy_pool.py +++ b/proxy_pool.py @@ -376,7 +376,7 @@ def from_args(args) -> Optional[ProxyPool]: # --------------------------------------------------------------------------- # Connecting to the Scraping Browser API — one policy for every engine # --------------------------------------------------------------------------- -# Measured in the family (template §26, 2026-09-24): a profile stays +# Measured 2026-09-24: a profile stays # `profile_locked` for 1.6-1.9 s after a CLEAN disconnect, so a second run # started straight after the first was refused with HTTP 500. Three attempts # 3 s apart ride that out, while a profile genuinely held by another run diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 9333a2e..69b2f4c 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -136,7 +136,7 @@ async def _open_browser(args, pool): # its receive loop, the caller sees only a timeout, and the orphaned # task prints a traceback AFTER a correct exit. So: a short timeout # per attempt, proxy_pool's retry policy, and that one traceback - # shape filtered (family template §26). + # shape filtered. _quiet_refused_handshakes() browser = None for attempt in range(1, CDP_CONNECT_ATTEMPTS + 1): @@ -279,11 +279,15 @@ async def _content(page) -> Optional[str]: return None -async def handle_captcha_if_present(page, args) -> None: - """Runs after EVERY navigation, for ANY page — not scoped to one URL.""" +async def handle_captcha_if_present(page, args, budget: page_flow.SolveBudget, + status_code: Optional[int] = None) -> bool: + """Runs after EVERY navigation, for ANY page — not scoped to one URL. + + Every purchase goes through `budget`, the page's one allowance. Returns + True when the page was reloaded.""" html = await _content(page) if html is None: - return + return False url = page.url # pyppeteer's evaluate is a coroutine, so the discovery script is awaited @@ -300,52 +304,62 @@ async def handle_captcha_if_present(page, args) -> None: challenge = reconcile_detections(detect_in_html(html, url), runtime) if not challenge: - return + return False - state = page_flow.classify(html, url=url) + state = page_flow.classify(html, status_code=status_code, url=url) if args.solve_captcha == "when-blocked" and not state.policy.may_solve: logger.info("%s detected via %s, but the page is %s — not paying to " "solve it.", challenge.kind, challenge.source, state) - return + return False logger.warning("%s detected via %s (sitekey=%s) — attempting to solve.", challenge.kind, challenge.source, challenge.sitekey) if not args.twocaptcha_key: logger.warning("No 2captcha API key, so this challenge cannot be solved " "— continuing with whatever the page already holds.") - return + return False + if not budget.spend(): + logger.warning("%s still present, but this page has already used its " + "%d paid solve(s) — not buying another.", + challenge.kind, budget.limit) + return False try: token = solve(challenge, args.twocaptcha_key, api_version=args.captcha_api, min_score=args.min_score) except Exception as e: # noqa: BLE001 — a solver failure is not a crash logger.error("Solving the challenge failed (%s) — continuing.", redact_secret_patterns(str(e))) - return + return False await page.evaluate(INJECT_TOKEN_FN, token) if state.policy.blocked: logger.info("Token injected. Reloading page to continue.") await asyncio.sleep(1.5) await page.reload({"waitUntil": "domcontentloaded", "timeout": GOTO_TIMEOUT_MS}) + return True else: # See the twin: a reload on a page that was not blocked discards the # token, and a form's token is only useful at submit time. logger.info("Token injected and left in the page's own response field. " "This page was not blocked, so it is NOT reloaded.") + return False async def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome: outcome = PageOutcome(page_num=page_num, url=url) block_retries = args.proxy_block_retries if (pool and len(pool) > 1) else 0 html, state, load_failed = None, None, False + # One allowance for the whole page: not per attempt, not per exit. + budget = page_flow.SolveBudget() for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) - load_failed, exit_failed = False, None + load_failed, exit_failed, status = False, None, None for attempt in range(1, args.retries + 1): try: - await session.page.goto( + response = await session.page.goto( url, {"waitUntil": "domcontentloaded", "timeout": GOTO_TIMEOUT_MS}) + status = response.status if response is not None else None load_failed = False break except (PPTimeout, PageError, NetworkError, @@ -382,16 +396,17 @@ async def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageO if load_failed: break - await handle_captcha_if_present(session.page, args) + if await handle_captcha_if_present(session.page, args, budget, status): + status = None # the reload replaced the document it described cards = await _wait_for_cards(session.page) html = await _content(session.page) or "" - state = page_flow.classify(html, url=session.page.url) + state = page_flow.classify(html, status_code=status, url=session.page.url) logger.info("Page %d is %s (%d card(s) rendered).", page_num, state, cards) if state.state == page_flow.UNPAINTED and state.policy.wait_first: await asyncio.sleep(RENDER_WAIT_MS / 1000.0) html = await _content(session.page) or html - state = page_flow.classify(html, url=session.page.url) + state = page_flow.classify(html, status_code=status, url=session.page.url) logger.info("After waiting, page %d is %s.", page_num, state) if not state.policy.rotate_exit: @@ -701,9 +716,12 @@ def parse_args(): if __name__ == "__main__": args = parse_args() if args.fingerprint: - logger.warning("--fingerprint is not implemented in the pyppeteer engine " - "— continuing without one. playwright_scraper.py applies " - "fingerprints; see fingerprint_client.py.") + # Refused, not ignored: a run that accepted the flag and then went + # without would look fingerprinted while it was not. + logger.error("--fingerprint is not implemented in the pyppeteer engine. " + "playwright_scraper.py applies fingerprints; see " + "fingerprint_client.py.") + sys.exit(2) try: sys.exit(scrape(args)) except ProxyError as e: diff --git a/scraper_api_client.py b/scraper_api_client.py index 67a1038..34ec137 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -27,14 +27,18 @@ Content-Type: application/json {"task_type": "scrape", "url": ..., "data_format": "raw", "format": "json", "timeout": 1..120, - "waitFor": "", + "waitFor": {"state": "load"}, # an OBJECT "cdpurl": "ws://user:pass@host:port" # optional } - -> 200 {"status": 200, "headers": {...}, "body": "..."} + -> 200 {"status": "...", "http_code": 200, "headers": {...}, + "body": "..."} -`waitFor` must be a JSON *string* (double-encoded), and the parameter is -spelled `cdpurl` (all lowercase) while `waitFor` is camelCase — the API's own -inconsistency, not a typo here. +`waitFor` is sent as an OBJECT. The JSON-string form this file used to send +is now answered with HTTP 422 — and the task is still billed. `waitFor.state` +takes only `load` or `domcontentloaded`. The target page's status is +`http_code`; `status` is the API's own verdict string and is not a number. +The parameter is spelled `cdpurl` (all lowercase) while `waitFor` is +camelCase — the API's own inconsistency, not a typo here. Usage ----- @@ -60,7 +64,7 @@ import cli_types import env_config import page_flow -from output_writer import finish_run, merge_pages +from output_writer import EXIT_API_ERROR, finish_run, merge_pages from product_parser import (category_from_url, locale_from_url, page_url, parse_products) from proxy_pool import redact_secret_patterns @@ -74,24 +78,26 @@ # The API caps `timeout` at 120s. MAX_API_TIMEOUT = 120 -# Kept distinct from 2 (bad usage) on purpose: a remote API failing is not the -# operator passing wrong arguments, and a harness that lumps them together -# sends you looking in the wrong place. -EXIT_API_ERROR = 5 +# EXIT_API_ERROR (5) comes from output_writer, where every engine's zero-row +# run that never obtained its content maps to it. Kept distinct from 2 (bad +# usage): a remote API failing is not the operator passing wrong arguments. -def _build_wait_for(args) -> Optional[str]: - """`waitFor` as the JSON STRING the API wants, or None. +def _build_wait_for(args) -> Optional[dict]: + """`waitFor` as the OBJECT the API takes, or None. Not needed on flippa.com (see the module docstring); kept because a future page kind, or a run routed through `--cdp-url`, may want it. """ if args.wait_text: - return json.dumps({"text": args.wait_text}) + return {"text": args.wait_text} if args.wait_element: - return json.dumps({"element": args.wait_element, "checkVisible": True}) + return {"element": args.wait_element, "checkVisible": True} if args.wait_state: - return json.dumps({"state": args.wait_state}) + if args.wait_state not in ("load", "domcontentloaded"): + raise ValueError(f"waitFor.state must be load or domcontentloaded, " + f"not {args.wait_state!r}") + return {"state": args.wait_state} return None @@ -128,7 +134,9 @@ def fetch_html(args, url: str) -> Tuple[str, Optional[int]]: # x-debug header — the only place the real cost of the call shows up. debug = resp.headers.get("x-debug") if debug: - logger.info("x-debug: %s", debug) + # Redacted: SECURITY.md lists this header as a place a credential can + # reach a log, and it is echoed verbatim from the vendor. + logger.info("x-debug: %s", redact_secret_patterns(debug)) if resp.status_code != 200: # 422 = the task ran and errored (an unreachable cdpurl does this); @@ -139,7 +147,9 @@ def fetch_html(args, url: str) -> Tuple[str, Optional[int]]: body = resp.json() html = body.get("body") or "" - upstream = body.get("status") + # The target's status is `http_code`. `status` is the API's own verdict + # string ("ok", "warn"), never the page's status. + upstream = body.get("http_code") logger.info("Upstream page status %s, %d bytes of HTML.", upstream, len(html)) return html, upstream if isinstance(upstream, int) else None diff --git a/selenium_scraper.py b/selenium_scraper.py index acbe99c..05a698d 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -216,34 +216,43 @@ def _discover_captcha(driver, url: str): return challenge_from_discovery(info, page_url=url) -def handle_captcha_if_present(driver, args) -> None: - """Runs after EVERY navigation, for ANY page — not scoped to one URL.""" +def handle_captcha_if_present(driver, args, budget: page_flow.SolveBudget) -> bool: + """Runs after EVERY navigation, for ANY page — not scoped to one URL. + + Every purchase goes through `budget`, the page's one allowance. Returns + True when the page was reloaded. Selenium exposes no HTTP status, so none + is passed to the classifier here or anywhere in this engine.""" html = driver.page_source url = driver.current_url challenge = reconcile_detections(detect_in_html(html, url), _discover_captcha(driver, url)) if not challenge: - return + return False - state = page_flow.classify(html, url=url) + state = page_flow.classify(html, status_code=None, url=url) if args.solve_captcha == "when-blocked" and not state.policy.may_solve: logger.info("%s detected via %s, but the page is %s — not paying to " "solve it.", challenge.kind, challenge.source, state) - return + return False logger.warning("%s detected via %s (sitekey=%s) — attempting to solve.", challenge.kind, challenge.source, challenge.sitekey) if not args.twocaptcha_key: logger.warning("No 2captcha API key, so this challenge cannot be solved " "— continuing with whatever the page already holds.") - return + return False + if not budget.spend(): + logger.warning("%s still present, but this page has already used its " + "%d paid solve(s) — not buying another.", + challenge.kind, budget.limit) + return False try: token = solve(challenge, args.twocaptcha_key, api_version=args.captcha_api, min_score=args.min_score) except Exception as e: # noqa: BLE001 — a solver failure is not a crash logger.error("Solving the challenge failed (%s) — continuing.", redact_secret_patterns(str(e))) - return + return False # The BODY form, not the arrow function: see captcha_solver's two # constants and _discover_captcha above. driver.execute_script(INJECT_TOKEN_BODY, token) @@ -251,11 +260,13 @@ def handle_captcha_if_present(driver, args) -> None: logger.info("Token injected. Reloading page to continue.") time.sleep(1.5) driver.refresh() + return True else: # See the twins: a reload on a page that was not blocked discards the # token, and a form's token is only useful at submit time. logger.info("Token injected and left in the page's own response field. " "This page was not blocked, so it is NOT reloaded.") + return False def _dump_debug(driver, args, page_num: int, html: Optional[str]) -> None: @@ -273,6 +284,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome = PageOutcome(page_num=page_num, url=url) block_retries = args.proxy_block_retries if (pool and len(pool) > 1) else 0 html, state, load_failed = None, None, False + # One allowance for the whole page: not per attempt, not per exit. + budget = page_flow.SolveBudget() for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) @@ -314,16 +327,18 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if load_failed: break - handle_captcha_if_present(session.driver, args) + handle_captcha_if_present(session.driver, args, budget) cards = _wait_for_cards(session.driver) html = session.driver.page_source - state = page_flow.classify(html, url=session.driver.current_url) + state = page_flow.classify(html, status_code=None, + url=session.driver.current_url) logger.info("Page %d is %s (%d card(s) rendered).", page_num, state, cards) if state.state == page_flow.UNPAINTED and state.policy.wait_first: time.sleep(RENDER_WAIT_MS / 1000.0) html = session.driver.page_source - state = page_flow.classify(html, url=session.driver.current_url) + state = page_flow.classify(html, status_code=None, + url=session.driver.current_url) logger.info("After waiting, page %d is %s.", page_num, state) if not state.policy.rotate_exit: @@ -578,9 +593,12 @@ def parse_args(): if __name__ == "__main__": args = parse_args() if args.fingerprint: - logger.warning("--fingerprint is not implemented in the Selenium engine " - "— continuing without one. playwright_scraper.py applies " - "fingerprints; see fingerprint_client.py.") + # Refused, not ignored: a run that accepted the flag and then went + # without would look fingerprinted while it was not. + logger.error("--fingerprint is not implemented in the Selenium engine. " + "playwright_scraper.py applies fingerprints; see " + "fingerprint_client.py.") + sys.exit(2) try: sys.exit(scrape(args)) except ProxyError as e: diff --git a/smoke_test.py b/smoke_test.py index 9bf482f..d3dd46c 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1628,7 +1628,7 @@ def check_output_contract(): # --------------------------------------------------------------------------- -# 5. The engines: parity, and the five checks §17 says to steal +# 5. The engines: parity, signatures, undefined names and imports # --------------------------------------------------------------------------- # The flag contract every engine in the family exposes, as # argparse destinations. Asserted in BOTH directions: a missing flag fails, @@ -1857,6 +1857,40 @@ def check_banned_wording(): "Scraping Browser API" in text) +def check_no_private_citations(): + print("\n[no citations of private instruction files]") + # The rules this repo follows are stated in its own words. A pointer into + # a file the public cannot open ("see section N of the template") reads as + # a reference nobody can follow. Assembled from pieces so this file does + # not match itself; covers this file too, and dotfiles like .gitignore. + section = chr(0xA7) + patterns = [re.compile(re.escape(section) + r"\s*\d"), + re.compile(re.escape("CLAUDE" + ".md"), re.I), + re.compile(re.escape("claude_" + "2scraper"), re.I), + re.compile(re.escape("2scraper_" + "repo_rules"), re.I), + re.compile(r"\b(?:play" + r"book|family temp" + r"late)\b", re.I)] + scanned, hits = 0, [] + for root, dirs, files in os.walk(REPO): + dirs[:] = [d for d in dirs + if d not in {".git", "__pycache__", "captures", "legacy"} + and not os.path.isfile(os.path.join(root, d, "pyvenv.cfg"))] + for filename in files: + if not filename.endswith((".py", ".md", ".yml", ".yaml", ".txt", + ".example", ".toml", ".cfg")) \ + and filename not in (".gitignore", ".dockerignore", "Dockerfile"): + continue + path = os.path.join(root, filename) + text = open(path, encoding="utf-8", errors="replace").read() + scanned += 1 + for pattern in patterns: + m = pattern.search(text) + if m: + line = text[:m.start()].count("\n") + 1 + hits.append(f"{os.path.relpath(path, REPO)}:{line}") + check(f"no shipped file cites a private instruction file ({scanned} files " + f"scanned)" + (f" — {hits}" if hits else ""), not hits and scanned > 10) + + def check_removed_flags_stay_removed(): print("\n[removed flags]") # Scoped to the ENGINES: --country is banned on a scraper (it could @@ -2210,7 +2244,7 @@ def fake_post(url, **kwargs): if "in.php" in url: return _FakeResponse({"status": 1, "request": "ID1"}) if "tasks/sync" in url: - return _FakeResponse({"status": 200, "body": PAGE_FIXTURE_HTML}, + return _FakeResponse({"status": "ok", "http_code": 200, "body": PAGE_FIXTURE_HTML}, headers={"x-debug": "cost=0.0005"}) raise AssertionError(url) @@ -2610,7 +2644,247 @@ def explode(f): eq("and no temporary file behind", leftovers, []) +def check_zero_rows_exit_codes(): + """0 rows is 'empty' (4) only when something PROVES the listing empty.""" + print("\n[zero-row exit codes]") + with tempfile.TemporaryDirectory() as tmp: + def run(reason, blocked=False): + with redirect_stdout(io.StringIO()): + return finish_run([], os.path.join(tmp, "z"), "both", False, + blocked=blocked, stop_reason=reason, + pages_requested=1, pages_completed=0, + start_url="u", final_url="u") + # Previously every one of these returned 4, "the catalogue is empty", + # for a run that never saw the catalogue. + for reason in ("content_unparsed", "page_never_painted", + "page_load_timeout", "worker_lost_page", "api_error", + "a_reason_added_later"): + eq(f"0 rows after {reason!r} is exit 5, content never obtained", + run(reason), output_writer.EXIT_CONTENT_NOT_OBTAINED) + for reason in ("no_results", "listing_exhausted", "no_new_listings"): + eq(f"0 rows after {reason!r} is exit 4, a genuinely empty listing", + run(reason), EXIT_NO_PRODUCTS) + eq("a block outranks both", run("page_load_timeout", blocked=True), + EXIT_BLOCKED) + eq("and nothing is written for any of them", sorted(os.listdir(tmp)), []) + with redirect_stdout(io.StringIO()): + rc = finish_run([_row(sku="1")], os.path.join(tmp, "p"), "json", + False, blocked=False, stop_reason="page_load_timeout", + pages_requested=2, pages_completed=1, + start_url="u", final_url="u") + eq("rows, then a failure, is still exit 6", rc, EXIT_PARTIAL) + eq("the engines' remote-API code is the same number", + output_writer.EXIT_API_ERROR, 5) + + +def check_output_file_modes(): + """Outputs get the mode a plain open() would give them, not mkstemp's 0600.""" + print("\n[output file modes]") + if os.name != "posix": + skip("output file modes", "not a POSIX system") + return + old = os.umask(0o022) + try: + with tempfile.TemporaryDirectory() as tmp: + for umask, want in ((0o022, 0o644), (0o002, 0o664), (0o077, 0o600)): + os.umask(umask) + path = os.path.join(tmp, f"new_{umask:o}.json") + output_writer.write_json([_row(sku="1")], path) + eq(f"a new file under umask {umask:03o} is {want:o}", + oct(os.stat(path).st_mode & 0o777), oct(want)) + os.umask(0o022) + path = os.path.join(tmp, "existing.csv") + open(path, "w").close() + os.chmod(path, 0o664) + output_writer.write_csv([_row(sku="1")], path) + eq("an existing file keeps its own mode (0664 under umask 022)", + oct(os.stat(path).st_mode & 0o777), oct(0o664)) + with redirect_stdout(io.StringIO()): + output_writer.write_run_meta(os.path.join(tmp, "run"), {"a": 1}) + eq("the sidecar is not owner-only either", + oct(os.stat(os.path.join(tmp, "run.meta.json")).st_mode & 0o777), + oct(0o644)) + finally: + os.umask(old) + + +def check_csv_escape_count_reaches_the_sidecar(): + print("\n[csv escapes are declared]") + eq("a leading newline is neutralised too", + output_writer.csv_safe("\nfoo"), "'\nfoo") + with tempfile.TemporaryDirectory() as tmp: + prefix = os.path.join(tmp, "run") + with redirect_stdout(io.StringIO()): + finish_run([_row(sku="1", title='=HYPERLINK("x")', description="@me"), + _row(sku="2", title="plain")], prefix, "both", False, + blocked=False, stop_reason="completed", + pages_requested=1, pages_completed=1, + start_url="u", final_url="u") + meta = json.load(open(prefix + ".meta.json", encoding="utf-8")) + eq("the sidecar counts the escaped CSV cells", meta.get("csv_cells_escaped"), 2) + rows = json.load(open(prefix + ".json", encoding="utf-8")) + eq("and the JSON keeps the site's own bytes", rows[0]["title"], + '=HYPERLINK("x")') + eq("a run field is never overwritten by the count", + meta.get("status"), "complete") + + +def _engine_sources(): + return {name: open(os.path.join(REPO, name + ".py"), encoding="utf-8").read() + for name in ("playwright_scraper", "puppeteer_scraper", "selenium_scraper")} + + +def check_one_solve_allowance_per_page(): + """Every paid solve goes through the page's one SolveBudget.""" + print("\n[captcha solve budget]") + budget = page_flow.SolveBudget() + eq("the first solve is allowed", budget.spend(), True) + eq("the second is refused, whatever retry or exit asks for it", + budget.spend(), False) + eq("the allowance is one per page", page_flow.SOLVES_PER_PAGE, 1) + # Read off disk, never behind an import: an import-gated version of this + # check would skip for exactly the engines whose driver is absent. + for name, src in _engine_sources().items(): + tree = ast.parse(src) + funcs = {n.name: n for n in ast.walk(tree) + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))} + handler = funcs.get("handle_captcha_if_present") + check(f"{name}: has handle_captcha_if_present", handler is not None) + if handler is None: + continue + # The guard must be exactly `if not budget.spend(): return ...` + # ahead of the solver call — a mention of budget.spend() that does + # not gate the purchase is not a budget. + def _is_guard(node): + t = node.test + return (isinstance(node, ast.If) and isinstance(t, ast.UnaryOp) + and isinstance(t.op, ast.Not) and isinstance(t.operand, ast.Call) + and getattr(t.operand.func, "attr", None) == "spend" + and getattr(t.operand.func.value, "id", None) == "budget" + and any(isinstance(b, ast.Return) for b in node.body)) + guards = [n.lineno for n in ast.walk(handler) + if isinstance(n, ast.If) and _is_guard(n)] + solves = [n.lineno for n in ast.walk(handler) if isinstance(n, ast.Call) + and getattr(n.func, "id", None) == "solve"] + check(f"{name}: the budget is claimed BEFORE the solver is called", + bool(guards) and bool(solves) and min(guards) < min(solves)) + calls = [c for c in ast.walk(tree) if isinstance(c, ast.Call) + and getattr(c.func, "id", getattr(c.func, "attr", None)) + == "handle_captcha_if_present"] + check(f"{name}: every call site passes the page's budget " + f"({len(calls)} call(s))", + calls and all(len(c.args) >= 3 and getattr(c.args[2], "id", None) + == "budget" for c in calls)) + fetch = funcs.get("_fetch_one_page") + fetch_src = ast.get_source_segment(src, fetch) if fetch else "" + before_loop = fetch_src.split("for block_attempt")[0] + check(f"{name}: the budget is created once per page, outside the " + f"retry/rotation loop", + "page_flow.SolveBudget()" in before_loop) + + +def check_status_reaches_the_classifier(): + print("\n[HTTP status reaches the classifier]") + for name, src in _engine_sources().items(): + tree = ast.parse(src) + calls = [c for c in ast.walk(tree) if isinstance(c, ast.Call) + and isinstance(c.func, ast.Attribute) and c.func.attr == "classify" + and getattr(c.func.value, "id", None) == "page_flow"] + bound = [c for c in calls if any(k.arg == "status_code" for k in c.keywords) + or len(c.args) >= 2] + check(f"{name}: every page_flow.classify call is given a status " + f"({len(bound)} of {len(calls)})", calls and len(bound) == len(calls)) + if name != "selenium_scraper": + check(f"{name}: the status is read from what goto() returned", + "response.status" in src) + eq("a refusal status with no listing on the page is blocked", + page_flow.classify("Error", status_code=403).state, + page_flow.BLOCKED) + + +def check_scraper_api_payload_and_logging(): + print("\n[Scraper API payload]") + import scraper_api_client as sac + + class Args: + wait_text = wait_element = None + wait_state = "load" + cdp_url = None + timeout = 60 + key = "k" + + wait_for = sac._build_wait_for(Args()) + eq("waitFor is an object, not a JSON string", wait_for, {"state": "load"}) + sent, records = {}, [] + + class Resp: + status_code = 200 + headers = {"x-debug": "cost=0.0005 key=0123456789abcdef0123456789abcdef"} + text = "" + + def json(self): + return {"status": "warn", "http_code": 403, "body": ""} + + def fake_post(url, **kw): + sent.update(kw["json"]) + return Resp() + + class Grab(logging.Handler): + def emit(self, record): + records.append(record.getMessage()) + + handler = Grab() + sac.logger.addHandler(handler) + level = sac.logger.level + sac.logger.setLevel(logging.INFO) + real = sac.requests.post + sac.requests.post = fake_post + try: + html, upstream = sac.fetch_html(Args(), "https://flippa.com/search") + finally: + sac.requests.post = real + sac.logger.removeHandler(handler) + sac.logger.setLevel(level) + check("the payload carries the object form", + isinstance(sent.get("waitFor"), dict)) + eq("the target's status is read from http_code, not the verdict string", + upstream, 403) + logged = " ".join(records) + check("the x-debug header is logged redacted", + "x-debug" in logged and "0123456789abcdef0123456789abcdef" not in logged) + + +def check_credential_scan_uses_relative_paths(): + print("\n[credential scan: relative paths]") + import importlib.util + path = os.path.join(REPO, ".github", "ci_checks.py") + if not os.path.exists(path): + skip("credential scan paths", "no .github/ci_checks.py in this tree") + return + spec = importlib.util.spec_from_file_location("ci_checks_paths", path) + ci = importlib.util.module_from_spec(spec) + spec.loader.exec_module(ci) + import pathlib + with tempfile.TemporaryDirectory() as tmp: + # A clone whose ANCESTOR is called like a skipped directory. + repo = pathlib.Path(tmp) / "legacy" / "clone" + repo.mkdir(parents=True) + (repo / "leak.py").write_text("KEY = '" + "ab" * 16 + "'\n") + ci.REPO = repo + ci._VENV_CACHE.clear() + with redirect_stdout(io.StringIO()): + failures = ci.secret_check() + check("a key in a clone under a 'legacy' ancestor is still found", + any("leak.py" in f for f in failures)) + (repo / "leak.py").unlink() + with redirect_stdout(io.StringIO()): + failures = ci.secret_check() + check("a scan that read 0 files fails instead of printing ok", + any("0 files" in f for f in failures)) + + def check_cli_refuses_impossible_values(): + """Bounds on the flags whose out-of-range values change what a run means.""" print("\n[cli bounds]") import argparse as _argparse @@ -2850,14 +3124,35 @@ def check_fingerprint_refusal_is_an_api_error(): if ENGINES.get("playwright_scraper") is None: skip("fingerprint refusal", "playwright not installed") return + # The run launches Chromium before it asks for a fingerprint, so without + # a browser binary it dies at launch (exit 1) and this check would be + # measuring the machine, not the code. CI installs the browser. + try: + from playwright.sync_api import sync_playwright + with sync_playwright() as pw: + browser_path = pw.chromium.executable_path + except Exception as e: # noqa: BLE001 + browser_path = None + print(f" (could not ask Playwright for its browser: {e})") + if not browser_path or not os.path.exists(browser_path): + skip("fingerprint refusal", "no Chromium binary for this Playwright " + "(run `playwright install chromium`)") + return import subprocess example = open(os.path.join(REPO, ".env.example"), encoding="utf-8").read() url = re.search(r"^[A-Z]+_URL=(https?://\S+)", example, re.M).group(1) fake_key = "f" + "a" * 31 - env = dict(os.environ, HTTPS_PROXY="http://127.0.0.1:9", - HTTP_PROXY="http://127.0.0.1:9", - **{name: "" for name in env_config.ENV_KEYS}) with tempfile.TemporaryDirectory() as tmp: + # A private HOME: the fingerprint client caches on disk under + # ~/.cache, and a developer's cached fingerprint would answer this run + # instead of the (unreachable) API — the run then goes on to the + # dead proxy and the check measures that. The browser path is kept. + env = dict(os.environ, HTTPS_PROXY="http://127.0.0.1:9", + HTTP_PROXY="http://127.0.0.1:9", HOME=tmp, + PLAYWRIGHT_BROWSERS_PATH=os.environ.get( + "PLAYWRIGHT_BROWSERS_PATH", + os.path.join(os.path.expanduser("~"), ".cache", "ms-playwright")), + **{name: "" for name in env_config.ENV_KEYS}) run = subprocess.run( [sys.executable, os.path.join(REPO, "playwright_scraper.py"), "--url", url, "--pages", "1", @@ -2899,7 +3194,7 @@ def check_family_sync_2026_09_29(): capture_output=True, text=True) check("`env_config.py --help` answers without reading .env", r.returncode == 0 and ".env file:" not in r.stdout) - # 4. The Scraping Browser connect policy (template §26): retry a lock or + # 4. The Scraping Browser connect policy: retry a lock or # an outage, never an expired login, and say which it was. eq("a 500 on the upgrade is retried", proxy_pool.cdp_retryable( "WebSocket error: ws://***:***@cb.2captcha.com:9222/ 500 Internal Server Error"), True) @@ -3051,7 +3346,8 @@ def main() -> int: check_readiness_wait_is_csp_safe, check_engine_imports_driver_at_module_level, check_no_undefined_names, check_shared_calls_bind, - check_banned_wording, check_removed_flags_stay_removed, + check_banned_wording, check_no_private_citations, + check_removed_flags_stay_removed, check_env_example_matches_code, check_env_precedence, check_policy_constants_have_consumers, check_dockerfile_copies_what_it_imports, check_sample_output, @@ -3062,7 +3358,13 @@ def main() -> int: check_concurrency_machinery, check_worker_pools_start_on_different_exits, check_run_integrity, check_csv_is_not_a_formula, - check_writes_are_atomic, + check_writes_are_atomic, check_zero_rows_exit_codes, + check_output_file_modes, + check_csv_escape_count_reaches_the_sidecar, + check_one_solve_allowance_per_page, + check_status_reaches_the_classifier, + check_scraper_api_payload_and_logging, + check_credential_scan_uses_relative_paths, check_cli_refuses_impossible_values, check_diff_refuses_a_window_comparison, check_captcha_javascript_actually_runs, From a83ac5c8e6d83718357d603dc80c7ee2fba383b4 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:19:47 +0000 Subject: [PATCH 2/2] Weekly dependency audit passes again; the README explains pyppeteer's urllib3 pip-audit had been red since 2026-10-05 on two new urllib3 1.x advisories in the pyppeteer lock, PYSEC-2026-4175 and PYSEC-2026-4177. Both are fixed only in urllib3 2.8.0; pyppeteer 2.0.0 still requires urllib3<2 and 1.26.20 is the last 1.x release, so no compatible pin clears them. They are ignored by ID beside the five already listed; any other advisory still fails. The README now says what that means for someone running the pyppeteer engine: requests in that virtualenv shares the vulnerable urllib3, so the proxy preflight and captcha API calls do too; prefer Playwright with proxy credentials. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +++++ README.md | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6be541a..6b80f05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,11 @@ parser but were not re-run live. - Public files no longer cite private instruction files by section number; the rules are stated in this repository's own words, and a suite check keeps it that way. +- The weekly `pip-audit` job had been red since 2026-10-05 on two new urllib3 + 1.x advisories (PYSEC-2026-4175, PYSEC-2026-4177) in the pyppeteer lock. + pyppeteer 2.0.0 still requires `urllib3<2` and the fix is only in 2.8.0, so + they are ignored by ID beside the five already listed, and the README says + so. ## [0.2.1] — 2026-09-29 diff --git a/README.md b/README.md index b04c577..a86d718 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ Install exactly one engine per environment: playwright and pyppeteer pin incompatible `pyee` versions, and pyppeteer and selenium collide on `urllib3`. Use a virtualenv per engine if you want more than one. +**The pyppeteer engine carries a known-vulnerable urllib3.** pyppeteer +requires `urllib3<2`, and seven published urllib3 advisories (as of +2026-10-15) are fixed only in 2.x — among them one where a cross-origin +redirect can forward `Proxy-Authorization` to the new host +(GHSA-qccp-gfcp-xxvc), and one where the TLS settings meant for an HTTPS +proxy and for the target server can be mixed up (PYSEC-2026-4175). In that +engine's virtualenv `requests` uses the same urllib3, so the proxy preflight +and the captcha API calls do too. The weekly `pip-audit` job ignores exactly +those seven, by ID; prefer Playwright if you run with proxy credentials. + Credentials live in `.env`, never on a command line — a secret in `argv` is readable by anything that can run `ps` and lands in shell history: