Count proxy traffic per run and stop at --proxy-budget-mb (default 10 MB)

.github/proxy_traffic_summary.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Canary step: one line about a run's proxy traffic for the job summary.
+
+    python3 .github/proxy_traffic_summary.py <out> <label> [budget_mb]
+
+Reads `<out>.meta.json` and `<out>.latest_attempt.meta.json` (the newer of
+the two when both exist), appends `**<label>**: proxy traffic X.X MB of
+<budget> MB` to $GITHUB_STEP_SUMMARY, and exits 1 with an `::error::` line
+when the run stopped at its proxy budget. A missing or unreadable sidecar,
+or a run that used no local proxy, is reported and is not a failure here:
+the canary's own check step judges the run itself.
+"""
+import json
+import os
+import sys
+from datetime import datetime
+
+
+def _load(path):
+    try:
+        with open(path, encoding="utf-8") as stream:
+            data = json.load(stream)
+        return data if isinstance(data, dict) else None
+    except FileNotFoundError:
+        return None
+    except (OSError, ValueError) as exc:
+        print("::warning::%s is unreadable: %s" % (path, exc))
+        return None
+
+
+def _finished_at(meta):
+    """A sidecar's finished_at as a number: epoch seconds or an ISO-8601
+    string (repos write either), 0.0 when absent or unreadable."""
+    value = meta.get("finished_at")
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        pass
+    try:
+        return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp()
+    except ValueError:
+        return 0.0
+
+
+def summarise(out, label, budget_mb=10.0):
+    """Return (line, whether the run stopped at its proxy budget)."""
+    metas = [m for m in (_load(out + ".meta.json"), _load(out + ".latest_attempt.meta.json")) if m]
+    meta = max(metas, key=_finished_at) if metas else None
+    budget = "%g MB" % budget_mb if budget_mb else "no"
+    if meta is None:
+        line = "**%s**: proxy traffic not recorded (no sidecar for %s)" % (label, out)
+    elif "proxy_bytes" not in meta:
+        line = "**%s**: proxy traffic not recorded (the sidecar has no proxy_bytes)" % label
+    elif meta.get("proxy_bytes") is None:
+        line = "**%s**: proxy traffic: none (no local proxy on this run)" % label
+    else:
+        line = "**%s**: proxy traffic %.1f MB of %s" % (label, (meta.get("proxy_bytes") or 0) / 1e6, budget)
+    exceeded = bool(meta and meta.get("proxy_budget_exceeded"))
+    return line, exceeded
+
+
+def main(argv):
+    if len(argv) < 3:
+        print("usage: proxy_traffic_summary.py <out> <label> [budget_mb]", file=sys.stderr)
+        return 2
+    try:
+        budget_mb = float(argv[3]) if len(argv) > 3 else 10.0
+    except ValueError:
+        budget_mb = 10.0
+    line, exceeded = summarise(argv[1], argv[2], budget_mb)
+    print(line)
+    summary = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary:
+        try:
+            with open(summary, "a", encoding="utf-8") as stream:
+                stream.write(line + "\n")
+        except OSError as exc:
+            print("::warning::could not write the job summary: %s" % exc)
+    if exceeded:
+        print("::error::stopped: proxy budget %g MB exceeded" % budget_mb)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

.github/workflows/canary.yml

@@ -171,9 +171,17 @@ jobs:
           set +e
           python3 playwright_scraper.py \
             --url "https://www.farfetch.com/shopping/kids/girls-clothing-4/items.aspx" \
-            --pages 3 --out canary_run --dump-html canary_run.html
+            --pages 3 --out canary_run --dump-html canary_run.html --proxy-budget-mb 10
           echo "exit_code=$?" >> "$GITHUB_OUTPUT"
 
+      - name: Proxy traffic
+        # What the run sent through FARFETCH_PROXY, read from its sidecar,
+        # into the job summary. Red when the run stopped at the 10 MB budget
+        # (stop_reason proxy_budget_exceeded); a missing sidecar is reported,
+        # not failed — the next step judges the run itself.
+        if: always() && env.HAVE_PROXY == 'true'
+        run: python3 .github/proxy_traffic_summary.py canary_run production-like 10
+
       - name: Interpret the exit code
         if: env.HAVE_PROXY == 'true'
         # No --allow-block here: from a residential exit, a block is news.
@@ -192,6 +200,7 @@ jobs:
             canary_run.json
             canary_run.csv
             canary_run.meta.json
+            canary_run.latest_attempt.meta.json
             canary_run.html*
             canary_run_page*_debug.*
           if-no-files-found: ignore

.github/workflows/tests.yml

@@ -251,8 +251,10 @@ jobs:
     # (which proves each requirements file is self-consistent, a thing nothing
     # checked before), and is exactly what the README tells users to do.
     #
-    # Browser BINARIES are still not installed: the offline suite imports the
-    # libraries and never launches one.
+    # Browser BINARIES: the offline suite imports the libraries and never
+    # launches one. The proxy-meter step at the end does launch a real
+    # browser against a stub proxy on 127.0.0.1 (Playwright's own Chromium,
+    # the runner's Chrome for pyppeteer and Selenium).
     strategy:
       fail-fast: false
       matrix:
@@ -327,6 +329,18 @@ jobs:
       - name: This engine's CLI answers --help
         run: .venv-${{ matrix.engine }}/bin/python ${{ matrix.engine }}_scraper.py --help
 
+      - name: Install Playwright's Chromium (playwright only)
+        if: matrix.engine == 'playwright'
+        run: .venv-playwright/bin/playwright install --with-deps chromium
+
+      - name: The proxy meter against a stub proxy, in a real browser
+        # tests/test_proxy_meter.py compares this engine's byte count with
+        # what a local stub proxy saw. PROXY_METER_REQUIRE makes a missing
+        # engine or browser a failure here instead of a skip.
+        env:
+          PROXY_METER_REQUIRE: ${{ matrix.library }}
+        run: .venv-${{ matrix.engine }}/bin/python -m unittest -v tests.test_proxy_meter
+
       - name: The console scripts install and resolve
         # A [project.scripts] entry naming a missing target installs happily
         # and fails only when a user runs it. Installing the package and

CHANGELOG.md

@@ -16,7 +16,12 @@ a promise that every flag and exit code is contractually frozen.
 > files and `<out>.meta.json` are left untouched. (2) Behind a local proxy,
 > the browser no longer downloads images, video/audio or web fonts.
 > `--load-media` restores the old behaviour. Nothing changes with
-> `--cdp-endpoint` or without a proxy.
+> `--cdp-endpoint` or without a proxy. (3) Behind a local proxy a run now
+> stops once about 10 MB have gone through it (`--proxy-budget-mb`, `0` =
+> unlimited): the products read so far are kept and the run exits 6 (5 with
+> none), `stop_reason: proxy_budget_exceeded`. A proxied run that writes no
+> products now writes `<out>.latest_attempt.meta.json` with its outcome and
+> byte count.
 
 ### Added
 
@@ -28,6 +33,19 @@ a promise that every flag and exit code is contractually frozen.
   (`context.route`) and pyppeteer (CDP `Fetch`), and by file extension in
   Selenium (`Network.setBlockedURLs`). `--load-media` opts out on all three.
   Playwright turns off its HTTP cache while a route is installed.
+- Proxy traffic meter (`proxy_meter.py`) in all three engines, active only
+  behind a local proxy: Playwright sums `request.sizes()`, pyppeteer and
+  Selenium read CDP `Network.loadingFinished.encodedDataLength` plus the
+  request sizes, and the pre-check's `HEAD` counts too. It is an estimate of
+  the tunnelled payload (TLS/`CONNECT` overhead not included). The sidecar
+  records `proxy_bytes`, `proxy_budget_bytes` and `proxy_budget_exceeded`, and
+  a proxied run ends with one `Proxy traffic: X.X MB of Y MB budget` line. New
+  flag `--proxy-budget-mb` (default 10) on every engine.
+- CI only: the production-like canary passes `--proxy-budget-mb 10`, writes
+  `**production-like**: proxy traffic X.X MB of 10 MB` to the job summary and
+  fails with `::error::stopped: proxy budget 10 MB exceeded` when the run hit
+  the budget. The engine jobs run the meter against a stub proxy in a real
+  browser.
 
 ### Changed
 

Dockerfile

@@ -24,7 +24,7 @@ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.tx
 # level, so the image died with ModuleNotFoundError on every invocation,
 # `--help` included. Nothing noticed, because CI never built the image.
 COPY arg_types.py captcha_solver.py notify.py run_state.py env_config.py fingerprint_client.py output_writer.py \
-     playwright_scraper.py product_parser.py product_detail_parser.py diff_runs.py proxy_pool.py ./
+     playwright_scraper.py product_parser.py product_detail_parser.py diff_runs.py proxy_pool.py proxy_meter.py ./
 
 ENTRYPOINT ["python3", "playwright_scraper.py"]
 CMD ["--help"]

README.md

@@ -250,6 +250,7 @@ The three browser engines share these:
 | `--cdp-endpoint` | – | Attach to a running browser instead of launching one |
 | `--proxy` | – | Proxy for a self-launched browser; ignored with `--cdp-endpoint`. Pre-checked before the browser starts ([traffic savers](#traffic-savers-pre-check-and-media-blocking)) |
 | `--load-media` | off | Load images, video and web fonts behind `--proxy` too (by default they are blocked there) |
+| `--proxy-budget-mb` | 10 | Stop once this many MB have gone through `--proxy` / `--proxy-file`; `0` = unlimited (see [Traffic budget](#traffic-budget)) |
 | `--retries` | `3` | Attempts per page load, pause doubling each time |
 | `--concurrency` | `1` | Playwright only — fetch pages through N parallel workers ([details](#concurrency)) |
 | `--twocaptcha-key` | – | 2Captcha API key (or set `TWOCAPTCHA_KEY`) |
@@ -284,8 +285,8 @@ A contract, not decoration — the harness and any pipeline can branch on these.
 | `2` | Bad usage |
 | `3` | **Blocked before parsing** — a challenge page, or an outright refusal |
 | `4` | Fetched the page fine, parsed **0 products**, and the page links to no products either |
-| `5` | **No content obtained** — navigation timeout, dead proxy, a proxy gateway that refused the pre-check with 407/402 (`stop_reason: proxy_auth_failed`), 4xx/5xx, a failed `--cdp-endpoint` connect, a Scraper API error, or a page that links to products and parsed to none (`stop_reason: parse_drift` — a parser problem, not a network one) |
-| `6` | **Partial run** — products written, but the page loop stopped early |
+| `5` | **No content obtained** — navigation timeout, dead proxy, a proxy gateway that refused the pre-check with 407/402 (`stop_reason: proxy_auth_failed`), 4xx/5xx, a failed `--cdp-endpoint` connect, a Scraper API error, or a page that links to products and parsed to none (`stop_reason: parse_drift` — a parser problem, not a network one), or the proxy budget ran out before any product was read (`stop_reason: proxy_budget_exceeded`) |
+| `6` | **Partial run** — products written, but the page loop stopped early (including at `--proxy-budget-mb`: `stop_reason: proxy_budget_exceeded`) |
 | `124` | Selenium only: chromedriver could not be **started** within the watchdog |
 
 **Exit 4 means the page loaded.** This is the one code that says something
@@ -841,6 +842,39 @@ off.
 Debug screenshots (`*_debug.png`) taken while blocking is on show the page
 without its images.
 
+#### Traffic budget
+
+Also only for a browser this repo launches behind `--proxy` / `--proxy-file`:
+every byte that browser sends or receives through the proxy is counted, and
+`--proxy-budget-mb` (default `10`; `0` = unlimited) caps a run. Once the
+running total passes the budget:
+
+- no further listing page, worker page or product page is opened, and the
+  browser aborts any request it still tries;
+- the products already read are kept, and the run ends with
+  `stop_reason: proxy_budget_exceeded` — exit 6 with products, exit 5
+  without. It never goes on silently, and a block or an empty page seen on
+  the way does not change that code.
+
+The sidecar records `proxy_bytes`, `proxy_budget_bytes` and
+`proxy_budget_exceeded` (`null` / `null` / `false` when no local proxy was
+used). A proxied run that writes no products records them in
+`<out>.latest_attempt.meta.json` beside the untouched data. A proxied run
+ends with one line such as `Proxy traffic: 3.2 MB of 10 MB budget`.
+
+The count is an estimate of the tunnelled payload — request line, headers
+and body sent, response headers and body as received — not the proxy's
+invoice: TLS records, the `CONNECT` exchange, WebSocket frames and the
+browser's own background requests are not visible to it, so the bill is a
+few per cent higher. 1 MB = 1,000,000 bytes.
+
+| Engine | Counted from |
+|---|---|
+| Playwright | `request.sizes()` of every finished or failed request |
+| pyppeteer | CDP `Network.loadingFinished.encodedDataLength` of every `http(s)` response that came over the network, plus each request's line, headers and body (`Network.requestWillBeSent`) |
+| Selenium | the same CDP events, read from Chrome's performance log |
+| pre-check | the `HEAD` itself (request + response headers) |
+
 Measured 2026-10-20 against a local stub page holding a png, a jpg, a woff2,
 an mp4 and a script. Without blocking, every engine fetched all of them. With
 blocking, Playwright and Selenium fetched only the page and the script, and

output_writer.py

@@ -12,6 +12,7 @@
 import uuid
 
 import notify
+import proxy_meter
 from dataclasses import dataclass, asdict, field
 from datetime import datetime, timezone
 from typing import Optional, List, Set
@@ -397,12 +398,15 @@ def write_failed_attempt(out_prefix: str, *, stop_reason: str,
                     products=0, mode=mode)
     meta.update(blocked=False, transport_error=transport_error,
                 data_updated=False)
+    # proxy_bytes / proxy_budget_bytes / proxy_budget_exceeded
+    meta.update(proxy_meter.sidecar())
     path = f"{out_prefix}{ATTEMPT_META_SUFFIX}"
     with _atomic(path) as f:
         json.dump(meta, f, ensure_ascii=False, indent=2)
     print(f"[+] Wrote attempt metadata -> {path} (status=failed, "
           f"stop_reason={stop_reason}); the data and {out_prefix}.meta.json "
           f"were left as they were.")
+    proxy_meter.report()
     return path
 
 
@@ -596,6 +600,11 @@ def finish_run(products: List[Product], out_prefix: str, fmt: str,
     # fix: 28 of 32 repos behaved this way. Same shape as the exit-code
     # unification this file already carries — a rule keyed on a list of
     # names has a hole for every name nobody added to it.
+    meter = proxy_meter.current()
+    if meter.exceeded:
+        # The run stopped itself at --proxy-budget-mb. That outranks whatever
+        # the pages it did fetch said: rows kept -> 6, none -> 5 (never 3/4).
+        stop_reason, blocked = proxy_meter.STOP_REASON, False
     complete = stop_reason in COMPLETE_STOP_REASONS and not pages_failed
     write_stats: dict = {}
     rc = save(products, out_prefix, fmt, allow_empty=allow_empty,
@@ -620,8 +629,17 @@ def finish_run(products: List[Product], out_prefix: str, fmt: str,
     # not overwrite a key the run itself set.
     for k, v in write_stats.items():
         meta.setdefault(k, v)
+    for k, v in proxy_meter.sidecar().items():
+        meta.setdefault(k, v)
     if wrote_output:
         write_run_meta(out_prefix, meta)
+    elif meter.active:
+        # Nothing written, but what the proxy carried is still recorded
+        # beside the untouched data: <out>.latest_attempt.meta.json.
+        attempt = dict(meta, data_updated=False)
+        with _atomic(f"{out_prefix}{ATTEMPT_META_SUFFIX}") as f:
+            json.dump(attempt, f, ensure_ascii=False, indent=2)
+    proxy_meter.report()
 
     rc = _finish_code(products, rc, blocked, stop_reason, complete,
                       out_prefix, pages_completed, pages_requested)