diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7a334da..c1f852a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,29 @@ discovered from a bill or a diff.
v2. Measured 2026-10-22 on Google's own demos in Playwright, Selenium and
pyppeteer; with the corrected kind both v2 demos were solved and accepted.
+- **A `--gl` Play does not serve is exit 2, not exit 3.** Play answers such a
+ code (`USA`, `xx`, `12`) with its own HTTP 400 page, which carries no store
+ images and was read as a block: fresh browsers, exit 3, and advice to buy a
+ proxy. It is now a usage error naming the flag, with no retries. `--gl` is
+ also checked for shape (two letters) before any browser starts.
+- **`screenshot_count` on app rows** counts the screenshots. The app page
+ wraps the list one level deeper than a grid tile, and the wrapper was
+ counted: 1 on every app row (WhatsApp has 9, Minecraft 8). Grid rows were
+ right and are unchanged.
+- **A run's last-page reason survives.** When the page that met `--pages` was
+ the one that added nothing, `stop_reason` was overwritten with `completed`;
+ it now says `no_new_products`. The run is still complete; only the reason
+ was wrong. The same line in the reviews loop is gone too.
+- **`--out x.json` writes `x.json`**, not `x.json.json`: a trailing `.json`,
+ `.csv` or `.meta.json` is dropped from the prefix.
+- **`--out` and `--dump-html` create missing directories**, and a path that
+ cannot be created is exit 2 before any browser starts. A `--dump-html` into
+ a missing directory used to fail mid-run and be reported as
+ `navigation_failed`, exit 5, about a page that had been fetched.
+- The check that every capture is committable now asks whether an ignore rule
+ matches (`git check-ignore --no-index`). Without that it never reported a
+ file already tracked, so it only worked until the fixture was committed.
+
### Changed
- CI only: the canary now runs weekly (Mondays 06:17 UTC) instead of daily, and on a manual dispatch.
diff --git a/README.md b/README.md
index db188cb..ccf5a7c 100644
--- a/README.md
+++ b/README.md
@@ -260,6 +260,13 @@ many cells differ.
it with a clean HTTP 404, and this scraper calls that `not_found` — an
answer, not a refusal, on all three engines. Check the id before you check
your proxy.
+* **A `--gl` Play does not serve is exit 2, not exit 3.** `--gl` must be two
+ letters; a well-formed code the store does not serve (`xx`, measured
+ 2026-10-22) gets Play's own HTTP 400 page, which this scraper reports as a
+ usage error naming the flag — with no retries and no proxy advice. `ZZ`, by
+ contrast, is answered with a full page.
+* **`--out` is a prefix.** `--out data/run.json` writes `data/run.json`,
+ `data/run.csv` and `data/run.meta.json`, creating `data/` if needed.
* **`genre_id` is null on every listing row, and that is the store.** A grid
tile publishes the genre only as display text ("Casual", "Arcade") — it
carries no locale-independent key at all, measured across the capture set.
diff --git a/captures/README.md b/captures/README.md
index 9cc9960..b1acc68 100644
--- a/captures/README.md
+++ b/captures/README.md
@@ -22,6 +22,7 @@ reads it; the earlier pair was in relevance order.
| `dev_us.html` | `/store/apps/dev?id=…` | a developer grid |
| `apps_home_us.html` | `/store/apps` | the store front, with the chart shelves |
| `cluster_us.html` | `/store/apps/collection/cluster?gsr=…` | an expanded shelf — what `--pages` follows |
+| `bad_request_gl.html` | `/store/apps/category/GAME?gl=USA` | Play's own HTTP 400 for a country it does not serve, verbatim (1.7 KB) — a usage error, and a second control for the asset-count signal |
| `notfound.html` | a package id that does not exist | HTTP 404, and the control for the asset-count signal |
| `reviews_p1.txt` | `batchexecute` RPC `UsvDTd`, `--sort newest` | 40 reviews, newest first, and a continuation token |
| `reviews_p2.txt` | the same, with that token | 40 more, sharing none of page 1's ids and continuing the date order |
diff --git a/captures/bad_request_gl.html b/captures/bad_request_gl.html
new file mode 100644
index 0000000..1760f07
--- /dev/null
+++ b/captures/bad_request_gl.html
@@ -0,0 +1 @@
+Error 400 (Bad Request)!!1400. That’s an error.
The server cannot process the request because it is malformed. It should not be retried. That’s all we know.
\ No newline at end of file
diff --git a/engine_core.py b/engine_core.py
index f889866..fe8f5ec 100644
--- a/engine_core.py
+++ b/engine_core.py
@@ -44,6 +44,7 @@ def close() -> None
"""
import argparse
+import os
import re
import sys
import time
@@ -98,6 +99,21 @@ def convert(text: str) -> float:
non_negative_float = _float_between(0.0)
+def country_code(text: str) -> str:
+ """`--gl`: two letters, upper-cased.
+
+ Only the SHAPE is checked here. Whether Play serves that country is the
+ store's answer, not a list this file could keep current — `ZZ` is two
+ letters and Play answers it with a full page — so a well-shaped code Play
+ refuses is caught from its HTTP 400 instead (see `fetch_page`).
+ """
+ if not (isinstance(text, str) and len(text) == 2 and text.isascii()
+ and text.isalpha()):
+ raise argparse.ArgumentTypeError(
+ f"{text!r} is not a two-letter country code (e.g. US, DE, JP)")
+ return text.upper()
+
+
def build_parser() -> argparse.ArgumentParser:
ap = argparse.ArgumentParser(
description="Scrape Google Play listings, apps, developers and reviews.")
@@ -111,9 +127,10 @@ def build_parser() -> argparse.ArgumentParser:
ap.add_argument("--query", help="a search query (--mode search)")
ap.add_argument("--app-id", help="an Android package id, e.g. com.whatsapp")
ap.add_argument("--developer", help="a developer id or name")
- ap.add_argument("--gl", default="US",
- help="the MARKET. Changes the rating, the currency and "
- "which apps are listed. Default US.")
+ ap.add_argument("--gl", default="US", type=country_code,
+ help="the MARKET, as a two-letter country code. Changes "
+ "the rating, the currency and which apps are listed. "
+ "A code Play does not serve is exit 2. Default US.")
ap.add_argument("--hl", default="en", help="the display language. Default en.")
ap.add_argument("--pages", type=positive_int, default=1,
help="how many fetches to make. On a grid this follows "
@@ -133,7 +150,10 @@ def build_parser() -> argparse.ArgumentParser:
help="drop reviewer name, profile id and avatar from "
"review rows")
ap.add_argument("--format", default="json", choices=("json", "csv", "both"))
- ap.add_argument("--out", default=DEFAULT_OUT_PREFIX)
+ ap.add_argument("--out", default=DEFAULT_OUT_PREFIX,
+ help="output prefix: writes PREFIX.json / .csv / "
+ ".meta.json. A trailing .json or .csv is dropped, "
+ "and missing directories are created.")
ap.add_argument("--delay", type=non_negative_float, default=1.5,
help="seconds between fetches")
ap.add_argument("--retries", type=non_negative_int, default=2,
@@ -177,7 +197,7 @@ def build_parser() -> argparse.ArgumentParser:
ap.add_argument("--allow-empty", action="store_true")
ap.add_argument("--dump-html", metavar="PREFIX",
help="write each fetched page; on success too, not only "
- "on failure")
+ "on failure. Missing directories are created.")
headless = ap.add_mutually_exclusive_group()
headless.add_argument("--headless", dest="headless", action="store_true",
default=True)
@@ -396,6 +416,16 @@ def fetch_page(session: Session, url: str, mode: str, page_num: int,
_dump(args, dump_name, html)
state, reason = page_flow.classify(html, status, url, mode)
+ if state == "bad_request":
+ # Play's own HTTP 400: the request is malformed, so no retry, no
+ # solve and no other exit will change the answer. It used to be
+ # read as a block — exit 3, fresh browsers, and advice to buy a
+ # proxy — for a country code Play does not serve.
+ raise UsageError(
+ f"Play refused the request as malformed (HTTP 400) — "
+ f"usually because --gl {args.gl} is not a country the store "
+ f"serves. Use one it does, e.g. US, DE, JP.")
+
if state != "content" and _wants_solve(args, state):
solved = _solve(session, html, url, mode, page_num)
if solved is not None:
@@ -537,8 +567,6 @@ def run_grid(session: Session, start_url: str, mode: str,
if out.pages_completed < args.pages and queue:
_sleep(args.delay)
- if out.pages_completed >= args.pages and not out.pages_failed:
- out.stop_reason = "completed"
out.extra["shelves_seen"] = len(visited)
return out
@@ -656,8 +684,6 @@ def run_reviews(session: Session, url: str, args: argparse.Namespace) -> RunOutc
break
_sleep(args.delay)
- if out.pages_completed >= args.pages and not out.pages_failed:
- out.stop_reason = "completed"
out.extra["reviews_per_call"] = args.reviews_per_call
out.extra["authors_included"] = not args.no_authors
return out
@@ -694,6 +720,50 @@ def _dump(args: argparse.Namespace, name: str, html: str) -> None:
print(f"[dump] {path} ({len(html)} bytes)", file=sys.stderr)
+_OUTPUT_SUFFIXES = (".meta.json", ".json", ".csv")
+
+
+def output_prefix(out: str) -> str:
+ """`--out x.json` writes `x.json`, `x.csv` and `x.meta.json`.
+
+ `--out` is a PREFIX — the run writes up to three files from it — and
+ passing a filename with its extension is the most common way people use
+ such a flag. Taken literally it produced `x.json.json`, `x.json.csv` and
+ `x.json.meta.json`. A recognised output suffix is now dropped; anything
+ else (`--out run.2026`) is left alone.
+ """
+ lowered = out.lower()
+ for suffix in _OUTPUT_SUFFIXES:
+ if lowered.endswith(suffix) and len(out) > len(suffix):
+ trimmed = out[:-len(suffix)]
+ print(f"[info] --out {out}: writing {trimmed}.json/.csv/.meta.json",
+ file=sys.stderr)
+ return trimmed
+ return out
+
+
+def ensure_parent_dir(prefix: Optional[str]) -> Optional[str]:
+ """Create the directory a path prefix writes into; say why if it cannot.
+
+ Checked BEFORE any browser starts. A `--dump-html` into a directory that
+ did not exist used to raise mid-run, after the page had been fetched, and
+ the run reported it as `navigation_failed`, exit 5 — "the content was
+ never obtained" — about content it had obtained.
+ """
+ if not prefix:
+ return None
+ parent = os.path.dirname(os.path.expanduser(prefix))
+ if not parent:
+ return None
+ try:
+ os.makedirs(parent, exist_ok=True)
+ except OSError as exc:
+ return f"cannot create {parent}: {exc.strerror or exc}"
+ if not os.access(parent, os.W_OK):
+ return f"{parent} is not writable"
+ return None
+
+
def main(launch: Callable[[argparse.Namespace, Optional[str]], Any],
argv: Optional[Sequence[str]] = None) -> int:
args = build_parser().parse_args(argv)
@@ -717,6 +787,17 @@ def main(launch: Callable[[argparse.Namespace, Optional[str]], Any],
return EXIT_USAGE
args.mode = mode
+ # Expanded once, here, so the directory created and the file written are
+ # the same path even when a `~` arrived quoted and the shell left it.
+ args.out = os.path.expanduser(output_prefix(args.out))
+ if args.dump_html:
+ args.dump_html = os.path.expanduser(args.dump_html)
+ for flag, prefix in (("--out", args.out), ("--dump-html", args.dump_html)):
+ problem = ensure_parent_dir(prefix)
+ if problem:
+ print(f"[error] {flag} {prefix}: {problem}", file=sys.stderr)
+ return EXIT_USAGE
+
if args.concurrency > 1:
cap = page_flow.concurrency_limit(mode, url)
if cap is not None:
diff --git a/page_flow.py b/page_flow.py
index 5bc9476..d93a085 100644
--- a/page_flow.py
+++ b/page_flow.py
@@ -133,6 +133,10 @@ def classify(html: Optional[str], status: Optional[int] = None,
# is not worth a retry, not worth a solve, and emphatically not a block —
# calling it one would send a reader to buy a proxy for a typo.
"not_found": {"parse": False, "retry": False, "solve": False, "blocked": False},
+ # Play's own HTTP 400: the request was malformed (a `gl` it does not
+ # serve). Nothing a retry, a solve or another exit can change — the run
+ # stops with a usage error naming the flag.
+ "bad_request": {"parse": False, "retry": False, "solve": False, "blocked": False},
"blocked": {"parse": False, "retry": True, "solve": True, "blocked": True},
# Served, and holds nothing we recognise. A real answer for a category
# with no apps or a search with no hits, so it is not retried: retrying an
diff --git a/product_parser.py b/product_parser.py
index 1b45e19..b59542c 100644
--- a/product_parser.py
+++ b/product_parser.py
@@ -189,6 +189,13 @@ class names are build hashes.
# images at all" and only where no HTTP status is available.
NOT_FOUND_MARKER = "the requested URL was not found on this server"
+# The heading of Play's own HTTP 400 page — what it answers when `gl` names a
+# country it does not serve (`USA`, `xx`, `12`; measured 2026-10-22). 1.7 KB,
+# no store images, and the phrase appears in none of the served captures. It
+# is a malformed REQUEST, not a refusal: reading it as blocked sent runs into
+# fresh-browser retries and advice to buy a proxy, for a typo in a flag.
+BAD_REQUEST_MARKER = "Error 400 (Bad Request)"
+
def asset_reference_count(markup: str) -> int:
"""How many times the page references Play's own image host."""
@@ -746,6 +753,30 @@ def _shelf_title(ancestors: List[list], records: List[list]) -> Optional[str]:
return None
+def _is_image(node: Any) -> bool:
+ """`[null, 2, [w, h], [null, null, ""], ...]` — one Play image."""
+ return (isinstance(node, list) and len(node) >= 4
+ and isinstance(node[2], list) and isinstance(node[3], list)
+ and any(isinstance(v, str) and v.startswith("http") for v in node[3]))
+
+
+def _screenshot_count(node: Any) -> Optional[int]:
+ """How many screenshots a screenshot slot holds.
+
+ The two routes wrap the list differently: a grid tile puts the images
+ directly in the slot, the app page puts them one level down —
+ `[[image, image, ...]]`. Counting the slot itself therefore reported 1 on
+ every app row (WhatsApp has 9, Minecraft 8 in the captures). Unwrap
+ single-element wrappers until the level that actually holds images.
+ """
+ while (isinstance(node, list) and len(node) == 1
+ and isinstance(node[0], list) and not _is_image(node[0])):
+ node = node[0]
+ if not isinstance(node, list):
+ return None
+ return sum(1 for item in node if _is_image(item))
+
+
def parse_tile(record: Sequence[Any], hl: str = "en", gl: str = "US",
page: Optional[int] = None, position: Optional[int] = None,
shelf: Optional[str] = None,
@@ -797,7 +828,7 @@ def parse_tile(record: Sequence[Any], hl: str = "en", gl: str = "US",
description=_strip_html(description[1] if isinstance(description, list)
and len(description) > 1
and isinstance(description[1], str) else _text(description)),
- screenshot_count=len(screenshots) if isinstance(screenshots, list) else None,
+ screenshot_count=_screenshot_count(screenshots),
shelf=shelf,
shelf_rank=shelf_rank,
)
@@ -966,7 +997,7 @@ def parse_detail(markup: str, package_id: Optional[str] = None,
developer_email=email,
developer_website=website,
privacy_policy_url=_first_url(_slot(body, DETAIL["privacy"]), "http"),
- screenshot_count=len(screenshots) if isinstance(screenshots, list) else None,
+ screenshot_count=_screenshot_count(screenshots),
chart_name=chart_name,
chart_rank=chart_rank,
)
@@ -1372,7 +1403,8 @@ def _developer_reply(node: Any) -> Tuple[Optional[str], Optional[str]]:
# What did we actually get?
# --------------------------------------------------------------------------
-PAGE_STATES = ("content", "not_found", "blocked", "empty", "unknown")
+PAGE_STATES = ("content", "not_found", "bad_request", "blocked", "empty",
+ "unknown")
def detect_page_state(markup: str, status: Optional[int] = None,
@@ -1421,6 +1453,15 @@ def detect_page_state(markup: str, status: Optional[int] = None,
and asset_reference_count(markup) == 0):
return "not_found", "not_found_template"
+ # A malformed request, which is the caller's to fix and never worth a
+ # retry. With a status it needs nothing else; without one (Selenium) the
+ # page's own heading AND the absence of every store image, as for 404.
+ if status == 400:
+ return "bad_request", "http_400"
+ if (status is None and BAD_REQUEST_MARKER in markup
+ and asset_reference_count(markup) == 0):
+ return "bad_request", "bad_request_template"
+
marker = challenge_marker(markup)
if marker:
return "blocked", f"challenge_marker:{marker}"
diff --git a/scraper_api_client.py b/scraper_api_client.py
index 8e52640..e352b7f 100644
--- a/scraper_api_client.py
+++ b/scraper_api_client.py
@@ -248,6 +248,13 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int:
# `(state, reason)`; comparing the tuple itself to "blocked" never
# matched, so a refusal fell through to "0 products".
state, reason = detect_page_state(html, status=upstream_status, url=args.url)
+ if state == "bad_request":
+ # Play's own HTTP 400 — a `gl=` in the URL that the store does not
+ # serve. The caller's to fix, so usage (2), not blocked or empty.
+ logger.error("Play refused the request as malformed (upstream HTTP %s): "
+ "usually a gl= country code the store does not serve. "
+ "Use one it does, e.g. gl=US.", upstream_status)
+ return 2
if state == "blocked":
dump = f"{args.out}_scraperapi_debug.html"
with open(dump, "w", encoding="utf-8") as f:
diff --git a/smoke_test.py b/smoke_test.py
index 046d11b..a67295d 100644
--- a/smoke_test.py
+++ b/smoke_test.py
@@ -193,6 +193,10 @@ def test_paid_app_values():
row.iap_price_range)
ok &= check("[paid] offers_iap follows the range", row.offers_iap is True)
ok &= check("[paid] genre_id", row.genre_id == "GAME_ARCADE", row.genre_id)
+ # The app page wraps its screenshots one level deeper than a grid tile;
+ # counting the wrapper reported 1 on every app row up to 0.2.0.
+ ok &= check("[paid] screenshot_count", row.screenshot_count == 8,
+ str(row.screenshot_count))
return ok
@@ -330,6 +334,11 @@ def test_page_state_on_every_capture():
return ok
+# Captures of Play's own error pages. Real responses, and the controls for
+# the structural signal: neither is built out of the store's images.
+_REFUSAL_CAPTURES = ("notfound.html", "bad_request_gl.html")
+
+
def test_markers_score_zero_on_good_pages():
"""A marker that matches a page you know is good is not a marker.
@@ -341,7 +350,7 @@ def test_markers_score_zero_on_good_pages():
"""
ok = True
served = [n for n in os.listdir(os.path.join(HERE, "captures"))
- if n.endswith(".html") and n != "notfound.html"]
+ if n.endswith(".html") and n not in _REFUSAL_CAPTURES]
if not served:
skip("markers", "no captures")
return True
@@ -354,11 +363,15 @@ def test_markers_score_zero_on_good_pages():
parser.asset_reference_count(html) >= parser.MIN_ASSET_REFERENCES)
# The 404 is the control: it is a real Play response and carries NONE of
# the image host, which is what makes the structural signal usable.
- nf = capture("notfound.html")
- if nf is not None:
- ok &= check("[marker] the store's 404 carries no image-host references",
- parser.asset_reference_count(nf) == 0,
- str(parser.asset_reference_count(nf)))
+ for name in _REFUSAL_CAPTURES:
+ page = capture(name)
+ if page is not None:
+ ok &= check(f"[marker] {name} carries no image-host references",
+ parser.asset_reference_count(page) == 0,
+ str(parser.asset_reference_count(page)))
+ ok &= check(f"[marker] {name} trips no refusal marker",
+ parser.challenge_marker(page) is None,
+ "it is the store answering, not refusing")
ok &= check("[marker] the set carries no reCAPTCHA spelling",
not any("recaptcha" in m.lower() or "sitekey" in m.lower()
for m in parser.BOT_CHALLENGE_MARKERS),
@@ -1286,7 +1299,8 @@ def close(self):
def _run_main(argv, pages, posts=(), discovery=None, after_solve=None,
- launch_error=None, env=None):
+ launch_error=None, env=None, out_name=None, dump=None,
+ prepare=None):
"""`(rc, world, out_prefix_files)` for one `engine_core.main` run."""
import engine_core
world = {"pages": list(pages), "posts": list(posts), "forms": [], "fetches": [],
@@ -1305,15 +1319,23 @@ def launch(args, proxy_url):
os.environ.update(env or {})
try:
with tempfile.TemporaryDirectory() as tmp:
- prefix = os.path.join(tmp, "o")
+ prefix = os.path.join(tmp, out_name or "o")
+ extra = ["--out", prefix, "--format", "both"]
+ if dump:
+ extra += ["--dump-html", os.path.join(tmp, dump)]
+ if prepare:
+ prepare(tmp)
buf = io.StringIO()
with redirect_stdout(buf), redirect_stderr(buf):
try:
- rc = engine_core.main(launch, list(argv) + ["--out", prefix,
- "--format", "both"])
+ rc = engine_core.main(launch, list(argv) + extra)
except SystemExit as exc:
rc = exc.code
+ prefix = engine_core.output_prefix(prefix)
world["log"] = buf.getvalue()
+ world["written"] = sorted(
+ os.path.relpath(os.path.join(d, f), tmp)
+ for d, _dirs, fs in os.walk(tmp) for f in fs)
files = {}
for ext in (".json", ".csv", ".meta.json"):
if os.path.exists(prefix + ext):
@@ -1614,12 +1636,141 @@ def test_captures_are_committable():
ok &= check("[captures] the directory is not empty", bool(names))
for name in names:
rel = os.path.join("captures", name)
- r = subprocess.run(["git", "check-ignore", "-q", rel], cwd=HERE)
+ # `--no-index`: ask whether an ignore RULE matches, whatever the
+ # index says. Without it a file already tracked is never reported, so
+ # the check only worked until the fixture was committed — and the
+ # trap is the rule, which will swallow the NEXT capture added.
+ r = subprocess.run(["git", "check-ignore", "-q", "--no-index", rel],
+ cwd=HERE)
ok &= check(f"[captures] {rel} is not gitignored", r.returncode != 0,
"a fixture git will not commit makes its check skip in CI")
return ok
+def test_screenshot_count_on_both_routes():
+ ok = True
+ for name, package, want in (("app_free_us.html", "com.whatsapp", 9),
+ ("app_iap_us.html", "com.mojang.minecraftpe", 8)):
+ html = capture(name)
+ if html is None:
+ skip("screenshots", f"captures/{name} absent")
+ continue
+ row = parser.parse_detail(html, package)
+ ok &= check(f"[screenshots] {name}: {want} on the app page",
+ row is not None and row.screenshot_count == want,
+ str(row and row.screenshot_count))
+ grid = capture("search_vpn_us.html")
+ if grid is not None:
+ rows = parser.parse_grid(grid)
+ ok &= check("[screenshots] a grid tile still counts its own list",
+ rows and all((r.screenshot_count or 0) > 1 for r in rows[:5]),
+ str([r.screenshot_count for r in rows[:5]]))
+ return ok
+
+
+def test_run_contract_last_page_reason_survives():
+ """A run's last page adding nothing is `no_new_products`, not `completed`.
+
+ A line after the loop set `completed` whenever the page budget was met,
+ which overwrote the true reason when the page that met it was the one
+ that added nothing (30 / 20 / 0 rows over three pages in the report).
+ """
+ ok = True
+ page1, shelf = capture("cat_game_us.html"), capture("cluster_us.html")
+ if None in (page1, shelf):
+ skip("run-contract", "category and cluster captures needed")
+ return True
+ rc, world, files = _run_main(
+ ["--mode", "listing", "--category", "GAME", "--pages", "3"],
+ [(page1, 200), (shelf, 200), (shelf, 200)])
+ meta = json.loads(files.get(".meta.json", "{}"))
+ ok &= check("[contract] three pages were fetched", len(world["fetches"]) == 3,
+ str(len(world["fetches"])))
+ ok &= check("[contract] the last page's reason survives",
+ meta.get("stop_reason") == "no_new_products",
+ str(meta.get("stop_reason")))
+ ok &= check("[contract] and the run is still complete",
+ rc == 0 and meta.get("status") == "complete",
+ f"rc={rc} {meta.get('status')}")
+ return ok
+
+
+def test_run_contract_bad_gl_is_usage():
+ """A country Play does not serve is the caller's mistake, not a block."""
+ ok = True
+ body = capture("bad_request_gl.html")
+ if body is None:
+ skip("run-contract", "captures/bad_request_gl.html absent")
+ return True
+ for status, why in ((400, "http_400"), (None, "bad_request_template")):
+ state, reason = parser.detect_page_state(body, status, "", "grid")
+ ok &= check(f"[gl] Play's 400 page with status {status} is bad_request",
+ (state, reason) == ("bad_request", why), f"{state} {reason}")
+
+ # Malformed shape: refused by argparse before any browser starts.
+ rc, world, _ = _run_main(["--mode", "listing", "--category", "GAME",
+ "--gl", "USA"], [(body, 400)])
+ ok &= check("[gl] --gl USA is exit 2", rc == 2, f"rc={rc}")
+ ok &= check("[gl] and launches nothing", world["launches"] == [],
+ str(world["launches"]))
+
+ # Well-shaped, but a code Play does not serve: its own 400.
+ rc, world, files = _run_main(["--mode", "listing", "--category", "GAME",
+ "--gl", "xx", "--retries", "2"], [(body, 400)])
+ ok &= check("[gl] Play's 400 is exit 2, not 3", rc == 2, f"rc={rc}")
+ ok &= check("[gl] and is not retried", len(world["fetches"]) == 1,
+ f"{len(world['fetches'])} fetches")
+ ok &= check("[gl] the message names the flag", "--gl XX" in world["log"],
+ world["log"][-300:])
+ ok &= check("[gl] no proxy advice for a typo", "[hint]" not in world["log"])
+ ok &= check("[gl] no output written", not files, str(list(files)))
+ return ok
+
+
+def test_run_contract_output_paths():
+ """`--out` with an extension, and paths into directories that do not exist."""
+ ok = True
+ page1 = capture("search_vpn_us.html")
+ if page1 is None:
+ skip("run-contract", "captures/search_vpn_us.html absent")
+ return True
+ argv = ["--mode", "search", "--query", "vpn"]
+
+ rc, world, files = _run_main(argv, [(page1, 200)],
+ out_name="nested/deeper/run.json",
+ dump="dumps/here/p")
+ written = world["written"]
+ ok &= check("[out] the run succeeds into directories it had to create",
+ rc == 0, f"rc={rc}")
+ ok &= check("[out] --out run.json writes run.json, run.csv, run.meta.json",
+ {"nested/deeper/run.json", "nested/deeper/run.csv",
+ "nested/deeper/run.meta.json"} <= set(written), str(written))
+ ok &= check("[out] and no doubled suffix",
+ not any(".json.json" in w or ".json.csv" in w for w in written),
+ str(written))
+ ok &= check("[out] --dump-html into a new directory writes its snapshot",
+ any(w.startswith("dumps/here/p_") and w.endswith(".html")
+ for w in written), str(written))
+
+ def make_blocker(tmp):
+ with open(os.path.join(tmp, "blocker"), "w") as fh:
+ fh.write("a file where a directory is wanted")
+ rc, world, _ = _run_main(argv, [(page1, 200)], out_name="blocker/run",
+ prepare=make_blocker)
+ ok &= check("[out] an --out that cannot be created is exit 2", rc == 2,
+ f"rc={rc}")
+ ok &= check("[out] and is refused before any browser starts",
+ world["launches"] == [], str(world["launches"]))
+
+ import engine_core
+ with redirect_stderr(io.StringIO()):
+ for given, want in (("x.json", "x"), ("x.CSV", "x"), ("x.meta.json", "x"),
+ ("run.2026", "run.2026"), (".json", ".json"), ("x", "x")):
+ got = engine_core.output_prefix(given)
+ ok &= check(f"[out] output_prefix({given!r}) == {want!r}", got == want, got)
+ return ok
+
+
def test_run_contract_captcha_budget():
"""At most one paid solve per page, whatever retries and rotations do."""
ok = True
@@ -1954,6 +2105,10 @@ def main():
test_run_contract_reviews_pages_chain_on_the_token,
test_recaptcha_variant_on_googles_demos,
test_captures_are_committable,
+ test_screenshot_count_on_both_routes,
+ test_run_contract_last_page_reason_survives,
+ test_run_contract_bad_gl_is_usage,
+ test_run_contract_output_paths,
test_run_contract_captcha_budget,
test_cli_rejects_counts_that_cannot_run,
test_env_url_does_not_override_a_typed_builder_flag,