diff --git a/.github/env_report.py b/.github/env_report.py new file mode 100644 index 0000000..f156a4d --- /dev/null +++ b/.github/env_report.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Print the versions a canary run actually had: `env_report.py [--chromium]`. + +Why it exists: when the canary goes red, the first question is whether the +SITE moved or the runner did (Python, the TLS library, the driver, the +browser). This answers it from the log of the run itself. It prints versions +and the kind of endpoint, never an endpoint: a Scraping Browser URL carries +its credentials in the login, and a log is a file people share. + +`--chromium` also launches the local Chromium to read its version, for the +job that drives one; the Scraping Browser job has no local browser to ask. +""" +import os +import platform +import ssl +import sys + + +def version_of(dist): + try: + from importlib.metadata import version + return version(dist) + except Exception: + return "not installed" + + +print("python %s" % platform.python_version()) +print("platform %s" % platform.platform()) +print("tls library %s" % ssl.OPENSSL_VERSION) +for dist in ("requests", "urllib3", "beautifulsoup4", "playwright"): + print("%-11s %s" % (dist, version_of(dist))) +print("endpoint %s" % ("remote CDP (Scraping Browser)" + if os.environ.get("MAKEMYTRIP_CDP_ENDPOINT") + else "local browser, no endpoint")) + +if "--chromium" in sys.argv: + try: + from playwright.sync_api import sync_playwright + with sync_playwright() as p: + browser = p.chromium.launch() + print("chromium %s" % browser.version) + browser.close() + except Exception as exc: + # A report must not fail the job it is describing. + print("chromium could not be launched: %s" % type(exc).__name__) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 76e1acd..48b6907 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -58,6 +58,8 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt -r requirements-playwright.txt playwright install --with-deps chromium + - name: Environment (so a red run says whether the site or the runner moved) + run: python .github/env_report.py --chromium - name: Run id: run run: | @@ -113,6 +115,8 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt -r requirements-playwright.txt + - name: Environment (so a red run says whether the site or the runner moved) + run: python .github/env_report.py # >= 3 pages, because with one page pagination is never exercised. - name: Run id: run diff --git a/CHANGELOG.md b/CHANGELOG.md index 45e5415..f572622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,19 @@ said so at the top of its release notes. - Comments, docs and the suite state their reasons in this repo's own words instead of pointing at section numbers in notes the public cannot read. A suite check keeps it that way. +- The README's `engines` badge no longer lists the three engines as equals: + it says pyppeteer is measured over CDP and Selenium locally only, which is + what the engine table below it already said. +- The README dates the last live checks beside the measurements table. + +### Added + +- `.github/env_report.py`, run by both canary jobs before the scrape: it + prints the Python, TLS library, driver and (for the local-browser job) + Chromium versions, and the kind of endpoint, never the endpoint. A red + canary then says from its own log whether the site or the runner moved. + Prompted by a third-party audit that met a TLS-library warning on an + older Python and could not tell it from the block. ## [0.1.1] — 2026-09-25 diff --git a/README.md b/README.md index c6eff55..6622f13 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![canary](https://github.com/2scraper/makemytrip-scraper/actions/workflows/canary.yml/badge.svg)](https://github.com/2scraper/makemytrip-scraper/actions/workflows/canary.yml) ![python](https://img.shields.io/badge/python-3.9%20%7C%203.12-blue) [![licence](https://img.shields.io/badge/licence-MIT-green)](LICENSE) -![engines](https://img.shields.io/badge/engines-playwright%20%7C%20selenium%20%7C%20pyppeteer-lightgrey) +![engines](https://img.shields.io/badge/engines-playwright%20%7C%20pyppeteer%20via%20CDP%20%7C%20selenium%20local%20only-lightgrey) ![account](https://img.shields.io/badge/makemytrip%20account-not%20needed-brightgreen) Scrapes [makemytrip.com](https://www.makemytrip.com)'s **hotel listings** @@ -117,6 +117,14 @@ Measured through a `country-in` Scraping Browser profile, 2026-09-24: | Goa `--url`, `--sort rating` | Playwright | 30 | 4.9, 5.0, 4.9, 4.8, 4.8 at the top | | Goa, one page | Scraper API + `cdpurl` | 5 | the page's server-rendered five; $0.0005 | +**Last live checks on file** (this line goes stale; the dates are the point): +a served three-page run through the Scraping Browser, 90 of 90 priced, +2026-09-24 (the canary's own first run); pyppeteer through residential +proxies, no rows, 2026-09-25; and from a datacentre address with no +endpoint, the run reported Akamai's refusal as exit 3 and wrote nothing, +2026-10-05, on Python 3.13 (the `datacentre` canary job's claim, re-checked +by hand). No served run has been measured since 2026-09-24. + --- ## Five things about MakeMyTrip that will look like bugs