diff --git a/.github/workflows/periodic_update.yml b/.github/workflows/periodic_update.yml index 47ba453d706..9b30b61e341 100644 --- a/.github/workflows/periodic_update.yml +++ b/.github/workflows/periodic_update.yml @@ -171,3 +171,9 @@ jobs: env: STEPS_CPR_OUTPUTS_PULL_REQUEST_NUMBER: ${{ steps.cpr.outputs.pull-request-number }} STEPS_CPR_OUTPUTS_PULL_REQUEST_URL: ${{ steps.cpr.outputs.pull-request-url }} + wxgui-screenshots: + # The update-i18n job builds without the GUI, so the screenshots are + # captured in a separate workflow (also run for changes to it). + permissions: + contents: read + uses: $/.github/workflows/wxgui_screenshots.yml diff --git a/.github/workflows/wxgui_screenshots.yml b/.github/workflows/wxgui_screenshots.yml new file mode 100644 index 00000000000..a241968cfb1 --- /dev/null +++ b/.github/workflows/wxgui_screenshots.yml @@ -0,0 +1,59 @@ +--- +name: wxGUI screenshots + +# Captures wxGUI screenshots with the translatable strings they show, for +# example to give translators visual context on Weblate. The result is only +# uploaded as an artifact; see utils/weblate_screenshots/weblate_upload.py for +# synchronizing it to Weblate. +on: + workflow_call: + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/wxgui_screenshots.yml" + - "utils/weblate_screenshots/**" + +permissions: {} + +jobs: + capture: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y ninja-build gettext python3-wxgtk4.0 \ + xvfb imagemagick librsvg2-common + LC_ALL=C.UTF-8 sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable + sudo apt-get update -y + xargs -a <(awk '! /^ *(#|$)/' ".github/workflows/apt.txt") -r -- \ + sudo apt-get install -y --no-install-recommends --no-install-suggests + - name: Configure + run: | + cmake -S "${GITHUB_WORKSPACE}" -B "${GITHUB_WORKSPACE}/build" -G Ninja -DWITH_DOCS=OFF + - name: Build + run: cmake --build build -j"$(nproc)" + - name: Capture screenshots + env: + GRASS_CONFIG_DIR: ${{ runner.temp }}/grass-config + LANG: C.UTF-8 + run: | + export PATH="${GITHUB_WORKSPACE}/build/output/bin:${PATH}" + mkdir -p "${GRASS_CONFIG_DIR}" + # A fixed project path keeps the screenshots identical between runs. + grass -e -c XY /tmp/grassdata/screenshots + xvfb-run -s "-screen 0 1600x1200x24" \ + grass /tmp/grassdata/screenshots/PERMANENT --exec \ + python3 utils/weblate_screenshots/wxgui_capture.py \ + --pot locale/templates/grasswxpy.pot --output wxgui-screenshots + - name: Upload screenshots + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wxgui-screenshots + path: wxgui-screenshots + retention-days: 30 diff --git a/utils/weblate_screenshots/tests/weblate_upload_test.py b/utils/weblate_screenshots/tests/weblate_upload_test.py new file mode 100644 index 00000000000..ff499ff56b1 --- /dev/null +++ b/utils/weblate_screenshots/tests/weblate_upload_test.py @@ -0,0 +1,390 @@ +"""Tests of weblate_upload.py against a fake Weblate API server. + +The responses model Weblate 5.16.2 and 2026.10.1, which do not differ for the +endpoints used. +""" + +import io +import json +import re +import sys +import threading +from email.parser import BytesParser +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest +from PIL import Image + +# The requests package is not a GRASS dependency. +pytest.importorskip("requests") +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import weblate_upload # noqa: E402 + +TOKEN = "secret-token" # noqa: S105 +PAGE_SIZE = 2 + +UNITS = [ + (1, "", "Layout settings"), + (2, "", "Cancel"), + (3, "Menu", "Cancel"), + (4, "", "Apply"), +] + + +def multipart_fields(handler): + """Return the form fields of a multipart request, files as bytes.""" + length = int(handler.headers["Content-Length"]) + message = BytesParser().parsebytes( + b"Content-Type: " + + handler.headers["Content-Type"].encode() + + b"\r\n\r\n" + + handler.rfile.read(length) + ) + return { + part.get_param("name", header="content-disposition"): part.get_payload( + decode=True + ) + for part in message.get_payload() + } + + +class FakeWeblate(BaseHTTPRequestHandler): + """Model the responses of the Weblate API for one component.""" + + def log_message(self, *args): + pass + + def reply(self, status, body=None, raw=None): + payload = raw if raw is not None else json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def paginate(self, items): + query = self.path.partition("?")[2] + page = int(query.partition("page=")[2] or 1) + chunk = items[(page - 1) * PAGE_SIZE : page * PAGE_SIZE] + base = f"{self.server.url}{self.path.partition('?')[0]}" + more = page * PAGE_SIZE < len(items) + self.reply( + 200, + { + "count": len(items), + "next": f"{base}?page={page + 1}" if more else None, + "previous": None, + "results": chunk, + }, + ) + + def serialize(self, shot_id): + shot = self.server.screenshots[shot_id] + url = self.server.url + return { + "id": shot_id, + "name": shot["name"], + "translation": f"{url}/api/translations/p/c/{shot['language']}/", + "file_url": f"{url}/api/screenshots/{shot_id}/file/", + "units": [f"{url}/api/units/{unit}/" for unit in sorted(shot["units"])], + "url": f"{url}/api/screenshots/{shot_id}/", + } + + def handle_request(self, method): + self.server.requests.append((method, self.path)) + if self.headers["Authorization"] != f"Token {TOKEN}": + return self.reply(403, {"detail": "Bad token"}) + path = self.path.partition("?")[0] + shots = self.server.screenshots + if method == "GET" and path == "/api/components/p/c/": + return self.reply(200, {"source_language": {"code": "en"}}) + if method == "GET" and path == "/api/components/p/c/screenshots/": + return self.paginate([self.serialize(shot_id) for shot_id in shots]) + if method == "GET" and path == "/api/translations/p/c/en/units/": + return self.paginate( + [ + {"id": unit_id, "context": context, "source": [source]} + for unit_id, context, source in UNITS + ] + ) + if method == "POST" and path == "/api/screenshots/": + fields = multipart_fields(self) + shot_id = self.server.next_id + self.server.next_id += 1 + shots[shot_id] = { + "name": fields["name"].decode(), + "language": fields["language_code"].decode(), + "image": fields["image"], + "units": set(), + } + # Like Weblate, the reply does not have the id field. + body = self.serialize(shot_id) + del body["id"] + return self.reply(201, body) + # Like the Weblate router, require the trailing slash. + match = re.fullmatch(r"/api/screenshots/(\d+)/(file/|units/(?:(\d+)/)?)?", path) + if not match or int(match[1]) not in shots: + return self.reply(404, {"detail": "Not found."}) + shot = shots[int(match[1])] + if method == "GET" and match[2] == "file/": + return self.reply(200, raw=shot["image"]) + if method == "POST" and match[2] == "file/": + shot["image"] = multipart_fields(self)["image"] + return self.reply(200, {"result": True}) + if method == "POST" and match[2] == "units/": + length = int(self.headers["Content-Length"]) + form = self.rfile.read(length).decode() + unit_id = int(form.partition("unit_id=")[2]) + if unit_id not in {unit[0] for unit in UNITS}: + return self.reply(400, {"unit_id": "Unit not found"}) + shot["units"].add(unit_id) + return self.reply(200, self.serialize(int(match[1]))) + if method == "DELETE" and match[3]: + shot["units"].discard(int(match[3])) + return self.reply(204, raw=b"") + if method == "DELETE" and match[2] is None: + del shots[int(match[1])] + return self.reply(204, raw=b"") + return self.reply(405, {"detail": "Method not allowed."}) + + def do_GET(self): + self.handle_request("GET") + + def do_POST(self): + self.handle_request("POST") + + def do_DELETE(self): + self.handle_request("DELETE") + + +@pytest.fixture +def weblate(): + server = HTTPServer(("127.0.0.1", 0), FakeWeblate) + server.url = f"http://127.0.0.1:{server.server_port}" + server.screenshots = {} + server.next_id = 10 + server.requests = [] + thread = threading.Thread(target=server.serve_forever) + thread.start() + yield server + server.shutdown() + server.server_close() + thread.join() + + +def png(path, color, size=(8, 6)): + Image.new("RGB", size, color).save(path) + + +def write_capture(directory, shots): + """Write a capture output directory. + + Shots are given as name: (color, [(msgctxt, msgid), ...]). + """ + directory.mkdir(exist_ok=True) + entries = [] + for name, (color, strings) in shots.items(): + file = name.replace("/", "__") + ".png" + png(directory / file, color) + entries.append( + { + "name": name, + "file": file, + "strings": [ + {"msgctxt": context, "msgid": msgid, "kind": "label"} + for context, msgid in strings + ], + "unmatched": [], + } + ) + (directory / "manifest.json").write_text(json.dumps({"screenshots": entries})) + + +@pytest.fixture +def run(weblate, monkeypatch, tmp_path, capsys): + """Run the uploader, return its output and the modifying requests.""" + monkeypatch.setenv("WEBLATE_TOKEN", TOKEN) + + def upload(*options): + weblate.requests.clear() + argv = [ + "weblate_upload.py", + "--url", + weblate.url, + "--project", + "p", + "--component", + "c", + "--input", + str(tmp_path / "shots"), + *options, + ] + monkeypatch.setattr(sys, "argv", argv) + capsys.readouterr() + weblate_upload.main() + changes = [r for r in weblate.requests if r[0] != "GET"] + return capsys.readouterr(), changes + + return upload + + +def linked(weblate): + return {shot["name"]: shot["units"] for shot in weblate.screenshots.values()} + + +SHOTS = { + "wxgui/a": ((255, 0, 0), [(None, "Layout settings"), (None, "Cancel")]), + "wxgui/b": ((0, 255, 0), [("Menu", "Cancel"), (None, "Apply")]), + "wxgui/c": ((0, 0, 255), [(None, "Apply")]), +} + + +def test_first_sync_creates_and_links(weblate, run, tmp_path): + write_capture(tmp_path / "shots", SHOTS) + output, changes = run() + assert linked(weblate) == {"wxgui/a": {1, 2}, "wxgui/b": {3, 4}, "wxgui/c": {4}} + assert {s["language"] for s in weblate.screenshots.values()} == {"en"} + assert "3 created, 0 updated, 0 unchanged, 0 deleted, 5 links added" in output.out + assert len(changes) == 3 + 5 + + +def test_second_sync_changes_nothing(weblate, run, tmp_path): + write_capture(tmp_path / "shots", SHOTS) + run() + output, changes = run() + assert changes == [] + assert "0 created, 0 updated, 3 unchanged, 0 deleted, 0 links added" in output.out + + +def test_changed_image_is_replaced(weblate, run, tmp_path): + write_capture(tmp_path / "shots", SHOTS) + run() + png(tmp_path / "shots" / "wxgui__a.png", (1, 2, 3)) + # Same pixels as before but a differently encoded file is not a change. + Image.new("RGB", (8, 6), (0, 255, 0)).save( + tmp_path / "shots" / "wxgui__b.png", compress_level=0 + ) + output, changes = run() + assert [m for m, _ in changes] == ["POST"] + assert changes[0][1].endswith("/file/") + assert "1 updated, 2 unchanged" in output.out + stored = Image.open(io.BytesIO(weblate.screenshots[10]["image"])) + assert stored.getpixel((0, 0)) == (1, 2, 3) + + +def test_size_change_is_replaced(weblate, run, tmp_path): + write_capture(tmp_path / "shots", SHOTS) + run() + png(tmp_path / "shots" / "wxgui__a.png", (255, 0, 0), size=(9, 6)) + output, changes = run() + assert len(changes) == 1 + assert "1 updated" in output.out + + +def test_removed_string_is_unlinked(weblate, run, tmp_path): + write_capture(tmp_path / "shots", SHOTS) + run() + shots = dict(SHOTS) + shots["wxgui/a"] = ((255, 0, 0), [(None, "Layout settings"), (None, "Apply")]) + write_capture(tmp_path / "shots", shots) + output, changes = run() + assert linked(weblate)["wxgui/a"] == {1, 4} + assert sorted(changes) == [ + ("DELETE", "/api/screenshots/10/units/2/"), + ("POST", "/api/screenshots/10/units/"), + ] + assert "1 links added, 1 links removed" in output.out + + +def test_prune_deletes_only_managed_stale(weblate, run, tmp_path): + write_capture(tmp_path / "shots", SHOTS) + run() + weblate.screenshots[50] = { + "name": "manual/other", + "language": "en", + "image": b"", + "units": {1}, + } + weblate.screenshots[51] = { + "name": "wxgui/gone", + "language": "en", + "image": b"", + "units": set(), + } + shots = {name: shot for name, shot in SHOTS.items() if name != "wxgui/c"} + write_capture(tmp_path / "shots", shots) + + _output, changes = run() + assert changes == [] + assert len(weblate.screenshots) == 5 + + output, changes = run("--prune") + assert sorted(changes) == [ + ("DELETE", "/api/screenshots/12/"), + ("DELETE", "/api/screenshots/51/"), + ] + assert set(linked(weblate)) == {"wxgui/a", "wxgui/b", "manual/other"} + assert linked(weblate)["manual/other"] == {1} + assert "2 deleted" in output.out + + +def test_dry_run_changes_nothing(weblate, run, tmp_path): + write_capture(tmp_path / "shots", SHOTS) + output, changes = run("--dry-run") + assert changes == [] + assert weblate.screenshots == {} + assert "Would create screenshot wxgui/a" in output.out + assert "3 created" in output.out + assert "5 links added" in output.out + + run() + shots = {name: shot for name, shot in SHOTS.items() if name != "wxgui/c"} + shots["wxgui/a"] = ((1, 2, 3), [(None, "Apply")]) + write_capture(tmp_path / "shots", shots) + before = {k: dict(v) for k, v in weblate.screenshots.items()} + output, changes = run("--dry-run", "--prune") + assert changes == [] + assert weblate.screenshots == before + assert "Would replace image" in output.out + assert "Would delete screenshot wxgui/c" in output.out + + +def test_unknown_string_is_warned(weblate, run, tmp_path): + shots = { + "wxgui/a": ( + (255, 0, 0), + [(None, "Layout settings"), (None, "Not in Weblate"), ("Ctx", "Cancel")], + ) + } + write_capture(tmp_path / "shots", shots) + output, _changes = run() + assert "'Not in Weblate'" in output.err + assert "'Ctx'" in output.err + assert "2 unknown strings" in output.out + assert linked(weblate) == {"wxgui/a": {1}} + + +def test_http_error_exits_without_token(weblate, run, tmp_path, monkeypatch): + write_capture(tmp_path / "shots", SHOTS) + monkeypatch.setenv("WEBLATE_TOKEN", "wrong-token") + with pytest.raises(SystemExit) as error: + run() + message = str(error.value) + assert "HTTP 403" in message + assert "Bad token" in message + assert "wrong-token" not in message + + +def test_missing_token_exits(run, tmp_path, monkeypatch): + write_capture(tmp_path / "shots", SHOTS) + monkeypatch.delenv("WEBLATE_TOKEN") + with pytest.raises(SystemExit, match="WEBLATE_TOKEN"): + run() + + +def test_name_outside_prefix_exits(weblate, run, tmp_path): + write_capture(tmp_path / "shots", {"other/a": ((0, 0, 0), [])}) + with pytest.raises(SystemExit, match="does not start with"): + run() + assert weblate.requests == [] diff --git a/utils/weblate_screenshots/weblate_upload.py b/utils/weblate_screenshots/weblate_upload.py new file mode 100644 index 00000000000..b81416b66e2 --- /dev/null +++ b/utils/weblate_screenshots/weblate_upload.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 + +"""Synchronize screenshots captured by wxgui_capture.py to Weblate. + +The PNG files in the capture output directory are uploaded to a Weblate +component as screenshots of its source language, so that they are shown with +the strings for all languages. The strings listed in manifest.json are linked +to the screenshots. Only the requests which change something are sent: a +screenshot is created when missing, its image is replaced when the pixels +differ, and its links to strings are added or removed to match the manifest. + +Only the screenshots with a name starting with the managed prefix are modified +or deleted, so screenshots uploaded by hand are left alone. Only REST API +endpoints available in both Weblate 5.16.2 and 2026.10.1 are used. Create an +API token in the Weblate user profile and pass it in the environment (not as +an argument, to keep it out of process lists and logs): + + export WEBLATE_TOKEN=wlu_... + python3 utils/weblate_screenshots/weblate_upload.py \ + --url https://weblate.osgeo.org --project grass-gis \ + --component --input shots --dry-run + +Strings which are not (yet) in Weblate, for example because it has not +updated the template, are reported as warnings and not linked. +""" + +import argparse +import io +import json +import os +import re +import sys +from collections import Counter +from pathlib import Path +from urllib.parse import quote + +import requests +from PIL import Image + +TIMEOUT = 60 + + +class Weblate: + """Minimal client of the Weblate REST API.""" + + def __init__(self, url, token, dry_run=False): + self.base = url.rstrip("/") + "/api/" + self.dry_run = dry_run + self.session = requests.Session() + self.session.headers["Authorization"] = f"Token {token}" + + def request(self, method, url, data=None, image=None): + """Send a request and return the response, exit on an HTTP error.""" + files = ( + {"image": (image.name, image.read_bytes(), "image/png")} if image else None + ) + url = url if url.startswith("http") else self.base + url + try: + response = self.session.request( + method, url, data=data, files=files, timeout=TIMEOUT + ) + except requests.RequestException as error: + sys.exit(f"{method} {url} failed: {error}") + if not response.ok: + sys.exit( + f"{method} {url} failed with HTTP {response.status_code}: " + f"{response.text[:300]}" + ) + return response + + def get_json(self, url): + return self.request("GET", url).json() + + def get_pages(self, url): + """Yield the results of a paginated list, following the next links.""" + while url: + page = self.get_json(url) + yield from page["results"] + url = page["next"] + + def change(self, description, method, url, **kwargs): + """Report a modifying request and send it unless in a dry run.""" + print(f"{'Would ' if self.dry_run else ''}{description}") + if self.dry_run: + return None + return self.request(method, url, **kwargs) + + +def object_id(url): + """Return the numeric ID at the end of an API object URL.""" + return int(re.search(r"/(\d+)/?$", url).group(1)) + + +def same_pixels(old, new): + """Tell if two images (as bytes) have the same size and pixels.""" + try: + old_image = Image.open(io.BytesIO(old)).convert("RGBA") + except OSError: + return False + new_image = Image.open(io.BytesIO(new)).convert("RGBA") + return ( + old_image.size == new_image.size and old_image.tobytes() == new_image.tobytes() + ) + + +def read_units(api, project, component): + """Return a map of (context, msgid) to the unit ID in the source language. + + Weblate gives an empty context for a message without msgctxt. + """ + base = f"{quote(project, safe='')}/{quote(component, safe='')}/" + language = api.get_json(f"components/{base}")["source_language"]["code"] + units = {} + for unit in api.get_pages(f"translations/{base}{quote(language, safe='')}/units/"): + # The source is a list of the plural forms, the first one is the msgid. + units[unit["context"], unit["source"][0]] = unit["id"] + return language, units + + +def sync_screenshot(api, entry, input_dir, existing, units, args, language, stats): + name = entry["name"] + path = input_dir / entry["file"] + wanted = set() + for string in entry["strings"]: + unit_id = units.get((string["msgctxt"] or "", string["msgid"])) + if unit_id is None: + context = f" (context {string['msgctxt']!r})" if string["msgctxt"] else "" + print( + f"Warning: {name}: string not in Weblate: {string['msgid']!r}{context}", + file=sys.stderr, + ) + stats["unknown strings"] += 1 + else: + wanted.add(unit_id) + + shot = existing.get(name) + if shot is None: + response = api.change( + f"create screenshot {name}", + "POST", + "screenshots/", + data={ + "name": name, + "project_slug": args.project, + "component_slug": args.component, + "language_code": language, + }, + image=path, + ) + stats["created"] += 1 + # The response of the creation has no id field, only URLs. + shot_id = object_id(response.json()["url"]) if response else None + current = set() + else: + shot_id = shot["id"] + old = api.request("GET", shot["file_url"]).content + if same_pixels(old, path.read_bytes()): + stats["unchanged"] += 1 + else: + api.change( + f"replace image of screenshot {name}", + "POST", + f"screenshots/{shot_id}/file/", + image=path, + ) + stats["updated"] += 1 + current = {object_id(url) for url in shot["units"]} + + for unit_id in sorted(wanted - current): + api.change( + f"link unit {unit_id} to screenshot {name}", + "POST", + f"screenshots/{shot_id}/units/", + data={"unit_id": unit_id}, + ) + stats["links added"] += 1 + for unit_id in sorted(current - wanted): + api.change( + f"unlink unit {unit_id} from screenshot {name}", + "DELETE", + f"screenshots/{shot_id}/units/{unit_id}/", + ) + stats["links removed"] += 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument("--url", required=True, help="Weblate base URL") + parser.add_argument("--project", required=True, help="project slug") + parser.add_argument("--component", required=True, help="component slug") + parser.add_argument("--input", required=True, help="wxgui_capture.py output") + parser.add_argument( + "--prefix", + default="wxgui/", + help="prefix of the names of managed screenshots (default: %(default)s)", + ) + parser.add_argument( + "--prune", + action="store_true", + help="delete managed screenshots which are not in the manifest", + ) + parser.add_argument( + "--dry-run", action="store_true", help="only print the planned changes" + ) + args = parser.parse_args() + + token = os.environ.get("WEBLATE_TOKEN") + if not token: + sys.exit("Set the API token in the WEBLATE_TOKEN environment variable.") + input_dir = Path(args.input) + manifest = json.loads((input_dir / "manifest.json").read_text(encoding="utf-8")) + for entry in manifest["screenshots"]: + if not entry["name"].startswith(args.prefix): + sys.exit( + f"Screenshot {entry['name']!r} does not start with {args.prefix!r}." + ) + + api = Weblate(args.url, token, args.dry_run) + project = quote(args.project, safe="") + component = quote(args.component, safe="") + language, units = read_units(api, args.project, args.component) + existing = {} + for shot in api.get_pages(f"components/{project}/{component}/screenshots/"): + if shot["name"].startswith(args.prefix): + existing.setdefault(shot["name"], shot) + + stats = Counter() + for entry in manifest["screenshots"]: + sync_screenshot(api, entry, input_dir, existing, units, args, language, stats) + if args.prune: + names = {entry["name"] for entry in manifest["screenshots"]} + for name, shot in existing.items(): + if name not in names: + api.change( + f"delete screenshot {name}", "DELETE", f"screenshots/{shot['id']}/" + ) + stats["deleted"] += 1 + + print( + ("Dry run: " if args.dry_run else "") + + ", ".join( + f"{stats[key]} {key}" + for key in ( + "created", + "updated", + "unchanged", + "deleted", + "links added", + "links removed", + "unknown strings", + ) + ) + ) + + +if __name__ == "__main__": + main() diff --git a/utils/weblate_screenshots/wxgui_capture.py b/utils/weblate_screenshots/wxgui_capture.py new file mode 100644 index 00000000000..25619d693ae --- /dev/null +++ b/utils/weblate_screenshots/wxgui_capture.py @@ -0,0 +1,716 @@ +#!/usr/bin/env python3 + +"""Capture wxGUI screenshots and the translatable strings they show. + +Each scenario opens a part of the GUI, saves a PNG of it, and records which +messages from a gettext template are visible in it. The result is meant to be +uploaded to Weblate as visual context for the strings. + +Strings are found without OCR: the translation function is wrapped to learn +which msgid produced each piece of text, and the widget tree of the captured +window is walked to collect the text actually drawn (no tooltips, nothing +hidden or scrolled out of view). + +Run in a GRASS session with a display, e.g., using Xvfb. The screen is read +with ImageMagick's import tool. A fixed project path (rather than a temporary +one) keeps the screenshots identical between runs: + + grass -e -c XY /tmp/grassdata/screenshots + xvfb-run -s "-screen 0 1600x1200x24" \ + grass /tmp/grassdata/screenshots/PERMANENT --exec \ + python3 utils/weblate_screenshots/wxgui_capture.py \ + --pot locale/templates/grasswxpy.pot --output shots + +Tooltips are not drawn, so each tool of icon-only toolbars gets a copy of +the screenshot with the tool framed, linked to its tooltip. The tool tree is +captured with each category expanded in turn, page by page. + +The output directory gets one PNG file per screenshot and manifest.json +listing the messages shown in each of them, with their source locations +(file:line) from the template as Weblate shows them. Screenshots of a +framed tool also give the screenshot they are based on and the frame. +""" + +import argparse +import ast +import builtins +import io +import json +import os +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path + +# ImageMagick's tool which reads the screen. +IMPORT = shutil.which("import") +PLACEHOLDER = re.compile(r"%(?:\([^)]*\))?[-#0 +]*\d*(?:\.\d+)?[sdifgxXr%]|\{[^{}]*\}") + + +def read_pot(path): + """Return a map of (msgctxt, msgid) pairs to their source locations. + + The msgctxt is None for messages without a context. The locations are the + file:line references of the template, as Weblate shows them. + """ + entries = {} + fields = {} + locations = [] + current = None + + def flush(): + if fields.get("msgid"): + entries[fields.get("msgctxt"), fields["msgid"]] = list(locations) + fields.clear() + locations.clear() + + for line in Path(path).read_text(encoding="utf-8").splitlines(): + line = line.strip() + keyword, _sep, value = line.partition(" ") + if not line: + flush() + current = None + elif keyword == "#:": + locations.extend(value.split()) + elif keyword in {"msgctxt", "msgid"}: + if "msgid" in fields: + flush() + current = keyword + fields[current] = ast.literal_eval(value) + elif line.startswith('"') and current: + fields[current] += ast.literal_eval(line) + else: + # Other comments, plural forms, and msgstr end the msgctxt and msgid. + current = None + flush() + return entries + + +def normalize(text): + """Reduce a label to the form used for matching. + + Mnemonic markers, accelerators after a tab, trailing colons and + ellipses are often added or removed between the msgid and the widget. + """ + text = text.split("\t", 1)[0] + text = re.sub(r"&(.)", r"\1", text) + text = " ".join(text.split()) + return text.rstrip(":").rstrip(".").rstrip("\u2026").strip() + + +class Recorder: + """Wrap translation functions to map rendered text back to messages. + + Messages are (msgctxt, msgid) pairs, the keys of the map returned by + read_pot(). + """ + + def __init__(self, known): + self.known = known + self.by_exact_text = {} + self.by_text = {} + self.patterns = {} + + def wrap(self, namespace, name): + """Record messages translated by namespace[name](message).""" + original = namespace[name] + + def translate(message): + result = original(message) + self.record((None, message), result) + return result + + namespace[name] = translate + + def wrap_with_context(self, namespace, name): + """Record messages translated by namespace[name](context, message).""" + original = namespace[name] + + def translate(context, message): + result = original(context, message) + self.record((context, message), result) + return result + + namespace[name] = translate + + def record(self, message, result): + if message not in self.known: + return + self.by_exact_text.setdefault(result, set()).add(message) + self.by_text.setdefault(normalize(result), set()).add(message) + if PLACEHOLDER.search(result) and message not in self.patterns: + parts = PLACEHOLDER.split(normalize(result)) + if not re.search(r"\w", "".join(parts)): + # A message made only of placeholders, such as "{name}", + # would match any text. + return + regex = ".+?".join(re.escape(part) for part in parts) + self.patterns[message] = re.compile(f"^{regex}$", re.DOTALL) + + def lookup(self, text): + """Return messages which could have produced the text. + + An exact match wins over a normalized one, so that, e.g., a "&File" + menu does not also match a "File" label elsewhere. Text made of parts + separated by several spaces, such as "Color tables [r.colors]" in + the tool tree, is otherwise looked up part by part. + """ + if text in self.by_exact_text: + return set(self.by_exact_text[text]) + key = normalize(text) + if not key: + return set() + found = set(self.by_text.get(key, ())) + if not found: + found = { + message for message, regex in self.patterns.items() if regex.match(key) + } + if not found: + parts = re.split(r"\s{2,}", text.strip()) + if len(parts) > 1: + found = set().union(*(self.lookup(part) for part in parts)) + return found + + +def visible_texts(window, clip=None): + """Yield (kind, text) for text drawn by the window and its children. + + Only text drawn on screen is collected: tooltips, labels of icon-only + toolbar tools, and widgets hidden or scrolled out of view by their + parents are left out. The clip is the visible screen area of the parent. + """ + import wx + import wx.aui + from core.globalvar import FN + from gui_core.wrap import CustomTreeCtrl + + if not window.IsShownOnScreen(): + return + rect = window.GetScreenRect() + if clip is not None: + rect = rect.Intersect(clip) + if rect.IsEmpty(): + return + if isinstance(window, wx.Frame): + # The title is drawn by the window manager, not in the screenshot. + if window.GetMenuBar(): + menubar = window.GetMenuBar() + for i in range(menubar.GetMenuCount()): + yield "menu", menubar.GetMenuLabel(i) + if window.GetStatusBar(): + statusbar = window.GetStatusBar() + for i in range(statusbar.GetFieldsCount()): + yield "statusbar", statusbar.GetStatusText(i) + elif isinstance(window, FN.FlatNotebook): + if hasattr(window, "GetAGWWindowStyleFlag"): + style = window.GetAGWWindowStyleFlag() + else: + style = window.GetWindowStyleFlag() + single = style & FN.FNB_HIDE_ON_SINGLE_TAB and window.GetPageCount() == 1 + if not (style & FN.FNB_HIDE_TABS or single): + for i in range(window.GetPageCount()): + yield "tab", window.GetPageText(i) + elif isinstance(window, (wx.Notebook, wx.aui.AuiNotebook)): + for i in range(window.GetPageCount()): + yield "tab", window.GetPageText(i) + elif isinstance(window, wx.ToolBar): + if window.HasFlag(wx.TB_TEXT): + for i in range(window.GetToolsCount()): + yield "label", window.GetToolByPos(i).GetLabel() + elif isinstance(window, wx.RadioBox): + yield "label", window.GetLabel() + for i in range(window.GetCount()): + yield "label", window.GetString(i) + elif isinstance(window, (wx.Choice, wx.ComboBox, wx.ListBox)): + # Only the selected item of a drop-down list is visible. + if isinstance(window, wx.ListBox): + yield from (("label", item) for item in window.GetItems()) + else: + yield "label", window.GetStringSelection() + elif isinstance(window, wx.TextCtrl): + # The hint is drawn only while the field is empty. + yield "label", window.GetValue() or window.GetHint() + elif isinstance(window, (wx.TreeCtrl, CustomTreeCtrl)): + yield from (("label", text) for text in tree_texts(window)) + elif isinstance(window, wx.ListCtrl) and window.InReportView(): + for i in range(window.GetColumnCount()): + yield "label", window.GetColumn(i).GetText() + elif isinstance(window, wx.Control): + yield "label", window.GetLabel() + + if isinstance(window, wx.TopLevelWindow): + # The screenshot covers the whole window, including bars outside + # its client area. + client = rect + else: + client = wx.Rect(window.ClientToScreen(wx.Point(0, 0)), window.GetClientSize()) + client = client.Intersect(rect) + for child in window.GetChildren(): + if not isinstance(child, wx.TopLevelWindow): + yield from visible_texts(child, client) + + +def item_in_view(tree, item): + """Tell if a tree item is entirely in view, not cut at the top or bottom.""" + if not tree.IsVisible(item): + return False + rect = tree.GetBoundingRect(item) + return rect is not None and 0 <= rect.GetTop() <= rect.GetBottom() < ( + tree.GetClientSize().height + ) + + +def tree_texts(tree, item=None): + """Yield the text of the tree items entirely in view.""" + + def valid(item): + # CustomTreeCtrl gives None where wx.TreeCtrl gives an invalid item. + return item is not None and item.IsOk() + + if item is None: + item = tree.GetRootItem() + if not valid(item): + return + if item_in_view(tree, item): + yield tree.GetItemText(item) + # A hidden root item is always expanded. + if item == tree.GetRootItem() or tree.IsExpanded(item): + child, cookie = tree.GetFirstChild(item) + while valid(child): + yield from tree_texts(tree, child) + child, cookie = tree.GetNextChild(item, cookie) + + +def settle(seconds=0.5): + """Process pending events so that the GUI is fully drawn.""" + import wx + + end = time.monotonic() + seconds + while time.monotonic() < end: + wx.GetApp().Yield(onlyIfNeeded=True) + time.sleep(0.02) + + +def read_screen(rect): + """Return the PNG data of a screen rectangle. + + The screen is read with ImageMagick because wx.ScreenDC returns stale + content with GTK 3. + """ + return subprocess.run( + [ + IMPORT, + "-window", + "root", + "-crop", + f"{rect.width}x{rect.height}+{rect.x}+{rect.y}", + "+repage", + "png:-", + ], + capture_output=True, + check=True, + ).stdout + + +def grab(window, path, target=None): + """Save the on-screen area of a window, or of a part of it, as a PNG file. + + The window is the top-level window to raise, and the target the widget + to capture, by default the whole window. + """ + window.Raise() + settle() + Path(path).write_bytes(read_screen((target or window).GetScreenRect())) + + +def icon_toolbars(window): + """Yield the shown toolbars in the window which draw no tool labels.""" + import wx + from wx.lib.agw import aui + + for child in window.GetChildren(): + if not child.IsShownOnScreen() or isinstance(child, wx.TopLevelWindow): + continue + if isinstance(child, wx.ToolBar): + if not child.HasFlag(wx.TB_TEXT): + yield child + elif isinstance(child, aui.AuiToolBar): + if not child.GetAGWWindowStyleFlag() & aui.AUI_TB_TEXT: + yield child + else: + yield from icon_toolbars(child) + + +def tool_rects(toolbar): + """Yield (tool, screen rectangle) for the tools of a toolbar. + + wx.ToolBar does not give the position of its tools with GTK, so each + tool is found by toggling its enabled state, which redraws its icon, + and comparing the toolbar before and after. + """ + import wx + from PIL import Image, ImageChops + from wx.lib.agw import aui + + if isinstance(toolbar, aui.AuiToolBar): + for index in range(toolbar.GetToolCount()): + tool = toolbar.FindToolByIndex(index) + if tool.GetKind() not in {wx.ITEM_NORMAL, wx.ITEM_CHECK, wx.ITEM_RADIO}: + continue + rect = toolbar.GetToolRect(tool.GetId()) + # Tools in the overflow menu have no rectangle. + if not rect.IsEmpty(): + position = toolbar.ClientToScreen(rect.GetPosition()) + yield tool, wx.Rect(position, rect.GetSize()) + return + + rect = toolbar.GetScreenRect() + + def image(): + return Image.open(io.BytesIO(read_screen(rect))).convert("RGB") + + before = image() + for index in range(toolbar.GetToolsCount()): + tool = toolbar.GetToolByPos(index) + if tool.IsSeparator() or tool.IsControl(): + continue + enabled = toolbar.GetToolEnabled(tool.GetId()) + toolbar.EnableTool(tool.GetId(), not enabled) + settle(0.2) + changed = ImageChops.difference(before, image()).getbbox() + toolbar.EnableTool(tool.GetId(), enabled) + settle(0.2) + if changed: + # Frame the whole height of the toolbar, with a small margin. + left = max(changed[0] - 3, 0) + right = min(changed[2] + 3, rect.width) + yield tool, wx.Rect(rect.x + left, rect.y, right - left, rect.height) + + +def slug(text): + return re.sub(r"[^a-z0-9]+", "-", normalize(text).lower()).strip("-") + + +class Capture: + def __init__(self, recorder, output): + self.recorder = recorder + self.output = Path(output) + self.output.mkdir(parents=True, exist_ok=True) + self.records = [] + # Sets of messages of the tools already highlighted, so that tools + # shown in several screenshots are highlighted only once. + self.highlighted = set() + + def shoot(self, name, window, target=None, texts=None): + """Save a screenshot of the window and the messages visible in it. + + The target is the widget to capture, by default the whole window. + The texts, (kind, text) pairs, restrict the strings to link to those + shown, by default all text drawn in the target. + Each tool of icon-only toolbars gets its own copy of the screenshot, + with the tool highlighted and linked to its tooltip. + """ + import wx + + target = target or window + # Other top-level windows could cover the captured one. + hidden = [ + other + for other in wx.GetTopLevelWindows() + if other is not window and other.IsShown() + ] + for other in hidden: + other.Hide() + path = self.output / f"{name.replace('/', '__')}.png" + grab(window, path, target) + for other in hidden: + other.Show() + + strings = {} + unmatched = set() + if texts is None: + texts = visible_texts(target) + for kind, text in texts: + if not text.strip(): + continue + messages = self.recorder.lookup(text) + if not messages: + unmatched.add(text) + for message in messages: + strings.setdefault(message, kind) + self.add_record(name, path, strings, sorted(unmatched)) + self.highlight_tools(name, target, path) + + def add_record(self, name, path, strings, unmatched, **extra): + """Add a screenshot and its strings, a map of messages to kinds.""" + self.records.append( + { + "name": name, + "file": path.name, + **extra, + "strings": [ + { + "msgctxt": context, + "msgid": msgid, + "kind": kind, + "locations": self.recorder.known[context, msgid], + } + for (context, msgid), kind in sorted( + strings.items(), key=lambda item: (item[0][0] or "", item[0][1]) + ) + ], + "unmatched": unmatched, + } + ) + print(f"{name}: {len(strings)} strings", file=sys.stderr) + + def highlight_tools(self, name, target, path): + """Save a copy of the screenshot per tool of icon-only toolbars. + + Each copy has the tool framed and is linked to its tooltip and + description, which are not drawn in the screenshot. + """ + from PIL import Image, ImageDraw + + origin = target.GetScreenPosition() + for toolbar in icon_toolbars(target): + for tool, rect in tool_rects(toolbar): + strings = {} + for text in (tool.GetShortHelp(), tool.GetLongHelp()): + for message in self.recorder.lookup(text) if text.strip() else (): + strings.setdefault(message, "tool") + if not strings or frozenset(strings) in self.highlighted: + continue + self.highlighted.add(frozenset(strings)) + box = (rect.x - origin.x, rect.y - origin.y, rect.width, rect.height) + image = Image.open(path).convert("RGB") + ImageDraw.Draw(image).rectangle( + (box[0], box[1], box[0] + box[2] - 1, box[1] + box[3] - 1), + outline=(255, 0, 0), + width=3, + ) + base = f"{name}/tool/{slug(min(msgid for _ctxt, msgid in strings))}" + tool_name = base + # Different tools can share a message, e.g., the same tooltip. + number = 1 + while any(record["name"] == tool_name for record in self.records): + number += 1 + tool_name = f"{base}-{number}" + tool_path = self.output / f"{tool_name.replace('/', '__')}.png" + image.save(tool_path) + self.add_record( + tool_name, + tool_path, + strings, + [], + base=name, + highlight=list(box), + ) + + def notebook_pages(self, prefix, window, notebook, names=None): + """Capture each page of a notebook. + + Screenshot names use the internal page name when available (stable + across languages), otherwise the page label. + """ + for index in range(notebook.GetPageCount()): + notebook.SetSelection(index) + settle() + page = names[index] if names else slug(notebook.GetPageText(index)) + self.shoot(f"{prefix}/{page}", window) + + def write_manifest(self): + manifest = self.output / "manifest.json" + manifest.write_text( + json.dumps({"screenshots": self.records}, indent=2, ensure_ascii=False) + + "\n", + encoding="utf-8", + ) + + +def fit_pages(window, notebook): + """Enlarge the window so that no notebook page needs to scroll. + + Otherwise, strings scrolled out of view would be linked to a screenshot + which does not show them. + """ + extra = max( + notebook.GetPage(index).GetVirtualSize().height + - notebook.GetPage(index).GetClientSize().height + for index in range(notebook.GetPageCount()) + ) + if extra > 0: + width, height = window.GetSize() + window.SetSize(width, height + extra) + settle() + + +def create_layer_manager(): + from lmgr.frame import GMFrame + + frame = GMFrame(parent=None) + # The frame redirects output to its console, which would hide errors. + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + settle(2) + return frame + + +def capture_layer_manager(capture, frame): + notebook = frame.notebook + names = {notebook.GetPageIndexByName(name): name for name in notebook.notebookPages} + capture.notebook_pages("wxgui/layer-manager", frame, notebook, names) + + +def capture_preferences(capture, frame): + from gui_core.preferences import PreferencesDialog + + dialog = PreferencesDialog(parent=frame, giface=frame._giface) + dialog.Show() + settle() + fit_pages(dialog, dialog.notebook) + capture.notebook_pages("wxgui/preferences", dialog, dialog.notebook) + dialog.Destroy() + + +def tree_children(tree, item): + """Yield the children of a CustomTreeCtrl item.""" + child, cookie = tree.GetFirstChild(item) + while child is not None: + yield child + child, cookie = tree.GetNextChild(item, cookie) + + +def expand_all(tree, item): + """Expand a CustomTreeCtrl item and its descendants. + + Each item is expanded before its children are listed, because virtual + trees, such as the tool tree, create the children when expanding. + """ + tree.Expand(item) + for child in tree_children(tree, item): + expand_all(tree, child) + + +def tree_descendants(tree, item): + """Yield a CustomTreeCtrl item and its descendants, in display order.""" + yield item + for child in tree_children(tree, item): + yield from tree_descendants(tree, child) + + +def capture_tool_tree(capture, frame): + """Capture each category of the tool tree, expanded, page by page. + + The window is made taller so that fewer pages are needed. Only the tree + is captured, as the rest of the window is in the Layer Manager scenario. + """ + from gui_core.wrap import CustomTreeCtrl + + frame.notebook.SetSelectionByName("search") + settle() + tree = next( + widget + for widget in descendants(frame.notebook.GetCurrentPage()) + if isinstance(widget, CustomTreeCtrl) + ) + width, height = frame.GetSize() + frame.SetSize(width, 1100) + settle() + unit = tree.GetScrollPixelsPerUnit()[1] + for category in list(tree_children(tree, tree.GetRootItem())): + expand_all(tree, category) + settle() + items = list(tree_descendants(tree, category)) + prefix = f"wxgui/tool-tree/{slug(tree.GetItemText(category))}" + first = 0 + page = 1 + while first < len(items): + # Scroll the first item not captured yet to the top. + tree.Scroll(-1, items[first].GetY() // unit) + settle() + shown = [i for i, item in enumerate(items) if item_in_view(tree, item)] + # Other categories are linked to the screenshot of the tree + # with all of them collapsed, not to every page. + texts = [("label", tree.GetItemText(items[i])) for i in shown] + capture.shoot(f"{prefix}/{page}", frame, target=tree, texts=texts) + if not shown or shown[-1] < first: + break + first = shown[-1] + 1 + page += 1 + tree.Collapse(category) + tree.Scroll(-1, 0) + frame.SetSize(width, height) + settle() + + +def descendants(window): + """Yield the child widgets of a window, recursively.""" + for child in window.GetChildren(): + yield child + yield from descendants(child) + + +# All scenarios get the Layer Manager because other windows need it as parent. +SCENARIOS = { + "layer-manager": capture_layer_manager, + "preferences": capture_preferences, + "tool-tree": capture_tool_tree, +} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument("--pot", required=True, help="gettext template to match") + parser.add_argument("--output", required=True, help="output directory") + parser.add_argument( + "--scenario", + action="append", + choices=sorted(SCENARIOS), + help="scenario to capture (default: all)", + ) + args = parser.parse_args() + + if not IMPORT: + sys.exit("The import tool of ImageMagick is required to read the screen.") + + sys.path.insert(0, os.path.join(os.environ["GISBASE"], "gui", "wxpython")) + import grass # noqa: F401 Installs the translation function. + + known = read_pot(args.pot) + recorder = Recorder(known) + recorder.wrap(builtins.__dict__, "_") + # Context-aware translation, if the GUI defines it, follows gettext.pgettext(). + if "pgettext" in builtins.__dict__: + recorder.wrap_with_context(builtins.__dict__, "pgettext") + + import wx + from core.settings import UserSettings + + # The Layer Manager is the main window only in the multi-window mode. + UserSettings.Set( + group="appearance", key="singleWindow", subkey="enabled", value=False + ) + + app = wx.App(False) + app.locale = wx.Locale(language=wx.LANGUAGE_DEFAULT) + capture = Capture(recorder, args.output) + scenarios = args.scenario or sorted(SCENARIOS) + + # Events are processed by settle(), so the main loop is not started. + # It would return at once anyway because there is no window yet. + try: + frame = create_layer_manager() + for name in scenarios: + SCENARIOS[name](capture, frame) + capture.write_manifest() + finally: + for window in wx.GetTopLevelWindows(): + window.Destroy() + settle() + + +if __name__ == "__main__": + main()