diff --git a/CLAUDE.md b/CLAUDE.md index 23cf40b4..7ffa08d9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,7 +72,7 @@ Guard tests for restructuring: - an online install/uninstall with harbourmaster as the plain platform (CFW hooks such as muOS's catalogue write outside the ports folder) - button and portmsg checks -It shows the results with pugwash's fifo dialogs (`pmtest/fifo.py`) and writes `pmtest/reports/--.zip`, scrubbed of MAC/private IP addresses, serials, Wi-Fi names and the hostname (`suite.scrub`). The tester's own `pugwash.txt`/`harbourmaster.txt`/`log.txt` are set aside in `pmtest/saved-logs/` for the run and put back afterwards (or by the next run after a crash), so a report only holds the run's logs. pugwash and the GUI tests run in the environment the installed launcher gives the GUI (`pm_launch.txt` with `PM_LAUNCH_ENV_ONLY=Y`). The build also writes `dist/pmtest.zip.md5`, for `harbourmaster install /pmtest.zip.md5`. `python3 -I tools/test_suite/matrix.py --out DIR` turns collected reports into `matrix.json` and a self-contained `matrix.html` coverage page, starting with a device × CFW release heat map. `tools/test_suite/devices.py` names the hardware behind each CFW's device string (`DEVICES`, `FAMILIES`) and reads a vendored copy of PortMaster-Info's `device_info.json` (`--refresh` updates it) for supported-but-untested cells; a report it can't place is printed and listed under Other; it and `receive.py` must keep running on Python 3.7 with only the standard library. A build made with `PM_PMTEST_UPLOAD=http://:8770/` carries `pmtest/upload.txt` and, if the tester agrees on the start screen, POSTs each finished report there, where `python3 -I tools/test_suite/receive.py ` saves it on the LAN. The public endpoint is the `pmtest-reports` Cloudflare Worker (its own repo, with `collect.py` to download reports); HTTPS uses PortMaster's certifi bundle. Builds without the variable send nothing. Tested in `tests/test_test_suite.py` and `tests/test_build.py`. +It shows the results with pugwash's fifo dialogs (`pmtest/fifo.py`) and writes `pmtest/reports/--.zip`, scrubbed of MAC/private IP addresses, serials, Wi-Fi names and the hostname (`suite.scrub`). The tester's own `pugwash.txt`/`harbourmaster.txt`/`log.txt` are set aside in `pmtest/saved-logs/` for the run and put back afterwards (or by the next run after a crash), so a report only holds the run's logs. pugwash and the GUI tests run in the environment the installed launcher gives the GUI (`pm_launch.txt` with `PM_LAUNCH_ENV_ONLY=Y`). The build lists `tree/` and `site/` in `pmtest/files.txt`, and the suite removes anything else there when it starts (`suite.prune_stale`): installing over an older suite keeps its files. The build also writes `dist/pmtest.zip.md5`, for `harbourmaster install /pmtest.zip.md5`. `python3 -I tools/test_suite/matrix.py --out DIR` turns collected reports into `matrix.json` and a self-contained `matrix.html` coverage page, starting with a device × CFW release heat map. `tools/test_suite/devices.py` names the hardware behind each CFW's device string (`DEVICES`, `FAMILIES`) and reads a vendored copy of PortMaster-Info's `device_info.json` (`--refresh` updates it) for supported-but-untested cells; a report it can't place is printed and listed under Other; it and `receive.py` must keep running on Python 3.7 with only the standard library. A build made with `PM_PMTEST_UPLOAD=http://:8770/` carries `pmtest/upload.txt` and, if the tester agrees on the start screen, POSTs each finished report there, where `python3 -I tools/test_suite/receive.py ` saves it on the LAN. The public endpoint is the `pmtest-reports` Cloudflare Worker (its own repo, with `collect.py` to download reports); HTTPS uses PortMaster's certifi bundle. Builds without the variable send nothing. Tested in `tests/test_test_suite.py` and `tests/test_build.py`. **Theme preview server** (`PortMaster/pylibs/theme_server/`, run from a checkout with `tools/theme_server/serve.py`; documented in `THEME.md`): - `serve.py` serves a page showing every screen rendered with a theme, at any resolution, color scheme and language, and exports PNGs with `--export`. diff --git a/do_release.sh b/do_release.sh index 1813110a..a6cb8366 100755 --- a/do_release.sh +++ b/do_release.sh @@ -239,6 +239,8 @@ if [ "$CHANNEL" = "testsuite" ]; then --exclude='PortMaster/portmsg.*' --exclude='PortMaster/*.so' --exclude='PortMaster/pylibs.zip' \ --exclude='NotoSans.tar.xz' --exclude='__pycache__' \ PortMaster tests tools pytest.ini | tar -xf - -C "$PMTEST/tree" + ## What the suite keeps when it starts: installing over an older suite leaves its files. + (cd "$PMTEST" && find tree site -type f ! -path '*/__pycache__/*' | LC_ALL=C sort > files.txt) (cd pmtest_build && zip -9qr ../pmtest.zip "PortMaster Test Suite.sh" pmtest) cp pmtest.zip "$DIST_DIR/" diff --git a/tests/conftest.py b/tests/conftest.py index 3bb690e7..390458ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -68,8 +68,9 @@ del TEST_ENV["SDL_VIDEODRIVER"] os.environ.pop("SDL_VIDEODRIVER", None) -for _dir in ("tools", "ports"): - (SESSION_DIR / _dir).mkdir() +## tools/PortMaster as installed: outside a git checkout (a device) some scenes write .pugwash-reboot there. +for _dir in ("tools/PortMaster", "ports"): + (SESSION_DIR / _dir).mkdir(parents=True) ## Run on a device (eg: by the PortMaster Test Suite port), the environment already ## describes that device and PortMaster (control.txt, device_info.txt, funcs.txt): diff --git a/tests/test_build.py b/tests/test_build.py index dec63317..04d74f46 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -211,6 +211,12 @@ def test_testsuite_port(tmp_path): ## Reports are only sent from builds made with PM_PMTEST_UPLOAD. assert "pmtest/upload.txt" not in names + ## The files the suite keeps when it starts (installing over an older suite leaves its files). + with zipfile.ZipFile(dist / "pmtest.zip") as zf: + listed = zf.read("pmtest/files.txt").decode().splitlines() + assert sorted(listed) == sorted(name[len("pmtest/"):] for name in names + if name.startswith(("pmtest/tree/", "pmtest/site/")) and not name.endswith("/")) + ## The build it installs is the one it was made with, and harbourmaster takes it as a port. with zipfile.ZipFile(dist / "pmtest.zip") as zf: inner = zf.read("pmtest/PortMaster.zip") diff --git a/tests/test_fuzz_port_json.py b/tests/test_fuzz_port_json.py index 0c457e59..980cdb8e 100644 --- a/tests/test_fuzz_port_json.py +++ b/tests/test_fuzz_port_json.py @@ -26,10 +26,26 @@ from harbourmaster.config import HM_SORT_ORDER from harbourmaster.util import net as hm_net +## Imported here, not in a fuzz case: on import it opens its log in HM_TOOLS_DIR/PortMaster +## (made, like an install has it, since the device test fixes), which the sandbox would count. +from harbourmaster import cli from conftest import DATA_DIR, basic_port_files, write_port_zip +def _json_keeps_non_finite(): + ## Some CFWs' Python can't recognise NaN (AmberELEC's: math.isnan(nan) is False), so json.dumps + ## writes it as a bare `nan` that json.loads refuses. The fuzzer's own round trips would fail + ## there, so it only makes NaN/infinity where they survive (PortMaster itself reads them fine). + try: + return json.loads(json.dumps([float('nan'), float('inf')]))[1] == float('inf') + except ValueError: + return False + + +NON_FINITE = [float('nan'), float('inf'), -float('inf')] if _json_keeps_non_finite() else [] + + ## Cases per test (the GUI test does a tenth, at least 20). N = int(os.environ.get('PM_FUZZ_N', '40')) SEED = int(os.environ.get('PM_FUZZ_SEED', '1')) @@ -58,7 +74,7 @@ def weird_strings(outside): def weird_values(rng, outside, depth=0): - simple = [None, True, False, 0, 1, -1, 2 ** 64, 1.5, float('nan'), float('inf'), -float('inf')] + simple = [None, True, False, 0, 1, -1, 2 ** 64, 1.5] + NON_FINITE pool = simple + weird_strings(outside) if depth > 2: return rng.choice(pool) @@ -264,7 +280,6 @@ def record(stage, err, case): def run_cli(hm, command, argv): import contextlib import io - from harbourmaster import cli with contextlib.redirect_stdout(io.StringIO()): getattr(cli, command)(hm, argv) diff --git a/tests/test_test_suite.py b/tests/test_test_suite.py index 10ecc69d..54178b90 100644 --- a/tests/test_test_suite.py +++ b/tests/test_test_suite.py @@ -1104,3 +1104,31 @@ def run(self, command, timeout=600, env=None, cwd=None, output=None): assert section.status == suite.PASS, section.details assert len(commands) == 4 assert not any("--no-check" in command for command in commands) + + +################################################################################ +## Installing over an older suite +def test_prune_stale_removes_what_an_older_suite_left(tmp_path): + ## harbourmaster only adds and replaces files: a test the new build dropped still ran. + for name in ("tree/tests/test_new.py", "tree/tests/test_old_snapshot.py", "tree/tests/data/old/msgids.txt", + "tree/tests/__pycache__/test_old.cpython-311.pyc", "site/pytest/__init__.py", "site/oldpkg/x.py", + "reports/last-run.txt", "suite.py"): + (tmp_path / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / name).write_text("x") + (tmp_path / "files.txt").write_text("tree/tests/test_new.py\nsite/pytest/__init__.py\n") + + removed = suite.prune_stale(tmp_path) + + assert sorted(removed) == ["site/oldpkg/x.py", "tree/tests/data/old/msgids.txt", "tree/tests/test_old_snapshot.py"] + assert (tmp_path / "tree/tests/test_new.py").is_file() and (tmp_path / "site/pytest/__init__.py").is_file() + assert not (tmp_path / "tree/tests/data").exists() and not (tmp_path / "site/oldpkg").exists() + ## Outside tree/ and site/, and compiled files, are left alone. + assert (tmp_path / "reports/last-run.txt").is_file() and (tmp_path / "suite.py").is_file() + assert (tmp_path / "tree/tests/__pycache__/test_old.cpython-311.pyc").is_file() + + +def test_prune_stale_without_a_list_keeps_everything(tmp_path): + (tmp_path / "tree").mkdir() + (tmp_path / "tree" / "test_x.py").write_text("x") + assert suite.prune_stale(tmp_path) == [] + assert (tmp_path / "tree" / "test_x.py").is_file() diff --git a/tests/test_theme_editor.py b/tests/test_theme_editor.py index 8529bb14..8880d149 100644 --- a/tests/test_theme_editor.py +++ b/tests/test_theme_editor.py @@ -83,8 +83,22 @@ def test_theme_files(tmp_path): assert files['image'] == ["a.png", "images/b.JPG"] assert files['sound'] == ["click.ogg"] - ## The theme's fonts, then PortMaster's (the Noto fonts even before they're unpacked). - assert files['font'][0] == "font.ttf" and "DejaVuSans.ttf" in files['font'] and "NotoSansJP-Regular.ttf" in files['font'] + ## The theme's fonts, then PortMaster's (the Noto fonts even before they're unpacked; + ## the PortMaster Test Suite leaves them out, so only where PortMaster has them). + assert files['font'][0] == "font.ttf" and "DejaVuSans.ttf" in files['font'] + if (schema.RESOURCES / "NotoSans.tar.xz").is_file() or (schema.RESOURCES / "NotoSansJP-Regular.ttf").is_file(): + assert "NotoSansJP-Regular.ttf" in files['font'] + + +def test_theme_files_lists_noto_from_the_archive(tmp_path, monkeypatch): + resources = tmp_path / "resources" + resources.mkdir() + (resources / "DejaVuSans.ttf").write_bytes(b"x") + (resources / "NotoSans.tar.xz").write_bytes(b"x") + monkeypatch.setattr(schema, "RESOURCES", resources) + assert schema.theme_files(None)['font'] == [ + "DejaVuSans.ttf", "NotoSansJP-Regular.ttf", "NotoSansKR-Regular.ttf", "NotoSansSC-Regular.ttf", + "NotoSansTC-Regular.ttf", "NotoSansHK-Regular.ttf"] def test_theme_names(): diff --git a/tests/test_theme_server_http.py b/tests/test_theme_server_http.py index 9d549f58..4d77a473 100644 --- a/tests/test_theme_server_http.py +++ b/tests/test_theme_server_http.py @@ -641,14 +641,19 @@ def test_nothing_starts_after_stop(slow_pool): def test_stop_gives_up_waiting(slow_pool, monkeypatch): ## A start that never ends: stop() still returns (PortMaster kills the server at 15 s). - monkeypatch.setattr(SlowZygote, "delay", 3) + monkeypatch.setattr(SlowZygote, "delay", 1.5) warm = threading.Thread(target=slow_pool.warm, daemon=True) warm.start() time.sleep(0.1) began = time.monotonic() slow_pool.stop(start_timeout=0.3) - assert time.monotonic() - began < 2 + assert time.monotonic() - began < 1.2 + + ## Let the start finish here: left running, it woke after monkeypatch had put the real + ## Worker back and failed in a later test's thread (seen on a slow device). + warm.join(10) + assert not warm.is_alive() def test_renderer_folders_need_the_servers(tmp_path): diff --git a/tests/test_update_check.py b/tests/test_update_check.py index eadaf339..a54bd2bc 100644 --- a/tests/test_update_check.py +++ b/tests/test_update_check.py @@ -251,6 +251,7 @@ def test_json_load_dict_default_is_copied(tmp_path): assert default == {"a": 1} +@pytest.mark.sdl def test_gui_get_config_broken(gui, monkeypatch, tmp_path): tools = tmp_path / "guitools" (tools / "PortMaster" / "config").mkdir(parents=True) diff --git a/tests/test_util_edges.py b/tests/test_util_edges.py index aaf0db12..6e8e95f3 100644 --- a/tests/test_util_edges.py +++ b/tests/test_util_edges.py @@ -270,8 +270,9 @@ def test_make_temp_directory_cleans_up_after_errors(): assert not temp_dir.exists() -def test_get_path_fs_testing(): - ## Never shells out from a checkout. +def test_get_path_fs_testing(monkeypatch): + ## Never shells out from a checkout. (The suite also runs outside one, on devices.) + monkeypatch.setattr(util_files, "HM_TESTING", True) assert util.get_path_fs("/") is None diff --git a/tools/test_suite/pmtest/suite.py b/tools/test_suite/pmtest/suite.py index 94758f01..b7b9ca03 100644 --- a/tools/test_suite/pmtest/suite.py +++ b/tools/test_suite/pmtest/suite.py @@ -40,6 +40,8 @@ ## PortMaster's own logs: the tester's are set aside during a run, so a report holds only the run's. PM_LOGS = ("pugwash.txt", "harbourmaster.txt", "log.txt") SAVED_LOGS = HERE / "saved-logs" +## Every file the build put in tree/ and site/: installing over an older suite keeps its files. +FILES_LIST = HERE / "files.txt" ## Only in builds made with PM_PMTEST_UPLOAD: where finished reports are sent. UPLOAD_FILE = HERE / "upload.txt" @@ -1195,6 +1197,35 @@ def run_suite(self): return 1 if any(section.status == FAIL for section in self.sections) else 0 +def prune_stale(here=HERE, roots=("tree", "site")): + """ + Removes what an older suite left in tree/ and site/: installing a port only adds and + replaces files, so a test file the new build dropped still ran (and failed). Only + files not in files.txt go, and folders left empty; nothing without files.txt. + """ + here = Path(here) + listing = here / FILES_LIST.name + if not listing.is_file(): + return [] + keep = set(line.strip() for line in listing.read_text(encoding="utf-8").splitlines() if line.strip()) + removed = [] + for root in roots: + top = here / root + if not top.is_dir(): + continue + for dirpath, dirnames, filenames in os.walk(str(top), topdown=False): + if "__pycache__" in Path(dirpath).parts: + continue + for name in filenames: + path = Path(dirpath) / name + if path.relative_to(here).as_posix() not in keep: + path.unlink() + removed.append(path.relative_to(here).as_posix()) + if dirpath != str(top) and not os.listdir(dirpath): + os.rmdir(dirpath) + return removed + + def main(argv=None): parser = argparse.ArgumentParser(description="PortMaster Test Suite") parser.add_argument("--no-ui", action="store_true", help="don't use the screen (pugwash)") @@ -1203,6 +1234,10 @@ def main(argv=None): options = parser.parse_args(argv) options.skip = [step for step in options.skip.split(",") if step] + removed = prune_stale() + if removed: + print(f"removed {len(removed)} files an older Test Suite left behind") + return Suite(options).main()