diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24c8f01..d8fb67d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,9 @@ jobs: # Python 3.9 is unavailable on Ubuntu 26.04; preserve the supported floor. - { os: ubuntu-24.04, python: '3.9', label: py3.9 } - { os: ubuntu-latest, python: '3.12', label: py3.12 } + # The oldest Pillow the images extra accepts, so an image API newer + # than that floor fails here rather than for a theme author. + - { os: ubuntu-latest, python: '3.12', label: 'py3.12, Pillow 10.1.0', pillow: '10.1.0' } - { os: ubuntu-latest, python: '3.14', label: py3.14 } - { os: macos-latest, python: '3.12', label: 'macOS, py3.12' } - { os: windows-latest, python: '3.12', label: 'Windows, py3.12' } @@ -95,6 +98,13 @@ jobs: with: python-version: ${{ matrix.python }} - run: python -m pip install --disable-pip-version-check ".[images]" + - name: Install the minimum Pillow + if: matrix.pillow + env: + PILLOW: ${{ matrix.pillow }} + run: | + python -m pip install --disable-pip-version-check "pillow==$PILLOW" + python -c "import os, PIL; assert PIL.__version__ == os.environ['PILLOW'], PIL.__version__" - run: python -m unittest discover -s tests -v build-package: @@ -230,67 +240,17 @@ jobs: - name: Optional stylesheets apply in install and try run: python -m tests.smoke_variants - # Three, not two. The flakes this has caught were intermittent, so a - # single pair passed often enough to be misleading. + # Three captures, not two, and every pair compared: the flakes this has + # caught were intermittent, and a run that differs from both others is + # told apart from a check-vs-shot difference that repeats. - name: Project check and three captures - shell: bash - run: | - fxcss check --theme fxcss/templates/starter --firefox "$FIREFOX_BIN" \ - --out "$GITHUB_WORKSPACE/out/check" - python - <<'PY' - import json, pathlib, shutil - report, = pathlib.Path("out/check").glob("run-*/summary.json") - result = json.loads(report.read_text()) - assert result["exit_code"] == 0, result - browser, = result["browsers"] - shutil.copytree(report.parent / browser["key"] / "shots", "shots/a") - PY - fxcss shot --theme fxcss/templates/starter --out shots/b --variants all - fxcss shot --theme fxcss/templates/starter --out shots/c --variants all + run: python -m tests.smoke_capture_states capture - name: Determinism — the same theme must render identically - shell: bash - run: | - set -e - for pair in b c; do - fxcss compare --base shots/a --head "shots/$pair" \ - --out "out/determinism-$pair" --platform ${{ matrix.label }} - done - python - <<'PY' - import json, pathlib, sys - failed = False - for pair in ("b", "c"): - s = json.loads(pathlib.Path(f"out/determinism-{pair}/summary.json").read_text()) - changed = s["changed_views"] - if s["any_change"]: - failed = True - print(f"run a vs run {pair}: rendered differently") - print(f" missing: {s['only_in_base']}; added: {s['only_in_head']}") - for v in changed: - print(f" {v['view']}: {v['percent']}% of pixels") - else: - print(f"run a vs run {pair}: all {len(s['views'])} views identical") - sys.exit(1 if failed else 0) - PY + run: python -m tests.smoke_capture_states determinism --platform ${{ matrix.label }} - name: Detects a real change - shell: bash - run: | - cp -r fxcss/templates/starter "$RUNNER_TEMP/changed-theme" - printf '\n#nav-bar { background-color: #ff2d55 !important; }\n' \ - >> "$RUNNER_TEMP/changed-theme/chrome/userChrome.css" - fxcss shot --theme "$RUNNER_TEMP/changed-theme" --out shots/changed - fxcss compare --base shots/a --head shots/changed --out out/changed \ - --platform ${{ matrix.label }} - python - <<'PY' - import json, sys, pathlib - s = json.loads(pathlib.Path("out/changed/summary.json").read_text()) - if not s["changed_views"]: - print("An obvious CSS change produced no pixel difference — " - "the comparison is not sensitive.") - sys.exit(1) - print(f"{len(s['changed_views'])} of {len(s['views'])} views changed, as expected") - PY + run: python -m tests.smoke_capture_states real-change --platform ${{ matrix.label }} # End to end and Linux-only (the code is platform-independent Python; # once is proof, three times is minutes): install the starter theme into @@ -326,67 +286,10 @@ jobs: print(f"upgrade --compare saw the change in: {', '.join(changed)}") PY + # Each view targeted by a state-only rule must change; falling back to + # an ordinary window must not pass. - name: State and surface views render their own states - shell: bash - run: | - # Each rule targets a distinct browser state. Falling back to an - # ordinary window must not pass the state-specific capture checks. - cp -r fxcss/templates/starter "$RUNNER_TEMP/state-theme" - cat >> "$RUNNER_TEMP/state-theme/chrome/userChrome.css" <<'CSS' - /* The revamped sidebar hides #sidebar-header; its container stays visible. */ - #sidebar-box { border: 6px solid #ff2d55 !important; } - #tabbrowser-tabs[orient="vertical"] { background-color: #34c759 !important; } - #home-button { background-color: #ff9500 !important; } - .tabbrowser-tab[soundplaying]:not([muted]) .tab-content { background: #e6194b !important; } - .tabbrowser-tab[muted] .tab-content { background: #4363d8 !important; } - /* Surfaces no full-window view shows: hover, the search engine - button and its menu, the panels. */ - #reload-button:hover > .toolbarbutton-icon { background-color: #af52de !important; } - #urlbar .searchmode-switcher:hover { outline: 4px solid #ff2d55 !important; } - panel:has(> .searchmode-switcher-panel-list) { --panel-background: #34c759 !important; } - .searchmode-switcher-panel-list { background-color: #34c759 !important; } - #appMenu-mainView { background-color: #ff9500 !important; } - #unified-extensions-view { background-color: #4363d8 !important; } - #PanelUI-fxa { background-color: #5856d6 !important; } - CSS - cat > "$RUNNER_TEMP/state-theme/chrome/userContent.css" <<'CSS' - @-moz-document url-prefix("about:preferences") { - body { background-color: #e6194b !important; } - } - CSS - fxcss shot --theme "$RUNNER_TEMP/state-theme" --out shots/state - fxcss compare --base shots/a --head shots/state --out out/state \ - --platform ${{ matrix.label }} - python - <<'PY' - import json, pathlib, sys - summary = json.loads(pathlib.Path("out/state/summary.json").read_text()) - coverage = json.loads(pathlib.Path("shots/state/capture-coverage.json").read_text()) - changed = {v["view"] for v in summary["changed_views"]} - wanted = {"extra-10-sidebar-bookmarks", "extra-11-sidebar-history", - "extra-14-vertical-tabs", "extra-15-toolbar", - "extra-04-audio", "extra-05-muted"} - wanted |= {f"{mode}-{view}" for mode in ("light", "dark") - for view in ("05-toolbar-hover", "06-search-engine", "07-search-menu", - "08-app-menu", "09-extensions-panel", "10-account-panel", - "11-settings")} - # A Firefox without the search engine button (an older ESR) cannot - # show it; nothing else may be skipped. - skipped = {name for name, view in coverage["views"].items() - if view.get("reason") == "no search engine button on this Firefox"} - for name in sorted(skipped & wanted): - print(f"{name}: skipped, no search engine button on this Firefox") - missing = sorted(wanted - changed - skipped) - if missing: - print("These views did not react to a rule that only they can " - "render, so they are not capturing the state they claim: " - + ", ".join(missing)) - for name in missing: - view = coverage["views"].get(name, {}) - if view.get("status") != "captured": - print(f" {name}: {view.get('status')} ({view.get('reason')})") - sys.exit(1) - print(f"all {len(wanted - skipped)} state views reacted; {len(changed)} views changed") - PY + run: python -m tests.smoke_capture_states states --platform ${{ matrix.label }} - name: catalogue run: fxcss catalogue --theme fxcss/templates/starter --out out/catalogue --self-contained @@ -397,3 +300,12 @@ jobs: name: fxcss-${{ matrix.label }} path: out/ retention-days: 7 + + # The original captures, unscaled, so a single changed pixel can be + # inspected; out/ holds only normalised copies of the head side. + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: fxcss-${{ matrix.label }}-shots + path: shots/ + retention-days: 7 diff --git a/.github/workflows/firefox-compatibility.yml b/.github/workflows/firefox-compatibility.yml index 12c00a7..2308478 100644 --- a/.github/workflows/firefox-compatibility.yml +++ b/.github/workflows/firefox-compatibility.yml @@ -18,7 +18,7 @@ jobs: channels: name: Firefox ${{ matrix.channel }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -78,9 +78,31 @@ jobs: --out "$GITHUB_WORKSPACE/out" --baseline "$RUNNER_TEMP/baseline" --update-baseline fxcss check --theme fxcss/templates/starter --firefox "$FIREFOX_BIN" \ --out "$GITHUB_WORKSPACE/out" --baseline "$RUNNER_TEMP/baseline" --max-changed-percent 0 + # The same repeatability and sensitivity checks pull-request CI runs on + # Stable and ESR, so a Beta or Nightly change to state capture shows here. + - name: Project check and three captures + run: python -m tests.smoke_capture_states capture + - name: Determinism — the same theme must render identically + run: python -m tests.smoke_capture_states determinism --platform "Linux-$CHANNEL" + env: + CHANNEL: ${{ matrix.channel }} + - name: Detects a real change + run: python -m tests.smoke_capture_states real-change --platform "Linux-$CHANNEL" + env: + CHANNEL: ${{ matrix.channel }} + - name: State and surface views render their own states + run: python -m tests.smoke_capture_states states --platform "Linux-$CHANNEL" + env: + CHANNEL: ${{ matrix.channel }} - uses: actions/upload-artifact@v7 if: always() with: name: firefox-${{ matrix.channel }} path: out/ retention-days: 14 + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: firefox-${{ matrix.channel }}-shots + path: shots/ + retention-days: 14 diff --git a/README.md b/README.md index 59ae037..45764ae 100644 --- a/README.md +++ b/README.md @@ -782,6 +782,13 @@ with the same name in different folders are told apart by folder (`custom-tabs`, `optional-tabs`); two in one folder that differ only in case or punctuation are reported as an error, since nothing distinguishes them. +Adding `optional/tabs.css` alongside an existing `custom/tabs.css` renames +the existing `tabs` variant to `custom-tabs` and names the new one +`optional-tabs`. Update explicit `--variants` selections and `--views` +filters accordingly. Existing baselines retain `variant-tabs`, so checks +report that view as missing until you review the renamed captures and +refresh the baseline with `--update-baseline`. + #### Browser states it captures `fxcss shot` renders a standard set of views, so a change is judged against the @@ -1233,14 +1240,16 @@ Updating fxcss locally does not update workflows already in a repository. | Feature | Enable with | When it runs | What readers see | | --- | --- | --- | --- | -| PR previews | `init` | Relevant pull request changes; macOS, Windows and Linux. | A comment showing before/after views and highlighted differences, including optional stylesheets. | +| PR previews | `init` | All pull request changes; macOS, Windows and Linux. | A comment showing before/after views and highlighted differences, including optional stylesheets. | | Firefox watch | `init --watch` | Weekly on Mondays, or manually; Release, Beta and Nightly on macOS. | Fix PRs or issues for actionable selector findings, plus a separate issue if screenshot capture fails. | -| README previews | `init --previews` | Matching theme changes on `main`/`master`, or manually; macOS. | Refreshed standard and variant screenshots plus cropped option comparisons on the `previews` branch. | +| README previews | `init --previews` | All repository changes on `main`/`master`, or manually; macOS. | Refreshed standard and variant screenshots plus cropped option comparisons on the `previews` branch. | | Release showcase | `init --showcase` | A release is published, or manually; macOS. | Light/dark screenshots against live websites on the `showcase` branch. | -The workflows download Firefox on their runners. Generated preview triggers -cover `chrome/`, `configuration/`, `custom/`, `optional/`, `options/`, `extras/` -and `variants/`. Review the filters if you keep theme assets elsewhere. +The workflows download Firefox on their runners. Generated PR and README +previews run for all repository changes, without path filters: imported CSS, +images, fonts and other dependencies can live anywhere in a theme repository. +This deliberately includes documentation-only edits so dependency-only changes +outside standard folders cannot silently skip previews. Re-run generation and review the changes to update the pinned fxcss version or adopt improvements to the workflow templates. New variants are included automatically. @@ -1677,6 +1686,12 @@ characters. Real Firefox checks cover Stable on all three platforms and ESR on Linux, repeated capture consistency, deliberate CSS changes, nested imports, content stylesheets and stacked optional stylesheets. +A Linux/Python 3.12 job also runs the suite with the minimum supported Pillow, +10.1.0. Capture determinism compares all three pairs from one `check` and two +`shot` runs, prints raw changed-pixel counts, and retains original screenshots +when a job fails. The scheduled compatibility jobs run the same real-change +and state-specific capture assertions on Stable, ESR, Beta and Nightly. + A separate compatibility workflow checks Stable, ESR, Beta and Nightly on Linux every Monday and Thursday, or on demand. Failed audits, incomplete captures and rendering differences fail the run; reports and screenshots are retained as @@ -1711,7 +1726,10 @@ package, or the image tests error instead of being skipped. If `pip install` refuses with `externally-managed-environment`, create a virtual environment first (`python3 -m venv .venv && . .venv/bin/activate`). The publisher workflow tests also use Node and bash when they are installed, and skip -otherwise. +otherwise. Offline release orchestration tests execute the workflow scripts +with stubbed pip, sleep and gh commands to check tag validation, the PyPI +installability retry gate and website dispatch without network access. They +also check that publication still requires CI and the built-distribution checks. ## How this was built diff --git a/fxcss/check.py b/fxcss/check.py index 20c91a4..f9b66cd 100644 --- a/fxcss/check.py +++ b/fxcss/check.py @@ -8,6 +8,7 @@ import shutil import sys import tempfile +from decimal import Decimal from pathlib import Path from . import capture, compare, core, __version__ @@ -120,7 +121,20 @@ def comparison_status(summary, threshold): # A theme cannot add a standard view; one missing from the baseline was # made by an older fxcss, so there is nothing to compare it with yet. added = [v for v in summary["only_in_head"] if not capture.is_standard_view(v)] - if added or any(v["percent"] > threshold for v in summary["views"]): + # Percent is rounded for display; even one changed pixel must exceed a + # zero limit. Older summaries may have only the displayed percentage. + # Use the configured decimal limit as a ratio so equality also survives + # binary float rounding (e.g. 57/10000 pixels at a 0.57% limit). + numerator, denominator = Decimal(str(threshold)).as_integer_ratio() + + def exceeds(view): + if ("changed_pixels" in view and "total_pixels" in view + and view["total_pixels"] > 0): + return (100 * view["changed_pixels"] * denominator + > numerator * view["total_pixels"]) + return view["percent"] > threshold + + if added or any(exceeds(v) for v in summary["views"]): return 1 return 0 diff --git a/fxcss/core.py b/fxcss/core.py index 50c4445..fb00445 100644 --- a/fxcss/core.py +++ b/fxcss/core.py @@ -44,6 +44,32 @@ class BrowserStartupError(MarionetteError): """Firefox opened its chrome window without an initial page document.""" +class FixtureNavigationStall(MarionetteError): + """A local fixture page did not load before Marionette's navigation deadline. + + Marionette reports only that its deadline passed. `state` records what the + tab looked like at that moment -- its remote type, content process, current + document and loading flags -- so a CI log alone can say whether the tab + never received a content process or a page was merely slow. None when that + state could not be collected. + """ + + def __init__(self, message, state=None): + super().__init__(message) + self.state = state + + @property + def unattached(self): + """Did the stalled tab never get a content process behind it? + + Only this case is the known Windows startup condition, where a browser + accepts commands but its child processes never come up. A tab with a + content process that did not finish loading is a different failure. + """ + return (isinstance(self.state, dict) and "process" in self.state + and self.state["process"] is None) + + def free_port(): """Pick an unused port for this session's Marionette listener. @@ -452,13 +478,17 @@ def _is_startup_race(exc): tabbrowser -- seen intermittently on Windows CI runners as `TypeError: can't access property "maybeCancelContentJSExecution", this._browser.frameLoader.remoteTab is null`. A newly created tab can also - retain a discarded content context even after re-selection. Fixture setup - may replace its temporary tabs for these failures; unrelated errors still - surface unchanged. + retain a discarded content context even after re-selection, or sit at its + navigation deadline without a content process ever attaching (a + windows-latest run timed out loading a local fixture page seconds after a + fresh start). Fixture setup may replace its temporary tabs for these + failures; unrelated errors, including a timeout on a tab that does have a + content process, still surface unchanged. """ message = str(exc) return ("remoteTab is null" in message or "frameLoader is null" in message - or _is_discarded_navigation(exc)) + or _is_discarded_navigation(exc) + or (isinstance(exc, FixtureNavigationStall) and exc.unattached)) def _is_discarded_navigation(exc): @@ -468,6 +498,12 @@ def _is_discarded_navigation(exc): and "Browsing context has been discarded" in message) +def _is_navigation_timeout(exc): + message = str(exc) + return (message.startswith("WebDriver:Navigate failed:") + and "'error': 'timeout'" in message) + + # Seconds to wait before each retry of the startup race, one entry per retry. # The original budget -- two retries, 2s apart -- was not enough: a # windows-latest run burned both and failed on the same race, so a slow @@ -684,6 +720,8 @@ def _navigate_fixture_tab(self, handle, url): self.m.command("WebDriver:Navigate", {"url": url}) return except MarionetteError as exc: + if _is_navigation_timeout(exc): + raise self._fixture_navigation_stall(url, exc) from exc if attempt == 2 or not _is_discarded_navigation(exc): raise print(" note: new tab context changed; refreshing it before navigation", @@ -692,6 +730,23 @@ def _navigate_fixture_tab(self, handle, url): self.m.set_context("chrome") time.sleep(0.5) + def _fixture_navigation_stall(self, url, exc): + # Marionette's timeout says nothing about why a local page never + # loaded. Record the tab's state while it is still selected, so the + # failure is diagnosable from a log, and so setup_window's retry can + # tell a tab without a content process from a page that was slow. + self.m.set_context("chrome") + try: + state = self.m.script(FIXTURE_TAB_STATE) + except MarionetteError as state_exc: + state = None + detail = f"tab state unavailable: {state_exc}" + else: + detail = f"tab state: {state}" + return FixtureNavigationStall( + f"fixture page {url} did not load before the navigation deadline " + f"({exc}); {detail}", state) + def _wait_for_fixture_pages(self, timeout=30): # A loading page can paint the same blank frame twice, so _shot's # visual stability check cannot establish that navigation finished. @@ -799,6 +854,28 @@ def set_dark(self, dark): return Array.from(win.gBrowser.tabs, tab => tab.linkedPanel); """ +# The selected tab at the moment a fixture navigation timed out. `process` is +# the content process behind the tab, null when none ever attached. +FIXTURE_TAB_STATE = """ +const win = Services.wm.getMostRecentWindow("navigator:browser"); +const gb = win.gBrowser; +const tab = gb.selectedTab; +const browser = tab.linkedBrowser; +const context = browser.browsingContext; +return { + url: browser.currentURI.spec, + remoteType: browser.remoteType, + process: browser.frameLoader?.remoteTab?.osPid ?? null, + document: context?.currentWindowGlobal?.documentURI?.spec ?? null, + discarded: context?.isDiscarded ?? null, + loading: browser.webProgress.isLoadingDocument, + busy: tab.hasAttribute("busy"), + tabs: gb.tabs.length, + contentProcesses: typeof ChromeUtils.getAllDOMProcesses === "function" + ? ChromeUtils.getAllDOMProcesses().length - 1 : null +}; +""" + FINALIZE_FIXTURE_LABELS = """ const done = arguments[arguments.length - 1]; const win = Services.wm.getMostRecentWindow("navigator:browser"); @@ -1339,8 +1416,8 @@ def _capture_audio_views(m, outdir): # fire-and-forget inside this one script, which returns immediately and # writes its result to a file for Python to poll -- no Marionette command is # sent while the dialog exists. Sleeps are nsITimer because the parent window -# is in a modal state, which suspends its own setTimeout callbacks. drawWindow -# sees the dialog because it is in-document top-layer DOM, not an OS popup. +# is in a modal state, which suspends its own setTimeout callbacks. A browser +# snapshot includes this in-document top-layer dialog; it is not an OS popup. DIALOG_VIEW = """ const outPath = arguments[0]; const win = Services.wm.getMostRecentWindow("navigator:browser"); @@ -1385,21 +1462,29 @@ def _capture_audio_views(m, outdir): const scale = win.devicePixelRatio; const width = win.innerWidth, height = win.innerHeight; - const draw = () => { + const draw = async () => { const canvas = win.document.createElementNS( "http://www.w3.org/1999/xhtml", "canvas"); canvas.width = width * scale; canvas.height = height * scale; const ctx = canvas.getContext("2d"); - ctx.scale(scale, scale); - ctx.drawWindow(win, 0, 0, width, height, "#ffffff"); + // Match Marionette's screenshot path. The legacy drawWindow repaint can + // rasterize text differently from the regular browser snapshots. The + // snapshot already has device-pixel dimensions, so do not scale it again. + const snapshot = await win.browsingContext.currentWindowGlobal.drawSnapshot( + new win.DOMRect(0, 0, width, height), scale, "#ffffff"); + try { + ctx.drawImage(snapshot, 0, 0); + } finally { + snapshot.close(); + } return canvas.toDataURL("image/png"); }; // The same contract _shot enforces: two consecutive identical captures. - let previous = draw(), shot = previous; + let previous = await draw(), shot = previous; for (let i = 0; i < 8; i++) { await sleep(500); - shot = draw(); + shot = await draw(); if (shot === previous) { break; } previous = shot; } @@ -2363,7 +2448,9 @@ def find_variant_sheets(theme: Path): folder (`custom-tabs` and `optional-tabs`) -- a stable name, since it only depends on the two files themselves. Sheets in one folder that differ only in case or punctuation have nothing to tell them apart, and are an error - that names both files. + that names both files. Adding optional/tabs.css beside custom/tabs.css + renames the existing `tabs` slug to `custom-tabs`; update explicit variant + selections and review and refresh baselines for the renamed captures. """ from .fetch import VARIANT_DIRS by_slug = {} diff --git a/fxcss/scaffold.py b/fxcss/scaffold.py index ff99d57..44eed4f 100644 --- a/fxcss/scaffold.py +++ b/fxcss/scaffold.py @@ -26,7 +26,6 @@ from urllib.parse import urlsplit from . import __version__, capture, core -from .fetch import VARIANT_DIRS PREVIEW_DEFAULTS = {"variants": "all", "views": "", "min_changed_percent": "0", "oldest_firefox": "esr"} @@ -142,9 +141,6 @@ def render_template(text, version, variant_alt, repo_url, preview=None): # Quote scalar defaults separately from the folded DEFAULT_URLS block. text = text.replace("'__FXCSS_REPO_URL__'", f"'{_yaml_quoted(repo_url)}'") text = text.replace("__FXCSS_REPO_URL__", repo_url) - paths = ["chrome", "configuration", *VARIANT_DIRS] - text = text.replace("__FXCSS_THEME_PATHS__", "\n".join( - f" - '{folder}/**'" for folder in paths)) return text diff --git a/fxcss/templates/pr-preview.yml b/fxcss/templates/pr-preview.yml index 695d3a0..b3fdcab 100644 --- a/fxcss/templates/pr-preview.yml +++ b/fxcss/templates/pr-preview.yml @@ -15,12 +15,11 @@ name: PR preview # now comes from a pinned release rather than from the pull request, a pull # request cannot change how it behaves either. +# Run for all repository changes: imported CSS, images and fonts can live +# anywhere in the repository. Folder filters can miss dependency-only edits. on: pull_request: types: [opened, synchronize, reopened] - paths: -__FXCSS_THEME_PATHS__ - - '.github/workflows/pr-preview.yml' permissions: contents: read diff --git a/fxcss/templates/readme-previews.yml b/fxcss/templates/readme-previews.yml index 307e04b..0f125b2 100644 --- a/fxcss/templates/readme-previews.yml +++ b/fxcss/templates/readme-previews.yml @@ -23,13 +23,12 @@ name: README previews # These are for looking at, not for testing -- the pass/fail check is # pr-preview.yml. +# Run for all repository changes: imported CSS, images and fonts can live +# anywhere in the repository. Folder filters can miss dependency-only edits. on: workflow_dispatch: push: branches: [master, main] - paths: -__FXCSS_THEME_PATHS__ - - '.github/workflows/readme-previews.yml' permissions: contents: write diff --git a/tests/smoke_capture_states.py b/tests/smoke_capture_states.py new file mode 100644 index 0000000..138fe99 --- /dev/null +++ b/tests/smoke_capture_states.py @@ -0,0 +1,265 @@ +"""Capture repeatability and sensitivity checks against a real Firefox. + +Run from the repository root, with Firefox in FIREFOX_BIN, one step at a time +so each failure is its own workflow step: + + python -m tests.smoke_capture_states capture + python -m tests.smoke_capture_states determinism --platform Linux + python -m tests.smoke_capture_states real-change --platform Linux + python -m tests.smoke_capture_states states --platform Linux + +capture writes shots/a (the screenshots `fxcss check` took), shots/b and +shots/c (two `fxcss shot` runs). determinism compares every pair, so a run that +differs from both others is told apart from a check-vs-shot difference that +repeats. real-change and states prove the comparison and the state-specific +views actually see the CSS they are meant to. Every view must match exactly: +no changed pixel is ever written off as noise here. +""" +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +from fxcss import capture + +STARTER = Path(__file__).resolve().parents[1] / "fxcss/templates/starter" +RUNS = ("a", "b", "c") +PAIRS = (("a", "b"), ("a", "c"), ("b", "c")) +RUN_NAMES = {"a": "fxcss check", "b": "fxcss shot", "c": "fxcss shot"} + +REAL_CHANGE_CSS = "\n#nav-bar { background-color: #ff2d55 !important; }\n" + +# Each rule targets a distinct browser state. Falling back to an ordinary +# window must not pass the state-specific capture checks. +STATE_CHROME_CSS = """ +/* The revamped sidebar hides #sidebar-header; its container stays visible. */ +#sidebar-box { border: 6px solid #ff2d55 !important; } +#tabbrowser-tabs[orient="vertical"] { background-color: #34c759 !important; } +#home-button { background-color: #ff9500 !important; } +.tabbrowser-tab[soundplaying]:not([muted]) .tab-content { background: #e6194b !important; } +.tabbrowser-tab[muted] .tab-content { background: #4363d8 !important; } +/* Surfaces no full-window view shows: hover, the search engine + button and its menu, the panels. */ +#reload-button:hover > .toolbarbutton-icon { background-color: #af52de !important; } +#urlbar .searchmode-switcher:hover { outline: 4px solid #ff2d55 !important; } +panel:has(> .searchmode-switcher-panel-list) { --panel-background: #34c759 !important; } +.searchmode-switcher-panel-list { background-color: #34c759 !important; } +#appMenu-mainView { background-color: #ff9500 !important; } +#unified-extensions-view { background-color: #4363d8 !important; } +#PanelUI-fxa { background-color: #5856d6 !important; } +""" +STATE_CONTENT_CSS = """@-moz-document url-prefix("about:preferences") { + body { background-color: #e6194b !important; } +} +""" +STATE_VIEWS = frozenset( + {"extra-10-sidebar-bookmarks", "extra-11-sidebar-history", "extra-14-vertical-tabs", + "extra-15-toolbar", "extra-04-audio", "extra-05-muted"} + | {f"{mode}-{view}" for mode in ("light", "dark") + for view in ("05-toolbar-hover", "06-search-engine", "07-search-menu", + "08-app-menu", "09-extensions-panel", "10-account-panel", "11-settings")}) + + +def fxcss(*args): + """Run the installed toolkit exactly as a theme author would.""" + print("+ fxcss " + " ".join(str(arg) for arg in args), flush=True) + subprocess.run([sys.executable, "-m", "fxcss", *map(str, args)], check=True) + + +def load(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def compare(root, base, head, out, platform): + fxcss("compare", "--base", root / base, "--head", root / head, "--out", root / out, + "--platform", platform) + return load(root / out / "summary.json") + + +def copy_theme(destination): + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(STARTER, destination) + return destination + + +def scratch(root): + return Path(os.environ.get("RUNNER_TEMP") or root / "out/tmp") + + +# Three, not two. The flakes this has caught were intermittent, so a single +# pair passed often enough to be misleading. +def capture_runs(root, firefox): + check = root / "out/check" + fxcss("check", "--theme", STARTER, "--firefox", firefox, "--out", check) + report, = check.glob("run-*/summary.json") + result = load(report) + if result["exit_code"] != 0: + raise SystemExit(f"fxcss check failed: {json.dumps(result, indent=2)}") + browser, = result["browsers"] + shutil.copytree(report.parent / browser["key"] / "shots", root / "shots/a") + for run in RUNS[1:]: + fxcss("shot", "--theme", STARTER, "--out", root / "shots" / run, "--variants", "all") + return 0 + + +def pixels(view): + return f"{view['changed_pixels']}/{view['total_pixels']} pixels ({view['percent']}%)" + + +def differing_views(summary): + """Every view this comparison found different, with any pixel at all.""" + names = {v["view"] for v in summary["views"] if v["changed_pixels"]} + names |= {v["view"] for v in summary["changed_views"]} + return names | set(summary["only_in_base"]) | set(summary["only_in_head"]) + + +def diagnose(differs): + """Name which run is the odd one out, from the pairs that saw a view differ. + + Every case still fails; this only says where to look. With just one check + capture, matching shot runs cannot establish whether the check command or + an intermittent first-run difference caused the mismatch. + """ + differs = frozenset(differs) + for odd in RUNS: + if differs == {pair for pair in PAIRS if odd in pair}: + if odd == "a": + return ("run a (fxcss check) differs from runs b and c (fxcss shot), which " + "match: investigate check-vs-shot setup and first-run effects; " + "the two shot captures show no shot-to-shot noise") + return (f"run {odd} differs from the other two runs, which match: an intermittent " + "difference between repeated captures") + if len(differs) == len(PAIRS): + return "all three runs differ from one another: the rendering is unstable" + pair, = differs + return (f"only runs {pair[0]} and {pair[1]} differ, and each matches the third: a difference " + "close to the per-pixel noise threshold") + + +def determinism_report(summaries): + """(passed, lines) for the a/b/c pairwise comparisons, keyed by pair.""" + lines, failed, differs = [], False, {} + for pair in PAIRS: + s = summaries[pair] + label = f"run {pair[0]} ({RUN_NAMES[pair[0]]}) vs run {pair[1]} ({RUN_NAMES[pair[1]]})" + names = differing_views(s) + if s["any_change"] or names: + failed = True + lines.append(f"{label}: rendered differently") + lines.append(f" missing: {s['only_in_base']}; added: {s['only_in_head']}") + for v in s["views"]: + if v["view"] in names: + lines.append(f" {v['view']}: {pixels(v)}") + else: + changed = sum(v["changed_pixels"] for v in s["views"]) + total = sum(v["total_pixels"] for v in s["views"]) + lines.append(f"{label}: all {len(s['views'])} views identical " + f"({changed}/{total} pixels changed)") + for name in names: + differs.setdefault(name, set()).add(pair) + if differs: + lines.append("Diagnosis:") + for name in sorted(differs): + lines.append(f" {name}: {diagnose(differs[name])}") + return not failed, lines + + +def determinism(root, platform): + summaries = {pair: compare(root, f"shots/{pair[0]}", f"shots/{pair[1]}", + f"out/determinism-{pair[0]}-{pair[1]}", platform) + for pair in PAIRS} + passed, lines = determinism_report(summaries) + print("\n".join(lines)) + return 0 if passed else 1 + + +def real_change_report(summary): + changed = summary["changed_views"] + if not changed: + return False, ["An obvious CSS change produced no pixel difference — " + "the comparison is not sensitive."] + lines = [f"{len(changed)} of {len(summary['views'])} views changed, as expected"] + lines += [f" {v['view']}: {pixels(v)}" for v in changed] + return True, lines + + +def real_change(root, platform): + theme = copy_theme(scratch(root) / "changed-theme") + with (theme / "chrome/userChrome.css").open("a", encoding="utf-8") as sheet: + sheet.write(REAL_CHANGE_CSS) + fxcss("shot", "--theme", theme, "--out", root / "shots/changed") + passed, lines = real_change_report(compare(root, "shots/a", "shots/changed", + "out/changed", platform)) + print("\n".join(lines)) + return 0 if passed else 1 + + +def search_button_skip(name, view): + """A Firefox without the search engine button (an older ESR) cannot show + it. Only the views capture.VIEWS allows to lack it may be skipped for it; + nothing else may be skipped at all.""" + info = capture.view_info(name) + return (bool(info) and capture.NO_SEARCH_BUTTON in info.unsupported + and view.get("status") == "unsupported" + and view.get("reason") == capture.NO_SEARCH_BUTTON) + + +def states_report(summary, coverage): + changed = {v["view"] for v in summary["changed_views"]} + views = coverage["views"] + skipped = {name for name in STATE_VIEWS if search_button_skip(name, views.get(name, {}))} + lines = [f"{name}: skipped, {capture.NO_SEARCH_BUTTON}" for name in sorted(skipped)] + missing = sorted(STATE_VIEWS - changed - skipped) + if missing: + lines.append("These views did not react to a rule that only they can render, so " + "they are not capturing the state they claim: " + ", ".join(missing)) + for name in missing: + view = views.get(name, {}) + if view.get("status") != "captured": + lines.append(f" {name}: {view.get('status')} ({view.get('reason')})") + return False, lines + lines.append(f"all {len(STATE_VIEWS - skipped)} state views reacted; " + f"{len(changed)} views changed") + by_name = {v["view"]: v for v in summary["views"]} + lines += [f" {name}: {pixels(by_name[name])}" for name in sorted(STATE_VIEWS & changed)] + return True, lines + + +def states(root, platform): + theme = copy_theme(scratch(root) / "state-theme") + with (theme / "chrome/userChrome.css").open("a", encoding="utf-8") as sheet: + sheet.write(STATE_CHROME_CSS) + (theme / "chrome/userContent.css").write_text(STATE_CONTENT_CSS, encoding="utf-8") + fxcss("shot", "--theme", theme, "--out", root / "shots/state") + passed, lines = states_report( + compare(root, "shots/a", "shots/state", "out/state", platform), + load(root / "shots/state" / capture.REPORT)) + print("\n".join(lines)) + return 0 if passed else 1 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + parser.add_argument("step", choices=("capture", "determinism", "real-change", "states")) + parser.add_argument("--platform", default="local") + parser.add_argument("--firefox", default=os.environ.get("FIREFOX_BIN")) + parser.add_argument("--root", type=Path, default=Path.cwd()) + args = parser.parse_args(argv) + root = args.root.resolve() + if args.step == "capture": + if not args.firefox: + parser.error("capture needs --firefox or FIREFOX_BIN") + return capture_runs(root, args.firefox) + if args.step == "determinism": + return determinism(root, args.platform) + if args.step == "real-change": + return real_change(root, args.platform) + return states(root, args.platform) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/smoke_startup.py b/tests/smoke_startup.py index 5cee7c2..5579f2b 100644 --- a/tests/smoke_startup.py +++ b/tests/smoke_startup.py @@ -43,6 +43,24 @@ def log_message(self, *args): worker.join(timeout=5) +def windows_processes_alive(pids, run=subprocess.run): + """The subset of `pids` that tasklist still reports as running. + + A session restarts Firefox when its first browser is unusable, so more + than one browser and launcher PID belong to it. Every one must be gone + once the session closes; checking only the last would hide a leak of the + browser that was replaced. + """ + alive = [] + for pid in pids: + listing = run(["tasklist", "/FI", f"PID eq {pid}", "/NH", "/FO", "CSV"], + check=True, capture_output=True, text=True) + if any(len(row) > 1 and row[1] == str(pid) + for row in csv.reader(io.StringIO(listing.stdout))): + alive.append(pid) + return alive + + def main(): theme = Path(core.__file__).parent / "templates" / "starter" @@ -51,9 +69,11 @@ class BrokenFirstProcess(core.Session): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.browser_pids = [] + self.launcher_pids = [] def _start_browser(self): super()._start_browser() + self.launcher_pids.append(self.proc.pid) self.browser_pids.append(self.m.script("return Services.appinfo.processID;")) if len(self.browser_pids) == 1: self.m.script(''' @@ -179,13 +199,14 @@ def late_label_direction(): assert corrected["overflow"] is False and corrected["direction"] == "rtl", corrected assert narrow["overflow"] is True, narrow browser_pid = session.m.script("return Services.appinfo.processID;") + assert browser_pid == session.browser_pids[-1], (browser_pid, session.browser_pids) + # On Windows firefox.exe is a launcher distinct from the browser it waits + # for; elsewhere the two PIDs coincide. + recorded = sorted(set(session.browser_pids + session.launcher_pids)) if os.name == "nt": - listing = subprocess.run( - ["tasklist", "/FI", f"PID eq {browser_pid}", "/NH", "/FO", "CSV"], - check=True, capture_output=True, text=True) - assert not any(len(row) > 1 and row[1] == str(browser_pid) - for row in csv.reader(io.StringIO(listing.stdout))), ( - "the Firefox browser outlived its session", listing.stdout) + alive = windows_processes_alive(recorded) + assert not alive, ("Firefox processes outlived their session", alive, + {"browsers": session.browser_pids, "launchers": session.launcher_pids}) print("Startup restarted a browser missing its initial document, recovered discarded contexts, waited for a slow page, " "finalised RTL labels and overflow, and kept setup idempotent", flush=True) diff --git a/tests/smoke_themes.py b/tests/smoke_themes.py index 5ad35be..3d0ce08 100644 --- a/tests/smoke_themes.py +++ b/tests/smoke_themes.py @@ -3,48 +3,95 @@ import time from pathlib import Path from fxcss import core, install -from tests.theme_fixtures import write_themes +from tests.theme_fixtures import EXPECTED_LAYOUTS, SHIPPED_CUSTOM_CHROME, write_themes + +CHROME_PROPERTY = ''' + const win = Services.wm.getMostRecentWindow("navigator:browser"); + return win.getComputedStyle(win.document.documentElement) + .getPropertyValue(arguments[0]).trim(); +''' + + +def chrome_property(session, name): + return session.m.script(CHROME_PROPERTY, [name]) + + +def wait_for_property(session, name, expected, attempts=40): + """Poll a computed custom property on the chrome root until it matches.""" + for _ in range(attempts): + actual = chrome_property(session, name) + if actual == expected: + break + time.sleep(0.25) + return actual + + +def check_installed(theme, expected, profile, firefox): + sheets = sorted((theme / "optional").glob("*.css")) + install.install_theme(theme, profile, "local-fixture", sheets=sheets) + with core.Session(profile, firefox) as session: + session.setup_window() + actual = wait_for_property(session, "--fxcss-layout", expected) + assert actual == expected, (theme.name, actual, expected) + if expected == "nested": + palette = chrome_property(session, "--fxcss-palette") + assert palette == "loaded", palette + if expected == "content": + session.m.set_context("content") + try: + content = session.m.script('return window.getComputedStyle(document.documentElement).getPropertyValue("--fxcss-content").trim();') + assert content == "loaded", content + finally: + session.m.set_context("chrome") + if expected == "custom": + shipped = chrome_property(session, "--fxcss-custom-chrome") + assert shipped == "shipped", (theme.name, shipped) + install.uninstall_theme(profile) + assert not (profile / "chrome").exists(), profile + assert not (profile / "user.js").exists(), profile + + +def check_custom_chrome_preview(theme, firefox): + """A theme's own customChrome.css must survive `try` previews and live reloads. + + build_profile and reload_theme both copy chrome/ and write a placeholder + customChrome.css for themes that only @import it. A theme that ships the + file used to have its rules overwritten by that placeholder, so the preview + showed a theme missing part of itself. + """ + with core.Session(theme, firefox) as session: + session.setup_window() + copied = (session.profile / "chrome" / "customChrome.css").read_text(encoding="utf-8") + assert copied == SHIPPED_CUSTOM_CHROME, copied + shipped = wait_for_property(session, "--fxcss-custom-chrome", "shipped") + assert shipped == "shipped", shipped + # watch mode leaves the profile's userChrome.css empty and loads each copy + # of chrome/ as a live sheet, so a reload is where an overwrite would show. + with core.Session(theme, firefox, empty_user_chrome=True) as session: + session.setup_window() + session.reload_theme() + shipped = wait_for_property(session, "--fxcss-custom-chrome", "shipped") + assert shipped == "shipped", shipped + (theme / "chrome" / "customChrome.css").write_text( + ':root { --fxcss-custom-chrome: reloaded; }\n', encoding="utf-8") + session.reload_theme() + reloaded = wait_for_property(session, "--fxcss-custom-chrome", "reloaded") + assert reloaded == "reloaded", reloaded def main(): firefox = core.find_firefox() with tempfile.TemporaryDirectory(prefix="fxcss-layout-smoke-") as td: root = Path(td) - for theme, expected in zip(write_themes(root), ("nested", "content", "last")): + themes = write_themes(root) + assert [theme.name for theme in themes] == list(EXPECTED_LAYOUTS), themes + for theme in themes: profile = root / (theme.name + " profile") profile.mkdir() - sheets = sorted((theme / "optional").glob("*.css")) - install.install_theme(theme, profile, "local-fixture", sheets=sheets) - with core.Session(profile, firefox) as session: - session.setup_window() - for _ in range(40): - actual = session.m.script(''' - const win = Services.wm.getMostRecentWindow("navigator:browser"); - return win.getComputedStyle(win.document.documentElement) - .getPropertyValue("--fxcss-layout").trim(); - ''') - if actual == expected: - break - time.sleep(0.25) - assert actual == expected, (theme.name, actual, expected) - if expected == "nested": - palette = session.m.script(''' - const win = Services.wm.getMostRecentWindow("navigator:browser"); - return win.getComputedStyle(win.document.documentElement) - .getPropertyValue("--fxcss-palette").trim(); - ''') - assert palette == "loaded", palette - if expected == "content": - session.m.set_context("content") - try: - content = session.m.script('return window.getComputedStyle(document.documentElement).getPropertyValue("--fxcss-content").trim();') - assert content == "loaded", content - finally: - session.m.set_context("chrome") - install.uninstall_theme(profile) - assert not (profile / "chrome").exists(), profile - assert not (profile / "user.js").exists(), profile + check_installed(theme, EXPECTED_LAYOUTS[theme.name], profile, firefox) print(f"{theme.name}: styling loaded and clean removal restored the empty profile", flush=True) + check_custom_chrome_preview(themes[-1], firefox) + print("shipped custom chrome: the theme's own customChrome.css survived preview and live reload", flush=True) if __name__ == "__main__": diff --git a/tests/smoke_variants.py b/tests/smoke_variants.py index b6df23a..7406395 100644 --- a/tests/smoke_variants.py +++ b/tests/smoke_variants.py @@ -1,37 +1,100 @@ """Check optional-sheet loading in a real Firefox, using disposable profiles.""" +import re import tempfile import time from pathlib import Path from fxcss import core, fetch, install +# Distinct sizes tell the two fixture images apart once decoded: a URL in a +# computed style only proves the reference resolved, not that the file at the +# other end is the one the theme shipped. +BASE_IMAGE_SIZE = (1, 1) +OPTION_IMAGE_SIZE = (7, 3) + +DECODE_IMAGE = """ +const done = arguments[arguments.length - 1]; +const win = Services.wm.getMostRecentWindow("navigator:browser"); +const img = new win.Image(); +img.onload = () => done([img.naturalWidth, img.naturalHeight]); +img.onerror = () => done("error: the image did not load"); +img.src = arguments[0]; +""" + +COMPUTED_STYLE = """ +const win = Services.wm.getMostRecentWindow("navigator:browser"); +const style = win.getComputedStyle(win.document.documentElement); +return arguments[0].map(name => style.getPropertyValue(name).trim()); +""" + + +def _svg(width, height): + return (f'') + + +def write_variant_theme(root): + """A theme whose optional sheet reaches outside chrome/ for its own files. + + The base sheet @imports a nested sheet and refers to an image, both inside + chrome/. The first optional sheet lives in custom/ beside chrome/, the way + themes ship optional components, and @imports a sheet of its own whose + rule refers to an image under custom/. `fxcss try` loads the optional + sheet in place, so those references resolve against the theme; `fxcss + install` must carry the nested sheet and image into the profile. + + Returns the theme and the optional sheets in cascade order. + """ + root = Path(root) + theme = root / "theme" + chrome = theme / "chrome" + custom = theme / "custom" + (chrome / "nested").mkdir(parents=True) + (chrome / "assets").mkdir() + (custom / "parts").mkdir(parents=True) + (custom / "assets").mkdir() + (chrome / "nested" / "base.css").write_text( + ":root { --fxcss-smoke-relative: loaded; }", encoding="utf-8") + (chrome / "assets" / "probe.svg").write_text(_svg(*BASE_IMAGE_SIZE), encoding="utf-8") + (chrome / "userChrome.css").write_text( + '@charset "UTF-8";\n@import "nested/base.css";\n' + '@namespace xul url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul");\n' + ':root { --fxcss-smoke-color: red; --fxcss-smoke-order: base;\n' + 'background-image: url("assets/probe.svg") !important; }\n', + encoding="utf-8") + first, last = (custom / name for name in ("blue option.css", "last.css")) + first.write_text('@import "parts/option extra.css";\n' + ":root { --fxcss-smoke-color: blue; " + "--fxcss-smoke-order: first; }\n", encoding="utf-8") + (custom / "parts" / "option extra.css").write_text( + ":root { --fxcss-smoke-nested: loaded;\n" + 'list-style-image: url("../assets/option.svg") !important; }\n', + encoding="utf-8") + (custom / "assets" / "option.svg").write_text(_svg(*OPTION_IMAGE_SIZE), encoding="utf-8") + last.write_text(":root { --fxcss-smoke-order: last; }", encoding="utf-8") + return theme, [first, last] + + +def image_url(computed): + """The URL inside a computed `url("...")` image value.""" + match = re.fullmatch(r'url\("(.*)"\)', computed) + assert match, computed + return match.group(1) + + +def check_image(session, computed, expected_uri, expected_size): + url = image_url(computed) + assert url == expected_uri, (url, expected_uri) + size = session.m.async_script(DECODE_IMAGE, [url]) + assert size == list(expected_size), (url, size, expected_size) + def main(): firefox = core.find_firefox() with tempfile.TemporaryDirectory(prefix="fxcss-variant-smoke-") as td: root = Path(td) - theme = root / "theme" - chrome = theme / "chrome" - (chrome / "nested").mkdir(parents=True) - (chrome / "assets").mkdir() - (theme / "custom").mkdir() - (chrome / "nested" / "base.css").write_text( - ":root { --fxcss-smoke-relative: loaded; }", encoding="utf-8") - (chrome / "assets" / "probe.svg").write_text( - '', - encoding="utf-8") - (chrome / "userChrome.css").write_text( - '@charset "UTF-8";\n@import "nested/base.css";\n' - '@namespace xul url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul");\n' - ':root { --fxcss-smoke-color: red; --fxcss-smoke-order: base;\n' - 'background-image: url("assets/probe.svg") !important; }\n', - encoding="utf-8") - first, last = (theme / "custom" / name - for name in ("blue option.css", "last.css")) - first.write_text(":root { --fxcss-smoke-color: blue; " - "--fxcss-smoke-order: first; }", encoding="utf-8") - last.write_text(":root { --fxcss-smoke-order: last; }", encoding="utf-8") + theme, (first, last) = write_variant_theme(root) for mode in ("install", "try"): if mode == "install": profile = root / "installed" @@ -42,21 +105,31 @@ def main(): session = core.Session(theme, firefox) fetch.apply_variants(session.profile, [first, last]) with session: - expected_image = (session.profile / "chrome" / "assets" / "probe.svg").as_uri() + session_chrome = session.profile / "chrome" + expected_base = (session_chrome / "assets" / "probe.svg").as_uri() + if mode == "install": + # install_theme copied the optional sheet's own files under + # chrome/custom/, keeping their layout relative to the sheet. + option_image = session_chrome / "custom" / "assets" / "option.svg" + assert option_image.is_file(), sorted(session_chrome.rglob("*")) + expected_option = option_image.as_uri() + else: + expected_option = (theme / "custom" / "assets" / "option.svg").resolve().as_uri() + assert not (session_chrome / "custom").exists(), "try copied optional files" + names = ["--fxcss-smoke-color", "--fxcss-smoke-order", + "--fxcss-smoke-relative", "--fxcss-smoke-nested", + "background-image", "list-style-image"] + loaded = ["blue", "last", "loaded", "loaded"] for _ in range(40): - values = session.m.script(""" - const win = Services.wm.getMostRecentWindow("navigator:browser"); - const style = win.getComputedStyle(win.document.documentElement); - return ["--fxcss-smoke-color", "--fxcss-smoke-order", - "--fxcss-smoke-relative", "background-image"] - .map(name => style.getPropertyValue(name).trim()); - """) - if values[:3] == ["blue", "last", "loaded"]: + values = session.m.script(COMPUTED_STYLE, [names]) + if values[:4] == loaded: break time.sleep(0.25) - assert values[:3] == ["blue", "last", "loaded"], (mode, values) - assert expected_image in values[3], (mode, values[3], expected_image) - print(f"{mode}: options override the base in order; relative imports and URLs resolve") + assert values[:4] == loaded, (mode, dict(zip(names, values))) + check_image(session, values[4], expected_base, BASE_IMAGE_SIZE) + check_image(session, values[5], expected_option, OPTION_IMAGE_SIZE) + print(f"{mode}: options override the base in order; nested imports and " + "image references resolve and decode inside and outside chrome/", flush=True) if __name__ == "__main__": diff --git a/tests/test_capture_state_checks.py b/tests/test_capture_state_checks.py new file mode 100644 index 0000000..31e10c9 --- /dev/null +++ b/tests/test_capture_state_checks.py @@ -0,0 +1,325 @@ +"""Exercise the CI capture-state smoke checks offline, with no Firefox.""" + +import contextlib +import io +import json +import re +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from fxcss import capture +from tests import smoke_capture_states as smoke + +try: + from PIL import Image +except ImportError: # pragma: no cover - the images extra is optional + Image = None + +REPO = Path(__file__).resolve().parents[1] +STEPS = ("capture", "determinism", "real-change", "states") +VIEWS = ("light-01-window", "dark-01-window") + + +def workflow(name): + return (REPO / ".github/workflows" / name).read_text(encoding="utf-8") + + +def summary(views, only_in_base=(), only_in_head=()): + """A compare summary.json for {view: changed_pixels}.""" + rows = [{"view": name, "changed_pixels": changed, "total_pixels": 1000, + "percent": round(100.0 * changed / 1000, 4), + "image": f"{name}.png" if changed else None} + for name, changed in views.items()] + return {"views": rows, "changed_views": [row for row in rows if row["image"]], + "only_in_base": list(only_in_base), "only_in_head": list(only_in_head), + "any_change": bool(any(views.values()) or only_in_base or only_in_head)} + + +def run_quietly(function, *args): + out = io.StringIO() + with contextlib.redirect_stdout(out): + result = function(*args) + return result, out.getvalue() + + +class WorkflowWiringTests(unittest.TestCase): + def assert_runs_every_step(self, text, platform): + commands = re.findall(r"^ run: python -m tests\.smoke_capture_states (.*)$", + text, re.M) + self.assertEqual([c.split()[0] for c in commands], list(STEPS)) + for command in commands[1:]: + self.assertIn(f"--platform {platform}", command) + + def test_pull_request_ci_runs_the_extracted_checks_in_order(self): + ci = workflow("ci.yml") + self.assert_runs_every_step(ci, "${{ matrix.label }}") + smoke_job = ci.split("\n smoke:\n", 1)[1] + self.assertNotIn("python - <<'PY'", smoke_job.split("upgrade --compare renders", 1)[0]) + self.assertNotIn("<<'PY'", smoke_job.split("State and surface views", 1)[1]) + + def test_compatibility_runs_them_on_every_channel_including_beta_and_nightly(self): + compat = workflow("firefox-compatibility.yml") + self.assert_runs_every_step(compat, '"Linux-$CHANNEL"') + for product in ("firefox-latest-ssl", "firefox-esr-latest-ssl", + "firefox-beta-latest-ssl", "firefox-nightly-latest-ssl"): + self.assertIn(f"product: {product}", compat) + # The existing zero-tolerance baseline repeat stays alongside. + self.assertIn("--max-changed-percent 0", compat) + + def test_original_shots_are_uploaded_when_a_check_fails(self): + for name, prefix in (("ci.yml", "fxcss-${{ matrix.label }}"), + ("firefox-compatibility.yml", "firefox-${{ matrix.channel }}")): + text = workflow(name) + self.assertRegex(text, r"if: always\(\)\n\s+with:\n\s+name: " + re.escape(prefix) + + r"\n\s+path: out/") + self.assertRegex(text, r"if: failure\(\)\n\s+with:\n\s+name: " + re.escape(prefix) + + r"-shots\n\s+path: shots/") + + def test_unit_matrix_adds_the_minimum_pillow_and_keeps_unconstrained_jobs(self): + ci = workflow("ci.yml") + units = ci.split("\n units:\n", 1)[1].split("\n build-package:\n", 1)[0] + entries = re.findall(r"^ - \{(.*)\}$", units, re.M) + floor = [e for e in entries if "pillow:" in e] + self.assertEqual(len(floor), 1, entries) + self.assertIn("python: '3.12'", floor[0]) + self.assertIn("ubuntu", floor[0]) + # Pinned to exactly the floor the images extra declares. + dependency, = re.findall(r'"pillow>=([\d.]+)"', (REPO / "pyproject.toml").read_text()) + pinned = re.search(r"pillow: '([\d.]+)'", floor[0])[1] + + def version(text): + parts = [int(part) for part in text.split(".")] + return parts + [0] * (3 - len(parts)) + + self.assertEqual(version(pinned), version(dependency)) + self.assertGreaterEqual(len(entries) - len(floor), 5) + self.assertIn("if: matrix.pillow", units) + self.assertIn('"pillow==$PILLOW"', units) + self.assertIn("PIL.__version__ == os.environ['PILLOW']", units) + # The whole suite runs there, not a hand-picked subset. + self.assertIn("python -m unittest discover -s tests -v", units) + + +class ReportTests(unittest.TestCase): + def test_sensitivity_check_requires_pixels_not_just_a_missing_view(self): + passed, lines = smoke.real_change_report(summary({}, only_in_base=["variant-old"])) + self.assertFalse(passed) + self.assertIn("no pixel difference", lines[0]) + passed, lines = smoke.real_change_report(summary({"light-01-window": 0})) + self.assertFalse(passed) + passed, lines = smoke.real_change_report(summary({"light-01-window": 7, + "dark-01-window": 0})) + self.assertTrue(passed) + self.assertIn("light-01-window: 7/1000 pixels (0.7%)", "\n".join(lines)) + + def test_every_pair_is_strict_and_reports_raw_pixel_counts(self): + clean = summary(dict.fromkeys(VIEWS, 0)) + passed, lines = smoke.determinism_report(dict.fromkeys(smoke.PAIRS, clean)) + self.assertTrue(passed) + self.assertEqual(len(lines), 3) + self.assertIn("0/2000 pixels changed", lines[0]) + # One pixel in one pair, the only pair a two-pair check did not run. + one = summary({"light-01-window": 1, "dark-01-window": 0}) + passed, lines = smoke.determinism_report( + {("a", "b"): clean, ("a", "c"): clean, ("b", "c"): one}) + self.assertFalse(passed) + self.assertIn("light-01-window: 1/1000 pixels (0.1%)", "\n".join(lines)) + + def test_a_missing_or_added_view_is_a_determinism_failure(self): + clean = summary(dict.fromkeys(VIEWS, 0)) + for missing in ({"only_in_base": ["light-02-urlbar"]}, + {"only_in_head": ["light-02-urlbar"]}): + passed, lines = smoke.determinism_report( + {("a", "b"): clean, ("a", "c"): summary(dict.fromkeys(VIEWS, 0), **missing), + ("b", "c"): clean}) + self.assertFalse(passed) + self.assertIn("light-02-urlbar", "\n".join(lines)) + + def test_the_odd_run_out_is_named(self): + ab, ac, bc = smoke.PAIRS + self.assertIn("check-vs-shot", smoke.diagnose({ab, ac})) + self.assertIn("first-run effects", smoke.diagnose({ab, ac})) + self.assertIn("run b differs", smoke.diagnose({ab, bc})) + self.assertIn("intermittent", smoke.diagnose({ab, bc})) + self.assertIn("run c differs", smoke.diagnose({ac, bc})) + self.assertIn("unstable", smoke.diagnose({ab, ac, bc})) + self.assertIn("noise threshold", smoke.diagnose({bc})) + + def coverage(self, **views): + records = {name: {"status": "captured"} for name in smoke.STATE_VIEWS} + records.update(views) + return {"views": records} + + def test_state_views_must_all_change(self): + everything = summary(dict.fromkeys(smoke.STATE_VIEWS, 5)) + passed, lines = smoke.states_report(everything, self.coverage()) + self.assertTrue(passed, lines) + self.assertIn(f"all {len(smoke.STATE_VIEWS)} state views reacted", "\n".join(lines)) + # A view that fell back to an ordinary window does not change. + fallback = summary(dict(dict.fromkeys(smoke.STATE_VIEWS, 5), **{"extra-04-audio": 0})) + passed, lines = smoke.states_report(fallback, self.coverage()) + self.assertFalse(passed) + self.assertIn("extra-04-audio", lines[0]) + + def test_only_search_views_may_be_skipped_and_only_for_the_missing_button(self): + unchanged = {"light-06-search-engine", "dark-07-search-menu"} + result = summary({name: 0 if name in unchanged else 5 for name in smoke.STATE_VIEWS}) + skip = {"status": "unsupported", "reason": capture.NO_SEARCH_BUTTON} + passed, lines = smoke.states_report( + result, self.coverage(**dict.fromkeys(unchanged, skip))) + self.assertTrue(passed, lines) + self.assertIn(f"light-06-search-engine: skipped, {capture.NO_SEARCH_BUTTON}", lines) + # Any other reason, a non-search view, or a captured view still fails. + for name, record in ( + ("light-06-search-engine", {"status": "unsupported", "reason": "other"}), + ("light-06-search-engine", {"status": "captured", + "reason": capture.NO_SEARCH_BUTTON}), + ("light-06-search-engine", {"status": "failed", + "reason": capture.NO_SEARCH_BUTTON}), + ("light-05-toolbar-hover", skip)): + with self.subTest(name=name, record=record): + broken = summary({view: 0 if view == name else 5 for view in smoke.STATE_VIEWS}) + passed, lines = smoke.states_report(broken, self.coverage(**{name: record})) + self.assertFalse(passed) + self.assertNotIn(f"{name}: skipped", lines) + self.assertIn(name, lines[0]) + + +@unittest.skipIf(Image is None, "the orchestration tests compare real images with Pillow") +class OrchestrationTests(unittest.TestCase): + """Run each step with `fxcss check` and `fxcss shot` replaced by files. + + Comparisons use the real `fxcss compare`, so the pixel counts, the + strictness and the diagnosis are exactly what CI would see. + """ + + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.calls = [] + self.altered = {} # run name -> {view: (x, y)} pixel to recolour + patcher = mock.patch.object(smoke, "fxcss", self.fake_fxcss) + patcher.start() + self.addCleanup(patcher.stop) + env = mock.patch.dict("os.environ", {"RUNNER_TEMP": str(self.root / "runner")}) + env.start() + self.addCleanup(env.stop) + + def write_shots(self, directory, run, coverage=None): + directory.mkdir(parents=True) + for name in VIEWS: + image = Image.new("RGB", (64, 32), (240, 240, 240)) + if name in self.altered.get(run, {}): + image.putpixel(self.altered[run][name], (0, 0, 0)) + image.save(directory / f"{name}.png") + (directory / capture.REPORT).write_text(json.dumps(coverage or { + "format": 1, "browser": {}, "views": {n: {"status": "captured"} for n in VIEWS}})) + + def fake_fxcss(self, command, *args): + from fxcss import compare + self.calls.append((command, *map(str, args))) + options = dict(zip(args[::2], args[1::2])) + if command == "check": + run = Path(options["--out"]) / "run-1" + self.write_shots(run / "stable" / "shots", "a") + run.joinpath("summary.json").write_text(json.dumps( + {"exit_code": self.check_exit, "browsers": [{"key": "stable"}]})) + elif command == "shot": + out = Path(options["--out"]) + self.write_shots(out, out.name) + elif command == "compare": + code = compare.run(Path(options["--base"]), Path(options["--head"]), + Path(options["--out"]), options["--platform"]) + self.assertEqual(code, 0) + else: + self.fail(command) + + check_exit = 0 + + def capture(self): + return run_quietly(smoke.main, ["capture", "--firefox", "/ff", "--root", str(self.root)]) + + def determinism(self): + return run_quietly(smoke.main, ["determinism", "--platform", "Test", + "--root", str(self.root)]) + + def test_capture_takes_the_check_screenshots_and_two_shot_runs_of_all_variants(self): + code, _ = self.capture() + self.assertEqual(code, 0) + self.assertEqual([c[0] for c in self.calls], ["check", "shot", "shot"]) + self.assertIn("/ff", self.calls[0]) + for call in self.calls[1:]: + self.assertEqual(call[-2:], ("--variants", "all")) + for run in smoke.RUNS: + self.assertTrue((self.root / "shots" / run / "light-01-window.png").is_file()) + + def test_a_failed_check_stops_before_any_shot(self): + self.check_exit = 1 + with self.assertRaises(SystemExit): + self.capture() + self.assertEqual([c[0] for c in self.calls], ["check"]) + + def test_identical_runs_pass_every_pair(self): + self.capture() + code, output = self.determinism() + self.assertEqual(code, 0, output) + compared = [(Path(c[2]).name, Path(c[4]).name) for c in self.calls if c[0] == "compare"] + self.assertEqual(compared, list(smoke.PAIRS)) + self.assertEqual(output.count("views identical (0/4096 pixels changed)"), 3) + + def test_a_repeatable_check_vs_shot_pixel_is_told_apart_from_shot_noise(self): + # One text-edge pixel that only the check capture has. + self.altered = {"a": {"light-01-window": (5, 5)}} + self.capture() + code, output = self.determinism() + self.assertEqual(code, 1) + self.assertIn("light-01-window: 1/2048 pixels (0.0488%)", output) + self.assertIn("run b (fxcss shot) vs run c (fxcss shot): all 2 views identical", output) + self.assertIn("light-01-window: run a (fxcss check) differs from runs b and c", output) + self.assertIn("check-vs-shot", output) + self.assertNotIn("intermittent", output) + + def test_one_shot_run_that_differs_is_reported_as_intermittent(self): + self.altered = {"b": {"dark-01-window": (9, 3)}} + self.capture() + code, output = self.determinism() + self.assertEqual(code, 1) + self.assertIn("dark-01-window: run b differs from the other two runs", output) + self.assertIn("intermittent", output) + self.assertNotIn("check-vs-shot", output) + + def test_real_change_and_state_themes_are_copies_with_the_rules_added(self): + starter = (smoke.STARTER / "chrome/userChrome.css").read_text(encoding="utf-8") + self.capture() + self.altered = {"changed": {"light-01-window": (1, 1)}} + code, output = run_quietly(smoke.main, ["real-change", "--root", str(self.root)]) + self.assertEqual(code, 0, output) + self.assertIn("1 of 2 views changed, as expected", output) + theme = self.root / "runner/changed-theme" + self.assertTrue((theme / "chrome/userChrome.css").read_text( + encoding="utf-8").endswith(smoke.REAL_CHANGE_CSS)) + self.assertEqual((smoke.STARTER / "chrome/userChrome.css").read_text(encoding="utf-8"), + starter) + + code, output = run_quietly(smoke.main, ["states", "--root", str(self.root)]) + # The fake captures lack every state view, so this must fail loudly. + self.assertEqual(code, 1) + self.assertIn("not capturing the state they claim", output) + state = self.root / "runner/state-theme/chrome" + self.assertIn(smoke.STATE_CHROME_CSS, (state / "userChrome.css").read_text(encoding="utf-8")) + self.assertEqual((state / "userContent.css").read_text(encoding="utf-8"), + smoke.STATE_CONTENT_CSS) + self.assertNotIn("#PanelUI-fxa", starter) + + def test_an_unchanged_real_change_capture_fails(self): + self.capture() + code, output = run_quietly(smoke.main, ["real-change", "--root", str(self.root)]) + self.assertEqual(code, 1) + self.assertIn("no pixel difference", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dialog_capture.py b/tests/test_dialog_capture.py new file mode 100644 index 0000000..5e07aa4 --- /dev/null +++ b/tests/test_dialog_capture.py @@ -0,0 +1,120 @@ +"""Run the modal capture script with fake Firefox objects, without a browser. + +These cover the asynchronous snapshot lifecycle, not Firefox's rasterization; +the real three-run CI comparison checks that on every supported platform. +""" +import json +import shutil +import subprocess +import unittest + +from fxcss import core + + +HARNESS = r""" +const fs = require("node:fs"); +const {source, scale, failDraw} = JSON.parse(fs.readFileSync(0, "utf8")); +const state = {snapshots: [], draws: [], canvases: [], closed: [], cancelled: 0}; +let finish; +const completed = new Promise(resolve => { finish = resolve; }); +let dismiss; +const prompt = new Promise(resolve => { dismiss = resolve; }); +const dialog = {getButton: () => ({click() { state.cancelled++; dismiss(); }})}; +const frame = { + contentDocument: {readyState: "complete", querySelector: () => dialog}, + getBoundingClientRect: () => ({x: 10, y: 20, width: 300, height: 150}), +}; +const win = { + devicePixelRatio: scale, innerWidth: 1264, innerHeight: 472, + DOMRect: class {constructor(x, y, width, height) { + Object.assign(this, {x, y, width, height}); + }}, + IOUtils: {async writeUTF8(path, text) { finish({path, ...JSON.parse(text)}); }}, + document: { + querySelector: () => frame, + createElementNS() { + const canvas = { + width: 0, height: 0, + getContext() { return { + drawImage(snapshot, ...coordinates) { + state.draws.push({id: snapshot.id, coordinates, dismissed: state.cancelled}); + if (failDraw) { throw Error("cannot draw snapshot"); } + canvas.pixels = snapshot.id === 0 ? "settling" : "stable"; + }, + }; }, + toDataURL() { return "data:image/png;base64," + canvas.pixels; }, + }; + state.canvases.push(canvas); + return canvas; + }, + }, + browsingContext: {currentWindowGlobal: {async drawSnapshot(rect, scale, background) { + const id = state.snapshots.length; + state.snapshots.push({rect, scale, background}); + // A real snapshot is asynchronous. Returning a promise instead of awaiting + // it must not be mistaken for two identical captures. + await new Promise(resolve => setImmediate(resolve)); + return {id, close() { state.closed.push(id); }}; + }}}, +}; +global.Services = {wm: {getMostRecentWindow: () => win}, prompt: { + MODAL_TYPE_INTERNAL_WINDOW: 1, BUTTON_POS_0: 1, BUTTON_POS_1: 256, + BUTTON_TITLE_IS_STRING: 127, BUTTON_POS_0_DEFAULT: 0, + asyncConfirmEx: () => prompt, +}}; +global.Ci = {nsITimer: {TYPE_ONE_SHOT: 0}}; +global.Cc = {"@mozilla.org/timer;1": {createInstance: () => ({ + initWithCallback(callback) { setImmediate(callback); }, +})}}; +(async () => { + const kicked = new Function(source)("dialog-handoff.json"); + const result = await completed; + process.stdout.write(JSON.stringify({kicked, result, ...state})); +})().catch(error => { console.error(error); process.exitCode = 1; }); +""" + + +@unittest.skipUnless(shutil.which("node"), "dialog script checks require Node") +class DialogSnapshotTests(unittest.TestCase): + def run_script(self, scale=1, fail_draw=False): + result = subprocess.run( + [shutil.which("node"), "-e", HARNESS], + input=json.dumps({"source": core.DIALOG_VIEW, "scale": scale, + "failDraw": fail_draw}), + text=True, capture_output=True, check=True, timeout=10, + ) + return json.loads(result.stdout) + + def test_waits_for_matching_snapshots_at_native_resolution(self): + for scale in (1, 2): + with self.subTest(scale=scale): + state = self.run_script(scale) + self.assertEqual(state["kicked"], "kicked") + self.assertEqual(state["result"], { + "path": "dialog-handoff.json", "done": True, + "shot": "data:image/png;base64,stable", "box": [10, 20, 300, 150], + }) + self.assertEqual(len(state["snapshots"]), 3) + for snapshot, canvas in zip(state["snapshots"], state["canvases"]): + self.assertEqual(snapshot, { + "rect": {"x": 0, "y": 0, "width": 1264, "height": 472}, + "scale": scale, "background": "#ffffff", + }) + self.assertEqual((canvas["width"], canvas["height"]), + (1264 * scale, 472 * scale)) + self.assertEqual(state["draws"], [ + {"id": i, "coordinates": [0, 0], "dismissed": 0} for i in range(3) + ]) + self.assertEqual(state["closed"], [0, 1, 2]) + self.assertEqual(state["cancelled"], 1) + + def test_draw_failure_releases_snapshot_and_dismisses_prompt(self): + state = self.run_script(fail_draw=True) + self.assertEqual(state["closed"], [0]) + self.assertEqual(state["cancelled"], 1) + self.assertNotIn("shot", state["result"]) + self.assertIn("cannot draw snapshot", state["result"]["skip"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_orchestration.py b/tests/test_release_orchestration.py new file mode 100644 index 0000000..28fa06e --- /dev/null +++ b/tests/test_release_orchestration.py @@ -0,0 +1,170 @@ +"""Offline regression tests for the actual release workflow scripts and gates.""" + +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from zipfile import ZipFile + +from tests.test_workflows import condition, script, step + + +WORKFLOWS = Path(__file__).resolve().parents[1] / '.github/workflows' + + +def job(text, name): + return re.search(r'^ ' + re.escape(name) + r':\n(.*?)(?=^ \S|\Z)', + text, re.M | re.S)[1] + + +@unittest.skipUnless(shutil.which('bash'), 'workflow scripts require bash') +class ReleaseOrchestrationTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.publish = (WORKFLOWS / 'publish.yml').read_text() + self.website = (WORKFLOWS / 'website-docs.yml').read_text() + (self.root / 'dist').mkdir() + self.wheel = self.root / 'dist/fxcss-9.8.7-py3-none-any.whl' + with ZipFile(self.wheel, 'w') as wheel: + wheel.writestr('fxcss-9.8.7.dist-info/METADATA', + 'Metadata-Version: 2.1\nName: fxcss\nVersion: 9.8.7\n') + + def run_script(self, body, **env): + return subprocess.run([shutil.which('bash'), '-c', body], cwd=self.root, + env=dict(os.environ, PYTHON=sys.executable, + RUNNER_TEMP=self.root.as_posix(), **env), + capture_output=True, text=True, timeout=15) + + def test_release_tag_matches_built_metadata_and_manual_branch_needs_no_tag(self): + body = 'python() { "$PYTHON" "$@"; }\n' + script( + self.publish, 'Validate release version') + for ref_type, ref_name, success in ( + ('tag', 'v9.8.7', True), ('tag', 'v9.8.6', False), + ('tag', '9.8.7', False), ('branch', 'main', True), + ('branch', 'release/preparation', True)): + with self.subTest(ref_type=ref_type, ref_name=ref_name): + result = self.run_script(body, RELEASE_REF_TYPE=ref_type, + RELEASE_REF_NAME=ref_name) + self.assertEqual(result.returncode == 0, success, result.stderr) + self.assertIn('Validated fxcss 9.8.7' if success else + 'does not match package version', result.stdout + result.stderr) + # Manual dispatch on a tag still validates the tag; a wheel filename + # alone must never override the package's own metadata. + self.wheel.rename(self.root / 'dist/fxcss-1.0.0-py3-none-any.whl') + result = self.run_script(body, RELEASE_REF_TYPE='tag', RELEASE_REF_NAME='v1.0.0') + self.assertNotEqual(result.returncode, 0) + self.assertIn("expected v9.8.7", result.stderr) + block = step(self.publish, 'Validate release version') + self.assertIn('RELEASE_REF_TYPE: ${{ github.ref_type }}', block) + self.assertIn('RELEASE_REF_NAME: ${{ github.ref_name }}', block) + + def test_installability_retry_immediate_delayed_and_exhausted(self): + # Functions intercept every pip/sleep invocation. Execute the real loop, + # including its exit status, without contacting an index or waiting. + stubs = r''' + attempts=0 + python3() { + attempts=$((attempts + 1)) + printf '%s\n' "$*" >> "$RUNNER_TEMP/pip.log" + [ "$attempts" -ge "$SUCCEED_AT" ] + } + sleep() { printf '%s\n' "$*" >> "$RUNNER_TEMP/sleep.log"; } + ''' + body = stubs + script(self.publish, 'Gate — wait until the release is installable') + for succeed_at, attempts, sleeps, success in ( + (1, 1, 0, True), (4, 4, 3, True), (40, 40, 39, True), + (41, 40, 40, False)): + with self.subTest(succeed_at=succeed_at): + for name in ('pip.log', 'sleep.log'): + (self.root / name).unlink(missing_ok=True) + result = self.run_script(body, SUCCEED_AT=str(succeed_at)) + self.assertEqual(result.returncode, 0 if success else 1, result.stderr) + calls = (self.root / 'pip.log').read_text().splitlines() + self.assertEqual(len(calls), attempts) + self.assertEqual(set(calls), { + '-m pip download --no-deps --no-cache-dir --quiet ' + f'--dest {self.root.as_posix()}/gate fxcss==9.8.7'}) + sleep_log = self.root / 'sleep.log' + self.assertEqual(sleep_log.read_text().splitlines() if sleep_log.exists() + else [], ['15'] * sleeps) + self.assertIn(f'installable (attempt {attempts},' if success else + '::error::fxcss==9.8.7 still not installable', result.stdout) + + def test_website_dispatch_events_and_target(self): + refresh = job(self.website, 'refresh') + guard = re.search(r'^ if: >-\n(.*?)(?=^ \S)', refresh, re.M | re.S)[1] + guard = ' '.join(line.strip() for line in guard.splitlines()) + block = step(self.website, 'Start Refresh docs on the website branch') + command = re.search(r'^ run: (.+)$', block, re.M)[1] + # Run the extracted command only when the extracted GitHub guard allows + # it. The gh stub records arguments, including the target branch. + stub = 'gh() { "$PYTHON" -c \'import json,sys; print(json.dumps(sys.argv[1:]))\' "$@"; }\n' + for repo in ('AdamXweb/fxcss', 'someone/fxcss'): + for event, conclusion in (('push', ''), ('workflow_run', 'success'), + ('workflow_run', 'failure'), ('workflow_run', 'cancelled'), + ('workflow_run', 'skipped'), ('workflow_run', 'timed_out')): + with self.subTest(repo=repo, event=event, conclusion=conclusion): + eligible = condition(guard, {'github.repository': repo, + 'github.event_name': event, + 'github.event.workflow_run.conclusion': conclusion}) + expected = repo == 'AdamXweb/fxcss' and ( + event == 'push' or conclusion == 'success') + self.assertEqual(eligible, expected) + result = self.run_script(stub + (command if eligible else ':'), + GITHUB_REPOSITORY=repo) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout) if result.stdout else [], + ['workflow', 'run', 'refresh-docs.yml', '--repo', repo, + '--ref', 'website'] if expected else []) + triggers = self.website.split('\non:\n', 1)[1].split('\npermissions:', 1)[0] + self.assertIn('workflows: [Publish]', triggers) + self.assertIn('types: [completed]', triggers) + self.assertIn('branches: [main]', triggers) + self.assertIn('paths: [README.md]', triggers) + self.assertIn('GH_TOKEN: ${{ github.token }}', block) + self.assertIn('permissions:\n actions: write', refresh) + + def test_publication_requires_ci_and_checks_of_the_uploaded_distribution(self): + checks = job(self.publish, 'checks') + build = job(self.publish, 'build') + package = job(self.publish, 'package-checks') + publish = job(self.publish, 'publish') + for guarded in (checks, build): + self.assertIn("if: github.repository == 'AdamXweb/fxcss'", guarded) + self.assertIn('uses: ./.github/workflows/ci.yml', checks) + self.assertIn('needs: build', package) + self.assertIn('uses: ./.github/workflows/package-check.yml', package) + self.assertIn('artifact-name: dist', package) + needs = re.search(r'^ needs: \[([^\]]+)\]', publish, re.M)[1] + self.assertEqual({name.strip() for name in needs.split(',')}, + {'build', 'checks', 'package-checks'}) + self.assertNotRegex(publish, r'(?m)^ if:') # retain implicit success() + self.assertNotIn('continue-on-error:', self.publish) + self.assertIn('environment: pypi', publish) + self.assertIn('id-token: write', publish) + self.assertIn('permissions:\n contents: read', self.publish) + self.assertIn('tags: [\'v*\']', self.publish) + self.assertIn('workflow_dispatch:', self.publish) + self.assertLess(build.index('Validate release version'), build.index('actions/upload-artifact@')) + self.assertRegex(build, r'actions/upload-artifact@[^\n]+\n with:\n name: dist\n path: dist/') + self.assertRegex(publish, r'actions/download-artifact@[^\n]+\n with:\n name: dist\n path: dist/') + self.assertLess(publish.index('gh-action-pypi-publish@'), publish.index('Gate —')) + package_workflow = (WORKFLOWS / 'package-check.yml').read_text() + self.assertIn('workflow_call:\n inputs:\n artifact-name:\n type: string\n required: true', package_workflow) + self.assertIn('name: ${{ inputs.artifact-name }}', package_workflow) + self.assertIn('path: package-dist/', package_workflow) + self.assertIn('python scripts/package_smoke.py --dist package-dist', package_workflow) + ci = (WORKFLOWS / 'ci.yml').read_text() + self.assertIn('workflow_call:', ci) + self.assertRegex(ci, r'compatibility:\n\s+description: [^\n]+\n\s+type: boolean\n\s+default: false') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_smoke_regressions.py b/tests/test_smoke_regressions.py new file mode 100644 index 0000000..590ab60 --- /dev/null +++ b/tests/test_smoke_regressions.py @@ -0,0 +1,277 @@ +"""Mocked coverage for the fixtures and startup diagnostics the smoke runs rely on. + +The smoke modules need a real Firefox. These tests pin the parts that do not: +what the fixture themes contain, how install and try place their files, how a +fixture navigation timeout is recorded and classified, and how the Windows +process check reads tasklist. +""" +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, call, patch + +from fxcss import core, fetch, install +from tests import smoke_startup, smoke_variants +from tests.theme_fixtures import EXPECTED_LAYOUTS, SHIPPED_CUSTOM_CHROME, write_themes + +# Verbatim from the windows-latest run of PR #85 at 1ebe0c9 (CI run +# 41004012303): the content café session's first fixture navigation timed out +# a few seconds after a fresh start, with no record of the tab's state. +CI_TIMEOUT = ("WebDriver:Navigate failed: {'error': 'timeout', 'message': " + "'Navigation timed out after 30000 ms', 'stacktrace': " + "'RemoteError@chrome://remote/content/shared/RemoteError.sys.mjs:14:8'}") +DISCARDED = ("WebDriver:Navigate failed: {'error': 'no such window', " + "'message': 'Browsing context has been discarded'}") +UNATTACHED = {"url": "about:blank", "remoteType": None, "process": None, + "document": "about:blank", "discarded": False, "loading": True, + "busy": True, "tabs": 2, "contentProcesses": 0} +ATTACHED = dict(UNATTACHED, remoteType="file", process=4321, contentProcesses=3) + + +class FixtureNavigationTimeoutTests(unittest.TestCase): + def navigate(self, outcomes, state=ATTACHED): + session = core.Session.__new__(core.Session) + session.m = Mock() + session.m.command.side_effect = outcomes + if isinstance(state, Exception): + session.m.script.side_effect = state + else: + session.m.script.return_value = state + with patch.object(core.time, "sleep"): + with self.assertRaises(core.MarionetteError) as raised: + session._navigate_fixture_tab("new-tab", "file:///start.html") + return session, raised.exception + + def test_a_timeout_records_the_tab_state_and_is_not_retried_in_place(self): + session, exc = self.navigate([None, core.MarionetteError(CI_TIMEOUT)], UNATTACHED) + self.assertIsInstance(exc, core.FixtureNavigationStall) + self.assertEqual(exc.state, UNATTACHED) + self.assertIn("file:///start.html", str(exc)) + self.assertIn("Navigation timed out after 30000 ms", str(exc)) + self.assertIn("'process': None", str(exc)) + # One navigation only: the deadline was already spent on it. + self.assertEqual(session.m.command.call_args_list, [ + call("WebDriver:SwitchToWindow", {"handle": "new-tab"}), + call("WebDriver:Navigate", {"url": "file:///start.html"}), + ]) + session.m.script.assert_called_once_with(core.FIXTURE_TAB_STATE) + # The state is read in chrome context and the caller gets chrome back. + self.assertEqual(session.m.set_context.call_args_list[-2:], + [call("chrome"), call("chrome")]) + + def test_unavailable_tab_state_is_reported_and_never_retried(self): + session, exc = self.navigate( + [None, core.MarionetteError(CI_TIMEOUT)], + core.MarionetteError("WebDriver:ExecuteScript failed: gone")) + self.assertIsInstance(exc, core.FixtureNavigationStall) + self.assertIsNone(exc.state) + self.assertIn("tab state unavailable", str(exc)) + self.assertFalse(exc.unattached) + self.assertFalse(core._is_startup_race(exc)) + session.m.set_context.assert_called_with("chrome") + + def test_discarded_contexts_still_refresh_the_same_tab(self): + session, exc = self.navigate([None, core.MarionetteError(DISCARDED)] * 3) + self.assertNotIsInstance(exc, core.FixtureNavigationStall) + self.assertEqual(session.m.command.call_count, 6) + session.m.script.assert_not_called() + + def test_other_navigation_errors_surface_unchanged(self): + error = core.MarionetteError("WebDriver:Navigate failed: {'error': 'unknown error'}") + session, exc = self.navigate([None, error]) + self.assertIs(exc, error) + self.assertEqual(session.m.command.call_count, 2) + session.m.script.assert_not_called() + + def test_only_a_tab_without_a_content_process_counts_as_a_startup_race(self): + self.assertTrue(core._is_navigation_timeout(core.MarionetteError(CI_TIMEOUT))) + self.assertFalse(core._is_navigation_timeout(core.MarionetteError(DISCARDED))) + self.assertFalse(core._is_navigation_timeout(core.MarionetteError( + "WebDriver:ExecuteScript failed: {'error': 'timeout'}"))) + stalled = core.FixtureNavigationStall("stalled", UNATTACHED) + self.assertTrue(stalled.unattached) + self.assertTrue(core._is_startup_race(stalled)) + for state in (ATTACHED, None, "unavailable", {}): + with self.subTest(state=state): + exc = core.FixtureNavigationStall("stalled", state) + self.assertFalse(exc.unattached) + self.assertFalse(core._is_startup_race(exc)) + # The bare timeout text, without tab state, is still not a race. + self.assertFalse(core._is_startup_race(core.MarionetteError(CI_TIMEOUT))) + + def test_setup_retries_an_unattached_stall_within_the_existing_budget(self): + stall = core.FixtureNavigationStall("stalled", UNATTACHED) + operation = Mock(side_effect=[stall, stall, "ready"]) + sleeps = [] + result = core._retry_startup_race(operation, sleep=sleeps.append) + self.assertEqual(result, "ready") + self.assertEqual(operation.call_count, 3) + self.assertEqual(sleeps, list(core.STARTUP_RACE_DELAYS[:2])) + + def test_a_stall_with_a_content_process_spends_no_retry_budget(self): + stall = core.FixtureNavigationStall("stalled", ATTACHED) + sleeps = [] + with self.assertRaises(core.FixtureNavigationStall) as raised: + core._retry_startup_race(Mock(side_effect=stall), sleep=sleeps.append) + self.assertIs(raised.exception, stall) + self.assertEqual(sleeps, []) + + def test_the_retry_budget_is_still_finite_for_a_persistent_stall(self): + stall = core.FixtureNavigationStall("stalled", UNATTACHED) + operation = Mock(side_effect=stall) + sleeps = [] + with self.assertRaises(core.FixtureNavigationStall): + core._retry_startup_race(operation, sleep=sleeps.append) + self.assertEqual(operation.call_count, len(core.STARTUP_RACE_DELAYS) + 1) + self.assertEqual(sleeps, list(core.STARTUP_RACE_DELAYS)) + + def test_a_stall_is_reported_as_a_marionette_error_without_a_traceback(self): + from fxcss import cli + import contextlib, io + stderr = io.StringIO() + error = core.FixtureNavigationStall("fixture page did not load", ATTACHED) + with patch.object(cli, "cmd_doctor", side_effect=error), contextlib.redirect_stderr(stderr): + self.assertEqual(cli.main(["doctor"]), 2) + self.assertIn("error: fixture page did not load", stderr.getvalue()) + self.assertNotIn("Traceback", stderr.getvalue()) + + +class ShippedCustomChromeFixtureTests(unittest.TestCase): + def setUp(self): + self.td = tempfile.TemporaryDirectory(prefix="fxcss-custom-chrome-") + self.root = Path(self.td.name) + self.themes = {theme.name: theme for theme in write_themes(self.root / "themes")} + self.theme = self.themes["shipped custom chrome"] + + def tearDown(self): + self.td.cleanup() + + def test_the_fixture_ships_and_imports_its_own_custom_chrome(self): + self.assertEqual(list(self.themes), list(EXPECTED_LAYOUTS)) + self.assertEqual((self.theme / "chrome" / "customChrome.css").read_text(encoding="utf-8"), + SHIPPED_CUSTOM_CHROME) + self.assertIn('@import "customChrome.css"', + (self.theme / "chrome" / "userChrome.css").read_text(encoding="utf-8")) + self.assertNotEqual(SHIPPED_CUSTOM_CHROME, core.CUSTOM_CHROME_PLACEHOLDER) + + def test_preview_profile_and_live_reload_copies_keep_the_shipped_rules(self): + profile = self.root / "preview" + core.build_profile(self.theme, profile) + self.assertEqual((profile / "chrome" / "customChrome.css").read_text(encoding="utf-8"), + SHIPPED_CUSTOM_CHROME) + live = self.root / "live-1" + import shutil + shutil.copytree(self.theme / "chrome", live) + core._ensure_custom_chrome(self.theme / "chrome", live) + self.assertEqual((live / "customChrome.css").read_text(encoding="utf-8"), + SHIPPED_CUSTOM_CHROME) + # The courtesy placeholder still appears for a theme that only imports it. + importing = self.themes["nested imports"] + (importing / "chrome" / "userChrome.css").write_text( + '@import "customChrome.css";\n', encoding="utf-8") + core.build_profile(importing, self.root / "importing") + self.assertEqual((self.root / "importing" / "chrome" / "customChrome.css").read_text( + encoding="utf-8"), core.CUSTOM_CHROME_PLACEHOLDER) + + def test_install_keeps_the_shipped_rules_and_uninstall_removes_them(self): + profile = self.root / "profile" + profile.mkdir() + install.install_theme(self.theme, profile, "local-fixture") + self.assertEqual((profile / "chrome" / "customChrome.css").read_text(encoding="utf-8"), + SHIPPED_CUSTOM_CHROME) + install.uninstall_theme(profile) + self.assertFalse((profile / "chrome").exists()) + + +class VariantFixtureTests(unittest.TestCase): + def setUp(self): + self.td = tempfile.TemporaryDirectory(prefix="fxcss-variant-fixture-") + self.root = Path(self.td.name) + self.theme, (self.first, self.last) = smoke_variants.write_variant_theme(self.root) + + def tearDown(self): + self.td.cleanup() + + def test_the_optional_sheet_reaches_outside_chrome(self): + self.assertEqual(self.first.parent, self.theme / "custom") + nested = self.theme / "custom" / "parts" / "option extra.css" + self.assertIn('@import "parts/option extra.css"', self.first.read_text(encoding="utf-8")) + self.assertIn('url("../assets/option.svg")', nested.read_text(encoding="utf-8")) + self.assertNotEqual(smoke_variants.BASE_IMAGE_SIZE, smoke_variants.OPTION_IMAGE_SIZE) + for path, size in (((self.theme / "chrome" / "assets" / "probe.svg"), smoke_variants.BASE_IMAGE_SIZE), + ((self.theme / "custom" / "assets" / "option.svg"), smoke_variants.OPTION_IMAGE_SIZE)): + text = path.read_text(encoding="utf-8") + self.assertIn(f'width="{size[0]}" height="{size[1]}"', text) + + def test_install_carries_the_nested_sheet_and_image_beside_the_copied_sheet(self): + profile = self.root / "installed" + profile.mkdir() + install.install_theme(self.theme, profile, "smoke", sheets=[self.first, self.last]) + chrome = profile / "chrome" + self.assertEqual((chrome / "custom" / "parts" / "option extra.css").read_bytes(), + (self.theme / "custom" / "parts" / "option extra.css").read_bytes()) + self.assertEqual((chrome / "custom" / "assets" / "option.svg").read_bytes(), + (self.theme / "custom" / "assets" / "option.svg").read_bytes()) + self.assertTrue((chrome / "custom" / "blue option.css").is_file()) + self.assertTrue((chrome / "custom" / "last.css").is_file()) + entry = (chrome / "userChrome.css").read_text(encoding="utf-8") + self.assertEqual([line for line in entry.splitlines() if line.startswith("@import")], [ + '@import "fxcss-base-userChrome.css";', + '@import "custom/blue%20option.css";', + '@import "custom/last.css";', + ]) + + def test_try_layers_the_sheets_in_place_without_copying_their_files(self): + profile = self.root / "preview" + core.build_profile(self.theme, profile) + fetch.apply_variants(profile, [self.first, self.last]) + chrome = profile / "chrome" + self.assertFalse((chrome / "custom").exists()) + entry = (chrome / "userChrome.css").read_text(encoding="utf-8") + self.assertEqual([line for line in entry.splitlines() if line.startswith("@import")], [ + '@import "fxcss-base-userChrome.css";', + f'@import "{self.first.resolve().as_uri()}";', + f'@import "{self.last.resolve().as_uri()}";', + ]) + + def test_image_urls_are_read_from_computed_values(self): + self.assertEqual(smoke_variants.image_url('url("file:///a/b%20c.svg")'), "file:///a/b%20c.svg") + with self.assertRaises(AssertionError): + smoke_variants.image_url("none") + + +class WindowsProcessCheckTests(unittest.TestCase): + def run_check(self, pids, alive): + calls = [] + + def fake_run(args, **kwargs): + calls.append(args) + pid = int(args[2].split()[-1]) + if pid in alive: + stdout = f'"firefox.exe","{pid}","Console","1","200,000 K"\r\n' + else: + stdout = "INFO: No tasks are running which match the specified criteria.\r\n" + return Mock(stdout=stdout, returncode=0) + + result = smoke_startup.windows_processes_alive(pids, run=fake_run) + return result, calls + + def test_every_recorded_pid_is_queried(self): + result, calls = self.run_check([111, 222, 333], alive=set()) + self.assertEqual(result, []) + self.assertEqual([args[2] for args in calls], ["PID eq 111", "PID eq 222", "PID eq 333"]) + self.assertTrue(all(args[:2] == ["tasklist", "/FI"] and args[3:] == ["/NH", "/FO", "CSV"] + for args in calls)) + + def test_a_replaced_browser_that_survives_is_reported(self): + result, _ = self.run_check([111, 222, 333], alive={111}) + self.assertEqual(result, [111]) + + def test_a_pid_that_merely_appears_in_another_column_does_not_count(self): + def fake_run(args, **kwargs): + return Mock(stdout='"firefox.exe","999","Console","111","1 K"\r\n') + self.assertEqual(smoke_startup.windows_processes_alive([111], run=fake_run), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_threshold_and_variant_migration.py b/tests/test_threshold_and_variant_migration.py new file mode 100644 index 0000000..67bbfd2 --- /dev/null +++ b/tests/test_threshold_and_variant_migration.py @@ -0,0 +1,151 @@ +"""Exact visual limits and migration of colliding optional-sheet names.""" + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from PIL import Image +from fxcss import capture, check, cli, compare, core + + +class ThresholdAndVariantMigrationTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.base, self.head, self.out = [self.root / n for n in ("base", "head", "out")] + self.base.mkdir() + self.head.mkdir() + + def image(self, directory, name="light-01-window", size=(32, 16), changed=0): + image = Image.new("RGB", size, "grey") + for pixel in range(changed): + image.putpixel((pixel % size[0], pixel // size[0]), (255, 0, 0)) + image.save(directory / f"{name}.png") + + def run_compare(self, **options): + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(compare.run(self.base, self.head, self.out, "test", **options), 0) + return json.loads((self.out / "summary.json").read_text()) + + def test_one_pixel_on_a_large_capture_exceeds_zero_and_small_limits(self): + size = (2560, 960) + self.image(self.base, size=size) + self.image(self.head, size=size, changed=1) + report = self.run_compare() + view, = report["views"] + self.assertEqual(view["percent"], 0) + self.assertEqual(view["changed_pixels"], 1) + self.assertEqual(view["total_pixels"], size[0] * size[1]) + self.assertTrue(report["any_change"]) + self.assertEqual(view["image"], "light-01-window.png") + self.assertTrue((self.out / view["image"]).is_file()) + for threshold, status in ((None, 0), (0, 1), (0.00004, 1), (0.00005, 0)): + with self.subTest(threshold=threshold): + self.assertEqual(check.comparison_status(report, threshold), status) + + def test_zero_changes_pass_a_zero_limit(self): + self.image(self.base) + self.image(self.head) + report = self.run_compare() + self.assertFalse(report["any_change"]) + self.assertEqual(check.comparison_status(report, 0), 0) + + def test_equality_passes_and_a_smaller_limit_fails(self): + for size, changed, equal, smaller in ( + ((1000, 100), 1, 0.001, 0.00099), + ((100, 100), 57, 0.57, 0.56999)): + with self.subTest(threshold=equal): + self.image(self.base, size=size) + self.image(self.head, size=size, changed=changed) + report = self.run_compare() + self.assertEqual(check.comparison_status(report, equal), 0) + self.assertEqual(check.comparison_status(report, smaller), 1) + + def test_display_minimum_does_not_hide_a_limit_violation(self): + self.image(self.base, size=(1000, 100)) + self.image(self.head, size=(1000, 100), changed=1) + report = self.run_compare(min_changed_percent=0.01) + self.assertEqual(len(report["below_threshold"]), 1) + self.assertEqual(report["changed_views"], []) + self.assertEqual(check.comparison_status(report, 0), 1) + + def test_legacy_summaries_fall_back_to_percent(self): + for counts in ({}, {"changed_pixels": 1}, {"total_pixels": 100}, + {"changed_pixels": 0, "total_pixels": 0}): + with self.subTest(counts=counts): + report = {"only_in_base": [], "only_in_head": [], + "views": [{"percent": 0.25, **counts}]} + self.assertEqual(check.comparison_status(report, 0), 1) + self.assertEqual(check.comparison_status(report, 0.25), 0) + + def test_missing_views_fail_even_without_a_limit(self): + self.image(self.base) + self.image(self.head) + self.image(self.base, "variant-removed") + report = self.run_compare() + self.assertEqual(report["only_in_base"], ["variant-removed"]) + for threshold in (None, 0, 0.001, 100): + with self.subTest(threshold=threshold): + self.assertEqual(check.comparison_status(report, threshold), 1) + + def migrated_theme(self): + theme = self.root / "theme" + (theme / "chrome").mkdir(parents=True) + (theme / "chrome/userChrome.css").write_text("/* theme */\n") + (theme / "custom").mkdir() + custom = theme / "custom/tabs.css" + custom.write_text("/* custom tabs */\n") + old = core.find_variant_sheets(theme) + self.assertEqual(old, {"tabs": custom}) + # Adding a cross-folder collision renames the existing slug too. Old + # selections and baselines must be migrated, never silently reassigned. + (theme / "optional").mkdir() + optional = theme / "optional/tabs.css" + optional.write_text("/* optional tabs */\n") + new = core.find_variant_sheets(theme) + self.assertEqual(new, {"custom-tabs": custom, "optional-tabs": optional}) + return theme, old, new + + def test_renamed_variant_fails_the_old_baseline(self): + _, old, new = self.migrated_theme() + for directory in (self.base, self.head): + self.image(directory) + for slug in old: + self.image(self.base, f"variant-{slug}") + for slug in new: + self.image(self.head, f"variant-{slug}") + report = self.run_compare() + self.assertEqual(report["only_in_base"], ["variant-tabs"]) + self.assertEqual(report["only_in_head"], ["variant-custom-tabs", "variant-optional-tabs"]) + for threshold in (None, 0, 100): + self.assertEqual(check.comparison_status(report, threshold), 1) + # An explicit old comparison filter still sees the missing base view. + selected = self.run_compare(views="variant-tabs") + self.assertEqual(selected["only_in_base"], ["variant-tabs"]) + self.assertEqual(check.comparison_status(selected, None), 1) + + def test_old_explicit_selections_fail_before_browser_lookup(self): + theme, _, new = self.migrated_theme() + for option, value, expected in ( + ("--variants", "tabs", "no optional stylesheet named tabs"), + ("--views", "variant-tabs", "no view matches variant-tabs")): + with self.subTest(option=option): + errors = io.StringIO() + with patch.object(check.core, "find_firefox") as browser: + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(errors): + status = cli.main(["check", "--theme", str(theme), option, value]) + browser.assert_not_called() + self.assertEqual(status, 2) + self.assertIn(expected, errors.getvalue()) + for slug in new: + self.assertIn(slug, errors.getvalue()) + self.assertIn("variant-custom-tabs", capture.expected_views(new)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workflows.py b/tests/test_workflows.py index f22f1d6..ce9820a 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -68,16 +68,39 @@ def setUp(self): def workflow(self, name): return (self.theme / ".github/workflows" / name).read_text() - def test_supported_options_and_preferences_trigger_both_preview_workflows(self): + def test_all_dependencies_trigger_both_preview_workflows(self): paths = [f"{name}/{name}-option.css" for name in VARIANT_DIRS] - paths += ["chrome/parts/toolbar.css", "configuration/user.js"] + paths += ["chrome/parts/toolbar.css", "configuration/user.js", + "shared/palette.css", "lib/deep/import.css", "theme.svg", + "art/background.png", "fonts/ui.woff2", "unusual directory/icon.svg", + "README.md"] for name in ("pr-preview.yml", "readme-previews.yml"): text = self.workflow(name) - patterns = re.findall(r"^ - '([^']+)'$", text, re.M) - self.assertNotIn("__FXCSS_", text) + triggers = text.split("\non:\n", 1)[1].split("\npermissions:", 1)[0] + self.assertNotRegex(triggers, r"(?m)^\s+paths(?:-ignore)?:") + self.assertIn("all repository changes", text) + self.assertNotIn("__FXCSS_THEME_PATHS__", text) + # No path filter means a single changed dependency triggers a run, + # even if none of the conventional theme folders changed. + patterns = re.findall(r"^ - '([^']+)'$", triggers, re.M) for path in paths: with self.subTest(workflow=name, path=path): - self.assertTrue(any(fnmatch.fnmatch(path, pattern) for pattern in patterns)) + self.assertTrue(not patterns or any( + fnmatch.fnmatch(path, pattern) for pattern in patterns)) + if name == "pr-preview.yml": + self.assertIn("types: [opened, synchronize, reopened]", triggers) + self.assertIn("permissions:\n contents: read", text) + self.assertNotIn("pull_request_target:", triggers) + else: + self.assertIn("branches: [master, main]", triggers) + self.assertIn("workflow_dispatch:", triggers) + + def test_preview_docs_explain_dependency_trigger_policy(self): + readme = (Path(__file__).resolve().parents[1] / "README.md").read_text() + self.assertIn("without path filters", readme) + self.assertIn("dependencies can live anywhere", readme) + self.assertNotIn("Review the filters if you keep theme assets elsewhere", readme) + self.assertNotIn("Matching theme changes", readme) def test_every_generated_workflow_has_its_placeholders_filled(self): for path in (self.theme / ".github/workflows").iterdir(): @@ -316,17 +339,12 @@ def test_support_findings_from_the_artifact_are_revalidated(self): {"format": 1, "status": "checked", "current": "158", "oldest": 153})) def test_sensitivity_check_requires_pixels_not_just_a_missing_view(self): - ci = (Path(__file__).resolve().parents[1] / ".github/workflows/ci.yml").read_text() - body = script(ci, "Detects a real change") - code = body.split("python - <<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] - output = self.root / "out/changed" - output.mkdir(parents=True) - (output / "summary.json").write_text(json.dumps({ - "any_change": True, "changed_views": [], "views": [], "only_in_base": ["variant-old"]})) - result = subprocess.run([sys.executable, "-c", code], cwd=self.root, - capture_output=True, text=True) - self.assertEqual(result.returncode, 1, result.stderr) - self.assertIn("no pixel difference", result.stdout) + from tests.smoke_capture_states import real_change_report + passed, lines = real_change_report({ + "any_change": True, "changed_views": [], "views": [], + "only_in_base": ["variant-old"]}) + self.assertFalse(passed) + self.assertIn("no pixel difference", "\n".join(lines)) @unittest.skipUnless(shutil.which("bash") and shutil.which("git"), "the CI scope step needs bash and Git") diff --git a/tests/theme_fixtures.py b/tests/theme_fixtures.py index 5944e86..fc42a3f 100644 --- a/tests/theme_fixtures.py +++ b/tests/theme_fixtures.py @@ -1,6 +1,20 @@ """Small original themes covering layouts used by Firefox CSS authors.""" from pathlib import Path +# The computed `--fxcss-layout` each fixture theme produces once its chrome/ +# and optional sheets are fully applied, in write_themes order. +EXPECTED_LAYOUTS = { + "nested imports": "nested", + "content café": "content", + "stacked options": "last", + "shipped custom chrome": "custom", +} + +# What a theme that ships its own customChrome.css puts in it. fxcss writes an +# empty placeholder for themes that @import the file without shipping it; this +# content must never be replaced by that placeholder. +SHIPPED_CUSTOM_CHROME = ':root { --fxcss-custom-chrome: shipped; }\n' + def write_themes(root): layouts = { @@ -19,7 +33,12 @@ def write_themes(root): "optional/first.css": ':root { --fxcss-layout: first; }\n', "optional/last.css": ':root { --fxcss-layout: last; }\n', }, + "shipped custom chrome": { + "chrome/userChrome.css": '@import "customChrome.css";\n:root { --fxcss-layout: custom; }\n', + "chrome/customChrome.css": SHIPPED_CUSTOM_CHROME, + }, } + assert list(layouts) == list(EXPECTED_LAYOUTS) result = [] for name, files in layouts.items(): theme = Path(root) / name