From fa12f99f474b86ffbe56ae9167e1e43b920f2524 Mon Sep 17 00:00:00 2001 From: adamXbot <111877622+adamXbot@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:58:01 +1100 Subject: [PATCH 1/2] Preserve theme customizations and plan maintenance across repository layouts Plan flat, nested, content-only, named and packaged layouts; preserve user additions and option identities; track immutable source revisions and explicit preferences; and generate workflows and patches for the original repository layout. Integrate the automation coverage now on main. Keep repository-wide preview triggers alongside nested command paths and toolkit commit pins. Remove obsolete path-filter plumbing and update optional-sheet smoke assertions to locate dependencies relative to the allocated option directory. Validation: 683 unit tests passed, one skipped. All 65 focused workflow, maintenance and smoke regressions passed with Pillow 10.1.0; Python 3.9 grammar and whitespace checks passed. Real Firefox validation remains in CI. --- README.md | 193 +++++- fxcss/__init__.py | 2 +- fxcss/adopt.py | 6 + fxcss/audit.py | 125 +++- fxcss/checklist.py | 307 ++++++++++ fxcss/cli.py | 779 +++++++++++++++++++------ fxcss/core.py | 24 +- fxcss/fetch.py | 648 ++++++++++++++++---- fxcss/install.py | 313 ++++++++-- fxcss/inventory.py | 449 ++++++++++++++ fxcss/layout.py | 767 ++++++++++++++++++++++++ fxcss/scaffold.py | 97 ++- fxcss/support.py | 6 +- tests/smoke_variants.py | 8 +- tests/test_connection_lifecycle.py | 12 +- tests/test_inventory.py | 217 +++++++ tests/test_layout.py | 250 ++++++++ tests/test_maintenance_cli.py | 475 +++++++++++++++ tests/test_maintenance_preservation.py | 254 ++++++++ tests/test_review_maintenance.py | 280 +++++++++ tests/test_smoke_regressions.py | 16 +- tests/test_units.py | 50 +- 22 files changed, 4819 insertions(+), 459 deletions(-) create mode 100644 fxcss/checklist.py create mode 100644 fxcss/inventory.py create mode 100644 fxcss/layout.py create mode 100644 tests/test_inventory.py create mode 100644 tests/test_layout.py create mode 100644 tests/test_maintenance_cli.py create mode 100644 tests/test_maintenance_preservation.py create mode 100644 tests/test_review_maintenance.py diff --git a/README.md b/README.md index 45764ae..229e900 100644 --- a/README.md +++ b/README.md @@ -107,18 +107,62 @@ It reports what it found before doing anything: Releases are preferred over branch tips, since that is what the author blessed; `--commit` takes the latest commit instead, and `--ref` takes any tag, branch or -SHA. `--with name,name` layers on the theme's optional stylesheets so you can see -a variant without hunting through install flags. `--shot dir` captures the -standard screenshots instead of opening a window, and `--keep dir` leaves the -download behind so you can start editing it with `watch`. +SHA. A browser URL for a branch or folder +(`https://github.com/owner/theme/tree/photon-style/themes/one`) is read as +meaning that branch and that folder, and the branch is then *tracked* rather +than pinned. `--with name,name` layers on the theme's optional stylesheets so +you can see a variant without hunting through install flags. `--shot dir` +captures the standard screenshots instead of opening a window, and `--keep dir` +leaves the download behind so you can start editing it with `watch`. -#### Private repositories and local folders +#### How the theme is found + +Theme repositories are laid out in more ways than `chrome/userChrome.css`. +`try`, `install` and `upgrade` share one *plan* for reading a source tree, and +say which layout they found: + +| Layout | Example | What happens | +| --- | --- | --- | +| `chrome/userChrome.css` at the root | WhiteSur Safari, ShyFox | Used where it is. | +| `userChrome.css` at the root, parts beside it | ArcWTF, Foxlim, the GNOME theme | The folder *is* `chrome/`; documentation, licences, screenshots and install scripts are left out unless a stylesheet refers to them. | +| Only `userContent.css` | GlassyFox | Installed as a content-only theme; the browser window is left alone. | +| One theme in a subfolder | `firefox/chrome/…`, `NewFox/profile/chrome/…` | Found, and the folder recorded so upgrades read the same place. | +| Several themes side by side | `1.Normal/`, `2.WithTST/`; Simplify's colour families | A real choice: a menu at a terminal, otherwise an error listing the `--subpath` values. Folders that look historical (`old/`, `Old Versions/`, screenshots) or belong to Thunderbird are listed but not chosen. | +| `src/userChrome-.css` files | WhiteSur | Named editions, chosen with `--subpath src/userChrome-WhiteSur.css`; a sibling `common/` folder is merged in the way the theme's installer does, by copying. | +| A committed `chrome.zip` | Win 7 Aero | Unpacked as data within size limits and read like any other tree. `.rar`/`.7z` are reported as unsupported with the reason. | +| No entry stylesheet at all | Brave-Fox, Firefox-Onebar | Unsupported, naming the sheets a README might tell you to `@import`; `--entry path.css` generates a `userChrome.css` that imports one. | + +The planner never selects a directory outside the download, and nothing from +the download is executed: a theme whose entry sheet imports files the source +does not contain (Material Fox's uncompiled `user-chrome.css`) is reported as +needing a build rather than installed half-empty. `install` stops there unless +`--force`; `try` only warns. + +`try --info` ends with a **setup checklist**: what the theme needs beyond its +files, read from the files it ships and its README — preference files outside +`configuration/user.js` (with their pref names), `.uc.js` scripts that need a +userChrome.js loader, extension settings to import, fonts, forks and platform +effects. Each item is *required*, *optional* or *unknown*, with the evidence. +Nothing on it is applied automatically; see `install --prefs` below. + +#### Private repositories, other hosts and local folders A private repository looks exactly like a missing one to anyone who cannot read it. fxcss sends `GITHUB_TOKEN` or `GH_TOKEN` when either is set; otherwise, if a -repository is not publicly visible, it asks the GitHub CLI for its login -(`gh auth token`) and tries again. So after `gh auth login`, private themes your -account can read work like public ones. A token also raises GitHub's rate limit. +repository is not publicly visible or an anonymous request hits GitHub's rate +limit, it asks the GitHub CLI for its login (`gh auth token`) and tries again. +So after `gh auth login`, private themes your account can read work like public +ones and a signed-in person is never told to sign in. The token is only ever +sent to GitHub's own hosts — never to another host, and never across a redirect +that leaves GitHub — and never appears in a message. + +Themes on **Codeberg**, **git.gay** (and other Forgejo instances listed in +`fetch.HOSTS`) and **GitLab** can be tried, installed and upgraded from their +URLs; their public APIs answer the same questions. `adopt` compares against +GitHub's tree API and says so when it meets another host. A repository that +has **moved** on GitHub resolves to its new owner/name, is downloaded from +there (the old name still answers on the API but not on the download host) and +is recorded under the new name. A theme that is not on GitHub, or not pushed yet, can be tried from its folder: `fxcss try ~/src/my-theme`. The folder is read where it is and never modified; @@ -158,6 +202,8 @@ profile. fxcss install owner/theme # into your default profile fxcss install owner/theme --with compact-tabs # optional sheets, permanently fxcss install ~/src/my-theme # a local checkout works too +fxcss install owner/theme --subpath 2.WithTST # one of several editions +fxcss install owner/theme --prefs user.js # also a shipped preference file fxcss install --list-profiles # see what it found first ``` @@ -284,9 +330,32 @@ The install is what a theme's install script does, done carefully: together with any `configuration/user.js` the theme ships, whose pref names are listed before you confirm. Everything outside the block is kept byte for byte; +- shipped preference files elsewhere (a root `user.js`, `config/user.js`, + `prefs/recommended.js`) appear on the setup checklist with their pref + names and are **not** applied unless you say `--prefs user.js`; then they + join the same removable block, as written, and upgrades carry the choice + forward (stopping if the file disappears upstream). Nothing is ever applied + from README prose or from `-moz-pref()` toggles in the CSS; - a manifest (`chrome/fxcss-install.json`) records every file written, its - sha256, and where the theme came from — which repo, which ref, and whether - that ref was a release or a branch. + sha256, where the theme came from — which host, repo and ref, whether that + ref was a release or a branch, and the full commit a branch pointed at — + plus the layout selected, the preference files applied and each optional + sheet's path in the repository. + +When several options have the same name, select their repository paths, for +example `--with custom/tabs.css,optional/tabs.css`. Each retains its identity +through upgrades. Options loaded after the base get separate dependency trees +when combined. If two selected files need the same native import slot, or +would overwrite different contents already there, installation refuses before +changing the profile, including with `--force`. + +Tracked branches are downloaded at the resolved commit SHA, and adoption +compares that exact tree. If the full SHA cannot be read, fxcss stops instead +of recording a branch name as a known revision. + +Installing over a profile fxcss already manages keeps the files you added +there active (see [upgrade](#fxcss-upgrade)); installing over a `chrome/` +folder fxcss did not write moves it aside whole, as the origin backup. Restart Firefox after installation; it reads `userChrome.css` at startup. See [uninstall](#fxcss-uninstall) to remove a theme later. @@ -332,7 +401,15 @@ Naming the repository is usually necessary. `adopt` checks `chrome/` for a git remote (definitive — someone cloned it there) and for GitHub URLs in the theme's own files (a hint worth confirming), but plenty of themes leave no trace at all once installed: WhiteSur's `chrome/` contains no URL anywhere. -That is normal, not a failure, and the message says so. +That is normal, not a failure, and the message says so. A repository that was +only *mentioned* — a credit, a palette, an upstream — is recorded only when at +least half of a version's files match; otherwise `adopt` refuses and asks you +to name the repository yourself, because recording the wrong project hands +every later upgrade to it. + +A match against a branch records the commit the branch pointed at, not its +name, so `upgrade --check` straight after adopting an exact branch tip says +"up to date" instead of announcing the same commit as an update. **Nothing is installed or replaced.** The `chrome/` already there is *copied* to a `chrome.backup-*` and then described, so `uninstall` has somewhere to put @@ -392,12 +469,15 @@ profile with no theme at all. Only the first can be described in detail; for the second, all fxcss honestly knows is that files are there and that installing would move them aside. -`--check` asks GitHub once per theme, not once per profile, and compares like +`--check` asks the host once per theme, not once per profile, and compares like for like: an install tracking releases is measured against the newest tag, one -tracking a branch against the commit that branch points at now. An install -pinned with `--ref` reports as pinned rather than as behind. Where the -manifest predates fxcss recording which of those applied, it says so instead -of guessing — "up to date" is never printed unless it was actually checked. +tracking a branch against the commit that branch points at now (a branch other +than the default is looked up by name). An install pinned with `--ref` reports +as pinned rather than as behind. A release-tracked install that is current +while the default branch has moved on says both — and stays on releases; +`upgrade --commit` is the way to follow the branch instead. Where the manifest +predates fxcss recording which of those applied, it says so instead of +guessing — "up to date" is never printed unless it was actually checked. The same reservation covers local edits: installs from before 0.16 recorded no file hashes, so `fxcss profiles` reports them as *not checked for edits* @@ -419,15 +499,25 @@ fxcss rollback # …and back again tag, one that followed a branch moves to that branch's current commit, and one pinned with `--ref` does not move at all unless you say so. -It stops rather than surprise you, in three places: +It stops rather than surprise you, in four places: - **Files you edited yourself.** Every install records a sha256 per file, so an upgrade knows which ones you have since changed and refuses to write over - them until `--force`. Files you *added* are never touched either way. + them until `--force`. +- **Files you added.** A stylesheet you dropped into `chrome/` — imported + from `customChrome.css`, say — is carried into the new tree so it stays + *active*, not merely kept in the backup where it would stop applying. The + manifest lists them under `user_files`. If the new version ships a file at + the same path with different contents, that is a collision: the upgrade + stops before anything moves and lists the paths; `--force` installs the + theme's copies and leaves yours in the backup, recorded as `displaced`. - **Options that vanished.** If you installed `--with theme-nord` and the new version renamed or dropped that sheet, the `@import` would simply stop resolving and the option would turn itself off. `upgrade` names the loss and - makes you choose instead. + makes you choose instead. Sheets are matched by their path in the repository + first, so a sheet that only exists inside `chrome/extras/` survives an + upgrade as reliably as one in `custom/`. The same goes for preference files + chosen with `--prefs`. - **Selectors the new version needs and your Firefox lacks** — with `--audit`, which runs the same check as [`fxcss audit`](#fxcss-audit) against the fetched copy *before* anything is installed. @@ -444,8 +534,17 @@ Firefox, like `shot`. `--check` changes nothing and answers with its exit code, for cron, launchd or CI: **0** up to date, **1** an upgrade is available, **2** it cannot be told (no install here, an unreachable repo, or a manifest too old to say what it -tracked). fxcss deliberately ships no scheduler of its own — this is the piece -you point yours at. +tracked). For a theme installed from a local folder there is no upstream to +ask, so the folder's files are compared with what was installed and "up to +date" includes the managed preference block. Explicit `--prefs` and `--with` +choices can also change an installation whose remote revision is current; +`--check` reports whether that proposed setup differs without applying it. +fxcss deliberately ships no scheduler of its own — +this is the piece you point yours at. + +The layout selected at install time (`--subpath`, `--entry`) is recorded and +reused, so a theme taken from `2.WithTST/` is upgraded from `2.WithTST/` and +one taken from `src/userChrome-WhiteSur.css` from the same edition. ```console $ fxcss upgrade @@ -520,6 +619,15 @@ If the profile originally had no `chrome/`, uninstalling after an upgrade returns it to that state when no files need to be kept. Restart Firefox to see the result. +A `chrome/` with no manifest in it is yours — what a rollback to the original +put back, or a theme installed by hand — and `uninstall` leaves it exactly as +it is, however many backups sit beside it: it removes a leftover fxcss block +from `user.js` if there is one, lists the backups, and changes nothing else. +So install → rollback → uninstall ends with your original files in place, and +running it again does the same. If `chrome/` is absent after an interrupted +swap or a completed uninstall, `uninstall` leaves that absent state alone too. +Use `fxcss rollback --to ` explicitly to restore a backup. + ## Building and inspecting themes Work on Firefox's tabs, toolbars and other browser UI with a live preview. @@ -1196,11 +1304,22 @@ exit for CI; `fxcss-keep` lines are exempt. #### Unused and unreachable code -`audit` also reports housekeeping, in its own section, separate from breakage: +`audit` also reports housekeeping, in its own section, separate from breakage. +It reads the theme through the same stylesheet inventory `install` and +`upgrade` use: quoted and unquoted `@import url(…)` forms, escaped paths, +imports behind `-moz-pref()` or media conditions, and the user hooks a theme +imports without shipping (`customChrome.css`, FlexFox's +`uc-user-settings.css`). Sheets reached only from `userContent.css` style web +and extension pages, and their selectors are reported as *other-document* +rather than as browser-window breakage. When an entry sheet is missing or a +required import resolves to nothing, the section opens with **INCOMPLETE** and +the reasons, instead of reading as a clean bill of health. - **Stylesheets nothing imports.** Files under `chrome/` unreachable by - following `@import` from `userChrome.css`. Sheets in a `custom/` or - `optional/` folder are excluded — being opt-in is the point of those. + following `@import` from `userChrome.css` or `userContent.css`. Sheets in a + `custom/`, `optional/`, `extras/` or `tweaks/` folder are excluded — being + opt-in is the point of those — and so are conditional imports that are + merely switched off. - **Custom properties used but never set**, where an unthemed Firefox does not provide them either. These are usually typos: the `var()` silently falls back. - **Custom properties set but read nowhere.** Reported cautiously — setting @@ -1232,11 +1351,27 @@ fxcss init --showcase --previews # also publish screenshots fxcss init --watch --showcase --previews # generate all six workflow files ``` -`init` writes to `.github/workflows/`. Existing files are preserved unless you -pass `--force`; review regenerated files before committing, especially if you -have customised them. Generation pins the installed fxcss version. The preview publisher validates -variant filenames and PNG headers, including options added by later PRs. -Updating fxcss locally does not update workflows already in a repository. +`init` writes to `.github/workflows/` **at the repository root** — found with +`git rev-parse --show-toplevel` from the theme folder — and points every +command in them at the theme's folder when that is not the root +(`fxcss shot --theme 'head/firefox'`). A repository that holds several themes +needs `--subpath` to say which. Flat and nested layouts both generate previews +for all repository changes, including dependencies outside the theme folder. +Existing files are preserved unless you pass `--force`; review regenerated +files before committing, especially if you have customised them. Generation +pins the installed fxcss version. These layout and patch-mapping changes need +fxcss **0.26.0 or later**; a PyPI pin only works after that version is published. +Before release, use `fxcss init --toolkit-ref ` +to install the reviewed toolkit commit directly in generated workflows. This +does not publish a release. The preview +publisher validates variant filenames and PNG headers, including options added +by later PRs. Updating fxcss locally does not update workflows already in a +repository. + +Audit patches for flat and nested themes address their original files relative +to the Git root, so the watch workflow can apply them there. Packaged ZIPs and +assembled editions have no direct source mapping: patch generation refuses +those layouts; audit the unpacked source folder to prepare a patch instead. | Feature | Enable with | When it runs | What readers see | | --- | --- | --- | --- | diff --git a/fxcss/__init__.py b/fxcss/__init__.py index 11563e2..2e2890a 100644 --- a/fxcss/__init__.py +++ b/fxcss/__init__.py @@ -1,3 +1,3 @@ """fxcss — a testing toolkit for Firefox userChrome.css themes.""" -__version__ = "0.25.0" +__version__ = "0.26.0" diff --git a/fxcss/adopt.py b/fxcss/adopt.py index 54f6477..95765b9 100644 --- a/fxcss/adopt.py +++ b/fxcss/adopt.py @@ -40,6 +40,12 @@ GITHUB_URL = re.compile(r"github\.com[/:]([\w.-]+)/([\w.-]+)") +# How much of a version's files must match before a repository that was only +# *mentioned* in the theme's text is recorded as its origin. Evaluation found +# 15 of 109 files matching a credited upstream and the wrong project recorded; +# half is a floor, not a proof, and an explicit owner/name bypasses it. +GUESS_MIN_SCORE = 0.5 + # Only text a human would have written a URL into, and not much of it: this # reads files from a directory fxcss did not create. READABLE_SUFFIXES = {".css", ".md", ".txt", ".json", ".cfg", ".ini"} diff --git a/fxcss/audit.py b/fxcss/audit.py index 2365326..447299a 100644 --- a/fxcss/audit.py +++ b/fxcss/audit.py @@ -161,12 +161,22 @@ def extract_tokens(theme: Path): by renaming individual ids and classes, and a whole selector containing `&` nesting or `::part()` cannot be handed to querySelectorAll anyway. """ + from . import inventory found = {} - for path in sorted((theme / "chrome").rglob("*.css")): + chrome = theme / "chrome" + # Sheets reached only from userContent.css style web and extension + # pages; their selectors are never about the browser window, so they + # are scoped out the way an @-moz-document block for another document + # is, with the reason recorded so the report can say which it was. + scanned = inventory.scan(chrome) if chrome.is_dir() else None + for path in sorted(chrome.rglob("*.css")): try: raw = path.read_text(encoding="utf-8", errors="replace") except OSError: continue + relative = path.relative_to(chrome).as_posix() + content_only = bool(scanned) and relative in scanned["sheets"] \ + and scanned["sheets"][relative]["scope"] == ["content"] # Blank comments across the whole file, not per line: a block comment # spans lines, and a selector named in prose is not a selector. Newlines @@ -200,7 +210,8 @@ def extract_tokens(theme: Path): "file": str(path.relative_to(theme)), "line": index + 1, "text": source_lines[index].rstrip(), - "scoped_out": scoped_out, + "scoped_out": scoped_out or content_only, + "document": "content" if content_only else "chrome", }) return found @@ -370,11 +381,21 @@ def _markup_home(pack, kind, name): return where.rsplit("/", 1)[-1] if where else None -def suggest(token, dom, pack=None, scoped_out=False): - """Infer a replacement for a token that matches nothing.""" +def suggest(token, dom, pack=None, scoped_out=False, document=None): + """Infer a replacement for a token that matches nothing. + + `document` says why a use is scoped out: "content" for a rule that + loads only through userContent.css, which styles web and extension + pages and is never evidence about the browser window. + """ kind, name = token[0], token[1:] ids, classes = dom["ids"], dom["classes"] + if scoped_out and document == "content": + return {"replacement": None, "confidence": "other-document", + "reason": "loaded only through userContent.css, which styles " + "web and extension pages, not the browser window"} + # Every rule using this token is scoped by @-moz-document to a document the # live audit never opens, so the running window's DOM is not evidence about # it either way -- and comparing against it invents renames between @@ -461,10 +482,14 @@ def audit(session, theme: Path, verbose=True, pack=None): # file:line sites a finding carries, so one shared verdict would let a # confident rename reach the sites this audit just admitted it cannot # judge -- the very failure the scope check exists to prevent. - for scoped_out in sorted({bool(use.get("scoped_out")) for use in uses}): + keys = sorted({(bool(use.get("scoped_out")), use.get("document") or "chrome") + for use in uses}) + for scoped_out, document in keys: group = [use for use in uses - if bool(use.get("scoped_out")) is scoped_out] - info = suggest(token, dom, pack=pack, scoped_out=scoped_out) + if bool(use.get("scoped_out")) is scoped_out + and (use.get("document") or "chrome") == document] + info = suggest(token, dom, pack=pack, scoped_out=scoped_out, + document=document) info.update({"token": token, "uses": group}) findings.append(info) @@ -678,7 +703,7 @@ def c(code, s): print() -def write_patch(result, theme: Path, out: Path): +def write_patch(result, theme: Path, out: Path, source_paths=None): """Emit a unified diff of the confident replacements, for review.""" _mark_duplicate_patches(result, theme) edits = {} @@ -703,10 +728,21 @@ def write_patch(result, theme: Path, out: Path): index = line_no - 1 if 0 <= index < len(updated): updated[index] = replace_in_line(updated[index], token, replacement) + target = source_paths[rel] if source_paths is not None else rel + def header(prefix): + name = f"{prefix}/{target}" + # Git's quoted-path syntax is needed for control characters, + # backslashes and literal double quotes in a repository filename. + if any(c in name for c in '\t\n\r\\"'): + import json + return json.dumps(name, ensure_ascii=False) + return name chunks.extend(difflib.unified_diff( - original, updated, fromfile=f"a/{rel}", tofile=f"b/{rel}", lineterm="")) + original, updated, fromfile=header("a"), tofile=header("b"), lineterm="")) - out.write_text("\n".join(chunks) + "\n", encoding="utf-8") + # Git patch syntax uses LF even when generated on Windows. Text-mode + # newline translation would add carriage returns to every context line. + out.write_bytes(("\n".join(chunks) + "\n").encode("utf-8")) return len(edits) @@ -739,30 +775,23 @@ def changelog(before, after, theme_tokens=None): # Directories of deliberately opt-in sheets. A theme ships these expecting its # installer (or the user) to enable them, so "nothing imports it" is the # intended state rather than a finding. -OPTIONAL_DIRS = {"custom", "optional", "options", "extras", "variants"} +from .inventory import OPTION_DIRS as _OPTION_DIRS # noqa: E402 +OPTIONAL_DIRS = set(_OPTION_DIRS) def import_graph(theme: Path): - """Every stylesheet reachable by following @import from the entry sheets.""" + """Every stylesheet reachable by following @import from the entry sheets. + + Read through the shared inventory, so quoted and unquoted imports, + escaped paths and conditional imports all count -- the same reading + install and upgrade use. Returns resolved absolute paths. + """ + from . import inventory chrome = theme / "chrome" - reachable, queue = set(), [] - for entry in ENTRY_SHEETS: - path = chrome / entry - if path.exists(): - queue.append(path.resolve()) - - while queue: - current = queue.pop() - if current in reachable or not current.exists(): - continue - reachable.add(current) - text = COMMENT.sub("", current.read_text(encoding="utf-8", errors="replace")) - for target in IMPORT_RE.findall(text): - target = target.strip() - if re.match(r"^(chrome:|resource:|https?:|data:)", target): - continue - queue.append((current.parent / target).resolve()) - return reachable + if not chrome.is_dir(): + return set() + found = inventory.scan(chrome) + return {inventory._locate(chrome, key).resolve() for key in found["sheets"]} def custom_properties(paths): @@ -829,11 +858,13 @@ def probe_properties(session, names): def collect_unused(theme: Path): """The parts that need no browser: reachability and property bookkeeping.""" + from . import inventory chrome = theme / "chrome" if not chrome.is_dir(): return None - reachable = import_graph(theme) + found = inventory.scan(chrome) + reachable = {(chrome / relative).resolve() for relative in found["sheets"]} orphans, optional = [], [] for sheet in sorted(chrome.rglob("*.css")): if sheet.resolve() in reachable: @@ -853,6 +884,19 @@ def collect_unused(theme: Path): "used": used, "kept": kept, "devtools": devtools, + # What the inventory could and could not settle: an unresolved + # required import or a missing entry means the picture is partial, + # and the report says so instead of reading as clean. + "analysis": { + "complete": found["complete"], + "reasons": list(found["reasons"]), + "entries": list(found["entries"]), + "unresolved": [f"{e['source']} → {e['target']}" for e in found["unresolved"]], + "conditional": [f"{e['source']} → {e['target']}" for e in found["conditional"]], + "hooks": sorted({Path(e["resolved"]).name for e in found["hooks"]}), + "content_only": sorted(rel for rel, s in found["sheets"].items() + if s["scope"] == ["content"]), + }, } @@ -957,6 +1001,7 @@ def _suggest(item, provided=False): "reachable": static["reachable"], "orphans": static["orphans"], "optional": static["optional"], + "analysis": static.get("analysis") or {"complete": True, "reasons": []}, "overrides": len(overrides), "kept": sorted(kept & (defined_only | devtools_only)), "packed": pack is not None, @@ -981,6 +1026,24 @@ def c(code, s): print(f" {c(DIM, 'Housekeeping, not breakage — nothing here stops the theme working.')}") print() + analysis = unused.get("analysis") or {} + if analysis and not analysis.get("complete", True): + print(f" {c(YELLOW, 'INCOMPLETE')} the import graph could not be read in full, " + "so the findings\n below describe only the part that resolved:") + for reason in analysis.get("reasons") or []: + print(f" {reason}") + print() + if analysis.get("conditional"): + count = len(analysis["conditional"]) + print(f" {c(DIM, f'{count} conditional import(s) point at files not shipped; off unless their pref or media applies.')}") + if analysis.get("hooks"): + print(f" {c(DIM, 'User hook(s) the theme imports but does not ship, as intended: ' + ', '.join(analysis['hooks']))}") + if analysis.get("content_only"): + count = len(analysis["content_only"]) + print(f" {c(DIM, f'{count} stylesheet(s) load only through userContent.css (web and extension pages, not the browser window).')}") + if analysis.get("conditional") or analysis.get("hooks") or analysis.get("content_only"): + print() + if unused["orphans"]: print(f" {c(YELLOW, 'NOT IMPORTED')} {len(unused['orphans'])} stylesheet" f"{'s' if len(unused['orphans']) != 1 else ''} nothing reaches") diff --git a/fxcss/checklist.py b/fxcss/checklist.py new file mode 100644 index 0000000..11b352d --- /dev/null +++ b/fxcss/checklist.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""What a theme needs beyond its files, stated as a checklist. + +Copying ``chrome/`` and enabling one pref is a complete installation for +many themes and a fraction of one for many others: a root ``user.js`` with +twenty prefs the README calls required, a Sidebery settings file to import, +a userChrome.js loader, a particular fork, fonts, a compositor. The 132-theme +evaluation found manifests recording hundreds of copied files while every +one of those stayed undone, and nothing in the output said so. + +This module reads the theme's own files and README and reports items in +three states, each with its evidence: + + required the theme's files or wording say it will not work without this + optional offered by the theme; the person decides + unknown mentioned, but whether it is needed cannot be read off the files + +Nothing here is applied. Preference files outside ``configuration/user.js`` +are listed with their pref names so a person can choose them with +``--prefs``; README prose and CSS ``-moz-pref`` toggles are only reported. +That is deliberate: a crawler that applies every pref-like string it finds +would set things nobody asked for, and a copy operation that calls itself +"configured" would be a lie a person acts on. +""" + +import re +from pathlib import Path + +from .install import PREF_NAME_RE, _strip_js_comments + +MAX_README_BYTES = 512 * 1024 +MAX_PREF_FILE_BYTES = 1024 * 1024 + +# Preference files themes ship outside the one location fxcss applies. +# Matched against repository-relative paths, case-insensitively. +PREF_FILE_RE = re.compile( + r"(^|/)((example[-_.]?)?user(\.js|\.js\.example|\.example\.js)" + r"|prefs?(\.js)?|[\w-]*\.user\.js|user[-_][\w-]+\.js)$", re.I) +PREF_DIRS_RE = re.compile(r"(^|/)(prefs?|config|configuration|preferences)/[^/]+\.js$", re.I) +REQUIRED_WORD_RE = re.compile(r"(required|mandatory|needed|must)", re.I) +INSTALL_VERB_RE = re.compile(r"\b(copy|place|put|move|drop|paste|add|save)\b", re.I) +OPTIONAL_WORD_RE = re.compile(r"(optional|recommended|example|suggested)", re.I) + +# Signals read from shipped files, not prose. +LOADER_FILE_RE = re.compile(r"\.(uc\.js|uc\.mjs|sys\.mjs|as\.css)$|(^|/)(config\.js|config-prefs\.js|userChrome\.js)$", re.I) +EXTENSION_CONFIG_RE = re.compile( + r"(^|/)[\w .-]*(sidebery|tst|tree[-_ ]?style[-_ ]?tab|tab[-_ ]?center|" + r"firefox[-_ ]?color|ublock|stylus)[\w .-]*\.(json|css|txt)$", re.I) +FONT_RE = re.compile(r"\.(ttf|otf|woff2?)$", re.I) + +# Things a README can say that files cannot show. Each is (pattern, item, +# state, note). Kept to names whose presence in a theme README is almost +# always about a requirement, and reported as "unknown" unless the sentence +# around it uses a required-word. +README_SIGNALS = ( + (r"\bfx-?autoconfig\b|\buserChrome\.js\b|\bautoconfig\b|\buserChromeJS\b", + "a userChrome.js loader (fx-autoconfig or similar)", + "fxcss does not install loaders; .uc.js scripts stay inert without one"), + (r"\bsidebery\b", "the Sidebery extension (and its settings)", + "fxcss does not install extensions or import their settings"), + (r"\btree ?style ?tabs?\b|\bTST\b", "the Tree Style Tab extension", + "fxcss does not install extensions"), + (r"\btab ?center ?reborn\b", "the Tab Center Reborn extension", + "fxcss does not install extensions"), + (r"\bfirefox ?color\b", "a Firefox Color theme", + "fxcss does not apply Firefox Color themes"), + (r"\bfloorp\b", "Floorp (a Firefox fork)", "verified only on the browser you run"), + (r"\bzen browser\b|\bzen-browser\b", "Zen Browser (a Firefox fork)", + "verified only on the browser you run"), + (r"\blibrewolf\b", "LibreWolf (a Firefox fork)", "verified only on the browser you run"), + (r"\bwaterfox\b", "Waterfox (a Firefox fork)", "verified only on the browser you run"), + (r"\bmica\b|\bwindows 11\b", "Windows 11 (Mica/compositor effects)", + "platform-specific appearance cannot be reproduced elsewhere"), + (r"\bgnome\b|\bkde\b|\bpicom\b|\bhyprland\b|\bcompositor\b", + "a Linux desktop/compositor setup", "transparency and blur depend on it"), + (r"\bvertical tabs?\b|\bsidebar\.verticalTabs\b", "Firefox's native vertical tabs", + "a setting in Firefox's own sidebar options"), + (r"\bfont\b|\bfonts\b|\bnerd font\b", "fonts the theme expects", + "fonts are not installed by fxcss"), + (r"\bdevPixelsPerPx\b|\bui\.scale\b", "a display scaling preference", + "listed because changing it affects every window, not only the theme"), +) +# Pref names in prose: "browser.tabs.allow_transparent_browser" and the like. +README_PREF_RE = re.compile( + r"\b((?:toolkit|browser|layout|svg|widget|ui|sidebar|gfx|devtools|privacy|" + r"network|dom|security|general|extensions|uc|userChrome|userchrome|full-screen-api|" + r"app|font|media|pdfjs|print|signon|places|services|nglayout|mousewheel|" + r"zoom|view_source|image|html5|intl|keyword|middlemouse|permissions|pref|" + r"reader|startup|accessibility|alerts|apz|clipboard|config|content|" + r"cookiebanners|datareporting|identity|javascript|lightweightThemes|" + r"mail|mailnews|findbar|urlclassifier|webgl|xpinstall)\.[A-Za-z0-9_.-]*[A-Za-z0-9_])\b") + +CSS_PREF_RE = re.compile(r"""-moz-(?:bool-)?pref\(\s*["']([^"']+)["']""") + + +def _read(path, limit): + try: + if path.stat().st_size > limit: + return "" + return path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + + +def pref_names_in(path): + """Pref names a JavaScript preference file sets, in order, deduplicated.""" + code = _strip_js_comments(_read(path, MAX_PREF_FILE_BYTES)) + names = [] + for match in PREF_NAME_RE.finditer(code): + name = match.group(2) + if name not in names: + names.append(name) + return names + + +def _walk(root, limit=20000): + root = Path(root) + count = 0 + for path in sorted(root.rglob("*")): + parts = path.relative_to(root).parts + if any(p in (".git", "node_modules", "__pycache__") for p in parts): + continue + count += 1 + if count > limit: + return + if path.is_file() and not path.is_symlink(): + yield path, "/".join(parts) + + +def pref_files(repo_root, theme_root=None): + """Shipped preference files, each with its names and a state guess. + + ``configuration/user.js`` under the theme root is what fxcss applies and + is reported as "applied". Anything else is "unknown" unless its name or + folder says required/recommended/example. The guess is a *label* for the + person; nothing is applied from it. + """ + repo_root = Path(repo_root) + theme_root = Path(theme_root) if theme_root else repo_root + applied = (theme_root / "configuration" / "user.js") + found = [] + for path, relative in _walk(repo_root): + lowered = relative.lower() + if not (PREF_FILE_RE.search(relative) or PREF_DIRS_RE.search(relative)): + continue + if path.suffix.lower() not in (".js", ".example") and not lowered.endswith(".js.example"): + continue + names = pref_names_in(path) + if not names: + continue + if applied.is_file() and path.resolve() == applied.resolve(): + state = "applied" + elif REQUIRED_WORD_RE.search(relative): + state = "required" + elif OPTIONAL_WORD_RE.search(relative): + state = "optional" + else: + state = "unknown" + found.append({"path": relative, "names": names, "state": state, + "count": len(names)}) + return found + + +def readme_text(repo_root): + for name in ("README.md", "readme.md", "README", "README.rst", "README.txt"): + path = Path(repo_root) / name + if path.is_file(): + return _read(path, MAX_README_BYTES) + return "" + + +def _sentences(text): + return re.split(r"(?<=[.!?\n])\s+", text) + + +def build(repo_root, theme_root=None, plan=None, applied_pref_files=()): + """The checklist: [{item, state, evidence, note}], required first. + + `applied_pref_files` are repository-relative preference files the + person chose with --prefs; those are reported as applied. + """ + repo_root = Path(repo_root) + theme_root = Path(theme_root) if theme_root else repo_root + items = [] + applied = {str(p).replace("\\", "/") for p in applied_pref_files or ()} + + readme = readme_text(repo_root) + sentences = _sentences(readme) if readme else [] + for entry in pref_files(repo_root, theme_root): + state = entry["state"] + if entry["path"] in applied or state == "applied": + state = "applied" + elif state == "unknown" and sentences: + # A README that tells the reader to copy the file into the + # profile is an installation step, which makes the file required. + base = re.escape(Path(entry["path"]).name) + hits = [s for s in sentences if re.search(rf"(? 5 else "") + items.append({ + "item": f"preferences in {entry['path']} ({entry['count']} pref(s): {preview})", + "state": state, + "evidence": entry["path"], + "note": ("applied inside the fxcss block of user.js" if state == "applied" else + f"not applied; `--prefs {entry['path']}` adds them to the managed block"), + "pref_file": entry["path"], + "names": entry["names"], + }) + + loaders, ext_configs, fonts = [], [], [] + for path, relative in _walk(repo_root): + if LOADER_FILE_RE.search(relative): + loaders.append(relative) + elif EXTENSION_CONFIG_RE.search(relative): + ext_configs.append(relative) + elif FONT_RE.search(relative): + fonts.append(relative) + if loaders: + shown = ", ".join(loaders[:3]) + (" …" if len(loaders) > 3 else "") + items.append({"item": f"a userChrome.js loader for {len(loaders)} script file(s)", + "state": "required", "evidence": shown, + "note": "fxcss copies the files but installs no loader; without " + "fx-autoconfig or similar they do nothing"}) + if ext_configs: + shown = ", ".join(ext_configs[:3]) + (" …" if len(ext_configs) > 3 else "") + items.append({"item": "extension settings the theme ships (import them in the extension)", + "state": "unknown", "evidence": shown, + "note": "fxcss does not install extensions or import their settings"}) + if fonts: + items.append({"item": f"{len(fonts)} font file(s) shipped with the theme", + "state": "unknown", "evidence": fonts[0], + "note": "fonts referenced from the stylesheets load from chrome/; " + "ones meant to be installed system-wide are not"}) + + for pattern, item, note in README_SIGNALS: + hits = [s for s in sentences if re.search(pattern, s, re.I)] + if not hits: + continue + state = "required" if any(REQUIRED_WORD_RE.search(s) for s in hits) else "unknown" + if state != "required" and any(OPTIONAL_WORD_RE.search(s) for s in hits): + state = "optional" + evidence = re.sub(r"\s+", " ", hits[0]).strip()[:120] + items.append({"item": item, "state": state, "evidence": f"README: {evidence}", + "note": note}) + + if readme: + mentioned = [] + known = {n for i in items for n in i.get("names", [])} + for name in README_PREF_RE.findall(readme): + if name not in mentioned and name not in known and name.count(".") >= 1 \ + and not name.lower().endswith((".css", ".js", ".json", ".md", ".com", ".org")): + mentioned.append(name) + if mentioned: + shown = ", ".join(mentioned[:6]) + (" …" if len(mentioned) > 6 else "") + items.append({"item": f"{len(mentioned)} preference name(s) mentioned in the README", + "state": "unknown", "evidence": shown, + "note": "read the README; fxcss never applies prefs from prose"}) + + toggles = set() + chrome = theme_root / "chrome" + if chrome.is_dir(): + for path, relative in _walk(chrome, limit=5000): + if path.suffix.lower() == ".css": + toggles.update(CSS_PREF_RE.findall(_read(path, MAX_PREF_FILE_BYTES))) + if toggles: + shown = ", ".join(sorted(toggles)[:6]) + (" …" if len(toggles) > 6 else "") + items.append({"item": f"{len(toggles)} -moz-pref() toggle(s) the stylesheets read", + "state": "optional", "evidence": shown, + "note": "set in about:config to switch features; nothing is set for you"}) + + if plan and plan.get("kind") == "content-only": + items.append({"item": "nothing styles the browser window", + "state": "unknown", "evidence": "userContent.css only", + "note": "this theme styles web and extension pages"}) + + order = {"required": 0, "applied": 1, "optional": 2, "unknown": 3} + items.sort(key=lambda i: order.get(i["state"], 4)) + return items + + +def render(items, indent=" "): + """Lines for a terminal. Empty list when there is nothing to say.""" + if not items: + return [] + counts = {} + for item in items: + counts[item["state"]] = counts.get(item["state"], 0) + 1 + summary = ", ".join(f"{n} {state}" for state, n in sorted( + counts.items(), key=lambda kv: {"required": 0, "applied": 1, "optional": 2, + "unknown": 3}.get(kv[0], 4))) + lines = [f"{indent}setup beyond the files ({summary}):"] + for item in items: + tag = {"required": "REQUIRED", "applied": "applied ", "optional": "optional", + "unknown": "unknown "}[item["state"]] + lines.append(f"{indent} {tag} {item['item']}") + if item.get("note"): + lines.append(f"{indent} {item['note']}") + return lines + + +def complete(items): + """Can the installation honestly be called complete? Only with no + required item left unapplied.""" + return not any(i["state"] == "required" for i in items) diff --git a/fxcss/cli.py b/fxcss/cli.py index 7650cf2..a051aa6 100644 --- a/fxcss/cli.py +++ b/fxcss/cli.py @@ -30,6 +30,7 @@ import argparse import os +import re import signal import subprocess import sys @@ -194,66 +195,225 @@ def _references(theme, selector): return css_references(theme, selector) -def cmd_try(args): - import shutil - import tempfile + +def _acquire(args, interactive, workdir, verb): + """Bring what `args.repo` names to disk: a local folder as it is, or a + repository from a supported host at the ref the person meant. + + Returns {repo_root, theme_id, source, info, why, remote} or None after + printing why not. A branch in a /tree/ URL is honoured and + tracked as a branch; its folder part becomes the default --subpath. + """ from . import fetch - _install_signal_handlers() - # A folder on this computer, as `install` accepts: a theme being written, - # or one that is not on GitHub. It is read where it is, never modified. local = Path(args.repo).expanduser() if local.is_dir(): - owner = name = None - if args.ref or args.commit: - print(" --ref and --commit choose a GitHub version; using the folder as it is.") - elif args.repo.startswith(("/", "~", ".")) or "\\" in args.repo: + repo_root = local.resolve() + if getattr(args, "ref", None) or getattr(args, "commit", False): + print(" --ref and --commit choose a hosted version; using the folder as it is.") + print(f"\n {verb} {repo_root}") + return {"repo_root": repo_root, "theme_id": str(repo_root), "info": None, + "why": None, "remote": False, + "source": {"kind": "local", "owner": None, "name": None, "ref": None, + "ref_kind": "unknown", "resolved": None, "path": str(repo_root)}} + if args.repo.startswith(("/", "~", ".")) or "\\" in args.repo: print(f"error: no folder at {local}", file=sys.stderr) - return 2 - else: - try: - owner, name = fetch.parse_repo(args.repo) - except fetch.RepoSpecError as exc: - print(f"error: {exc} Or pass the path of a theme folder.", file=sys.stderr) - return 2 + return None + try: + parsed = fetch.parse_repo_spec(args.repo) + except fetch.RepoSpecError as exc: + print(f"error: {exc} Or pass the path of a theme folder.", file=sys.stderr) + return None + owner, name, host = parsed["owner"], parsed["name"], parsed["host"] + branch = parsed["ref"] + if parsed["subpath"] and not getattr(args, "subpath", None): + args.subpath = parsed["subpath"] + print(f" folder from the URL: {parsed['subpath']}") - if owner is None: - ref, why, info = None, None, None - elif args.ref: + info = None + if args.ref: ref, why = args.ref, f"ref {args.ref}" - info = None else: - info = fetch.resolve(owner, name) + info = fetch.resolve(owner, name, host=host, branch=branch) + owner, name = info["owner"], info["name"] print() for line in fetch.humanise(info): print(" " + line) - prefer = "commit" if args.commit else "release" - interactive = (sys.stdin.isatty() and sys.stdout.isatty() - and not os.environ.get("CI")) - ref, why = _choose_ref(info, prefer, interactive) + if branch: + tracked = info.get("tracked") or {} + if tracked.get("missing"): + print(f"\n error: {owner}/{name} has no branch named {branch!r}; " + "the URL may be stale. --ref chooses another.", file=sys.stderr) + return None + ref, why = branch, f"branch {branch}" + else: + prefer = "commit" if getattr(args, "commit", False) else "release" + ref, why = _choose_ref(info, prefer, interactive) + + print(f"\n fetching {why} …") + repo_root = fetch.download(owner, name, fetch.snapshot_ref(ref, info), workdir / "src", host=host) + label = f"{owner}/{name}@{ref}" if host == "github.com" else f"{host}/{owner}/{name}@{ref}" + return {"repo_root": repo_root, "theme_id": label, "info": info, "why": why, + "remote": True, + "source": _install_source(owner, name, ref, info, explicit=bool(args.ref), + host=host, branch=branch)} + + +def _plan_source(repo_root, args, interactive, workdir, remote=False): + """Decide what to install from a source tree, asking only when it is a + real choice. Returns (plan, theme_root) or None after printing.""" + from . import layout + + subpath = getattr(args, "subpath", None) + entry = getattr(args, "entry", None) + plan = layout.discover(repo_root, subpath=subpath, entry=entry, workdir=workdir / "layout") + if plan["status"] == "ambiguous" and interactive and plan["editions"]: + candidates = ([e for e in plan["editions"] if not e["demoted"] and not e["misnamed"]] + or plan["editions"]) + print("\n This repository holds several themes:") + for number, edition in enumerate(candidates, 1): + print(f" {number}. {edition['label']}") + try: + raw = input(f" Which one [1-{len(candidates)}]: ") + except EOFError: + raw = "" + chosen = candidates[_parse_choice(raw, len(candidates), 0)] + plan = layout.discover(repo_root, subpath=chosen["key"], entry=entry, + workdir=workdir / "layout") + if plan["status"] == "ambiguous": + print(f"\n {plan['reason']}", file=sys.stderr) + return None + if plan["status"] != "ok": + if not plan["editions"] and not plan["archives"]: + where = "repository" if remote else "folder" + print(f"\n No userChrome.css anywhere in this {where}, so there is\n" + f" nothing for Firefox to load. Is it a userChrome theme?\n" + f" ({plan['reason']})", file=sys.stderr) + else: + print(f"\n Nothing fxcss can install here: {plan['reason']}", file=sys.stderr) + return None + theme_root = layout.materialize(plan, workdir / "layout") + for line in layout.describe_plan(plan): + print(f" {line}") + return plan, theme_root + + +def _inventory_report(theme_root, facts, strict, force): + """Say what loads and what does not; False when `strict` forbids going on.""" + from . import inventory + + found = inventory.scan(theme_root / "chrome", + option_names={o["path"].name for o in facts.get("options", [])}) + for line in inventory.summary(found): + print(f" {line}") + if found["entry_unresolved"]: + names = ", ".join(f"{e['source']} → {e['target']}" for e in found["entry_unresolved"][:4]) + print("\n The entry stylesheet imports file(s) this source does not contain:" + f"\n {names}\n That usually means the theme needs building — a release " + "package, or its\n own installer's output — rather than a copy of its sources.", + file=sys.stderr) + if strict and not force: + print(" Nothing was installed. Pass --force to install it as it is.", + file=sys.stderr) + return None + return found + + +def _match_sheets(raw, facts): + """--with names (stems or relative paths) to option paths, or an error.""" + wanted = [w.strip() for w in raw.split(",") if w.strip()] + options = facts.get("options", []) + chosen = [] + for name in wanted: + exact = [o for o in options if o["id"] == name] + matches = exact or [o for o in options if o["stem"].lower() == name.lower()] + if len(matches) != 1: + label = "ambiguous" if matches else "unknown" + available = ", ".join(o["id"] for o in (matches or options)) or "none" + print(f"error: {label} optional sheet {name!r}; choose a relative path: " + f"{available}", file=sys.stderr) + return None + if matches[0]["path"] not in chosen: + chosen.append(matches[0]["path"]) + return chosen + + +def _sheet_paths(chosen, facts): + """Legacy labels mapped to IDs; clashing stems use the full path as label.""" + from collections import Counter + ids = {o["path"].resolve(): o["id"] for o in facts.get("options", [])} + counts = Counter(Path(sheet).stem for sheet in chosen) + return {(Path(sheet).stem if counts[Path(sheet).stem] == 1 else ids[Path(sheet).resolve()]): + ids.get(Path(sheet).resolve(), Path(sheet).name) for sheet in chosen} + + +def _sheet_ids(chosen, facts): + return list(_sheet_paths(chosen, facts).values()) + +def _choose_prefs(raw, repo_root, theme_root): + """--prefs values to repository-relative preference files, or an error.""" + from . import checklist + + wanted = [w.strip().replace("\\", "/") for w in (raw or "").split(",") if w.strip()] + if not wanted: + return [] + available = {entry["path"]: entry for entry in checklist.pref_files(repo_root, theme_root)} + missing = [w for w in wanted if w not in available] + if missing: + print(f"error: no preference file at {', '.join(missing)}; the theme ships: " + f"{', '.join(sorted(available)) or 'none'}", file=sys.stderr) + return None + for path in wanted: + names = available[path]["names"] + shown = ", ".join(names[:8]) + (f", … and {len(names) - 8} more" if len(names) > 8 else "") + print(f" --prefs {path}: {len(names)} pref(s) will join the fxcss block: {shown}") + return wanted + + +def _report_checklist(repo_root, theme_root, plan, pref_files): + from . import checklist + + items = checklist.build(repo_root, theme_root, plan, applied_pref_files=pref_files) + for line in checklist.render(items): + print(line) + return items + + +def _layout_record(plan): + return {"kind": plan["kind"], "subpath": plan["subpath"], "entry": plan["entry"]} + + +def cmd_try(args): + import shutil + import tempfile + from . import fetch + + _install_signal_handlers() + interactive = (sys.stdin.isatty() and sys.stdout.isatty() + and not os.environ.get("CI")) workdir = Path(tempfile.mkdtemp(prefix="fxcss-try-")) keep = args.keep.resolve() if args.keep else None try: - if owner is None: - repo_root = local.resolve() - print(f"\n trying {repo_root}") - else: - print(f"\n fetching {why} …") - repo_root = fetch.download(owner, name, ref, workdir / "src") - theme_root = fetch.find_theme_root(repo_root) - if theme_root is None: - where = "folder" if owner is None else "repository" - print(f"\n No userChrome.css anywhere in this {where}, so there is\n" - " nothing for Firefox to load. Is it a userChrome theme?", - file=sys.stderr) + got = _acquire(args, interactive, workdir, verb="trying") + if got is None: return 2 - - facts = fetch.describe(repo_root, theme_root) - where = theme_root.relative_to(repo_root) - top = "the top of the folder" if owner is None else "the repository root" - print(f" theme found at {where if str(where) != '.' else top}" - f" ({facts['stylesheets']} stylesheets, {facts['bytes'] // 1024} KB)") + repo_root = got["repo_root"] + planned = _plan_source(repo_root, args, interactive, workdir, remote=got["remote"]) + if planned is None: + return 2 + plan, theme_root = planned + + facts = fetch.describe(plan["repo_root"], theme_root, plan) + if plan["kind"] == "standard": + where = theme_root.relative_to(plan["repo_root"]) + top = "the repository root" if got["remote"] else "the top of the folder" + print(f" theme found at {where if str(where) != '.' else top}" + f" ({facts['stylesheets']} stylesheets, {facts['bytes'] // 1024} KB)") + else: + print(f" laid out from a {plan['kind']} source" + f" ({facts['stylesheets']} stylesheets, {facts['bytes'] // 1024} KB)") + _inventory_report(theme_root, facts, strict=False, force=True) if facts["scripts"]: print(f"\n This theme ships {', '.join(facts['scripts'])}. fxcss does not run " @@ -267,17 +427,14 @@ def cmd_try(args): names = [v.stem for v in facts["variants"]] print(f"\n Optional stylesheets you can layer on with --with:") print(" " + ", ".join(names)) + print() + _report_checklist(plan["repo_root"], theme_root, plan, []) chosen = [] if args.with_sheets: - wanted = {w.strip().lower() for w in args.with_sheets.split(",") if w.strip()} - by_name = {v.stem.lower(): v for v in facts["variants"]} - missing = wanted - set(by_name) - if missing: - print(f"\n error: no optional sheet named {', '.join(sorted(missing))}", - file=sys.stderr) + chosen = _match_sheets(args.with_sheets, facts) + if chosen is None: return 2 - chosen = [by_name[w] for w in sorted(wanted)] # A test drive is throwaway, so this says what will happen and # goes ahead: seeing two colour themes cancel out is a perfectly # good way to learn that they do. @@ -323,7 +480,6 @@ def cmd_try(args): print(f" kept the download at {keep}") shutil.rmtree(workdir, ignore_errors=True) - def _short_path(path): """Home-relative, because the interesting part is the tail.""" try: @@ -455,9 +611,12 @@ def survey_profiles(): def _theme_label(manifest): """`owner/name @ ref`, or the raw theme string when that is all there is.""" + from .fetch import REMOTE_KINDS source = manifest.get("source") or {} - if source.get("kind") == "github" and source.get("owner"): - return f"{source['owner']}/{source['name']} @ {source.get('ref')}" + if source.get("kind") in REMOTE_KINDS and source.get("owner"): + host = source.get("host") or "github.com" + prefix = "" if host == "github.com" else f"{host}/" + return f"{prefix}{source['owner']}/{source['name']} @ {source.get('ref')}" if source.get("kind") == "local": return source.get("path") or "a local directory" return manifest.get("theme") or "unknown" @@ -499,11 +658,14 @@ def cmd_profiles(args): for row in rows: source = (row["manifest"] or {}).get("source") or {} key = (source.get("owner"), source.get("name")) - if source.get("kind") != "github" or key in updates: + if source.get("kind") not in fetch.REMOTE_KINDS or key in updates: continue try: updates[key] = fetch.update_state( - source, fetch.resolve(source["owner"], source["name"])) + source, fetch.resolve( + source["owner"], source["name"], + host=source.get("host") or "github.com", + branch=source.get("ref") if source.get("ref_kind") == "branch" else None)) except RuntimeError as exc: updates[key] = {"state": "unknown", "ref": None, "label": "could not ask GitHub", @@ -541,7 +703,7 @@ def cmd_profiles(args): # ref_kind only means something for a repo: "unknown" against a # local directory would read as a gap rather than as N/A. tracking = (f" (tracking the {source['ref_kind']})" - if source.get("kind") == "github" + if source.get("kind") in fetch.REMOTE_KINDS and source.get("ref_kind") in ("release", "branch") else "") print(f" installed {manifest.get('installed', '?')}" @@ -569,6 +731,11 @@ def cmd_profiles(args): print(" update up to date") elif update: print(f" update {update['label']}") + if update and update.get("also"): + print(f" {update['also']}") + if manifest.get("user_files"): + print(f" yours {len(manifest['user_files'])} file(s) you added, " + "kept active") elif row["state"] == "unmanaged": print(f" chrome/ {row['files']} file(s), not installed by " "fxcss") @@ -584,37 +751,50 @@ def cmd_profiles(args): if any(r["default"] for r in rows): print(" ● the profile Firefox opens by default") checkable = any(((r["manifest"] or {}).get("source") or {}).get("kind") - == "github" for r in rows) + in fetch.REMOTE_KINDS for r in rows) if not args.check and checkable: print(" Pass --check to ask GitHub whether anything newer exists.") return 0 -def _install_source(owner, name, ref, info, explicit): + +def _install_source(owner, name, ref, info, explicit, host="github.com", branch=None): """Record where a theme came from, precisely enough to fetch it again. The distinction that matters later is `ref_kind`: re-fetching a release means looking for a newer tag, while re-fetching a branch means looking at what that branch points at now. The string "main" cannot tell those apart, so it is settled here, where the answer is actually known, and written - down. Nothing infers it afterwards. + down. Nothing infers it afterwards. A branch records the full commit it + pointed at, so an exact match later reads as up to date. """ - source = {"kind": "github", "owner": owner, "name": name, "ref": ref, + from . import fetch + + source = {"kind": fetch.HOSTS.get(host, "github"), "host": host, + "owner": owner, "name": name, "ref": ref, "ref_kind": "explicit" if explicit else "unknown", "resolved": ref, "path": None} if not info: return source + if explicit: + source["resolved"] = fetch.snapshot_ref(ref, info) + return source + if info.get("renamed_from"): + source["renamed_from"] = info["renamed_from"] release = (info.get("release") or {}).get("tag") + tracked = info.get("tracked") or {} if ref == release: source["ref_kind"] = "release" elif ref == info.get("default_branch"): source["ref_kind"] = "branch" # A branch has no version, so the commit it pointed at is the only # thing a later run can compare against to see it has moved. - source["resolved"] = (info.get("commit") or {}).get("sha") or ref + source["resolved"] = fetch.snapshot_ref(ref, info) + elif tracked.get("ref") == ref and not tracked.get("missing"): + source["ref_kind"] = "branch" + source["resolved"] = fetch.snapshot_ref(ref, info) return source - def profile_state(profile): """"managed" (fxcss installed a theme here), "unmanaged" or "empty".""" from . import install @@ -861,6 +1041,7 @@ def _confirm(question, interactive, assume_yes): return not raw.strip().lower().startswith("n") + def cmd_install(args): import shutil import tempfile @@ -876,60 +1057,25 @@ def cmd_install(args): interactive = (sys.stdin.isatty() and sys.stdout.isatty() and not os.environ.get("CI")) - workdir = None + workdir = Path(tempfile.mkdtemp(prefix="fxcss-install-")) try: - local = Path(args.repo).expanduser() - if local.is_dir(): - repo_root = local.resolve() - theme_id = str(repo_root) - source = {"kind": "local", "owner": None, "name": None, - "ref": None, "ref_kind": "unknown", "resolved": None, - "path": str(repo_root)} - print(f"\n installing from {repo_root}") - else: - try: - owner, name = fetch.parse_repo(args.repo) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - info = None - if args.ref: - ref, why = args.ref, f"ref {args.ref}" - else: - info = fetch.resolve(owner, name) - print() - for line in fetch.humanise(info): - print(" " + line) - prefer = "commit" if args.commit else "release" - ref, why = _choose_ref(info, prefer, interactive) - workdir = Path(tempfile.mkdtemp(prefix="fxcss-install-")) - print(f"\n fetching {why} …") - repo_root = fetch.download(owner, name, ref, workdir / "src") - theme_id = f"{owner}/{name}@{ref}" - source = _install_source(owner, name, ref, info, - explicit=bool(args.ref)) - - theme_root = fetch.find_theme_root(repo_root) - if theme_root is None: - print("\n No chrome/userChrome.css anywhere in this theme, so " - "there is\n nothing for Firefox to load. Is it a " - "userChrome theme?") + got = _acquire(args, interactive, workdir, verb="installing from") + if got is None: + return 2 + planned = _plan_source(got["repo_root"], args, interactive, workdir, + remote=got["remote"]) + if planned is None: + return 2 + plan, theme_root = planned + facts = fetch.describe(plan["repo_root"], theme_root, plan) + if _inventory_report(theme_root, facts, strict=True, force=args.force) is None: return 2 - facts = fetch.describe(repo_root, theme_root) chosen = [] if args.with_sheets: - wanted = {w.strip().lower() - for w in args.with_sheets.split(",") if w.strip()} - by_name = {v.stem.lower(): v for v in facts["variants"]} - missing = wanted - set(by_name) - if missing: - print(f"error: no optional sheet named " - f"{', '.join(sorted(missing))}; available: " - f"{', '.join(sorted(by_name)) or 'none'}", - file=sys.stderr) + chosen = _match_sheets(args.with_sheets, facts) + if chosen is None: return 2 - chosen = [by_name[w] for w in sorted(wanted)] else: chosen = _choose_sheets(facts["variants"], interactive) @@ -944,6 +1090,10 @@ def cmd_install(args): file=sys.stderr) return 2 + pref_files = _choose_prefs(args.prefs, plan["repo_root"], theme_root) + if pref_files is None: + return 2 + picked = _choose_profile(args.profile, interactive) paint_c = paint() from .audit import BOLD as _BOLD @@ -956,14 +1106,29 @@ def cmd_install(args): if prefs: print(f" user.js: the theme also sets {len(prefs)} pref(s): " f"{', '.join(prefs)}") + print() + items = _report_checklist(plan["repo_root"], theme_root, plan, pref_files) + previous = install.read_manifest(picked["path"]) + if previous: + _report_additions(install.user_additions(picked["path"], previous)) if not _confirm(" Install into this profile?", interactive, args.yes): print(" nothing installed.") return 1 - result = install.install_theme(theme_root, picked["path"], theme_id, - sheets=chosen, source=source) + extra = {"layout": _layout_record(plan), "pref_files": pref_files, + "sheet_paths": _sheet_paths(chosen, facts), + "sheet_ids": _sheet_ids(chosen, facts)} + try: + result = install.install_theme( + theme_root, picked["path"], got["theme_id"], sheets=chosen, + source=got["source"], on_collision="theme" if args.force else "stop", + extra=extra, repo_root=plan["repo_root"]) + except install.CollisionError as exc: + _report_collision(exc) + return 2 if result["backup"]: print(f" existing chrome/ saved as {result['backup']}") + _report_preserved(result) if result["sheets"]: print(f" optional sheets: " f"{paint_c(_BOLD, ', '.join(result['sheets']))}") @@ -971,6 +1136,10 @@ def cmd_install(args): print(f" note: {note}") print(f" installed {len(result['files'])} file(s) into " f"{picked['path'] / 'chrome'}") + from . import checklist + if not checklist.complete(items): + print(" the files are in place; the REQUIRED items above are not " + "something fxcss can do for you") print("\n Restart Firefox to see it. `fxcss uninstall` puts " "everything back.") return 0 @@ -978,9 +1147,7 @@ def cmd_install(args): print(f"error: {exc}", file=sys.stderr) return 2 finally: - if workdir: - shutil.rmtree(workdir, ignore_errors=True) - + shutil.rmtree(workdir, ignore_errors=True) def cmd_uninstall(args): from . import install @@ -1006,6 +1173,15 @@ def cmd_uninstall(args): except RuntimeError as exc: print(f"error: {exc}", file=sys.stderr) return 2 + if result.get("noop"): + print(" no fxcss-installed theme is active here, so nothing was " + "removed or restored.") + if result.get("user_js_cleaned"): + print(" user.js: removed a leftover fxcss block") + if result.get("backups"): + print(f" backups kept: {', '.join(result['backups'])} " + "(`fxcss rollback --to ` restores one)") + return 0 if result["removed"]: print(f" removed {result['removed']} file(s)") if result["restored"]: @@ -1020,6 +1196,45 @@ def cmd_uninstall(args): return 0 +def _report_additions(additions, indent=" "): + """Say which of the person's own files the upgrade will keep active.""" + files, skipped = additions["files"], additions["skipped"] + if files: + print(f"{indent}{len(files)} file(s) you added will stay active:") + for name in files[:6]: + print(f"{indent} {name}") + if len(files) > 6: + print(f"{indent} … and {len(files) - 6} more") + if skipped: + print(f"{indent}{len(skipped)} added item(s) are not regular files " + "(links, git metadata) and stay only in the backup") + + +def _report_preserved(result, indent=" "): + """After an install: what of the person's own is active, what is not.""" + if result.get("user_files"): + print(f"{indent}kept {len(result['user_files'])} file(s) you added " + "active in the new chrome/") + if result.get("displaced"): + print(f"{indent}{len(result['displaced'])} file(s) you added were " + f"replaced by the theme's own (your copies are in " + f"{result['backup']}):") + for name in result["displaced"][:6]: + print(f"{indent} {name}") + + +def _report_collision(exc, indent=" "): + print(f"\n{indent}The new version ships files at paths you added " + f"yourself, with different\n{indent}contents. Nothing was changed. " + "Move or rename yours, or pass --force to\n" + f"{indent}install the theme's copies (yours stay in the backup):", + file=sys.stderr) + for name in exc.collisions[:12]: + print(f"{indent} {name}", file=sys.stderr) + if len(exc.collisions) > 12: + print(f"{indent} … and {len(exc.collisions) - 12} more", file=sys.stderr) + + def _report_drift(moved, indent=" "): """Say what has changed under chrome/ since the install, if anything.""" if moved["modified"]: @@ -1032,7 +1247,8 @@ def _report_drift(moved, indent=" "): print(f"{indent}{len(moved['missing'])} file(s) deleted since install") if moved["extra"]: print(f"{indent}{len(moved['extra'])} file(s) added by hand " - "(these are left alone)") + "(kept active across upgrades)") + def cmd_upgrade(args): @@ -1053,23 +1269,34 @@ def cmd_upgrade(args): source = manifest.get("source") or {} print(f" installed: {_theme_label(manifest)}") + recorded_layout = manifest.get("layout") if isinstance(manifest.get("layout"), dict) else {} + if args.subpath is None and recorded_layout.get("subpath"): + args.subpath = recorded_layout["subpath"] + if args.entry is None and recorded_layout.get("entry"): + args.entry = recorded_layout["entry"] # What is there to move to? info = None + local_root = None if source.get("kind") == "local": local_root = Path(source.get("path") or "") - if not (local_root / "chrome").is_dir(): + if not local_root.is_dir(): print(f"\n error: the directory this was installed from is gone " f"({local_root})", file=sys.stderr) return 2 target_ref, why = None, f"the current contents of {local_root}" - state = {"state": "unsupported"} - elif source.get("kind") == "github": + state = {"state": "unsupported", "also": ""} + elif source.get("kind") in fetch.REMOTE_KINDS: + host = source.get("host") or "github.com" + branch = source.get("ref") if source.get("ref_kind") == "branch" else None try: - info = fetch.resolve(source["owner"], source["name"]) + info = fetch.resolve(source["owner"], source["name"], host=host, branch=branch) except RuntimeError as exc: print(f"\n error: {exc}", file=sys.stderr) return 2 + if info.get("renamed_from"): + print(f" upstream moved: {info['renamed_from']} is now " + f"{info['owner']}/{info['name']}; recorded under the new name") state = fetch.update_state(source, info) if args.ref: target_ref, why = args.ref, f"ref {args.ref}" @@ -1083,6 +1310,8 @@ def cmd_upgrade(args): state.get("label") or "what is installed" print(f" upstream: {state['label']}" + (f" — {state['detail']}" if state.get("detail") else "")) + if state.get("also"): + print(f" also: {state['also']}") else: print("\n error: this install has no record of where it came from, " "so fxcss\n cannot fetch a newer copy. Reinstall it with " @@ -1102,9 +1331,11 @@ def cmd_upgrade(args): "hashes, so\n whether it has been edited since cannot be " "checked.") + remote = source.get("kind") in fetch.REMOTE_KINDS + setup_requested = args.prefs is not None or args.with_sheets is not None nothing_to_do = (state["state"] == "current" and not args.ref - and not args.commit and source.get("kind") == "github") - if args.check: + and not args.commit and remote and not setup_requested) + if args.check and remote and not setup_requested: if nothing_to_do: print("\n Up to date.") return 0 @@ -1119,34 +1350,39 @@ def cmd_upgrade(args): "anyway.") return 0 - if (moved["modified"] or moved["missing"]) and not args.force: - print("\n Refusing to upgrade over edits fxcss did not make — the " - "new version\n would overwrite them. Save what you want, then " - "pass --force.\n (Files you *added* are never touched; only " - "the ones above.)", file=sys.stderr) - return 2 - - workdir = None + workdir = Path(tempfile.mkdtemp(prefix="fxcss-upgrade-")) + keep_pin = (source.get("ref_kind") == "explicit" and not args.ref and not args.commit + and bool(re.fullmatch(r"[0-9a-fA-F]{40}", str(source.get("resolved") or "")))) try: - if source.get("kind") == "local": + if local_root is not None: repo_root = local_root else: - workdir = Path(tempfile.mkdtemp(prefix="fxcss-upgrade-")) print(f"\n fetching {why} …") - repo_root = fetch.download(source["owner"], source["name"], - target_ref, workdir / "src") - - theme_root = fetch.find_theme_root(repo_root) - if theme_root is None: - print("\n error: no chrome/userChrome.css in that version", - file=sys.stderr) + revision = source["resolved"] if keep_pin else fetch.snapshot_ref(target_ref, info) + repo_root = fetch.download(info["owner"], info["name"], revision, + workdir / "src", host=info["host"]) + + planned = _plan_source(repo_root, args, interactive, workdir) + if planned is None: + print(" (the recorded layout no longer matches this version; choose " + "with --subpath)", file=sys.stderr) + return 2 + plan, theme_root = planned + facts = fetch.describe(plan["repo_root"], theme_root, plan) + if _inventory_report(theme_root, facts, strict=True, force=args.force) is None: return 2 - facts = fetch.describe(repo_root, theme_root) - wanted = (args.with_sheets.split(",") if args.with_sheets is not None - else manifest.get("sheets") or []) - wanted = [w.strip() for w in wanted if w.strip()] - continuity = fetch.sheet_continuity(wanted, facts["variants"]) + if args.with_sheets is not None: + selected = _match_sheets(args.with_sheets, facts) + if selected is None: + return 2 + continuity = {"kept": selected, "missing": []} + else: + recorded = manifest.get("sheet_paths") if isinstance(manifest.get("sheet_paths"), dict) else {} + wanted = manifest.get("sheet_ids") + if not isinstance(wanted, list) or not all(isinstance(p, str) for p in wanted): + wanted = [recorded.get(stem, stem) for stem in (manifest.get("sheets") or [])] + continuity = fetch.sheet_continuity(wanted, facts["variants"], facts.get("options")) if continuity["missing"]: print(f"\n error: the new version has no optional sheet named " f"{', '.join(continuity['missing'])}.\n It was renamed or " @@ -1171,6 +1407,64 @@ def cmd_upgrade(args): "pass --force.", file=sys.stderr) return 2 + if args.prefs is not None: + pref_files = _choose_prefs(args.prefs, plan["repo_root"], theme_root) + if pref_files is None: + return 2 + else: + pref_files = [str(p) for p in (manifest.get("pref_files") or []) if isinstance(p, str)] + from . import checklist + shipped = {entry["path"] for entry in checklist.pref_files(plan["repo_root"], theme_root)} + gone = [p for p in pref_files if p not in shipped] + if gone: + print(f"\n error: the new version has no preference file at " + f"{', '.join(gone)}.\n Its prefs would silently stop being " + "applied. Re-run with --prefs to choose\n from: " + f"{', '.join(sorted(shipped)) or 'none'}, or --prefs '' for none.", + file=sys.stderr) + return 2 + if pref_files: + print(f" keeping preference file(s): {', '.join(pref_files)}") + + if local_root is not None or setup_requested: + # A local source has no upstream to ask; the files themselves are + # the answer. Compare what an install would write with what the + # last one wrote, and say "up to date" only when they agree. + preview = install.preview_digests(theme_root, continuity["kept"], + repo_root=plan["repo_root"]) + recorded = {k: v for k, v in (manifest.get("digests") or {}).items() + if k in set(manifest.get("files") or [])} + changed = sorted(k for k in set(preview) | set(recorded) + if preview.get(k) != recorded.get(k)) + block = install.user_js_body(theme_root, pref_files, + repo_root=plan["repo_root"]) + if block != manifest.get("user_js_block"): + changed.append("user.js (fxcss preference block)") + if pref_files != (manifest.get("pref_files") or []) or ( + isinstance(manifest.get("sheet_ids"), list) and + _sheet_ids(continuity["kept"], facts) != manifest["sheet_ids"]): + changed.append("setup selection") + same_source = local_root is not None or state["state"] in ("current", "pinned") + if not changed and not args.ref and same_source: + if args.check: + print("\n Up to date: the source and setup match what is installed.") + return 0 + print("\n Already up to date: the source and setup match what is " + "installed.") + return 0 + print(f"\n {len(changed)} file(s) differ from what is installed" + + (": " + ", ".join(changed[:5]) if changed else "")) + if args.check: + print(" Run `fxcss upgrade` to take it.") + return 1 + + if (moved["modified"] or moved["missing"]) and not args.force: + print("\n Refusing to upgrade over edits fxcss did not make — the " + "new version\n would overwrite them. Save what you want, then " + "pass --force.\n (Files you *added* are carried over; only " + "the ones above.)", file=sys.stderr) + return 2 + if args.audit: from . import audit as audit_mod firefox = choose_firefox(args.firefox) @@ -1196,7 +1490,6 @@ def cmd_upgrade(args): from . import compare as compare_mod except ImportError as exc: return _needs_pillow("upgrade --compare", exc) - import tempfile firefox = choose_firefox(args.firefox) outdir = Path(tempfile.mkdtemp(prefix="fxcss-upgrade-compare-")) print("\n rendering what is installed now …") @@ -1213,27 +1506,45 @@ def cmd_upgrade(args): return status print(f" before/after images: {outdir / 'diff'}") + print() + _report_checklist(plan["repo_root"], theme_root, plan, pref_files) if install.firefox_running(picked["path"]): print("\n note: Firefox appears to be running — the change will " "only show after a restart.") + _report_additions(install.user_additions(picked["path"], manifest)) if not _confirm(f"\n Upgrade to {why}?", interactive, args.yes): print(" nothing changed.") return 1 - theme_id = (str(repo_root) if source.get("kind") == "local" - else f"{source['owner']}/{source['name']}@{target_ref}") - new_source = (dict(source) if source.get("kind") == "local" - else _install_source(source["owner"], source["name"], - target_ref, info, - explicit=bool(args.ref))) - result = install.install_theme( - theme_root, picked["path"], theme_id, - sheets=continuity["kept"], source=new_source, - # The chrome/ from before fxcss ever ran here, carried forward so - # `uninstall` still reaches it however many upgrades happen. - origin_backup=manifest.get("origin_backup")) + if local_root is not None: + theme_id, new_source = str(repo_root), dict(source) + else: + theme_id = (f"{info['owner']}/{info['name']}@{target_ref}" if info["host"] == "github.com" + else f"{info['host']}/{info['owner']}/{info['name']}@{target_ref}") + new_source = _install_source(info["owner"], info["name"], target_ref, info, + explicit=bool(args.ref) or (source.get("ref_kind") == "explicit" + and not args.commit), host=info["host"], + branch=source.get("ref") if source.get("ref_kind") == "branch" else None) + if keep_pin: + new_source["resolved"] = source["resolved"] + extra = {"layout": _layout_record(plan), "pref_files": pref_files, + "sheet_paths": _sheet_paths(continuity["kept"], facts), + "sheet_ids": _sheet_ids(continuity["kept"], facts)} + try: + result = install.install_theme( + theme_root, picked["path"], theme_id, + sheets=continuity["kept"], source=new_source, + # The chrome/ from before fxcss ever ran here, carried forward so + # `uninstall` still reaches it however many upgrades happen. + origin_backup=manifest.get("origin_backup"), + on_collision="theme" if args.force else "stop", + extra=extra, repo_root=plan["repo_root"]) + except install.CollisionError as exc: + _report_collision(exc) + return 2 print(f"\n upgraded to {why}") print(f" the previous version is kept as {result['backup']}") + _report_preserved(result) keep = args.keep if keep is not None and keep < 1: # This command has just promised a way back; honouring --keep 0 @@ -1254,9 +1565,7 @@ def cmd_upgrade(args): print(f"\n error: {exc}", file=sys.stderr) return 2 finally: - if workdir: - shutil.rmtree(workdir, ignore_errors=True) - + shutil.rmtree(workdir, ignore_errors=True) def cmd_adopt(args): from . import adopt, fetch, install @@ -1324,7 +1633,12 @@ def cmd_adopt(args): best, checked = None, 0 for entry in refs: try: - listing = fetch.tree(owner, name, entry["ref"]) + revision = entry["ref"] + if entry["kind"] == "branch": + revision = fetch.require_commit(entry.get("sha") or + fetch.branch_head(owner, name, entry["ref"]), entry["ref"]) + entry["sha"] = revision + listing = fetch.tree(owner, name, revision) except RuntimeError as exc: print(f" {exc}", file=sys.stderr) break @@ -1332,6 +1646,7 @@ def cmd_adopt(args): continue result = adopt.compare(chrome, listing) result["ref"], result["kind"] = entry["ref"], entry["kind"] + result["sha"] = entry.get("sha") checked += 1 print(f" {adopt.describe(result)}") if adopt.better(result, best): @@ -1351,6 +1666,17 @@ def cmd_adopt(args): return 2 print(f"\n best match: {adopt.describe(best)}") + if not args.repo and best["score"] < adopt.GUESS_MIN_SCORE: + # The repository was only *mentioned* in the theme's files -- a + # credit, a palette, an upstream -- and the files mostly disagree + # with it. Recording that would hand every later upgrade to the + # wrong project. + print(f"\n Only {best['matched']}/{best['files']} of that version's " + "files match, and the repository was\n guessed from a mention " + "in the theme's own text, not from anything definite.\n Not " + "recording it. If it really is this theme, say so:\n" + f" fxcss adopt {owner}/{name}", file=sys.stderr) + return 2 if not adopt.exact(best): print(" Recorded as that version plus local differences, so an " "upgrade knows\n not to overwrite them without being told.") @@ -1364,8 +1690,13 @@ def cmd_adopt(args): kind = {"release": "release", "tag": "release", "branch": "branch"}.get(best["kind"], "explicit") - source = {"kind": "github", "owner": owner, "name": name, - "ref": best["ref"], "ref_kind": kind, "resolved": best["ref"], + resolved = best["ref"] + if kind == "branch": + # The commit the branch points at, never its name: an exact match + # recorded as "main" announced an upgrade on the very next check. + resolved = fetch.require_commit(best.get("sha"), best["ref"]) + source = {"kind": "github", "host": "github.com", "owner": owner, "name": name, + "ref": best["ref"], "ref_kind": kind, "resolved": resolved, "path": None} try: manifest = install.adopt_theme( @@ -1522,28 +1853,68 @@ def cmd_tweaks(args): return 0 + def cmd_init(args): - from . import scaffold + from . import layout, scaffold theme = args.theme.resolve() - if not (theme / "chrome" / "userChrome.css").exists(): + plan = layout.discover(theme, subpath=args.subpath) + if plan["status"] == "ambiguous": + print(f"error: {plan['reason']}", file=sys.stderr) + return 2 + if plan["status"] != "ok": print(f"error: no chrome/userChrome.css under {theme} - run this from " - f"your theme's root, or pass --theme", file=sys.stderr) + f"your theme's root, or pass --theme. {plan['reason']}", file=sys.stderr) + return 2 + if plan["kind"] in ("named", "fragment"): + print(f"error: this layout ({plan['kind']}) needs an explicit edition or " + "entry every time it is read, which the generated workflows cannot " + "pass yet; lay the theme out as chrome/userChrome.css to use them", + file=sys.stderr) return 2 + selected = plan["selected"] + # The real folder the workflows should name: where chrome/ lives for a + # standard layout, the folder with the flat files otherwise. + theme_dir = theme / selected["subpath"] if selected["subpath"] else theme + repo_root = scaffold.repository_root(theme_dir) + if repo_root.resolve() == theme_dir.resolve() and theme_dir != theme: + # No git checkout to ask: the folder passed as --theme is the + # repository, and the edition sits inside it. + repo_root = theme try: - slugs = sorted(core.find_variant_sheets(theme)) - preview = scaffold.preview_settings(theme, args.variants, args.views, + standard = layout.standard_root(theme, subpath=args.subpath) \ + if plan["kind"] != "standard" or args.subpath else theme_dir + slugs = sorted(core.find_variant_sheets(standard)) + preview = scaffold.preview_settings(standard, args.variants, args.views, args.min_changed_percent, args.oldest_firefox) except ValueError as exc: print(f"error: {exc}", file=sys.stderr) return 2 - written, skipped = scaffold.write_workflows( - theme, slugs, watch=args.watch, showcase=args.showcase, - previews=args.previews, force=args.force, preview=preview) + for line in layout.describe_plan(plan): + print(f" {line}") + if repo_root.resolve() != theme_dir.resolve(): + try: + rel = theme_dir.resolve().relative_to(repo_root.resolve()).as_posix() + except ValueError: + rel = str(theme_dir) + print(f" workflows go to {repo_root / '.github' / 'workflows'} and point at {rel}/") + toolkit_ref = getattr(args, "toolkit_ref", None) + if toolkit_ref: + print(f" workflows install fxcss from commit {toolkit_ref}") + else: + print(f" workflows pin fxcss {__version__}; that version must be published on PyPI " + "before they run. Use --toolkit-ref to test an unreleased commit.") + try: + written, skipped = scaffold.write_workflows( + theme_dir, slugs, watch=args.watch, showcase=args.showcase, + previews=args.previews, force=args.force, preview=preview, + repo_root=repo_root, toolkit_ref=toolkit_ref) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 print(scaffold.next_steps(written, skipped, slugs, args.watch, args.showcase, args.previews)) return 0 - def cmd_watch(args): theme = args.theme.resolve() firefox = choose_firefox(args.firefox) @@ -1668,7 +2039,13 @@ def _run_audit(args): if unused and not args.no_unused: audit.report_unused(unused, colour=not args.no_colour, show_all=args.all) if args.patch: - files = audit.write_patch(result, theme, args.patch.resolve()) + source_paths = None + if getattr(args, "_layout_plan", None): + from . import layout + uses = {use["file"] for finding in result["findings"] + if finding["confidence"] == "renamed" for use in finding["uses"]} + source_paths = layout.patch_source_paths(args._layout_plan, theme, uses) + files = audit.write_patch(result, theme, args.patch.resolve(), source_paths=source_paths) if files: print(f" wrote {args.patch} ({files} file(s)); review it, then:") print(f" git apply {args.patch}\n") @@ -1909,6 +2286,16 @@ def _bool(v): return v.lower() not in ("false", "0", "no") +def _layout_options(p): + p.add_argument("--subpath", default=None, metavar="DIR-OR-FILE", + help="which theme to take from a repository that holds several: " + "a folder (1.Normal), a named entry file " + "(src/userChrome-WhiteSur.css) or an archive (chrome.zip)") + p.add_argument("--entry", default=None, metavar="FILE", + help="for a theme with no userChrome.css: the stylesheet to " + "load through a generated one, e.g. Brave-Fox/Import.css") + + def _common(p, theme=True, firefox_help=None): if theme: p.add_argument("--theme", type=Path, default=Path.cwd(), @@ -1950,6 +2337,7 @@ def build_parser(): help="use the latest commit rather than the latest release") tr.add_argument("--with", dest="with_sheets", default=None, metavar="NAME[,NAME]", help="also load named optional stylesheets") + _layout_options(tr) tr.add_argument("--dark", action="store_true", help="start in dark mode") tr.add_argument("--shot", type=Path, default=None, help="capture screenshots instead of opening interactively") @@ -1976,6 +2364,11 @@ def build_parser(): help="use the latest commit rather than the latest release") ins.add_argument("--with", dest="with_sheets", default=None, metavar="NAME[,NAME]", help="also install named optional stylesheets") + _layout_options(ins) + ins.add_argument("--prefs", default=None, metavar="FILE[,FILE]", + help="also apply these shipped preference files (paths relative " + "to the repository, e.g. user.js or config/user.js) inside " + "the removable fxcss block; names are previewed first") ins.add_argument("--profile", default=None, metavar="NAME-OR-PATH", help="which Firefox profile (default: the one Firefox itself opens)") ins.add_argument("--list-profiles", action="store_true", @@ -2029,6 +2422,10 @@ def build_parser(): metavar="NAME[,NAME]", help="optional sheets to install (default: the ones " "already installed)") + _layout_options(up) + up.add_argument("--prefs", default=None, metavar="FILE[,FILE]", + help="shipped preference files to apply (default: the ones " + "already applied; '' for none)") up.add_argument("--keep", type=int, default=3, metavar="N", help="how many backups to keep (default 3; the original " "chrome/ is never pruned)") @@ -2108,6 +2505,8 @@ def build_parser(): ini = sub.add_parser("init", help="add PR previews and CI checks to your theme repo") ini.add_argument("--theme", type=Path, default=Path.cwd(), help="theme root, the folder containing chrome/ (default: cwd)") + ini.add_argument("--toolkit-ref", metavar="SHA", + help="pin workflows to a full fxcss Git commit instead of a PyPI release") ini.add_argument("--watch", action="store_true", help="also add the weekly Firefox release/beta/nightly audit " "and capture canary") @@ -2129,6 +2528,9 @@ def build_parser(): help="PR previews and the watch: say whether a change drops styling " "for any Firefox back to this one; 'esr' for the current ESR " "(default), a major version such as 140, or '' to skip") + ini.add_argument("--subpath", default=None, metavar="DIR", + help="which theme in a repository that holds several " + "(a folder relative to --theme)") ini.add_argument("--force", action="store_true", help="replace workflow files that already exist") ini.set_defaults(func=cmd_init) @@ -2303,6 +2705,30 @@ def build_parser(): return ap +# Commands that only *read* a theme. For these a flat or otherwise +# non-standard layout is materialised into a temporary chrome/ tree, so +# `fxcss shot --theme .` works in a repository that keeps userChrome.css at +# its root. Commands that write beside the theme (init, check) or watch its +# files (watch) resolve layouts themselves or need the standard shape. +LAYOUT_AWARE = {"shot", "audit", "doctor", "snapshot", "changelog", "tweaks", + "pick", "inspect", "catalogue"} + + +def _resolve_layouts(args): + from . import layout + if getattr(args, "cmd", None) in LAYOUT_AWARE and getattr(args, "theme", None): + if args.cmd == "audit" and args.patch: + args.theme, args._layout_plan = layout.standard_root(args.theme, with_plan=True) + layout.patch_source_paths(args._layout_plan, args.theme, []) + else: + args.theme = layout.standard_root(args.theme) + if getattr(args, "cmd", None) == "support": + for key in ("base", "head"): + value = getattr(args, key, None) + if value is not None and Path(value).is_dir(): + setattr(args, key, layout.standard_root(value)) + + def main(argv=None): raw = list(sys.argv[1:] if argv is None else argv) if raw and raw[0] == "__complete": @@ -2321,8 +2747,13 @@ def main(argv=None): except AttributeError: pass from .fetch import FetchError + from .layout import LayoutError try: + _resolve_layouts(args) return args.func(args) + except LayoutError as exc: + print(f"\nerror: {exc}", file=sys.stderr) + return 2 except (FetchError, core.MarionetteError) as exc: # Expected fetch and browser failures get a concise diagnostic rather # than a traceback; programming errors still propagate. diff --git a/fxcss/core.py b/fxcss/core.py index fb00445..bf02453 100644 --- a/fxcss/core.py +++ b/fxcss/core.py @@ -430,6 +430,13 @@ def build_profile(repo: Path, profile: Path, dark=False, native_menus=None, exact fidelity even when a rule is deleted. """ profile.mkdir(parents=True, exist_ok=True) + if not (repo / "chrome").is_dir(): + # A flat, content-only or packaged layout: plan it and lay it out + # the standard way first, so the copy below is the same for every + # theme. The materialised root lives in the throwaway profile. + from . import layout + repo = layout.materialize(layout.discover(repo, workdir=profile / ".fxcss-layout"), + profile / ".fxcss-layout") shutil.copytree(repo / "chrome", profile / "chrome", dirs_exist_ok=True) _ensure_custom_chrome(repo / "chrome", profile / "chrome") if empty_user_chrome: @@ -2452,15 +2459,16 @@ def find_variant_sheets(theme: Path): renames the existing `tabs` slug to `custom-tabs`; update explicit variant selections and review and refresh baselines for the renamed captures. """ - from .fetch import VARIANT_DIRS + from . import inventory by_slug = {} - for folder in VARIANT_DIRS: - directory = theme / folder - if directory.is_dir(): - for css in sorted(directory.glob("*.css")): - slug = re.sub(r"[^a-z0-9-]+", "-", css.stem.lower()).strip("-") - if slug: - by_slug.setdefault(slug, []).append((folder, css)) + # The same option inventory install and upgrade use: option folders + # beside chrome/ and inside it, minus sheets an entry already imports. + for option in inventory.options(theme): + css = option["path"] + folder = f"chrome-{option['folder']}" if option["inside_chrome"] else option["folder"] + slug = re.sub(r"[^a-z0-9-]+", "-", css.stem.lower()).strip("-") + if slug: + by_slug.setdefault(slug, []).append((folder, css)) def clash(name, first, second): first, second = (p.relative_to(theme).as_posix() for p in (first, second)) diff --git a/fxcss/fetch.py b/fxcss/fetch.py index c01d276..3978756 100644 --- a/fxcss/fetch.py +++ b/fxcss/fetch.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Fetch a theme from GitHub so it can be test-driven. +"""Fetch a theme from a public forge so it can be test-driven or installed. `fxcss try owner/repo` downloads a theme, installs it into a throwaway profile and opens it, so someone can see what a theme actually looks like before putting @@ -16,6 +16,13 @@ file. Firefox does not execute a .js file sitting in a profile's chrome folder -- that needs an autoconfig hook in the *application* directory, which fxcss does not create. + +Hosts: GitHub is first-class (releases, branch tips, the git-tree API that +`adopt` compares against, and tokens from GITHUB_TOKEN / GH_TOKEN / `gh`). +Codeberg and other Forgejo instances, and GitLab, are supported for +resolving, downloading and update checks through their public APIs; `adopt` +says so when it meets one of them rather than guessing. A GitHub token is +only ever sent to GitHub hosts, including across redirects. """ import io @@ -27,6 +34,7 @@ import tarfile import tempfile import urllib.error +import urllib.parse import urllib.request from pathlib import Path @@ -38,7 +46,9 @@ class FetchError(RuntimeError): class RepoSpecError(FetchError, ValueError): - """The argument is neither owner/name nor a github.com URL.""" + """The argument is neither owner/name nor a URL on a supported host.""" + + USER_AGENT = "fxcss (+https://github.com/AdamXweb/fxcss)" # A theme is small. Anything wildly bigger than this is not one, and unpacking it @@ -46,16 +56,38 @@ class RepoSpecError(FetchError, ValueError): MAX_ARCHIVE_BYTES = 80 * 1024 * 1024 MAX_UNPACKED_BYTES = 200 * 1024 * 1024 +# Hosts a repository can come from, and which API dialect each speaks. +# "forgejo" covers Codeberg, git.gay and any other Forgejo/Gitea instance +# listed here; adding one is a line. +HOSTS = { + "github.com": "github", + "codeberg.org": "forgejo", + "git.gay": "forgejo", + "gitlab.com": "gitlab", +} +REMOTE_KINDS = ("github", "forgejo", "gitlab") +# Where a GitHub token may travel. Anywhere else gets an anonymous request. +GITHUB_TOKEN_HOSTS = ("api.github.com", "codeload.github.com", "github.com", + "raw.githubusercontent.com", "objects.githubusercontent.com") + REPO_SPEC = re.compile( r"^(?:https?://)?(?:www\.)?(?:github\.com/)?([\w.-]+)/([\w.-]+?)(?:\.git)?/?$") +# A browser URL for a branch or folder: github.com/o/n/tree/[/], +# codeberg.org/o/n/src/branch/[/], gitlab.com/o/n/-/tree/[/]. +TREE_URL = re.compile( + r"^(?:https?://)?(?:www\.)?(?P[\w.-]+)/(?P[\w.-]+)/(?P[\w.-]+?)" + r"(?:\.git)?(?:/(?:-/)?(?:tree|src/branch|src/tag|blob|src)/(?P[^/]+)" + r"(?:/(?P.*?))?)?/?$") INSTALL_SCRIPTS = [ "install.sh", "install.bat", "install.ps1", "install.py", "setup.sh", "Makefile", "makefile", "justfile", ] -# Themes commonly keep optional tweaks in one of these. -VARIANT_DIRS = ["custom", "optional", "options", "extras", "variants"] +# Themes commonly keep optional tweaks in one of these (beside chrome/ or +# inside it). One list for installation, captures and generated workflows. +from .inventory import OPTION_DIRS as _OPTION_DIRS # noqa: E402 +VARIANT_DIRS = list(_OPTION_DIRS) # "- `-c` Left hand side tab close button" and similar list items. Inside # backticks the flag keeps its argument: "- `-t ` Colour theme". @@ -64,20 +96,52 @@ class RepoSpecError(FetchError, ValueError): r"\s*[—:-]?\s+(.{4,120}?)\s*$") +def parse_repo_spec(spec: str): + """Take a repository argument apart. + + Returns {host, kind, owner, name, ref, subpath}. `ref` and `subpath` + are set only for a branch/folder URL such as + ``https://github.com/o/n/tree/photon-style/themes/x``; the catalogue + carries such URLs, and the branch in them is part of what the author + means by "this theme". A bare ``owner/name`` is GitHub. + """ + text = (spec or "").strip() + match = REPO_SPEC.match(text) + if match: + return {"host": "github.com", "kind": "github", "owner": match.group(1), + "name": match.group(2), "ref": None, "subpath": None} + match = TREE_URL.match(text) + if match and match.group("host") in HOSTS: + name = match.group("name") + subpath = (match.group("subpath") or "").strip("/") or None + return {"host": match.group("host"), "kind": HOSTS[match.group("host")], + "owner": match.group("owner"), "name": name, + "ref": match.group("ref") or None, "subpath": subpath} + if match and "." in match.group("host"): + raise RepoSpecError( + f"{match.group('host')} is not a host fxcss can fetch from; supported: " + + ", ".join(sorted(HOSTS)) + ". Clone the theme and pass its folder.") + raise RepoSpecError( + f"could not read {spec!r} as a repository. Use owner/name, a github.com " + "URL (optionally /tree//), or a codeberg.org / gitlab.com URL.") + + def parse_repo(spec: str): - match = REPO_SPEC.match(spec.strip()) - if not match: + """(owner, name) for a GitHub repository spec; the long-standing contract.""" + parsed = parse_repo_spec(spec) + if parsed["kind"] != "github": raise RepoSpecError( - f"could not read {spec!r} as a GitHub repo. " - "Use owner/name or a github.com URL.") - return match.group(1), match.group(2) + f"{parsed['host']} repositories need the full URL form here; " + "this operation compares against GitHub's API and supports GitHub only.") + return parsed["owner"], parsed["name"] # A token is optional. GITHUB_TOKEN or GH_TOKEN, when set, is sent with every -# request: it raises the anonymous rate limit and opens private repositories. -# Without one, fxcss asks the GitHub CLI for its login, but only after an -# anonymous request found nothing, since a private repository looks exactly -# like a missing one to anyone who cannot read it. +# GitHub request: it raises the anonymous rate limit and opens private +# repositories. Without one, fxcss asks the GitHub CLI for its login, but +# only after an anonymous request found nothing or was rate-limited, since a +# private repository looks exactly like a missing one to anyone who cannot +# read it. The token itself is never printed; messages name its source. _cli_login = {"token": None, "tried": False} @@ -111,22 +175,61 @@ def _use_cli_login(): return True +def _github_host(url): + host = (urllib.parse.urlsplit(url).hostname or "").lower() + return host in GITHUB_TOKEN_HOSTS + + +class _NoLeakRedirects(urllib.request.HTTPRedirectHandler): + """Drop the Authorization header when a redirect leaves GitHub. + + urllib copies every header onto the redirected request; a token meant for + api.github.com must not follow a 302 to an arbitrary host. + """ + + def redirect_request(self, req, fp, code, msg, headers, newurl): + new = super().redirect_request(req, fp, code, msg, headers, newurl) + if new is not None and not _github_host(newurl): + new.remove_header("Authorization") + return new + + +# Installed as the module-wide opener so that every urlopen in this process +# -- and anything a test patches -- goes through the same redirect rule. +urllib.request.install_opener(urllib.request.build_opener(_NoLeakRedirects)) + + def _request(url, accept="application/vnd.github+json"): headers = {"User-Agent": USER_AGENT, "Accept": accept} token = _token()[0] - if token: + if token and _github_host(url): headers["Authorization"] = f"Bearer {token}" return urllib.request.Request(url, headers=headers) -def _unreachable(exc): - return FetchError(f"could not reach GitHub ({getattr(exc, 'reason', exc)}). " +def _open(request, timeout): + return urllib.request.urlopen(request, timeout=timeout) + + +def _unreachable(exc, what="GitHub"): + return FetchError(f"could not reach {what} ({getattr(exc, 'reason', exc)}). " "Check the connection and try again.") +def _rate_limited(exc): + remaining = exc.headers.get("X-RateLimit-Remaining") if exc.headers else None + return exc.code == 429 or (exc.code == 403 and remaining in (None, "0")) + + def _api(path): + """One GitHub API call; None for 404, FetchError for anything else. + + An anonymous request that hits the rate limit is retried once with the + GitHub CLI's login when one exists: a signed-in person should not be + told to sign in. + """ try: - with urllib.request.urlopen(_request(API + path), timeout=45) as response: + with _open(_request(API + path), timeout=45) as response: return json.loads(response.read().decode("utf-8")) except urllib.error.HTTPError as exc: if exc.code == 404: @@ -137,18 +240,46 @@ def _api(path): "It may have expired or been revoked: replace it, or unset it to " "use public repositories without one.") from exc if exc.code in (403, 429): - raise FetchError( - "GitHub refused the request (rate limit). Set GITHUB_TOKEN, or " - "sign in with `gh auth login`, to raise it; or pass --ref to " - "skip the lookups.") from exc + if _rate_limited(exc) and _use_cli_login(): + return _api(path) + token, source = _token() + hint = ("The request was made with the token from " + f"{source}; wait for the limit to reset, or pass --ref to " + "skip the lookups." if token else + "Set GITHUB_TOKEN, or sign in with `gh auth login`, to raise " + "it; or pass --ref to skip the lookups.") + raise FetchError(f"GitHub refused the request (rate limit). {hint}") from exc raise FetchError(f"GitHub answered {exc.code} {exc.reason} for {path}") from exc except urllib.error.URLError as exc: raise _unreachable(exc) from exc -def not_found(owner, name): +def _api_url(url, what): + """One call to another host's JSON API; None for 404.""" + try: + with _open(_request(url, accept="application/json"), timeout=45) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None + if exc.code in (403, 429): + raise FetchError(f"{what} refused the request (rate limit); try again later " + "or pass --ref to skip the lookups.") from exc + raise FetchError(f"{what} answered {exc.code} {exc.reason} for {url}") from exc + except urllib.error.URLError as exc: + raise _unreachable(exc, what) from exc + + +def not_found(owner, name, host="github.com"): """What to say when a repository cannot be read, and what to try next.""" token, source = _token() + if host != "github.com": + return FetchError("\n".join([ + f"{host} has no public repository {owner}/{name}.", + f" Check the name at https://{host}/{owner}/{name}", + " For a theme on this computer, pass its folder instead, " + "such as ~/src/my-theme", + ])) if token: first = (f"no repository {owner}/{name} that the GitHub token from " f"{source} can read.") @@ -167,38 +298,159 @@ def not_found(owner, name): ])) -def resolve(owner, name): - """Report what is available: the default branch tip and the latest release.""" +def _commit_info(sha, date, message): + sha = sha or "" + return {"sha": sha[:7], "full_sha": sha, "date": date or "", + "message": (message or "").splitlines()[0][:72] if message else ""} + + +def resolve(owner, name, host="github.com", branch=None): + """Report what is available: the default branch tip and the latest release. + + `branch` names a branch the person asked for (from a /tree/ URL + or a manifest); its tip is reported under `tracked` so an update check + compares against the right thing. Renamed repositories resolve to their + canonical owner/name, reported under `renamed_from`, because the old + name still answers on the API but not on the download host. + """ + kind = HOSTS.get(host, "github") + if kind == "forgejo": + return _resolve_forgejo(owner, name, host, branch) + if kind == "gitlab": + return _resolve_gitlab(owner, name, host, branch) + repo = _api(f"/repos/{owner}/{name}") if repo is None and _use_cli_login(): repo = _api(f"/repos/{owner}/{name}") if repo is None: raise not_found(owner, name) - branch = repo.get("default_branch", "main") - commit = _api(f"/repos/{owner}/{name}/commits/{branch}") or {} + renamed_from = None + full_name = repo.get("full_name") or "" + if "/" in full_name and full_name.lower() != f"{owner}/{name}".lower(): + renamed_from = f"{owner}/{name}" + owner, name = full_name.split("/", 1) + + default = repo.get("default_branch", "main") + commit = _api(f"/repos/{owner}/{name}/commits/{urllib.parse.quote(default, safe='')}") or {} release = _api(f"/repos/{owner}/{name}/releases/latest") + tracked = None + if branch and branch != default: + data = _api(f"/repos/{owner}/{name}/branches/{urllib.parse.quote(branch, safe='')}") + if data and isinstance(data.get("commit"), dict): + inner = data["commit"].get("commit") or {} + tracked = dict(_commit_info(data["commit"].get("sha"), + ((inner.get("author") or {}).get("date")), + inner.get("message")), ref=branch) + else: + tracked = {"ref": branch, "missing": True} info = { - "owner": owner, "name": name, + "owner": owner, "name": name, "host": host, "kind": "github", + "renamed_from": renamed_from, "description": repo.get("description") or "", "stars": repo.get("stargazers_count", 0), "licence": (repo.get("license") or {}).get("spdx_id"), - "default_branch": branch, - "commit": { - "sha": (commit.get("sha") or "")[:7], - "date": (commit.get("commit", {}).get("author", {}) or {}).get("date", ""), - "message": (commit.get("commit", {}).get("message") or "").splitlines()[0][:72], - } if commit else None, + "default_branch": default, + "commit": _commit_info( + commit.get("sha"), + ((commit.get("commit", {}) or {}).get("author", {}) or {}).get("date", ""), + (commit.get("commit", {}) or {}).get("message")) if commit else None, "release": { "tag": release.get("tag_name"), "name": release.get("name") or release.get("tag_name"), "date": release.get("published_at", ""), } if release else None, + "tracked": tracked, } return info +def _resolve_forgejo(owner, name, host, branch): + base = f"https://{host}/api/v1/repos/{owner}/{name}" + repo = _api_url(base, host) + if repo is None: + raise not_found(owner, name, host) + renamed_from = None + full_name = repo.get("full_name") or "" + if "/" in full_name and full_name.lower() != f"{owner}/{name}".lower(): + renamed_from = f"{owner}/{name}" + owner, name = full_name.split("/", 1) + base = f"https://{host}/api/v1/repos/{owner}/{name}" + default = repo.get("default_branch") or "main" + + def branch_tip(ref): + data = _api_url(f"{base}/branches/{urllib.parse.quote(ref, safe='')}", host) + if not data or not isinstance(data.get("commit"), dict): + return None + commit = data["commit"] + return _commit_info(commit.get("id"), commit.get("timestamp"), commit.get("message")) + + release = _api_url(f"{base}/releases/latest", host) + tip = branch_tip(default) + tracked = None + if branch and branch != default: + tracked = branch_tip(branch) + tracked = dict(tracked, ref=branch) if tracked else {"ref": branch, "missing": True} + return { + "owner": owner, "name": name, "host": host, "kind": "forgejo", + "renamed_from": renamed_from, + "description": repo.get("description") or "", + "stars": repo.get("stars_count", 0), + "licence": None, + "default_branch": default, + "commit": tip, + "release": {"tag": release.get("tag_name"), + "name": release.get("name") or release.get("tag_name"), + "date": release.get("published_at", "")} if release else None, + "tracked": tracked, + } + + +def _resolve_gitlab(owner, name, host, branch): + project = urllib.parse.quote(f"{owner}/{name}", safe="") + base = f"https://{host}/api/v4/projects/{project}" + repo = _api_url(base, host) + if repo is None: + raise not_found(owner, name, host) + renamed_from = None + full_name = repo.get("path_with_namespace") or "" + if "/" in full_name and full_name.lower() != f"{owner}/{name}".lower(): + renamed_from = f"{owner}/{name}" + owner, name = full_name.rsplit("/", 1) + base = f"https://{host}/api/v4/projects/{urllib.parse.quote(full_name, safe='')}" + default = repo.get("default_branch") or "main" + + def branch_tip(ref): + data = _api_url(f"{base}/repository/branches/{urllib.parse.quote(ref, safe='')}", host) + if not data or not isinstance(data.get("commit"), dict): + return None + commit = data["commit"] + return _commit_info(commit.get("id"), commit.get("committed_date"), + commit.get("title") or commit.get("message")) + + releases = _api_url(f"{base}/releases?per_page=1", host) or [] + release = releases[0] if isinstance(releases, list) and releases else None + tip = branch_tip(default) + tracked = None + if branch and branch != default: + tracked = branch_tip(branch) + tracked = dict(tracked, ref=branch) if tracked else {"ref": branch, "missing": True} + return { + "owner": owner, "name": name, "host": host, "kind": "gitlab", + "renamed_from": renamed_from, + "description": repo.get("description") or "", + "stars": repo.get("star_count", 0), + "licence": None, + "default_branch": default, + "commit": tip, + "release": {"tag": release.get("tag_name"), + "name": release.get("name") or release.get("tag_name"), + "date": release.get("released_at", "")} if release else None, + "tracked": tracked, + } + + def choose_ref(info, prefer): """Pick what to download. Releases are what a theme's author blessed.""" if prefer == "commit": @@ -262,24 +514,56 @@ def known_refs(owner, name, limit=10): Releases first, because a theme installed by hand was almost certainly installed from one, then any remaining tags. The default branch goes last -- someone who cloned rather than downloaded is on it, but it is a moving - target and a poor thing to pin a profile to without being asked. + target and a poor thing to pin a profile to without being asked. A + branch entry carries the commit it points at, so what gets recorded is + the commit and not the name. """ refs, seen = [], set() - def add(ref, kind): + def add(ref, kind, sha=None): if ref and ref not in seen: seen.add(ref) - refs.append({"ref": ref, "kind": kind}) + refs.append({"ref": ref, "kind": kind, "sha": sha}) for release in _api(f"/repos/{owner}/{name}/releases?per_page={limit}") or []: add(release.get("tag_name"), "release") for tag in _api(f"/repos/{owner}/{name}/tags?per_page={limit}") or []: add(tag.get("name"), "tag") repo = _api(f"/repos/{owner}/{name}") or {} - add(repo.get("default_branch"), "branch") + default = repo.get("default_branch") + if default: + head = _api(f"/repos/{owner}/{name}/commits/{urllib.parse.quote(default, safe='')}") or {} + add(default, "branch", head.get("sha")) return refs[:limit] +def branch_head(owner, name, branch): + """The full commit SHA a GitHub branch points at now, or None.""" + data = _api(f"/repos/{owner}/{name}/commits/{urllib.parse.quote(branch, safe='')}") + return (data or {}).get("sha") or None + + +def require_commit(sha, ref): + """Require an immutable Git commit before asserting branch provenance.""" + if not isinstance(sha, str) or not re.fullmatch(r"[0-9a-fA-F]{40}", sha): + raise FetchError(f"could not resolve branch {ref!r} to a full commit SHA; " + "nothing was installed or adopted") + return sha.lower() + + +def snapshot_ref(ref, info): + """Pin a known branch while retaining its name separately for tracking.""" + if not info: + return ref # explicit refs deliberately skip discovery + if ref == info.get("default_branch"): + commit = info.get("commit") or {} + elif ref == (info.get("tracked") or {}).get("ref"): + commit = info["tracked"] + else: + return ref # release/tag or an explicitly requested ref + return require_commit(commit.get("full_sha") or commit.get("sha"), ref) + + def tree(owner, name, ref): """{path: blob sha} for every file in a repository at a ref. @@ -298,11 +582,26 @@ def tree(owner, name, ref): if entry.get("type") == "blob" and entry.get("path")} +def same_commit(recorded, current): + """Do two commit ids name the same commit? + + Older manifests recorded the seven-character short form; a full SHA + recorded now still has to compare equal to it. Either way an empty value + matches nothing. + """ + recorded, current = (recorded or "").lower(), (current or "").lower() + if not recorded or not current: + return False + if len(recorded) < 7 or len(current) < 7: + return recorded == current + return recorded.startswith(current) or current.startswith(recorded) + + def update_state(source, info): """Is what a profile has installed still what upstream offers? `source` is a manifest's source dict, `info` what `resolve()` reports now. - Returns {state, ref, label, detail}, where state is one of: + Returns {state, ref, label, detail, also}, where state is one of: current installed ref is upstream's newest of the kind tracked available something newer exists; `ref` is what to move to @@ -310,51 +609,66 @@ def update_state(source, info): unknown the manifest predates recording how the ref was chosen unsupported a local directory install: there is no upstream to ask - Kept separate from any fetching so it can be tested without a network, and - stated as five outcomes rather than a boolean because "fxcss cannot tell" - has to be renderable -- an install whose ref_kind was never recorded is - not the same as one that is up to date, and showing it as up to date - would be a lie a user would act on. + `also` carries the other side of the picture without acting on it: a + release-tracked install that is current while the branch has moved on + says so, and stays on releases. Kept separate from any fetching so it + can be tested without a network, and stated as outcomes rather than a + boolean because "fxcss cannot tell" has to be renderable. """ source = source or {} kind = source.get("kind") if kind == "local": return {"state": "unsupported", "ref": None, "label": "local install", - "detail": source.get("path") or ""} - if kind != "github": + "detail": source.get("path") or "", "also": ""} + if kind not in REMOTE_KINDS: return {"state": "unknown", "ref": None, "label": "unknown source", - "detail": "nothing recorded about where this came from"} + "detail": "nothing recorded about where this came from", "also": ""} ref_kind = source.get("ref_kind") if ref_kind == "explicit": return {"state": "pinned", "ref": None, "label": f"pinned to {source.get('ref')}", - "detail": "installed with --ref, so nothing is checked"} + "detail": "installed with --ref, so nothing is checked", "also": ""} if ref_kind == "release": latest = (info.get("release") or {}).get("tag") if not latest: return {"state": "unknown", "ref": None, "label": "no releases upstream", - "detail": "the theme published none to compare against"} + "detail": "the theme published none to compare against", "also": ""} + also = "" + if commit_is_newer(info): + commit = info.get("commit") or {} + also = (f"{info.get('default_branch')} has moved on since the release " + f"({(commit.get('date') or '')[:10]}); `--commit` would follow it") if latest == source.get("ref"): return {"state": "current", "ref": latest, "label": latest, - "detail": ""} + "detail": "", "also": also} return {"state": "available", "ref": latest, "label": latest, - "detail": (info.get("release") or {}).get("date", "")[:10]} + "detail": (info.get("release") or {}).get("date", "")[:10], "also": also} if ref_kind == "branch": - commit = info.get("commit") or {} - sha = commit.get("sha") + ref = source.get("ref") + tracked = info.get("tracked") or {} + if tracked.get("ref") == ref and ref != info.get("default_branch"): + if tracked.get("missing"): + return {"state": "unknown", "ref": None, + "label": f"branch {ref} no longer exists upstream", + "detail": "choose another with --ref", "also": ""} + commit = tracked + else: + commit = info.get("commit") or {} + sha = commit.get("full_sha") or commit.get("sha") if not sha: return {"state": "unknown", "ref": None, - "label": "branch tip unreadable", "detail": ""} - if sha == source.get("resolved"): - return {"state": "current", "ref": source.get("ref"), - "label": f"{source.get('ref')} @ {sha}", "detail": ""} - return {"state": "available", "ref": source.get("ref"), - "label": f"{source.get('ref')} @ {sha}", - "detail": commit.get("message", "")[:52]} + "label": "branch tip unreadable", "detail": "", "also": ""} + short = sha[:7] + if same_commit(source.get("resolved"), sha): + return {"state": "current", "ref": ref, + "label": f"{ref} @ {short}", "detail": "", "also": ""} + return {"state": "available", "ref": ref, + "label": f"{ref} @ {short}", + "detail": (commit.get("message") or "")[:52], "also": ""} return {"state": "unknown", "ref": None, "label": "ref kind not recorded", - "detail": "installed before fxcss recorded this; reinstall to fix"} + "detail": "installed before fxcss recorded this; reinstall to fix", "also": ""} def _safe_members(archive, destination: Path): @@ -375,79 +689,166 @@ def _safe_members(archive, destination: Path): yield member -def download(owner, name, ref, into: Path): - """Download and unpack a repository at a ref. Returns the unpacked root.""" - url = f"https://codeload.github.com/{owner}/{name}/tar.gz/{ref}" +def archive_url(owner, name, ref, host="github.com"): + """Where a host serves a tar.gz of a repository at a ref.""" + kind = HOSTS.get(host, "github") + quoted = urllib.parse.quote(ref, safe="/") + if kind == "forgejo": + return f"https://{host}/{owner}/{name}/archive/{quoted}.tar.gz" + if kind == "gitlab": + flat = ref.replace("/", "-") + return f"https://{host}/{owner}/{name}/-/archive/{quoted}/{name}-{flat}.tar.gz" + return f"https://codeload.github.com/{owner}/{name}/tar.gz/{quoted}" - def fetch(): - with urllib.request.urlopen(_request(url, accept="*/*"), timeout=180) as response: + +def download(owner, name, ref, into: Path, host="github.com"): + """Download and unpack a repository at a ref. Returns the unpacked root. + + On GitHub a renamed repository still answers on the API but 404s on the + download host under its old name, so a 404 here is followed by one + canonical-name lookup before giving up. + """ + url = archive_url(owner, name, ref, host) + what = "GitHub" if host == "github.com" else host + + def fetch(target): + with _open(_request(target, accept="*/*"), timeout=180) as response: return response.read(MAX_ARCHIVE_BYTES + 1) try: try: - payload = fetch() + payload = fetch(url) except urllib.error.HTTPError as exc: - # With --ref there was no lookup first, so this may be the first - # sign that the repository is private. - if exc.code != 404 or not _use_cli_login(): + if exc.code != 404 or host != "github.com": raise - payload = fetch() + # With --ref there was no lookup first, so this may be the first + # sign that the repository is private -- or that it was renamed. + if _use_cli_login(): + try: + payload = fetch(url) + except urllib.error.HTTPError as again: + if again.code != 404: + raise + payload = None + else: + payload = None + if payload is None: + repo = _api(f"/repos/{owner}/{name}") or {} + canonical = repo.get("full_name") or "" + if "/" in canonical and canonical.lower() != f"{owner}/{name}".lower(): + owner, name = canonical.split("/", 1) + payload = fetch(archive_url(owner, name, ref, host)) + else: + raise except urllib.error.HTTPError as exc: if exc.code == 404: raise FetchError( f"could not download {owner}/{name} at {ref!r}: that repository " "or ref was not found.\n" - f" {not_found(owner, name).args[0].splitlines()[0]}") from exc + f" {not_found(owner, name, host).args[0].splitlines()[0]}") from exc raise FetchError(f"could not download {owner}/{name}@{ref}: {exc}") from exc except urllib.error.URLError as exc: - raise _unreachable(exc) from exc + raise _unreachable(exc, what) from exc if len(payload) > MAX_ARCHIVE_BYTES: raise FetchError("archive is larger than 80 MB; refusing") into.mkdir(parents=True, exist_ok=True) - with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as archive: - members = _safe_members(archive, into) - try: - # Python 3.12+: the 'data' filter also strips absolute paths and - # rejects links, complementing the checks in _safe_members. - archive.extractall(into, members=members, filter="data") - except TypeError: # Python < 3.12 has no filter parameter - archive.extractall(into, members=members) + try: + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as archive: + members = _safe_members(archive, into) + try: + # Python 3.12+: the 'data' filter also strips absolute paths and + # rejects links, complementing the checks in _safe_members. + archive.extractall(into, members=members, filter="data") + except TypeError: # Python < 3.12 has no filter parameter + archive.extractall(into, members=members) + except tarfile.TarError as exc: + raise FetchError(f"the download from {what} was not a readable archive: {exc}") from exc entries = [p for p in into.iterdir() if p.is_dir()] if len(entries) == 1: - return entries[0] # GitHub wraps everything in one directory + return entries[0] # hosts wrap everything in one directory return into def find_theme_root(unpacked: Path): """Locate the folder that holds chrome/userChrome.css. - Not always the repository root: plenty of themes keep sources under src/ or - ship several variants side by side. + Delegates to the layout planner, so the answer is never a directory + outside `unpacked` and is None for every layout that needs a plan + (flat, content-only, several editions) rather than a guess. """ - candidates = sorted(unpacked.rglob("userChrome.css"), - key=lambda p: len(p.relative_to(unpacked).parts)) - for sheet in candidates: - if sheet.parent.name == "chrome": - return sheet.parent.parent - if candidates: - # userChrome.css not inside a folder called chrome; treat its parent as - # the chrome folder anyway. - return candidates[0].parent.parent - return None - - -def describe(repo_root: Path, theme_root: Path): - """What this theme offers, and how its author says to install it.""" + from .layout import find_theme_root as planned + return planned(unpacked) + + +def _inside(path, root): + try: + Path(path).resolve().relative_to(Path(root).resolve()) + return True + except ValueError: + return False + + +def describe(repo_root: Path, theme_root: Path, plan=None): + """What this theme offers, and how its author says to install it. + + `variants` keeps its old shape (a list of sheet paths) for callers that + show names; `options` carries the same sheets with their stable relative + identities, from the shared inventory. With `plan` (a layout plan whose + theme root was materialised elsewhere) those identities are still + repository-relative, so a manifest written from a flat layout names + ``tweaks/x.css`` and not a temporary folder. + """ + from . import inventory + scripts = [name for name in INSTALL_SCRIPTS if (repo_root / name).is_file()] - variants = [] - for folder in VARIANT_DIRS: - directory = repo_root / folder - if directory.is_dir(): - for sheet in sorted(directory.glob("*.css")): - variants.append(sheet) + options = [] + seen = set() + roots = [theme_root] if theme_root else [] + if theme_root is None or (_inside(theme_root, repo_root) + and Path(theme_root).resolve() != Path(repo_root).resolve()): + roots.append(repo_root) + selected = (plan or {}).get("selected") or {} + chrome_dir = selected.get("chrome_dir", "") if (plan or {}).get("kind") != "standard" else None + for root in roots: + chrome = (root / "chrome") if root else None + for option in inventory.options(root, chrome if chrome and chrome.is_dir() else None): + key = option["path"].resolve() + if key in seen: + continue + seen.add(key) + if _inside(option["path"], repo_root): + option["id"] = option["path"].resolve().relative_to( + Path(repo_root).resolve()).as_posix() + elif chrome_dir is not None and chrome and _inside(option["path"], chrome): + inner = option["path"].resolve().relative_to(chrome.resolve()).as_posix() + option["id"] = f"{chrome_dir}/{inner}" if chrome_dir else inner + options.append(option) + + if (plan or {}).get("kind") == "named" and theme_root: + # A named edition's loose sheets -- WhiteSur's src/common/hide-single-tab.css, + # which its installer copies in on request -- are options too. Only + # stylesheets nothing reaches count, under their repository path. + source_dir = Path(plan["repo_root"]) / (selected.get("chrome_dir") or "") + reachable = inventory.reachable_paths(theme_root / "chrome") + for folder in ("", "common", "shared"): + directory = source_dir / folder if folder else source_dir + if not directory.is_dir(): + continue + for css in sorted(directory.glob("*.css")): + if not css.is_file() or inventory._is_hook(css.name): + continue + if any(css.name == Path(r).name for r in reachable): + continue + from .layout import NAMED_ENTRY_RE + if NAMED_ENTRY_RE.match(css.name) or css.resolve() in seen: + continue + seen.add(css.resolve()) + options.append({"id": css.relative_to(repo_root).as_posix(), "path": css, + "stem": css.stem, "folder": folder or "edition", + "inside_chrome": False}) flags = [] for readme in ("README.md", "readme.md", "README", "README.rst"): @@ -463,7 +864,8 @@ def describe(repo_root: Path, theme_root: Path): sheets = list(theme_root.rglob("*.css")) if theme_root else [] return { "scripts": scripts, - "variants": variants, + "variants": [option["path"] for option in options], + "options": options, "flags": flags[:14], "stylesheets": len(sheets), "bytes": sum(p.stat().st_size for p in sheets), @@ -471,20 +873,30 @@ def describe(repo_root: Path, theme_root: Path): } -def sheet_continuity(wanted, variants): +def sheet_continuity(wanted, variants, options=None): """Do the optional sheets someone installed still exist upstream? - Returns {kept: [Path], missing: [str]}, matching on the stem the manifest - records. A sheet that has been renamed or dropped between versions is the - quiet way an upgrade takes away a setting: the @import simply stops - resolving and the option turns itself off. Naming the loss is the whole - job here -- the caller decides whether that stops the upgrade. + Returns {kept: [Path], missing: [str]}, matching each recorded name + against the new version: by relative path first when `options` (the + inventory's option dicts) are given, then by stem. A sheet that has been + renamed or dropped between versions is the quiet way an upgrade takes + away a setting: the @import simply stops resolving and the option turns + itself off. Naming the loss is the whole job here -- the caller decides + whether that stops the upgrade. """ - by_stem = {sheet.stem.lower(): sheet for sheet in variants} + from . import inventory + + if options is None: + options = [{"id": sheet.as_posix(), "path": sheet, "stem": sheet.stem, + "folder": sheet.parent.name, "inside_chrome": False} + for sheet in variants] kept, missing = [], [] for name in wanted or []: - sheet = by_stem.get(str(name).lower()) - (kept.append(sheet) if sheet else missing.append(str(name))) + option = inventory.match_option(name, options) + if option is None: + missing.append(str(name)) + elif option["path"] not in kept: + kept.append(option["path"]) return {"kept": kept, "missing": missing} @@ -509,11 +921,15 @@ def humanise(info): """Lines describing what was found upstream.""" out = [] header = f"{info['owner']}/{info['name']}" + if info.get("host") and info["host"] != "github.com": + header = f"{info['host']}/{header}" if info["stars"]: header += f" ★{info['stars']}" if info["licence"] and info["licence"] != "NOASSERTION": header += f" {info['licence']}" out.append(header) + if info.get("renamed_from"): + out.append(f" (moved from {info['renamed_from']}; recorded under its new name)") if info["description"]: out.append(f" {info['description'][:96]}") if info["release"]: @@ -524,4 +940,10 @@ def humanise(info): if info["commit"]: out.append(f" latest commit {info['commit']['sha']}" f" ({info['commit']['date'][:10]}) {info['commit']['message']}") + tracked = info.get("tracked") + if tracked and not tracked.get("missing"): + out.append(f" branch {tracked['ref']:<10} {tracked['sha']}" + f" ({tracked['date'][:10]}) {tracked['message']}") + elif tracked: + out.append(f" branch {tracked['ref']}: not found upstream") return out diff --git a/fxcss/install.py b/fxcss/install.py index 5f28149..e0e4ed6 100644 --- a/fxcss/install.py +++ b/fxcss/install.py @@ -35,6 +35,10 @@ MANIFEST_NAME = "fxcss-install.json" +# The stylesheets Firefox loads from chrome/ by name. A theme needs at least +# one of them; a userContent.css-only theme is a real theme too. +ENTRY_SHEETS = ("userChrome.css", "userContent.css") + # Bumped when the manifest's shape changes. Old manifests are still read -- # `read_manifest` normalises them -- but they are never silently rewritten: # whatever wrote a manifest is the thing that gets to own its contents. @@ -52,24 +56,30 @@ STYLESHEET_PREF = ( 'user_pref("toolkit.legacyUserProfileCustomizations.stylesheets", true);') -# One pass over a stylesheet: comments and strings are consumed whole, so an -# @import or url() inside either is never mistaken for a live reference, and -# a "/*" inside a string never starts a comment. -CSS_TOKEN_RE = re.compile(r""" - /\*.*?(?:\*/|\Z) - | @import\s+(?:url\(\s*)?(?P["'])(?P(?:\\.|(?!(?P=q1))[^\\\n])*)(?P=q1) - | @import\s+url\(\s*(?P[^"'()\s]+)\s*\) - | (?["'])(?P(?:\\.|(?!(?P=q2))[^\\\n])*)(?P=q2)\s*\) - | (?[^"'()\s]+)\s*\) - | (?P["'])(?:\\.|(?!(?P=q3))[^\\\n])*(?:(?P=q3)|(?=\n)|\Z) -""", re.S | re.I | re.X) - # How much a selected optional sheet may pull in alongside itself. Generous # for icon sets, small enough that a theme cannot fill a profile with files. MAX_SHEET_DEPENDENCIES = 256 MAX_SHEET_DEPENDENCY_BYTES = 32 * 1024 * 1024 +class CollisionError(RuntimeError): + """An upgrade would put an upstream file where the user added their own. + + Raised before anything in the profile moves: the replacement is still in + staging. `collisions` lists the chrome/-relative paths, so the caller can + show exactly what is in the way and let the person decide. + """ + + def __init__(self, collisions): + self.collisions = list(collisions) + shown = ", ".join(self.collisions[:6]) + if len(self.collisions) > 6: + shown += f", … and {len(self.collisions) - 6} more" + super().__init__( + f"the new version ships {len(self.collisions)} file(s) at paths " + f"you added yourself: {shown}") + + # --- finding profiles ------------------------------------------------------- def profile_roots(): @@ -405,18 +415,46 @@ def user_js_with_block(existing, body): return base + BLOCK_BEGIN + "\n" + body + "\n" + BLOCK_END + "\n" -def user_js_body(theme_root): +def user_js_body(theme_root, pref_files=None, repo_root=None): """What goes inside the block: the pref that turns userChrome.css on, - plus whatever configuration/user.js the theme ships.""" + plus whatever configuration/user.js the theme ships. + + `pref_files` are further preference files the person chose explicitly + (relative to `repo_root`, or to the theme root without one, as `fxcss + install --prefs` records them). Each is + included as shipped, under a comment naming it, so the block can be read + back and removed as one unit. A chosen file that is missing is skipped + here -- the caller has already checked continuity and said so. + """ body = STYLESHEET_PREF shipped = Path(theme_root) / "configuration" / "user.js" if shipped.is_file(): body += "\n\n" + shipped.read_text(encoding="utf-8", errors="replace").rstrip("\n") + for relative in pref_files or []: + path = safe_theme_path(repo_root or theme_root, relative) + if path is None or not path.is_file(): + continue + text = path.read_text(encoding="utf-8", errors="replace").rstrip("\n") + body += f"\n\n/* fxcss: from the theme's {_printable(str(relative))} */\n{text}" # Sanitised here too, so the manifest records the block as written. return sanitize_block_body(body) +def safe_theme_path(theme_root, relative): + """`theme_root/relative`, or None when `relative` strays outside it.""" + text = str(relative or "") + parts = Path(text).parts + if not parts or ".." in parts or Path(text).is_absolute(): + return None + candidate = Path(theme_root).joinpath(*parts) + try: + candidate.resolve().relative_to(Path(theme_root).resolve()) + except ValueError: + return None + return candidate + + PREF_NAME_RE = re.compile( r"""\b(?:user_pref|pref|sticky_pref|lockPref)\(\s*(["'])(.+?)\1""") @@ -457,15 +495,11 @@ def _strip_js_comments(text): def css_references(text): """(kind, target) for each live @import ("import") and url() ("url"). - Commented-out references are skipped and strings are respected, but CSS - escapes inside a target are not decoded. + Use the shared inventory grammar, including CSS escapes. """ - for match in CSS_TOKEN_RE.finditer(text): - for kind, group in (("import", "imp"), ("import", "bimp"), - ("url", "url"), ("url", "burl")): - if match.group(group) is not None: - yield kind, match.group(group) - break + from .inventory import references + for reference in references(text): + yield reference["kind"], reference["target"] def _local_reference(target): @@ -475,13 +509,8 @@ def _local_reference(target): leading slash, a drive letter -- or a bare #fragment resolves the same wherever the sheet sits, so only relative paths need carrying along. """ - from urllib.parse import unquote - - target = target.strip() - if not target or target.startswith(("#", "/", "\\")) \ - or re.match(r"^[A-Za-z][\w+.-]*:", target): - return None - return unquote(re.split(r"[?#]", target, maxsplit=1)[0]) or None + from .inventory import local_target + return local_target(target) def _inside(path, root): @@ -536,6 +565,9 @@ def _copy_sheet_dependencies(sheet, dest, chrome, source_root, budget): if placed.exists(): if not placed.is_file() or \ placed.read_bytes() != resolved.read_bytes(): + if placed in budget.get("written", set()): + raise RuntimeError(f"selected optional sheets need different files at " + f"chrome/{placed.relative_to(chrome).as_posix()}") notes.append( f"{label} (the theme's own chrome/" f"{placed.relative_to(chrome).as_posix()} kept)") @@ -556,6 +588,7 @@ def _copy_sheet_dependencies(sheet, dest, chrome, source_root, budget): "not copied)") continue shutil.copy2(resolved, placed) + budget.setdefault("written", set()).add(placed) if kind == "import" or placed.suffix.lower() == ".css": queue.append((origin, placed)) return notes @@ -578,7 +611,7 @@ def variant_destination(chrome_dir, filename): for css in sorted(chrome_dir.rglob("*.css")): text = css.read_text(encoding="utf-8", errors="replace") for kind, target in css_references(text): - target = target.strip() + target = _local_reference(target) if kind != "import" or not target: continue if not (target.lower().endswith("/" + needle) @@ -714,6 +747,73 @@ def drift(profile, manifest=None): return result +def user_additions(profile, manifest=None): + """Files the person added to a managed chrome/ since fxcss wrote it. + + Returns {"files": [relative paths], "skipped": [relative paths]} where + paths are profile-relative ("chrome/custom/mine.css"). These are the + files an upgrade must keep *active*, not merely keep in a backup: a + stylesheet imported from customChrome.css stops applying the moment it + is only in chrome.backup-*. Git metadata and anything that is not a + regular file is listed under `skipped` -- a clone's .git/ belongs to the + old tree, and a symlink could point anywhere. + """ + profile = Path(profile) + manifest = read_manifest(profile) if manifest is None else manifest + if not manifest: + return {"files": [], "skipped": []} + files, skipped = [], [] + for relative in drift(profile, manifest)["extra"]: + parts = Path(relative).parts + path = profile.joinpath(*parts) + if ".git" in parts or path.is_symlink() or not path.is_file(): + skipped.append(relative) + else: + files.append(relative) + return {"files": files, "skipped": skipped} + + +def preservation_plan(profile, additions, staged_chrome): + """How each user-added file fares against a staged replacement chrome/. + + Returns {carry, identical, collisions}: `carry` are paths the new tree + does not have, so the user's bytes go across untouched; `identical` are + paths the new version now ships with exactly the user's bytes (upstream + adopted the file -- nothing to carry); `collisions` are paths where the + new version ships different bytes, which no rule can settle silently. + Nothing is copied here; this is the reviewable part. + """ + profile, staged_chrome = Path(profile), Path(staged_chrome) + plan = {"carry": [], "identical": [], "collisions": []} + for relative in additions: + parts = Path(relative).parts[1:] # drop the leading "chrome" + staged = staged_chrome.joinpath(*parts) + if staged.is_file(): + if staged.read_bytes() == profile.joinpath(*Path(relative).parts).read_bytes(): + plan["identical"].append(relative) + else: + plan["collisions"].append(relative) + elif staged.exists() or staged.is_symlink() or any( + parent.is_file() for parent in staged.parents + if parent != staged_chrome and staged_chrome in parent.parents): + # A directory where the user had a file, or a file where the user + # had a directory: the user's bytes cannot land there either. + plan["collisions"].append(relative) + else: + plan["carry"].append(relative) + return plan + + +def _carry_additions(profile, carry, staged_chrome): + """Copy user-added files into staging at the same chrome/-relative place.""" + profile, staged_chrome = Path(profile), Path(staged_chrome) + for relative in carry: + parts = Path(relative).parts + dest = staged_chrome.joinpath(*parts[1:]) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(profile.joinpath(*parts), dest) + + # --- install ---------------------------------------------------------------- def _timestamp(): @@ -798,7 +898,7 @@ def _sheet_source_root(theme_root, sheet): return sheet_dir if common == Path(common.anchor) else common -def _prepare_chrome(theme_root, chrome, sheets, notes=None): +def _prepare_chrome(theme_root, chrome, sheets, notes=None, repo_root=None): """Build a complete replacement in staging, without touching live files. Notes about optional-sheet references that could not be carried along @@ -811,34 +911,84 @@ def _prepare_chrome(theme_root, chrome, sheets, notes=None): # Some themes @import customChrome.css without shipping it; an empty file # keeps the import resolving (same courtesy core.build_profile extends). placeholder = chrome / "customChrome.css" - if not placeholder.exists() and "customChrome.css" in \ - (chrome / "userChrome.css").read_text(encoding="utf-8", - errors="replace"): + entry = chrome / "userChrome.css" + if not placeholder.exists() and entry.is_file() and "customChrome.css" in \ + entry.read_text(encoding="utf-8", errors="replace"): placeholder.write_text("/* placeholder created by fxcss */\n", encoding="utf-8") + if sheets and not entry.exists(): + # A userContent.css-only theme has nothing to layer optional sheets + # onto; an empty entry gives the wrapper below something to import. + entry.write_text("/* entry created by fxcss for optional sheets */\n", + encoding="utf-8") installed_sheets, extra_imports = [], [] budget = {"files": MAX_SHEET_DEPENDENCIES, - "bytes": MAX_SHEET_DEPENDENCY_BYTES} + "bytes": MAX_SHEET_DEPENDENCY_BYTES, "written": set()} + # Allocate from the pristine tree, before a selected option's own imports + # can be mistaken for a native slot for the next option. + sheets = list(dict.fromkeys(Path(sheet) for sheet in sheets)) + destinations = [] for sheet in sheets: - sheet = Path(sheet) - dest = variant_destination(chrome, sheet.name) + source_root = _sheet_source_root(theme_root, sheet) + if repo_root is not None and _inside(sheet.resolve(), Path(repo_root).resolve()): + source_root = Path(repo_root).resolve() + source_chrome = (Path(theme_root) / "chrome").resolve() + if _inside(sheet.resolve(), source_chrome): + dest = chrome / sheet.resolve().relative_to(source_chrome) + extra_imports.append(quote(dest.relative_to(chrome).as_posix())) + else: + dest = variant_destination(chrome, sheet.name) if dest is None: - # No import site exists: load it after the base through a wrapper. - dest = chrome / "custom" / sheet.name + if len(sheets) > 1: + # Independent option trees retain ../ dependencies without + # borrowing a sibling option's asset with the same filename. + relative = sheet.resolve().relative_to(source_root) + key = hashlib.sha256(relative.as_posix().encode()).hexdigest()[:16] + dest = chrome / "fxcss-options" / key / relative + else: + dest = chrome / "custom" / sheet.name extra_imports.append(quote(dest.relative_to(chrome).as_posix())) + if any(dest == previous for _, previous, _ in destinations): + raise RuntimeError(f"selected optional sheets share the native import slot " + f"chrome/{dest.relative_to(chrome).as_posix()}; choose one") + destinations.append((sheet, dest, source_root)) + for sheet, dest, source_root in destinations: + if dest.exists() and (not dest.is_file() or dest.read_bytes() != sheet.read_bytes()): + raise RuntimeError(f"optional sheet {sheet.name} would overwrite a different file at " + f"chrome/{dest.relative_to(chrome).as_posix()}") dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(sheet, dest) + budget["written"].add(dest) installed_sheets.append(sheet.stem) found = _copy_sheet_dependencies( - sheet, dest, chrome, _sheet_source_root(theme_root, sheet), budget) + sheet, dest, chrome, source_root, budget) if notes is not None: notes.extend(found) layer_stylesheets(chrome / "userChrome.css", extra_imports) return installed_sheets +def preview_digests(theme_root, sheets=(), repo_root=None): + """What an install from `theme_root` would record, without installing. + + Returns {relative path: sha256} for the files the theme itself would put + under chrome/ (optional sheets included), computed from a staging copy + that is discarded. Comparing this with a manifest's `digests` answers + "has the local source changed since it was installed?" exactly, where + the old answer -- "a local source might have changed" -- announced an + upgrade every time. + """ + theme_root = Path(theme_root) + with tempfile.TemporaryDirectory(prefix="fxcss-preview-") as td: + chrome = Path(td) / "chrome" + _prepare_chrome(theme_root, chrome, sheets, [], repo_root=repo_root) + return {("chrome/" + p.relative_to(chrome).as_posix()): _digest(p) + for p in chrome.rglob("*") if p.is_file() and p.name != MANIFEST_NAME} + + def install_theme(theme_root, profile, theme_id, sheets=(), stamp=None, - source=None, origin_backup=_UNSET): + source=None, origin_backup=_UNSET, on_collision="stop", + extra=None, repo_root=None): """Prepare a theme, then swap it into the profile with a way back. All copies, optional imports, prefs and the manifest are prepared on the @@ -849,14 +999,27 @@ def install_theme(theme_root, profile, theme_id, sheets=(), stamp=None, `origin_backup` is carried through upgrades, including None for a profile that originally had no chrome/. `source` records how to fetch the theme again; older callers can still supply that information in `theme_id`. + + When the active chrome/ is one fxcss manages, files the person added to + it are carried into the new tree so they stay active (`user_files` in + the manifest). A path the new version now ships with different bytes is + a collision: with `on_collision="stop"` (the default) CollisionError is + raised before anything moves; with "theme" the new version's file is + installed and the person's copy stays in the backup. `extra` is a dict + of additional manifest keys (layout, pref files) the caller wants kept. """ from . import __version__ theme_root, profile = Path(theme_root), Path(profile) - if not (theme_root / "chrome" / "userChrome.css").is_file(): - raise RuntimeError(f"no chrome/userChrome.css under {theme_root}") + entries = [name for name in ENTRY_SHEETS + if (theme_root / "chrome" / name).is_file()] + if not entries: + raise RuntimeError(f"no chrome/userChrome.css or chrome/userContent.css " + f"under {theme_root}") if not profile.is_dir(): raise RuntimeError(f"profile directory does not exist: {profile}") + if on_collision not in ("stop", "theme"): + raise ValueError(f"on_collision must be 'stop' or 'theme', not {on_collision!r}") active = profile / "chrome" user_js = profile / "user.js" @@ -869,10 +1032,29 @@ def install_theme(theme_root, profile, theme_id, sheets=(), stamp=None, chrome = staging / "chrome" sheet_notes = [] installed_sheets = _prepare_chrome(theme_root, chrome, sheets, - sheet_notes) + sheet_notes, repo_root=repo_root) + # Upstream-owned files are recorded before the person's own are + # merged in, so `files`/`digests` describe only what the theme + # shipped and uninstall never removes something it did not write. + files = sorted(p.relative_to(staging).as_posix() + for p in chrome.rglob("*") + if p.is_file() and p != chrome / MANIFEST_NAME) + digests = {name: digest for name, digest in + ((name, _digest(staging / name)) for name in files) + if digest} + user_files, displaced = [], [] + if previous: + additions = user_additions(profile, previous) + plan = preservation_plan(profile, additions["files"], chrome) + if plan["collisions"] and on_collision == "stop": + raise CollisionError(plan["collisions"]) + displaced = plan["collisions"] + _carry_additions(profile, plan["carry"], chrome) + user_files = sorted(plan["carry"]) created = not user_js.exists() existing = "" if created else read_prefs(user_js) - block = user_js_body(theme_root) + block = user_js_body(theme_root, (extra or {}).get("pref_files"), + repo_root=repo_root) staged_prefs = staging / "user.js" if not created: shutil.copy2(user_js, staged_prefs) @@ -881,9 +1063,6 @@ def install_theme(theme_root, profile, theme_id, sheets=(), stamp=None, # not something an upgrade should reset just because it exists. if origin_backup is not _UNSET and previous: created = previous.get("user_js_created", created) - files = sorted(p.relative_to(staging).as_posix() - for p in chrome.rglob("*") - if p.is_file() and p != chrome / MANIFEST_NAME) manifest = { "schema": MANIFEST_SCHEMA, "fxcss": __version__, @@ -898,10 +1077,15 @@ def install_theme(theme_root, profile, theme_id, sheets=(), stamp=None, # References in those sheets that could not be installed. "sheet_notes": sheet_notes, "files": files, - "digests": {name: digest for name, digest in - ((name, _digest(staging / name)) for name in files) - if digest}, + "digests": digests, + # The person's own files, kept active across this install; + # and the ones the new version displaced (still in `backup`). + "user_files": user_files, + "displaced": displaced, } + for key, value in (extra or {}).items(): + if key not in manifest: + manifest[key] = value (chrome / MANIFEST_NAME).write_text( json.dumps(manifest, indent=1) + "\n", encoding="utf-8") @@ -1222,16 +1406,27 @@ def uninstall_theme(profile, stamp=None): backups = [b for b in list_backups(profile) if not b["path"].is_symlink()] stamp = stamp or _timestamp() summary = {"removed": 0, "restored": None, "moved_aside": None, - "kept": [], "theme": manifest.get("theme") if manifest else None} - if not manifest and not backups: - raise RuntimeError( - f"nothing fxcss installed in {profile}: no " - f"chrome/{MANIFEST_NAME} and no chrome.backup-* to restore") + "kept": [], "theme": manifest.get("theme") if manifest else None, + "noop": False} with _profile_work(profile, "uninstall") as work: prefs = _clean_prefs(work, profile, manifest) if prefs is not None: summary["user_js_cleaned"] = True outgoing = work / "previous" + if not manifest: + # A chrome/ without a manifest is the person's own: what a + # rollback to the original put back, or a theme installed by + # hand. There is no managed installation to remove, so nothing + # moves -- restoring the newest backup here would reinstall the + # theme the rollback just took out. The same holds when chrome/ + # is absent after uninstall. Only a stray pref block is ours to clean up. + summary["noop"] = True + summary["backups"] = [b["name"] for b in backups] + if prefs is _REMOVE_PREFS: + (profile / "user.js").unlink() + elif prefs is not None: + prefs.replace(profile / "user.js") + return summary if manifest: # Preserve links as links; never traverse them while removing files. shutil.copytree(chrome, work / "chrome", symlinks=True) @@ -1255,11 +1450,5 @@ def uninstall_theme(profile, stamp=None): if backup.is_dir() and not backup.is_symlink(): replacement = backup summary["restored"] = recorded - else: - replacement = backups[0]["path"] - summary["restored"] = replacement.name - outgoing = _spare_name(profile, f"chrome.removed-{stamp}") - if chrome.exists() or chrome.is_symlink(): - summary["moved_aside"] = outgoing.name _swap_profile(profile, replacement, outgoing, prefs, work) return summary diff --git a/fxcss/inventory.py b/fxcss/inventory.py new file mode 100644 index 0000000..b147adc --- /dev/null +++ b/fxcss/inventory.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +"""One inventory of a theme's stylesheets: what loads, from where, and why. + +Install, upgrade, audit and the generated workflows used to each read a +theme's ``@import`` lines with their own regular expression, and they +disagreed: one missed unquoted ``url()`` imports and reported twenty live +sheets as orphans, another missed ``@import "x.css" screen and +-moz-pref("…")`` and called the conditional sheet dead. This module is the +single reading they now share. + +It is syntax-aware in the ways that matter for userChrome themes -- comments +and strings are consumed whole, quoted and unquoted targets are both read, +CSS escapes in a target are decoded, and the condition after an import is +kept -- and it states what it could not settle rather than guessing: an +import that resolves to nothing is *missing*, *conditional* (behind a +``-moz-pref`` or media query), a *hook* (a conventional user-override file +the theme does not ship) or *external* (``chrome://``, ``data:``, a URL), +and those are reported as different things. + +Scope is tracked from the entry sheet: ``userChrome.css`` reaches the +browser window, ``userContent.css`` reaches web content and extension +pages, and a sheet reached only from the latter is never evidence of +browser-chrome breakage. ``complete`` is False whenever an entry sheet is +missing or a required import is unresolved, so a half-shipped package reads +as an incomplete analysis rather than a clean bill of health. +""" + +import os +import re +from pathlib import Path +from urllib.parse import unquote + +# The stylesheets Firefox loads from chrome/ by name, with the documents +# each one reaches. +ENTRY_SCOPES = {"userChrome.css": "chrome", "userContent.css": "content"} +ENTRY_SHEETS = tuple(ENTRY_SCOPES) + +# Conventional names for a user's own override sheet, which themes @import +# without shipping: the import of a missing file fails silently and that is +# the whole on/off mechanism. Lower-case, compared against the file name. +HOOK_NAMES = frozenset({ + "customchrome.css", "customcontent.css", "custom.css", "user.css", + "local.css", "overrides.css", "override.css", "my.css", "mine.css", + "userchrome.local.css", "usercontent.local.css", "usercustom.css", +}) + +# Beyond the exact names, a missing import whose stem says it is the +# person's own file (FlexFox's components/uc-user-settings.css, "create it +# to modify the interface") is a hook too. Deliberately narrow: "user-chrome" +# is Material Fox's *build output*, and calling that a hook would hide the +# fact that the source tree needs compiling. +HOOK_STEM_RE = re.compile(r"(custom|settings|local|overrides?|mine|my[-_])", re.I) + +# Directories of deliberately opt-in sheets. Searched both beside chrome/ +# and inside it; a sheet an entry already reaches is active, not optional. +# A missing import that points into one of these (Safari Golden Gate's +# userContent.css imports ThemeCore/custom/