From d1877b84bb10ff7caa6bd12ab4e75498ba760790 Mon Sep 17 00:00:00 2001 From: Bill Schnurr Date: Fri, 30 Oct 2026 10:18:30 -0700 Subject: [PATCH] Share Python extension localization tooling and submission action Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/actions/localization/action.yml | 99 ++++++ README.md | 68 ++++ noxfile.py | 10 + scripts/localization.py | 285 +++++++++++++++++ scripts/localization_test.py | 399 ++++++++++++++++++++++++ 5 files changed, 861 insertions(+) create mode 100644 .github/actions/localization/action.yml create mode 100644 scripts/localization.py create mode 100644 scripts/localization_test.py diff --git a/.github/actions/localization/action.yml b/.github/actions/localization/action.yml new file mode 100644 index 0000000..a66cea6 --- /dev/null +++ b/.github/actions/localization/action.yml @@ -0,0 +1,99 @@ +name: Submit English localization +description: Export and submit English strings using the shared Python localization tooling. + +inputs: + app-id: + description: Python localization GitHub App ID. + required: true + private-key: + description: Python localization GitHub App private key. + required: true + source-paths: + description: JSON array of runtime source directories relative to the extension root. + default: '["src"]' + submodules: + description: Whether the extension needs recursive submodule checkout. + default: 'false' + node-version: + description: Node.js version for the localization CLI. + default: '22.21.1' + npm-registry: + description: Package feed used for the pinned localization CLI. + default: 'https://packagefeedproxy.microsoft.io/npm/' + localization-branch: + description: Existing localization-drop branch for English submissions. + default: dev + +runs: + using: composite + steps: + - name: Validate localization credentials + shell: python + env: + LOCALIZATION_APP_ID: ${{ inputs.app-id }} + LOCALIZATION_PRIVATE_KEY: ${{ inputs.private-key }} + run: | + import os + if not os.environ["LOCALIZATION_APP_ID"] or not os.environ["LOCALIZATION_PRIVATE_KEY"]: + raise ValueError("Configure PYTHON_LOCALIZATION_APP_ID and PYTHON_LOCALIZATION_PRIVATE_KEY.") + + - name: Checkout extension + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + submodules: ${{ inputs.submodules }} + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: ${{ inputs.node-version }} + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.10' + + - name: Extract English strings + shell: bash + env: + LOCALIZATION_ACTION_PATH: ${{ github.action_path }} + LOCALIZATION_SOURCE_PATHS: ${{ inputs.source-paths }} + NPM_CONFIG_REGISTRY: ${{ inputs.npm-registry }} + run: >- + python "$LOCALIZATION_ACTION_PATH/../../../scripts/localization.py" + --operation export + --extension-root "$GITHUB_WORKSPACE" + --source-paths "$LOCALIZATION_SOURCE_PATHS" + --output-directory "$RUNNER_TEMP/localization-english" + + - name: Authenticate as python-localization + id: app-token + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2 + with: + app-id: ${{ inputs.app-id }} + private-key: ${{ inputs.private-key }} + owner: microsoft + repositories: vscode-python-extensions-loc + permission-contents: write + + - name: Checkout localization repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: microsoft/vscode-python-extensions-loc + ref: ${{ inputs.localization-branch }} + token: ${{ steps.app-token.outputs.token }} + path: localization-translations + fetch-depth: 0 + + - name: Submit changed English XLF + shell: bash + env: + LOCALIZATION_ACTION_PATH: ${{ github.action_path }} + LOCALIZATION_BRANCH: ${{ inputs.localization-branch }} + run: >- + python "$LOCALIZATION_ACTION_PATH/../../../scripts/localization.py" + --operation submit + --extension-root "$GITHUB_WORKSPACE" + --output-directory "$RUNNER_TEMP/localization-english" + --drop-root "$GITHUB_WORKSPACE/localization-translations" + --branch "$LOCALIZATION_BRANCH" diff --git a/README.md b/README.md index 7d96d91..8fbb65e 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,74 @@ Use the Node.js version pinned in `.nvmrc` for development. ## Consuming in Extensions +### Shared localization automation + +The composite action at `.github/actions/localization` and the standard-library +Python CLI at `scripts/localization.py` share localization automation across the +Python extensions. This tooling does not require adopting either runtime library. +Pin the action and Azure tooling checkout to the same reviewed commit. + +Each extension's English-only GitHub workflow calls: + +```yaml +permissions: + contents: read + +jobs: + localization: + runs-on: ubuntu-latest + steps: + - uses: microsoft/vscode-common-python-lsp/.github/actions/localization@ + with: + app-id: ${{ vars.PYTHON_LOCALIZATION_APP_ID }} + private-key: ${{ secrets.PYTHON_LOCALIZATION_PRIVATE_KEY }} + submodules: recursive + source-paths: '["src", "external/vscode-common-python-lsp/typescript/src"]' +``` + +Keep trigger paths and concurrency in the caller workflow. Omit the submodule +input and use the default `["src"]` for standalone runtime extensions. Extensions +without a `package.json` `l10n` declaration export and import manifest strings +only, even if their source contains runtime localization calls. + +The action bundles checkout, Node/Python setup, extraction with pinned +`@vscode/l10n-dev@0.0.35`, app authentication, and English submission. It loads the +helper from its own action revision. Configure the app ID variable and private-key +secret in each caller (or grant organization-level access); the app needs Contents +write access only to `microsoft/vscode-python-extensions-loc`. The default npm feed +is `https://packagefeedproxy.microsoft.io/npm/`. Inputs never appear in generated +shell commands: source paths and branch values are passed through environment +variables. + +English goes to `src/en/./.-new.xlf` on the +existing `dev` branch. Unchanged XLF is a successful no-op. Submissions never +force-push; concurrent updates retry rejected pushes with a bounded rebase. + +Pylance-Eng stable/pre-release templates orchestrate **pull-only** imports using +the same CLI, with source, tools, and translations in separate workspace folders: + +```text +python /scripts/localization.py --operation import \ + --extension-root --translations-root /out \ + --output-directory +``` + +The importer derives identity/runtime metadata from the extension manifest, +requires all 14 configured locales, and validates every table before copying. +Each required real-language manifest/runtime table must contain translation +evidence: a nonempty target differing from English, or a target explicitly marked +translated/final/signed-off. Individual untranslated strings are allowed; +pseudo-localization is checked for coverage but is not real-language evidence. +Missing, empty, incomplete, or wholly untranslated handbacks fail explicitly. +Do not publish while waiting for service onboarding or real translated output. + +Generated root `package.nls..json` and declared runtime locale files should +be ignored by Git but included by VSCE. Keep English `package.nls.json` tracked. +Run dependency-free regression tests with +`python -m unittest discover -s scripts -p "*_test.py"`. + +### Shared runtime libraries + Consume this repository as a git submodule so the Python and TypeScript libraries are pinned to the same commit. The VS Code Python tool extensions place the checkout under `external/`: diff --git a/noxfile.py b/noxfile.py index 0aa5e33..70e48ab 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,6 +14,9 @@ def install(session: nox.Session) -> None: @nox.session() def tests(session: nox.Session) -> None: """Run Python tests.""" + session.run( + "python", "-m", "unittest", "discover", "-s", "scripts", "-p", "*_test.py" + ) session.install("-e", "./python[dev]") session.run("pytest", "python/tests", "-v") @@ -31,3 +34,10 @@ def lint(session: nox.Session) -> None: session.run("isort", "--check", "python/vscode_common_python_lsp") session.run("isort", "--check", "python/tests") + session.run("flake8", "scripts/localization.py", "scripts/localization_test.py") + session.run( + "black", "--check", "scripts/localization.py", "scripts/localization_test.py" + ) + session.run( + "isort", "--check", "scripts/localization.py", "scripts/localization_test.py" + ) diff --git a/scripts/localization.py b/scripts/localization.py new file mode 100644 index 0000000..1a3fb30 --- /dev/null +++ b/scripts/localization.py @@ -0,0 +1,285 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""Submit English localization inputs or import validated release handbacks.""" + +import argparse +import json +import pathlib +import re +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET + +CLI = "@vscode/l10n-dev@0.0.35" +LOCALES = ( + "cs", + "de", + "es", + "fr", + "it", + "ja", + "ko", + "pl", + "pt-br", + "qps-ploc", + "ru", + "tr", + "zh-cn", + "zh-tw", +) +LOCALE_ALIASES = {"zh-hans": "zh-cn", "zh-hant": "zh-tw"} +TRANSLATED_STATES = {"translated", "final", "signed-off"} + + +def inside(root: pathlib.Path, value: str) -> pathlib.Path: + """Resolve a non-root path without allowing escape from the extension.""" + path = (root / value).resolve() + if path == root or not path.is_relative_to(root): + raise ValueError(f"Expected a path inside the extension: {value}") + return path + + +def metadata(root: pathlib.Path) -> tuple[str, pathlib.Path | None]: + """Read the canonical extension ID and optional runtime output directory.""" + package = json.loads((root / "package.json").read_text(encoding="utf-8-sig")) + identity = [package.get("publisher"), package.get("name")] + if not all( + isinstance(part, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", part) + for part in identity + ): + raise ValueError("package.json must declare a valid publisher and name.") + runtime = package.get("l10n") + if runtime is not None and (not isinstance(runtime, str) or not runtime): + raise ValueError("package.json l10n must be a nonempty relative directory.") + return ".".join(identity), inside(root, runtime) if runtime is not None else None + + +def run_cli(root: pathlib.Path, *arguments: str) -> None: + """Run the pinned localization CLI through the configured npm feed.""" + executable = shutil.which("npx") + if executable is None: + raise FileNotFoundError("npx is required to run the localization CLI.") + subprocess.run([executable, "--yes", CLI, *arguments], cwd=root, check=True) + + +def read_table(path: pathlib.Path, *, comments: bool = False) -> dict: + """Read a nonempty string table, optionally allowing export message metadata.""" + table = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(table, dict) or not table: + raise ValueError(f"Invalid or empty localization table: {path}") + for value in table.values(): + if comments and isinstance(value, dict): + value = value.get("message") + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"Invalid or empty localization string in {path}") + return table + + +def create_staging(root: pathlib.Path, output: pathlib.Path) -> None: + """Keep staging outside source until the complete import is validated.""" + if output.resolve().is_relative_to(root.resolve()): + raise ValueError("Localization staging must be outside extension source.") + output.mkdir(parents=True, exist_ok=False) + + +def export_english( + root: pathlib.Path, output: pathlib.Path, source_paths: list[str] +) -> pathlib.Path: + """Export manifest and declared runtime strings to a new staging directory.""" + extension_id, runtime = metadata(root) + read_table(root / "package.nls.json") + create_staging(root, output) + inputs = [] + if runtime is not None: + if not source_paths or not all(isinstance(p, str) for p in source_paths): + raise ValueError("Runtime export requires a nonempty list of source paths.") + sources = [inside(root, path) for path in source_paths] + for source in sources: + if not source.is_dir(): + raise FileNotFoundError( + f"Missing extraction source: {source}. " + "Initialize required submodules." + ) + run_cli(root, "export", "-o", str(output), *(str(p) for p in sources)) + bundle = output / "bundle.l10n.json" + read_table(bundle, comments=True) + inputs.append(str(bundle)) + xlf = output / f"{extension_id}-new.xlf" + run_cli(root, "generate-xlf", "-o", str(xlf), *inputs, "package.nls.json") + content = xlf.read_text(encoding="utf-8-sig") + with xlf.open("w", encoding="utf-8", newline="\n") as destination: + destination.write(content) + return xlf + + +def validate_handback(files: list[pathlib.Path], runtime: bool) -> None: + """Require coverage and translation evidence in each required locale table.""" + kinds = ("package", "bundle") if runtime else ("package",) + tables = {} + for path in files: + document = ET.parse(path) + for table in document.findall(".//{*}file"): + locale = table.get("target-language", "").lower() + locale = LOCALE_ALIASES.get(locale, locale) + kind = table.get("original") + if locale not in LOCALES or kind not in kinds: + continue + key = (locale, kind) + if key in tables: + raise ValueError(f"Duplicate translation table: {locale}/{kind}") + units = table.findall(".//{*}trans-unit") + if not units: + raise ValueError(f"Empty translation table: {locale}/{kind}") + translated = False + for unit in units: + source = unit.find("{*}source") + target = unit.find("{*}target") + if source is None or not "".join(source.itertext()).strip(): + raise ValueError(f"Missing English source in {locale}/{kind}") + if target is not None: + text = "".join(target.itertext()).strip() + translated |= bool(text) and ( + text != "".join(source.itertext()).strip() + or target.get("state") in TRANSLATED_STATES + ) + tables[key] = translated + for locale in LOCALES: + for kind in kinds: + if (locale, kind) not in tables: + raise ValueError( + f"Incomplete translation handback: missing {locale}/{kind}" + ) + if locale != "qps-ploc" and not tables[locale, kind]: + raise ValueError( + f"Translation handback is not ready: {locale}/{kind} is wholly " + "untranslated. Wait for translated output from the " + "localization service." + ) + + +def import_translations( + root: pathlib.Path, translations: pathlib.Path, output: pathlib.Path +) -> None: + """Validate all handbacks in staging before placing any generated locale files.""" + extension_id, runtime = metadata(root) + if not translations.is_dir(): + raise FileNotFoundError( + f"Translated output is unavailable: {translations}. " + "Submit English strings and wait for localization-service handbacks." + ) + files = sorted(translations.rglob(f"{extension_id}-new.xlf")) + if not files: + raise FileNotFoundError(f"No {extension_id} handbacks found in {translations}") + validate_handback(files, runtime is not None) + create_staging(root, output) + run_cli(root, "import-xlf", "-o", str(output), *(str(file) for file in files)) + names = [f"package.nls.{locale}.json" for locale in LOCALES] + if runtime is not None: + names.extend(f"bundle.l10n.{locale}.json" for locale in LOCALES) + for name in names: + read_table(output / name) + if runtime is not None: + runtime.mkdir(parents=True, exist_ok=True) + for name in names: + destination = root if name.startswith("package.nls.") else runtime + shutil.copyfile(output / name, destination / name) + + +def git(root: pathlib.Path, *arguments: str) -> str: + """Run a Git command without shell interpolation.""" + result = subprocess.run( + ["git", *arguments], cwd=root, check=False, capture_output=True, text=True + ) + if result.returncode: + print(result.stderr, file=sys.stderr) + result.check_returncode() + return result.stdout.strip() + + +def submit_english( + root: pathlib.Path, output: pathlib.Path, drop: pathlib.Path, branch: str +) -> None: + """Commit only this extension's changed English XLF on the configured branch.""" + extension_id, _ = metadata(root) + git(drop, "check-ref-format", "--branch", branch) + if git(drop, "branch", "--show-current") != branch: + raise ValueError(f"Localization checkout must be on the {branch} branch.") + if git(drop, "status", "--porcelain"): + raise ValueError("Localization checkout must be clean before submission.") + git(drop, "pull", "--rebase", "origin", branch) + git(drop, "config", "user.name", "github-actions[bot]") + git( + drop, + "config", + "user.email", + "41898282+github-actions[bot]@users.noreply.github.com", + ) + git(drop, "config", "core.autocrlf", "input") + source = output / f"{extension_id}-new.xlf" + destination = drop / "src" / "en" / extension_id / source.name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + relative = str(destination.relative_to(drop)) + git(drop, "add", "--", relative) + if not git(drop, "diff", "--cached", "--name-only", "--", relative): + print("English localization strings are already up to date.") + return + git(drop, "commit", "-m", f"Update English strings for {extension_id}") + for attempt in range(3): + git(drop, "pull", "--rebase", "origin", branch) + pushed = subprocess.run( + ["git", "push", "origin", f"HEAD:refs/heads/{branch}"], + cwd=drop, + capture_output=True, + text=True, + check=False, + ) + if pushed.returncode == 0: + print(f"Submitted English strings for {extension_id} to {branch}.") + return + if attempt == 2 or not ( + "[rejected]" in pushed.stderr + and ("fetch first" in pushed.stderr or "non-fast-forward" in pushed.stderr) + ): + print(pushed.stderr, file=sys.stderr) + pushed.check_returncode() + print("Localization branch advanced; rebasing before retrying submission.") + + +def main(argv=None) -> None: + """Run the same explicit Python interface from GitHub Actions and Azure.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--operation", choices=("export", "import", "submit"), required=True + ) + parser.add_argument( + "--extension-root", type=pathlib.Path, default=pathlib.Path.cwd() + ) + parser.add_argument("--output-directory", type=pathlib.Path, required=True) + parser.add_argument("--source-paths", default='["src"]') + parser.add_argument("--translations-root", type=pathlib.Path) + parser.add_argument("--drop-root", type=pathlib.Path) + parser.add_argument("--branch", default="dev") + args = parser.parse_args(argv) + root = args.extension_root.resolve() + output = args.output_directory.resolve() + if args.operation == "export": + paths = json.loads(args.source_paths) + if not isinstance(paths, list): + raise ValueError("--source-paths must be a JSON array.") + export_english(root, output, paths) + elif args.operation == "import": + if args.translations_root is None: + parser.error("--translations-root is required for import") + extension_id, _ = metadata(root) + import_translations(root, args.translations_root / extension_id, output) + else: + if args.drop_root is None: + parser.error("--drop-root is required for submission") + submit_english(root, output, args.drop_root.resolve(), args.branch) + + +if __name__ == "__main__": + main() diff --git a/scripts/localization_test.py b/scripts/localization_test.py new file mode 100644 index 0000000..97adc45 --- /dev/null +++ b/scripts/localization_test.py @@ -0,0 +1,399 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""Dependency-free localization core and Git submission regression tests.""" + +import json +import pathlib +import shutil +import subprocess +import tempfile +import unittest +import xml.etree.ElementTree as ET +from unittest import mock + +import localization + + +class LocalizationTests(unittest.TestCase): + """Exercise generic exports and fully staged, translation-ready imports.""" + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="shared localization ") + self.addCleanup(self.temporary.cleanup) + self.base = pathlib.Path(self.temporary.name).resolve() + self.root = self.base / "extension" + self.root.mkdir() + (self.root / "src").mkdir() + self.output = self.base / "staging" + self.translations = self.base / "handbacks" + self.translations.mkdir() + self.configure() + self.bad_json = None + self.cli = mock.patch.object(localization, "run_cli", side_effect=self.run_cli) + self.calls = self.cli.start() + self.addCleanup(self.cli.stop) + + def configure(self, name="tool", runtime="./l10n"): + package = {"publisher": "ms-python", "name": name} + if runtime is not None: + package["l10n"] = runtime + (self.root / "package.json").write_text(json.dumps(package), encoding="utf-8") + (self.root / "package.nls.json").write_text( + '{"description":"English source"}\n', encoding="utf-8" + ) + + def handbacks(self, *, runtime=True, untranslated=None, same=False, state="new"): + extension_id, _ = localization.metadata(self.root) + files = [] + for locale in localization.LOCALES: + folder = self.translations / locale + folder.mkdir(exist_ok=True) + path = folder / f"{extension_id}-new.xlf" + document = ET.Element( + "xliff", + {"xmlns": "urn:oasis:names:tc:xliff:document:1.2", "version": "1.2"}, + ) + language = {"zh-cn": "zh-Hans", "zh-tw": "zh-Hant"}.get(locale, locale) + for kind in (("package", "bundle") if runtime else ("package",)): + table = ET.SubElement( + document, "file", {"original": kind, "target-language": language} + ) + body = ET.SubElement(table, "body") + for index in range(2): + unit = ET.SubElement(body, "trans-unit", {"id": str(index)}) + ET.SubElement(unit, "source").text = f"English {index}" + target = ET.SubElement(unit, "target", {"state": state}) + target.text = ( + f"English {index}" + if same or untranslated == (locale, kind) or index == 1 + else f"{locale} translation" + ) + ET.ElementTree(document).write(path, encoding="utf-8", xml_declaration=True) + files.append(path) + return files + + def run_cli(self, root, operation, _flag, output, *inputs): + output = pathlib.Path(output) + self.assertEqual(root, self.root) + if operation == "export": + (output / "bundle.l10n.json").write_text( + '{"Open logs":{"message":"Open logs","comment":["Button"]}}', + encoding="utf-8", + ) + elif operation == "generate-xlf": + output.write_bytes(b"\r\n\r\n") + elif operation == "import-xlf": + _, runtime = localization.metadata(root) + for locale in localization.LOCALES: + kinds = ("package.nls", "bundle.l10n") if runtime else ("package.nls",) + for kind in kinds: + name = f"{kind}.{locale}.json" + content = json.dumps({"description": f"{locale} translation"}) + if name == self.bad_json: + content = "{}" + (output / name).write_text(content, encoding="utf-8-sig") + else: + self.fail(f"Unexpected operation: {operation}, {inputs}") + + def test_generic_runtime_export(self): + for name in ("debugpy", "mypy-type-checker", "vscode-python-envs"): + with self.subTest(name=name): + self.configure(name) + output = self.base / name + xlf = localization.export_english(self.root, output, ["src"]) + self.assertEqual(xlf.name, f"ms-python.{name}-new.xlf") + self.assertEqual(xlf.read_bytes(), b"\n\n") + + def test_shared_source_paths(self): + source = self.root / "external" / "shared" / "src" + source.mkdir(parents=True) + localization.export_english( + self.root, self.output, ["src", "external/shared/src"] + ) + self.assertEqual( + self.calls.call_args_list[0].args[4:], + (str(self.root / "src"), str(source)), + ) + + def test_manifest_only_export(self): + self.configure("autopep8", runtime=None) + xlf = localization.export_english(self.root, self.output, []) + self.assertEqual(xlf.name, "ms-python.autopep8-new.xlf") + self.assertEqual(self.calls.call_count, 1) + self.assertEqual(self.calls.call_args.args[1], "generate-xlf") + self.assertFalse((self.output / "bundle.l10n.json").exists()) + + def test_missing_source_and_invalid_source_input(self): + for paths in (["missing"], ["../outside"], [], [1]): + with self.subTest(paths=paths): + with self.assertRaises((ValueError, FileNotFoundError)): + localization.export_english(self.root, self.output, paths) + self.assertFalse((self.output / "ms-python.tool-new.xlf").exists()) + if self.output.exists(): + shutil.rmtree(self.output) + + def test_bad_metadata(self): + for runtime in ("../outside", ".", "", 4): + with self.subTest(runtime=runtime): + self.configure(runtime=runtime) + with self.assertRaises(ValueError): + localization.metadata(self.root) + self.configure("../escape") + with self.assertRaises(ValueError): + localization.metadata(self.root) + + def test_cli_failure_is_propagated(self): + self.calls.side_effect = subprocess.CalledProcessError(1, ["npx"]) + with self.assertRaises(subprocess.CalledProcessError): + localization.export_english(self.root, self.output, ["src"]) + + def test_missing_and_empty_export_bundle(self): + for content in (None, "{}"): + with self.subTest(content=content): + + def empty_export(_root, _operation, _flag, output, *_inputs): + if content is not None: + (pathlib.Path(output) / "bundle.l10n.json").write_text(content) + + self.calls.side_effect = empty_export + with self.assertRaises((ValueError, FileNotFoundError)): + localization.export_english(self.root, self.output, ["src"]) + shutil.rmtree(self.output) + + def test_runtime_import_and_english_preservation(self): + self.handbacks() + english = (self.root / "package.nls.json").read_bytes() + localization.import_translations(self.root, self.translations, self.output) + self.assertEqual((self.root / "package.nls.json").read_bytes(), english) + self.assertEqual(len(list(self.root.glob("package.nls.*.json"))), 14) + self.assertEqual(len(list((self.root / "l10n").glob("*.json"))), 14) + + def test_manifest_only_import(self): + self.configure("autopep8", runtime=None) + self.handbacks(runtime=False) + localization.import_translations(self.root, self.translations, self.output) + self.assertEqual(len(list(self.root.glob("package.nls.*.json"))), 14) + self.assertFalse((self.root / "l10n").exists()) + + def test_readiness_checked_per_table(self): + for kind in ("package", "bundle"): + with self.subTest(kind=kind): + self.handbacks(untranslated=("de", kind)) + existing = self.root / "package.nls.de.json" + existing.write_text('{"description":"keep"}', encoding="utf-8") + with self.assertRaisesRegex(ValueError, f"de/{kind} is wholly"): + localization.import_translations( + self.root, self.translations, self.output + ) + self.assertEqual(existing.read_text(), '{"description":"keep"}') + self.assertFalse(self.output.exists()) + self.calls.assert_not_called() + + def test_partial_translation_is_allowed(self): + files = self.handbacks() + localization.validate_handback(files, runtime=True) + + def test_source_identical_marked_translated_is_allowed(self): + for state in ("translated", "final", "signed-off"): + with self.subTest(state=state): + files = self.handbacks(same=True, state=state) + localization.validate_handback(files, runtime=True) + + def test_all_new_english_is_rejected(self): + files = self.handbacks(same=True) + with self.assertRaisesRegex(ValueError, "wholly untranslated"): + localization.validate_handback(files, runtime=True) + + def test_pseudo_localization_is_not_translation_evidence(self): + files = self.handbacks(untranslated=("qps-ploc", "package")) + localization.validate_handback(files, runtime=True) + + def test_missing_and_duplicate_locale_tables(self): + files = self.handbacks() + with self.assertRaisesRegex(ValueError, "missing zh-tw/package"): + localization.validate_handback(files[:-1], runtime=True) + with self.assertRaisesRegex(ValueError, "Duplicate"): + localization.validate_handback([*files, files[0]], runtime=True) + + def test_missing_runtime_tables(self): + files = self.handbacks(runtime=False) + with self.assertRaisesRegex(ValueError, "missing cs/bundle"): + localization.validate_handback(files, runtime=True) + + def test_missing_and_wrong_identity_handbacks(self): + with self.assertRaisesRegex(FileNotFoundError, "No ms-python.tool handbacks"): + localization.import_translations(self.root, self.translations, self.output) + with self.assertRaisesRegex(FileNotFoundError, "unavailable"): + localization.import_translations( + self.root, self.base / "absent", self.output + ) + + def test_invalid_cli_output_copies_nothing(self): + self.handbacks() + self.bad_json = "bundle.l10n.zh-tw.json" + existing = self.root / "package.nls.de.json" + existing.write_text('{"description":"keep"}', encoding="utf-8") + with self.assertRaisesRegex(ValueError, "Invalid or empty"): + localization.import_translations(self.root, self.translations, self.output) + self.assertEqual(existing.read_text(), '{"description":"keep"}') + self.assertFalse((self.root / "l10n").exists()) + + def test_invalid_string_tables(self): + table = self.base / "invalid.json" + for value in ([], {}, {"key": 1}, {"key": ""}, {"key": {"message": "text"}}): + with self.subTest(value=value): + table.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaises(ValueError): + localization.read_table(table) + + def test_existing_staging_is_rejected(self): + self.handbacks() + self.output.mkdir() + with self.assertRaises(FileExistsError): + localization.import_translations(self.root, self.translations, self.output) + self.calls.assert_not_called() + + def test_staging_cannot_be_a_generated_source_directory(self): + self.handbacks() + with self.assertRaisesRegex(ValueError, "outside extension source"): + localization.import_translations( + self.root, self.translations, self.root / "l10n" + ) + self.assertFalse((self.root / "l10n").exists()) + self.calls.assert_not_called() + + def test_malformed_xml_and_empty_tables(self): + files = self.handbacks() + files[0].write_text("\n") + subprocess.run( + ["git", "init", "--bare", "--quiet", str(self.remote)], check=True + ) + subprocess.run( + ["git", "clone", "--quiet", str(self.remote), str(self.drop)], + check=True, + capture_output=True, + ) + self.git("config", "user.name", "Fixture") + self.git("config", "user.email", "fixture@example.invalid") + self.git("checkout", "-b", "main") + self.git("commit", "--allow-empty", "-m", "Initial") + self.git("push", "origin", "main") + self.main = self.git("rev-parse", "HEAD") + self.git("checkout", "-b", "dev") + self.git("push", "-u", "origin", "dev") + + def git(self, *args): + return localization.git(self.drop, *args) + + def submit(self): + localization.submit_english(self.root, self.output, self.drop, "dev") + + def test_push_then_noop_preserves_main(self): + self.submit() + submitted = self.git("rev-parse", "HEAD") + self.assertNotEqual(submitted, self.main) + self.assertEqual(self.git("rev-parse", "origin/main"), self.main) + self.assertEqual(self.git("rev-parse", "origin/dev"), submitted) + self.assertEqual( + self.git("diff", "--name-only", self.main, submitted), + "src/en/ms-python.tool/ms-python.tool-new.xlf", + ) + self.submit() + self.assertEqual(self.git("rev-parse", "HEAD"), submitted) + + def test_wrong_branch_is_rejected(self): + self.git("checkout", "main") + with self.assertRaisesRegex(ValueError, "must be on the dev branch"): + self.submit() + + def test_missing_remote_branch_is_rejected(self): + self.git("push", "origin", "--delete", "dev") + with self.assertRaises(subprocess.CalledProcessError): + self.submit() + + def test_dirty_drop_is_rejected(self): + (self.drop / "unrelated").write_text("do not commit") + with self.assertRaisesRegex(ValueError, "must be clean"): + self.submit() + + def test_racing_push_rebases_and_retries(self): + run = subprocess.run + rejected = False + + def race(command, **kwargs): + nonlocal rejected + if command[:3] == ["git", "push", "origin"] and not rejected: + rejected = True + return subprocess.CompletedProcess( + command, 1, "", "! [rejected] HEAD -> dev (fetch first)" + ) + return run(command, **kwargs) + + with mock.patch.object(localization.subprocess, "run", side_effect=race): + self.submit() + self.assertTrue(rejected) + self.assertEqual( + self.git("rev-parse", "origin/dev"), self.git("rev-parse", "HEAD") + ) + + +if __name__ == "__main__": + unittest.main()