From 3874491214ded1ed249b221841107d8efcc43316 Mon Sep 17 00:00:00 2001 From: xcv58 Date: Fri, 23 Oct 2026 15:22:36 -0400 Subject: [PATCH 1/5] Reuse locked internal compiler output for fallback plugin packaging --- CONTRIBUTING.md | 2 + docs/plugins/local-native-plugins.md | 2 + scripts/plugins/build-local-plugins.sh | 24 +++- scripts/plugins/with-build-cache.py | 158 +++++++++++++++++++++++ scripts/tests/test_plugin_build_cache.py | 152 ++++++++++++++++++++++ 5 files changed, 333 insertions(+), 5 deletions(-) create mode 100644 scripts/plugins/with-build-cache.py create mode 100644 scripts/tests/test_plugin_build_cache.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46facefec..c4c0aa0fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,6 +103,8 @@ Use `TEST_FILTER=ClassName/testMethod` for one method, or `make test` for all re Unsigned XCTest builds use `build/DerivedDataTests` by default, separate from the signed app used by `make run`. Override `TEST_DERIVED_DATA` when needed, keeping it separate from `DERIVED_DATA` to avoid leaving a test bundle inside the installed app. +Fallback plugin packaging compiles reuse a locked internal compiler cache, while packages and per-run logs remain in the requested output directory. Use `MACTOOLS_PLUGIN_FRESH_BUILD=1 make build-plugin PLUGIN=` for a cold validation build. See [local plugin build storage](docs/plugins/local-native-plugins.md#development-steps) for cache roots and probe isolation; normal Makefile app/test reuse and aggregate-product packaging keep their existing paths. + For copy and catalog changes, run `make validate-localization` to check English resources, complete coverage of all 11 supported languages, and translation format contracts. `make script-tests` includes this audit and fails when a translation is missing. To inspect per-key findings, run `python3 scripts/audit-localization.py --require-complete --json`. Follow the [plugin localization rules](docs/plugins/development-guidelines.md#protocols-and-ownership), and manually check language switching and affected UI layout. Action-symbol changes must keep manifests and runtime definitions consistent; verify them with `make test TEST_FILTER=PluginRuntimeActionSnapshotTests`. | Change | Verification scope | diff --git a/docs/plugins/local-native-plugins.md b/docs/plugins/local-native-plugins.md index 5c85546ac..43b41a311 100644 --- a/docs/plugins/local-native-plugins.md +++ b/docs/plugins/local-native-plugins.md @@ -108,6 +108,8 @@ A full Debug sync mirrors the current checkout: packages missing from its local Debug package copies normalize `minHostVersion` to the locally built app version because the host and plugin bundles come from the same checkout. Release packages preserve each source manifest's declared minimum host version. +Fallback compiles in `build-local-plugins.sh` reuse internal compiler workspaces under `~/Library/Caches/MacTools/PluginBuilds`. The cache separates source checkout, project/scheme, Xcode/SDK, host architecture, configuration, destination, signing mode and compiler flags. A lock covers compilation and copying bundles into packages. Each run retains its log and build record under `/BuildReports/`; packaged bundles remain under `/Packages` and do not depend on the compiler cache. Set `MACTOOLS_PLUGIN_FRESH_BUILD=1` for a separate cold workspace, `MACTOOLS_PLUGIN_CACHE_VARIANT` to isolate a probe, or `MACTOOLS_PLUGIN_CACHE_ROOT` for another internal root. Existing compiler output is preserved. Prebuilt packages and `--products-dir` aggregate products keep their existing packaging path. + If the plugin needs extra frameworks, private include paths, bundle resources, helper/tool targets, or target dependencies, add only those differences in `Plugins//project.yml`. If the plugin package contains an extra executable inside the bundle resources, declare it in `plugin.json.package.signPaths` so release packaging signs it before signing the bundle. Plugin UI copy should be localized by the plugin itself. Put plugin string catalogs under `Plugins//Resources`, then look them up from the plugin bundle, for example: diff --git a/scripts/plugins/build-local-plugins.sh b/scripts/plugins/build-local-plugins.sh index 815736cf8..3d4451e18 100755 --- a/scripts/plugins/build-local-plugins.sh +++ b/scripts/plugins/build-local-plugins.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +ORIGINAL_ARGS=("$@") if [[ -z "${PYTHON3:-}" ]]; then if [[ -x /usr/bin/python3 ]]; then @@ -133,7 +134,7 @@ if [[ -z "$SOURCE_DIR" || ! -d "$SOURCE_DIR" ]]; then exit 1 fi -REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" OFFICIAL_PLUGIN_SOURCE_DIR="$(cd "$REPO_ROOT/Plugins" && pwd)" THIRD_PARTY_NOTICE_MANIFEST="$REPO_ROOT/Sources/Resources/ThirdPartyNotices/manifest.json" THIRD_PARTY_NOTICE_GENERATOR="$REPO_ROOT/scripts/licenses/generate-third-party-notices.py" @@ -142,11 +143,11 @@ if [[ -z "$XCODEBUILD_COMMAND" ]]; then fi OUTPUT_DIR="$(mkdir -p "$OUTPUT_DIR" && cd "$OUTPUT_DIR" && pwd)" PACKAGES_DIR="$OUTPUT_DIR/Packages" -DERIVED_DATA_DIR="$OUTPUT_DIR/DerivedData" +DERIVED_DATA_DIR="${MACTOOLS_PLUGIN_DERIVED_ROOT:-}" CATALOG_PATH="$OUTPUT_DIR/catalog.dev.json" rm -rf "$PACKAGES_DIR" -mkdir -p "$PACKAGES_DIR" "$DERIVED_DATA_DIR" +mkdir -p "$PACKAGES_DIR" json_value() { local file="$1" @@ -309,6 +310,14 @@ build_bundle_if_needed() { return fi + # Prebuilt/aggregate products keep their existing path; only fallback compiles need a cache owner. + if [[ "${MACTOOLS_PLUGIN_CACHE_CHECKOUT:-}" != "$REPO_ROOT" ]]; then + local cache_args=(--source-dir "$SOURCE_DIR" --output-dir "$OUTPUT_DIR" + --configuration "$CONFIGURATION" --destination "$DESTINATION" --xcodebuild "$XCODEBUILD_COMMAND") + [[ "$UNSIGNED_BUILD" == "1" ]] && cache_args+=(--unsigned-build) + exec "$PYTHON3" "$REPO_ROOT/scripts/plugins/with-build-cache.py" "${cache_args[@]}" -- "${ORIGINAL_ARGS[@]}" + fi + local project project_file_for "$root" "$manifest" project if [[ -z "$project" ]]; then @@ -324,8 +333,13 @@ build_bundle_if_needed() { fi local derived_data - derived_data="$DERIVED_DATA_DIR/$(basename "$root")" - rm -rf "$derived_data" + local project_key + project_key="$("$PYTHON3" - "$project" "$scheme" <<'PY' +import hashlib, json, pathlib, sys +print(hashlib.sha256(json.dumps([str(pathlib.Path(sys.argv[1]).resolve()), sys.argv[2]]).encode()).hexdigest()[:24]) +PY +)" + derived_data="$DERIVED_DATA_DIR/$project_key" local build_args=( -project "$project" diff --git a/scripts/plugins/with-build-cache.py b/scripts/plugins/with-build-cache.py new file mode 100644 index 000000000..366fe4e35 --- /dev/null +++ b/scripts/plugins/with-build-cache.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Own a reusable compiler workspace until local plugin packaging has finished.""" +import argparse +from datetime import datetime, timezone +import fcntl +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import signal +import subprocess +import sys +import uuid + +REPO = Path(__file__).resolve().parents[2] +OUTPUT_SETTINGS = ('OBJROOT', 'SYMROOT', 'CONFIGURATION_BUILD_DIR', 'CONFIGURATION_TEMP_DIR', + 'PROJECT_TEMP_DIR', 'SHARED_PRECOMPS_DIR', 'CLANG_MODULE_CACHE_PATH', + 'SWIFT_MODULE_CACHE_PATH', 'XCODE_XCCONFIG_FILE') +INPUT_SETTINGS = ('ARCHS', 'EXCLUDED_ARCHS', 'ONLY_ACTIVE_ARCH', 'SDKROOT', 'SWIFT_VERSION', + 'TOOLCHAINS', 'SWIFT_EXEC', 'CC', 'CXX', 'LD', 'ENABLE_TESTABILITY', + 'SWIFT_OPTIMIZATION_LEVEL', 'GCC_OPTIMIZATION_LEVEL', + 'OTHER_SWIFT_FLAGS', 'OTHER_CFLAGS', 'OTHER_CPLUSPLUSFLAGS', 'OTHER_LDFLAGS', + 'SWIFT_ACTIVE_COMPILATION_CONDITIONS', 'GCC_PREPROCESSOR_DEFINITIONS', + 'DEVELOPMENT_TEAM', 'CODE_SIGN_IDENTITY', 'CODE_SIGNING_ALLOWED', + 'CODE_SIGNING_REQUIRED', 'MACTOOLS_PLUGIN_CACHE_VARIANT') + + +def internal_path(value): + path = Path(value).expanduser() + for candidate in (path.absolute(), path.resolve()): + if candidate == Path('/Volumes') or Path('/Volumes') in candidate.parents: + raise ValueError('External storage is deferred; choose an internal path.') + return path.resolve() + + +def query(*command): + return subprocess.check_output(command, text=True).strip() + + +def run(args): + overrides = [name for name in OUTPUT_SETTINGS if os.environ.get(name)] + if overrides: + raise ValueError('Compiler output overrides are unmanaged: ' + ', '.join(overrides)) + cache = internal_path(os.environ.get('MACTOOLS_PLUGIN_CACHE_ROOT', + str(Path.home() / 'Library/Caches/MacTools/PluginBuilds'))) + output = internal_path(args.output_dir) + if cache == output or cache in output.parents or output in cache.parents: + raise ValueError('Plugin packages/reports and compiler cache must have separate roots.') + executable = shutil.which(args.xcodebuild) + if executable is None: + raise ValueError('xcodebuild executable not found: ' + args.xcodebuild) + identity = dict(checkout=str(REPO), source=str(Path(args.source_dir).resolve()), + configuration=args.configuration, destination=args.destination, + unsigned=args.unsigned_build, host_architecture=platform.machine(), + developer=os.environ.get('DEVELOPER_DIR') or query('xcode-select', '-p'), + xcodebuild=str(Path(executable).resolve()), xcode_version=query(executable, '-version'), + sdk_build=query('xcrun', '--sdk', 'macosx', '--show-sdk-build-version'), + settings={name: os.environ[name] for name in INPUT_SETTINGS if name in os.environ}) + key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:24] + fresh = os.environ.get('MACTOOLS_PLUGIN_FRESH_BUILD') == '1' + workspace = cache / (key + ('-fresh-' + uuid.uuid4().hex[:8] if fresh else '')) + report = output / 'BuildReports' / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ-') + uuid.uuid4().hex[:8]) + report.mkdir(parents=True, exist_ok=False) + workspace.mkdir(parents=True, exist_ok=True) + temporary = workspace / 'Temporary' + temporary.mkdir(exist_ok=True) + env = dict(os.environ, MACTOOLS_PLUGIN_CACHE_CHECKOUT=str(REPO), + MACTOOLS_PLUGIN_DERIVED_ROOT=str(workspace / 'DerivedData'), TMPDIR=str(temporary) + '/') + command = ['/bin/bash', str(REPO / 'scripts/plugins/build-local-plugins.sh'), *args.arguments] + record = dict(identity=identity, cache_key=key, workspace=str(workspace), report=str(report), + command=command, fresh=fresh, state='waiting-for-lock') + + def save(): + (report / 'build.json').write_text(json.dumps(record, indent=2) + '\n') + + save() + print('Plugin build report: ' + str(report), flush=True) + with (workspace / '.build.lock').open('a+') as lock: + def cancel_wait(signum, frame): + record.update(state='failed', exit_code=128 + signum) + save() + raise SystemExit(128 + signum) + + waiting = {sig: signal.signal(sig, cancel_wait) for sig in (signal.SIGINT, signal.SIGTERM)} + try: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + record['lock_contended'] = True + save() + fcntl.flock(lock, fcntl.LOCK_EX) + finally: + for sig, handler in waiting.items(): + signal.signal(sig, handler) + record.update(state='running', started_at=datetime.now(timezone.utc).isoformat()) + save() + child = None + received = [] + + def forward(signum, frame): + received.append(signum) + if child is not None: + try: + os.killpg(child.pid, signum) + except ProcessLookupError: + pass + + previous = {sig: signal.signal(sig, forward) for sig in (signal.SIGINT, signal.SIGTERM)} + try: + with (report / 'build.log').open('wb') as log: + child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True, pass_fds=(lock.fileno(),)) + if received: + forward(received[-1], None) + code = child.wait() + except OSError as error: + code = 2 + record['error'] = str(error) + finally: + if child is not None: + # A failed/cancelled shell must leave no background compiler writers. + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.wait() + for sig, handler in previous.items(): + signal.signal(sig, handler) + code = 128 + received[-1] if received else (128 - code if code < 0 else code) + record.update(state='passed' if code == 0 else 'failed', exit_code=code, + finished_at=datetime.now(timezone.utc).isoformat()) + save() + try: + print((report / 'build.log').read_text(errors='replace'), end='', flush=True) + except BrokenPipeError: + pass + return code + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source-dir', required=True) + parser.add_argument('--output-dir', required=True) + parser.add_argument('--configuration', required=True) + parser.add_argument('--destination', default='') + parser.add_argument('--xcodebuild', required=True) + parser.add_argument('--unsigned-build', action='store_true') + parser.add_argument('arguments', nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.arguments[:1] == ['--']: + args.arguments = args.arguments[1:] + try: + sys.exit(run(args)) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + print('Plugin build cache: ' + str(error), file=sys.stderr) + sys.exit(2) diff --git a/scripts/tests/test_plugin_build_cache.py b/scripts/tests/test_plugin_build_cache.py new file mode 100644 index 000000000..86747f08c --- /dev/null +++ b/scripts/tests/test_plugin_build_cache.py @@ -0,0 +1,152 @@ +"""Fallback plugin compiles reuse output without racing the packaging consumer.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest + +from test_copy_plugin_manifest import runtime_envelope + +REPO = Path(__file__).resolve().parents[2] +SCRIPT = REPO / 'scripts/plugins/build-local-plugins.sh' + + +class PluginBuildCacheTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.source = self.root / 'source/Example' + self.source.mkdir(parents=True) + (self.source / 'Example.xcodeproj').mkdir() + (self.source / 'plugin.json').write_text(json.dumps(runtime_envelope( + build={'project': 'Example.xcodeproj', 'scheme': 'Example'}))) + self.bin = self.root / 'bin' + self.bin.mkdir() + self.env = {key: value for key, value in os.environ.items() if not key.startswith('MACTOOLS_PLUGIN_')} + self.env.update(PATH=str(self.bin) + os.pathsep + self.env['PATH'], + PYTHON3=sys.executable, DEVELOPER_DIR=str(self.root / 'Xcode/Developer'), + MACTOOLS_PLUGIN_CACHE_ROOT=str(self.root / 'compiler'), CALLS=str(self.root / 'calls')) + self.tool('xcrun', "import os\nprint(os.environ.get('SDK_BUILD','fixture-sdk'))\n") + self.builder = self.tool('xcodebuild', '''import os,pathlib,sys +if '-version' in sys.argv: + print('fixture-xcode') + sys.exit(0) +derived=pathlib.Path(sys.argv[sys.argv.index('-derivedDataPath')+1]) +with open(os.environ['CALLS'],'a') as stream: stream.write(str(derived)+'\\n') +bundle=derived/'Build/Products'/sys.argv[sys.argv.index('-configuration')+1]/'Example.bundle' +bundle.mkdir(parents=True,exist_ok=True) +(bundle/'payload').write_text('built plugin') +print('Fixture build log',flush=True) +sys.exit(int(os.environ.get('BUILD_EXIT','0'))) +''') + self.tool('ditto', '''import os,pathlib,subprocess,sys,time +if os.environ.get('HOLD_COPY'): + pathlib.Path(os.environ['HOLD_COPY']).touch() + deadline=time.monotonic()+8 + while not pathlib.Path(os.environ['RELEASE_COPY']).exists() and time.monotonic() Date: Fri, 23 Oct 2026 15:27:51 -0400 Subject: [PATCH 2/5] Validate plugin cache storage before resetting packages --- scripts/plugins/build-local-plugins.sh | 8 +++++--- scripts/plugins/with-build-cache.py | 19 ++++++++++++++----- scripts/tests/test_plugin_build_cache.py | 10 ++++++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/scripts/plugins/build-local-plugins.sh b/scripts/plugins/build-local-plugins.sh index 3d4451e18..f700dafb2 100755 --- a/scripts/plugins/build-local-plugins.sh +++ b/scripts/plugins/build-local-plugins.sh @@ -141,6 +141,11 @@ THIRD_PARTY_NOTICE_GENERATOR="$REPO_ROOT/scripts/licenses/generate-third-party-n if [[ -z "$XCODEBUILD_COMMAND" ]]; then XCODEBUILD_COMMAND="$REPO_ROOT/scripts/xcodebuild-filtered.sh" fi +cache_args=(--source-dir "$SOURCE_DIR" --output-dir "$OUTPUT_DIR" + --configuration "$CONFIGURATION" --destination "$DESTINATION" --xcodebuild "$XCODEBUILD_COMMAND") +[[ "$UNSIGNED_BUILD" == "1" ]] && cache_args+=(--unsigned-build) +# Validate before resetting Packages, including when a cache root was misconfigured inside it. +"$PYTHON3" "$REPO_ROOT/scripts/plugins/with-build-cache.py" --check-paths "${cache_args[@]}" OUTPUT_DIR="$(mkdir -p "$OUTPUT_DIR" && cd "$OUTPUT_DIR" && pwd)" PACKAGES_DIR="$OUTPUT_DIR/Packages" DERIVED_DATA_DIR="${MACTOOLS_PLUGIN_DERIVED_ROOT:-}" @@ -312,9 +317,6 @@ build_bundle_if_needed() { # Prebuilt/aggregate products keep their existing path; only fallback compiles need a cache owner. if [[ "${MACTOOLS_PLUGIN_CACHE_CHECKOUT:-}" != "$REPO_ROOT" ]]; then - local cache_args=(--source-dir "$SOURCE_DIR" --output-dir "$OUTPUT_DIR" - --configuration "$CONFIGURATION" --destination "$DESTINATION" --xcodebuild "$XCODEBUILD_COMMAND") - [[ "$UNSIGNED_BUILD" == "1" ]] && cache_args+=(--unsigned-build) exec "$PYTHON3" "$REPO_ROOT/scripts/plugins/with-build-cache.py" "${cache_args[@]}" -- "${ORIGINAL_ARGS[@]}" fi diff --git a/scripts/plugins/with-build-cache.py b/scripts/plugins/with-build-cache.py index 366fe4e35..dbb374b4d 100644 --- a/scripts/plugins/with-build-cache.py +++ b/scripts/plugins/with-build-cache.py @@ -39,15 +39,20 @@ def query(*command): return subprocess.check_output(command, text=True).strip() -def run(args): - overrides = [name for name in OUTPUT_SETTINGS if os.environ.get(name)] - if overrides: - raise ValueError('Compiler output overrides are unmanaged: ' + ', '.join(overrides)) +def storage_paths(output_dir): cache = internal_path(os.environ.get('MACTOOLS_PLUGIN_CACHE_ROOT', str(Path.home() / 'Library/Caches/MacTools/PluginBuilds'))) - output = internal_path(args.output_dir) + output = internal_path(output_dir) if cache == output or cache in output.parents or output in cache.parents: raise ValueError('Plugin packages/reports and compiler cache must have separate roots.') + return cache, output + + +def run(args): + overrides = [name for name in OUTPUT_SETTINGS if os.environ.get(name)] + if overrides: + raise ValueError('Compiler output overrides are unmanaged: ' + ', '.join(overrides)) + cache, output = storage_paths(args.output_dir) executable = shutil.which(args.xcodebuild) if executable is None: raise ValueError('xcodebuild executable not found: ' + args.xcodebuild) @@ -147,11 +152,15 @@ def forward(signum, frame): parser.add_argument('--destination', default='') parser.add_argument('--xcodebuild', required=True) parser.add_argument('--unsigned-build', action='store_true') + parser.add_argument('--check-paths', action='store_true', help=argparse.SUPPRESS) parser.add_argument('arguments', nargs=argparse.REMAINDER) args = parser.parse_args() if args.arguments[:1] == ['--']: args.arguments = args.arguments[1:] try: + if args.check_paths: + storage_paths(args.output_dir) + sys.exit(0) sys.exit(run(args)) except (OSError, ValueError, subprocess.CalledProcessError) as error: print('Plugin build cache: ' + str(error), file=sys.stderr) diff --git a/scripts/tests/test_plugin_build_cache.py b/scripts/tests/test_plugin_build_cache.py index 86747f08c..48c28d581 100644 --- a/scripts/tests/test_plugin_build_cache.py +++ b/scripts/tests/test_plugin_build_cache.py @@ -147,6 +147,16 @@ def test_external_cache_and_output_overrides_are_rejected_before_compile(self): self.assertEqual(result.returncode, 2, result.stdout + result.stderr) self.assertFalse((self.root / 'calls').exists()) + def test_misconfigured_overlapping_cache_is_preserved_before_package_reset(self): + cache = self.root / 'output/Packages/compiler' + cache.mkdir(parents=True) + marker = cache / 'compiler-output' + marker.write_text('preserve') + result = self.invoke('output', env=dict(self.env, MACTOOLS_PLUGIN_CACHE_ROOT=str(cache))) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(marker.read_text(), 'preserve') + self.assertFalse((self.root / 'calls').exists()) + if __name__ == '__main__': unittest.main() From b02e7922099f188e397c25a5440b16d9547c19f8 Mon Sep 17 00:00:00 2001 From: xcv58 Date: Fri, 23 Oct 2026 15:50:55 -0400 Subject: [PATCH 3/5] Validate current plugin products and finalize hangup cleanup --- scripts/plugins/build-local-plugins.sh | 26 ++++++++++----- scripts/plugins/with-build-cache.py | 31 ++++++++++------- scripts/tests/test_plugin_build_cache.py | 42 ++++++++++++++++++++++-- 3 files changed, 77 insertions(+), 22 deletions(-) diff --git a/scripts/plugins/build-local-plugins.sh b/scripts/plugins/build-local-plugins.sh index f700dafb2..a71aa7ec6 100755 --- a/scripts/plugins/build-local-plugins.sh +++ b/scripts/plugins/build-local-plugins.sh @@ -359,19 +359,29 @@ PY CODE_SIGN_IDENTITY= ) fi - build_args+=(build -quiet) - - if ! "$XCODEBUILD_COMMAND" "${build_args[@]}"; then + if ! "$XCODEBUILD_COMMAND" "${build_args[@]}" build -quiet; then echo "Failed to build plugin bundle '$scheme' for $root." >&2 return 1 fi + local settings_file="$MACTOOLS_PLUGIN_BUILD_REPORT/$project_key-build-settings.json" + "$XCODEBUILD_COMMAND" "${build_args[@]}" -showBuildSettings -json > "$settings_file" local built_bundle - built_bundle="$(find "$derived_data/Build/Products" -name "$bundle_name" -type d -print | sort | head -1)" - if [[ -z "$built_bundle" ]]; then - echo "Built bundle not found: $bundle_name" >&2 - return 1 - fi + # Retained products may include obsolete targets or paths. Use the current scheme's declared product. + built_bundle="$("$PYTHON3" - "$settings_file" "$bundle_name" "$derived_data" <<'PY' +import json, pathlib, sys +settings_file, name, derived = sys.argv[1:] +settings = json.load(open(settings_file)) +paths = {pathlib.Path(entry['buildSettings']['TARGET_BUILD_DIR']) / name + for entry in settings if entry.get('buildSettings', {}).get('FULL_PRODUCT_NAME') == name} +if len(paths) != 1: + sys.exit('Current scheme must declare exactly one product for ' + name) +product = paths.pop().resolve() +if pathlib.Path(derived).resolve() not in product.parents or not product.is_dir(): + sys.exit('Current product is missing or outside managed compiler storage: ' + str(product)) +print(product) +PY +)" printf -v "$output_var" '%s' "$built_bundle" } diff --git a/scripts/plugins/with-build-cache.py b/scripts/plugins/with-build-cache.py index dbb374b4d..f1d43de30 100644 --- a/scripts/plugins/with-build-cache.py +++ b/scripts/plugins/with-build-cache.py @@ -39,6 +39,20 @@ def query(*command): return subprocess.check_output(command, text=True).strip() +def signal_group(pid, signum): + try: + os.killpg(pid, signum) + except ProcessLookupError: + pass + except PermissionError: + # Teardown can race an empty group on macOS. Never ignore an EPERM with live members. + rows = subprocess.check_output(['/bin/ps', '-axo', 'pgid=,stat='], text=True) + for line in rows.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[0] == str(pid) and not parts[1].startswith('Z'): + raise + + def storage_paths(output_dir): cache = internal_path(os.environ.get('MACTOOLS_PLUGIN_CACHE_ROOT', str(Path.home() / 'Library/Caches/MacTools/PluginBuilds'))) @@ -72,7 +86,8 @@ def run(args): temporary = workspace / 'Temporary' temporary.mkdir(exist_ok=True) env = dict(os.environ, MACTOOLS_PLUGIN_CACHE_CHECKOUT=str(REPO), - MACTOOLS_PLUGIN_DERIVED_ROOT=str(workspace / 'DerivedData'), TMPDIR=str(temporary) + '/') + MACTOOLS_PLUGIN_DERIVED_ROOT=str(workspace / 'DerivedData'), + MACTOOLS_PLUGIN_BUILD_REPORT=str(report), TMPDIR=str(temporary) + '/') command = ['/bin/bash', str(REPO / 'scripts/plugins/build-local-plugins.sh'), *args.arguments] record = dict(identity=identity, cache_key=key, workspace=str(workspace), report=str(report), command=command, fresh=fresh, state='waiting-for-lock') @@ -88,7 +103,7 @@ def cancel_wait(signum, frame): save() raise SystemExit(128 + signum) - waiting = {sig: signal.signal(sig, cancel_wait) for sig in (signal.SIGINT, signal.SIGTERM)} + waiting = {sig: signal.signal(sig, cancel_wait) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} try: try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) @@ -107,12 +122,9 @@ def cancel_wait(signum, frame): def forward(signum, frame): received.append(signum) if child is not None: - try: - os.killpg(child.pid, signum) - except ProcessLookupError: - pass + signal_group(child.pid, signum) - previous = {sig: signal.signal(sig, forward) for sig in (signal.SIGINT, signal.SIGTERM)} + previous = {sig: signal.signal(sig, forward) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} try: with (report / 'build.log').open('wb') as log: child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT, @@ -126,10 +138,7 @@ def forward(signum, frame): finally: if child is not None: # A failed/cancelled shell must leave no background compiler writers. - try: - os.killpg(child.pid, signal.SIGKILL) - except ProcessLookupError: - pass + signal_group(child.pid, signal.SIGKILL) child.wait() for sig, handler in previous.items(): signal.signal(sig, handler) diff --git a/scripts/tests/test_plugin_build_cache.py b/scripts/tests/test_plugin_build_cache.py index 48c28d581..ca8382952 100644 --- a/scripts/tests/test_plugin_build_cache.py +++ b/scripts/tests/test_plugin_build_cache.py @@ -2,6 +2,7 @@ import json import os from pathlib import Path +import signal import subprocess import sys import tempfile @@ -31,15 +32,20 @@ def setUp(self): PYTHON3=sys.executable, DEVELOPER_DIR=str(self.root / 'Xcode/Developer'), MACTOOLS_PLUGIN_CACHE_ROOT=str(self.root / 'compiler'), CALLS=str(self.root / 'calls')) self.tool('xcrun', "import os\nprint(os.environ.get('SDK_BUILD','fixture-sdk'))\n") - self.builder = self.tool('xcodebuild', '''import os,pathlib,sys + self.builder = self.tool('xcodebuild', '''import json,os,pathlib,sys if '-version' in sys.argv: print('fixture-xcode') sys.exit(0) derived=pathlib.Path(sys.argv[sys.argv.index('-derivedDataPath')+1]) +products=derived/'Build/Products'/os.environ.get('PRODUCT_DIRECTORY','')/sys.argv[sys.argv.index('-configuration')+1] +name=os.environ.get('PRODUCT_NAME','Example.bundle') +if '-showBuildSettings' in sys.argv: + print(json.dumps([{'buildSettings':{'TARGET_BUILD_DIR':str(products),'FULL_PRODUCT_NAME':name}}])) + sys.exit(0) with open(os.environ['CALLS'],'a') as stream: stream.write(str(derived)+'\\n') -bundle=derived/'Build/Products'/sys.argv[sys.argv.index('-configuration')+1]/'Example.bundle' +bundle=products/name bundle.mkdir(parents=True,exist_ok=True) -(bundle/'payload').write_text('built plugin') +(bundle/'payload').write_text(os.environ.get('BUNDLE_PAYLOAD','built plugin')) print('Fixture build log',flush=True) sys.exit(int(os.environ.get('BUILD_EXIT','0'))) ''') @@ -140,6 +146,36 @@ def test_failure_retains_report_and_allows_following_run(self): self.assertEqual(self.invoke('following').returncode, 0) self.assertEqual(self.latest()['workspace'], record['workspace']) + def test_old_bundle_is_rejected_when_current_scheme_no_longer_produces_it(self): + self.assertEqual(self.invoke('first').returncode, 0) + workspace = self.latest()['workspace'] + moved = self.invoke('moved', env=dict(self.env, PRODUCT_DIRECTORY='ZZ-current', BUNDLE_PAYLOAD='current moved plugin')) + self.assertEqual(moved.returncode, 0, moved.stdout + moved.stderr) + self.assertEqual((self.root / 'moved/Packages/example.mactoolsplugin/Example.bundle/payload').read_text(), 'current moved plugin') + self.assertEqual(self.latest()['workspace'], workspace) + result = self.invoke('renamed', env=dict(self.env, PRODUCT_NAME='Renamed.bundle')) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.latest()['state'], 'failed') + self.assertEqual(self.latest()['workspace'], workspace) + self.assertFalse((self.root / 'renamed/Packages/example.mactoolsplugin').exists()) + + def test_terminal_hangup_stops_packaging_and_finalizes_failure(self): + held, release = self.root / 'copy-started', self.root / 'copy-release' + console = self.root / 'hangup.log' + with console.open('wb') as stream: + child = subprocess.Popen(self.command('cancelled'), env=dict(self.env, HOLD_COPY=str(held), RELEASE_COPY=str(release)), + stdout=stream, stderr=subprocess.STDOUT) + self.addCleanup(self.stop, child) + try: + self.wait_for(held.exists) + child.send_signal(signal.SIGHUP) + self.assertEqual(child.wait(timeout=10), 129, console.read_text()) + record = self.latest() + self.assertEqual((record['state'], record['exit_code']), ('failed', 129)) + self.assertEqual(self.invoke('following').returncode, 0) + finally: + release.touch() + def test_external_cache_and_output_overrides_are_rejected_before_compile(self): for changes in [{'MACTOOLS_PLUGIN_CACHE_ROOT': '/Volumes/MacTools-deferred-cache'}, {'OBJROOT': str(self.root / 'unmanaged')}]: From e18b9214cf60fe7784ec48016ccb27e276212211 Mon Sep 17 00:00:00 2001 From: xcv58 Date: Fri, 23 Oct 2026 15:58:13 -0400 Subject: [PATCH 4/5] Preserve plugin notices through checkout symlinks --- scripts/plugins/build-local-plugins.sh | 4 ++-- scripts/tests/test_plugin_build_cache.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/plugins/build-local-plugins.sh b/scripts/plugins/build-local-plugins.sh index a71aa7ec6..339c26c61 100755 --- a/scripts/plugins/build-local-plugins.sh +++ b/scripts/plugins/build-local-plugins.sh @@ -128,14 +128,14 @@ if [[ -z "$SOURCE_DIR" || -z "$OUTPUT_DIR" ]]; then exit 1 fi -SOURCE_DIR="$(cd "$SOURCE_DIR" 2>/dev/null && pwd || true)" +SOURCE_DIR="$(cd "$SOURCE_DIR" 2>/dev/null && pwd -P || true)" if [[ -z "$SOURCE_DIR" || ! -d "$SOURCE_DIR" ]]; then echo "Plugin source directory not found. Expected: $SOURCE_DIR" >&2 exit 1 fi REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" -OFFICIAL_PLUGIN_SOURCE_DIR="$(cd "$REPO_ROOT/Plugins" && pwd)" +OFFICIAL_PLUGIN_SOURCE_DIR="$(cd "$REPO_ROOT/Plugins" && pwd -P)" THIRD_PARTY_NOTICE_MANIFEST="$REPO_ROOT/Sources/Resources/ThirdPartyNotices/manifest.json" THIRD_PARTY_NOTICE_GENERATOR="$REPO_ROOT/scripts/licenses/generate-third-party-notices.py" if [[ -z "$XCODEBUILD_COMMAND" ]]; then diff --git a/scripts/tests/test_plugin_build_cache.py b/scripts/tests/test_plugin_build_cache.py index ca8382952..f739d83d2 100644 --- a/scripts/tests/test_plugin_build_cache.py +++ b/scripts/tests/test_plugin_build_cache.py @@ -3,6 +3,7 @@ import os from pathlib import Path import signal +import shutil import subprocess import sys import tempfile @@ -110,6 +111,29 @@ def test_reuses_compiler_across_outputs_and_keeps_packaged_products_and_logs(sel self.assertIn('Fixture build log', Path(report['report'], 'build.log').read_text()) self.assertFalse((self.root / label / 'DerivedData').exists()) + def test_symlinked_checkout_keeps_official_plugin_notices(self): + checkout = self.root / 'checkout' + for relative in ['scripts/plugins/build-local-plugins.sh', 'scripts/plugins/with-build-cache.py', + 'scripts/plugins/copy-plugin-manifest.py', 'scripts/plugins/plugin_source_manifest.py', + 'Configs/AppVersion.xcconfig']: + target = checkout / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(REPO / relative, target) + shutil.copytree(self.source, checkout / 'Plugins/Example') + generator = checkout / 'scripts/licenses/generate-third-party-notices.py' + generator.parent.mkdir(parents=True) + generator.write_text('import pathlib,sys\npathlib.Path(sys.argv[sys.argv.index("--output")+1]).write_text("Required notice")\n') + alias = self.root / 'checkout-alias' + alias.symlink_to(checkout, target_is_directory=True) + result = subprocess.run( + ['/bin/bash', '-c', 'cd "$1" && exec ./scripts/plugins/build-local-plugins.sh "${@:2}"', + 'fixture', str(alias), '--source-dir', 'Plugins', '--output-dir', str(self.root / 'aliased'), + '--xcodebuild', str(self.builder), '--skip-catalog'], + env=self.env, text=True, capture_output=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.root / 'aliased/Packages/example.mactoolsplugin/THIRD_PARTY_NOTICES.txt').read_text(), + 'Required notice') + def test_configuration_signing_sdk_probe_and_fresh_workspaces_are_separate(self): cases = [([], {}), (['--configuration', 'Release'], {}), (['--unsigned-build'], {}), ([], {'SDK_BUILD': 'changed-sdk'}), ([], {'MACTOOLS_PLUGIN_CACHE_VARIANT': 'probe'}), From 7b32e52ee0b94e7eb9fd3338726cb2ea6dff4515 Mon Sep 17 00:00:00 2001 From: xcv58 Date: Fri, 23 Oct 2026 19:11:53 -0400 Subject: [PATCH 5/5] Publish plugin build reports atomically --- scripts/plugins/with-build-cache.py | 4 +++- scripts/tests/test_plugin_build_cache.py | 28 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/scripts/plugins/with-build-cache.py b/scripts/plugins/with-build-cache.py index f1d43de30..460ce22f5 100644 --- a/scripts/plugins/with-build-cache.py +++ b/scripts/plugins/with-build-cache.py @@ -93,7 +93,9 @@ def run(args): command=command, fresh=fresh, state='waiting-for-lock') def save(): - (report / 'build.json').write_text(json.dumps(record, indent=2) + '\n') + temporary_report = report / 'build.json.tmp' + temporary_report.write_text(json.dumps(record, indent=2) + '\n') + temporary_report.replace(report / 'build.json') save() print('Plugin build report: ' + str(report), flush=True) diff --git a/scripts/tests/test_plugin_build_cache.py b/scripts/tests/test_plugin_build_cache.py index f739d83d2..b44b34d0d 100644 --- a/scripts/tests/test_plugin_build_cache.py +++ b/scripts/tests/test_plugin_build_cache.py @@ -170,6 +170,34 @@ def test_failure_retains_report_and_allows_following_run(self): self.assertEqual(self.invoke('following').returncode, 0) self.assertEqual(self.latest()['workspace'], record['workspace']) + def test_report_remains_readable_while_an_update_is_being_written(self): + shim = self.root / 'shim' + shim.mkdir() + (shim / 'sitecustomize.py').write_text('''import json,os,pathlib,time +original=pathlib.Path.write_text +def write_text(path,data,*args,**kwargs): + if path.name in {'build.json','build.json.tmp'} and json.loads(data).get('state')=='running': + with path.open('w') as stream: + pathlib.Path(os.environ['REPORT_WRITE_STARTED']).touch() + deadline=time.monotonic()+8 + while not pathlib.Path(os.environ['RELEASE_REPORT_WRITE']).exists() and time.monotonic()