Reuse internal compiler output for local plugin packaging

CONTRIBUTING.md

@@ -103,6 +103,8 @@ Use `TEST_FILTER=ClassName/testMethod` for one method, or `make test` for all re
 
 Unsigned XCTest builds use `build/DerivedDataTests` by default, separate from the signed app used by `make run`. Override `TEST_DERIVED_DATA` when needed, keeping it separate from `DERIVED_DATA` to avoid leaving a test bundle inside the installed app.
 
+Fallback plugin packaging compiles reuse a locked internal compiler cache, while packages and per-run logs remain in the requested output directory. Use `MACTOOLS_PLUGIN_FRESH_BUILD=1 make build-plugin PLUGIN=<name>` for a cold validation build. See [local plugin build storage](docs/plugins/local-native-plugins.md#development-steps) for cache roots and probe isolation; normal Makefile app/test reuse and aggregate-product packaging keep their existing paths.
+
 For copy and catalog changes, run `make validate-localization` to check English resources, complete coverage of all 11 supported languages, and translation format contracts. `make script-tests` includes this audit and fails when a translation is missing. To inspect per-key findings, run `python3 scripts/audit-localization.py --require-complete --json`. Follow the [plugin localization rules](docs/plugins/development-guidelines.md#protocols-and-ownership), and manually check language switching and affected UI layout. Action-symbol changes must keep manifests and runtime definitions consistent; verify them with `make test TEST_FILTER=PluginRuntimeActionSnapshotTests`.
 
 | Change | Verification scope |

docs/plugins/local-native-plugins.md

@@ -108,6 +108,8 @@ A full Debug sync mirrors the current checkout: packages missing from its local
 
 Debug package copies normalize `minHostVersion` to the locally built app version because the host and plugin bundles come from the same checkout. Release packages preserve each source manifest's declared minimum host version.
 
+Fallback compiles in `build-local-plugins.sh` reuse internal compiler workspaces under `~/Library/Caches/MacTools/PluginBuilds`. The cache separates source checkout, project/scheme, Xcode/SDK, host architecture, configuration, destination, signing mode and compiler flags. A lock covers compilation and copying bundles into packages. Each run retains its log and build record under `<output-dir>/BuildReports/`; packaged bundles remain under `<output-dir>/Packages` and do not depend on the compiler cache. Set `MACTOOLS_PLUGIN_FRESH_BUILD=1` for a separate cold workspace, `MACTOOLS_PLUGIN_CACHE_VARIANT` to isolate a probe, or `MACTOOLS_PLUGIN_CACHE_ROOT` for another internal root. Existing compiler output is preserved. Prebuilt packages and `--products-dir` aggregate products keep their existing packaging path.
+
 If the plugin needs extra frameworks, private include paths, bundle resources, helper/tool targets, or target dependencies, add only those differences in `Plugins/<PluginName>/project.yml`. If the plugin package contains an extra executable inside the bundle resources, declare it in `plugin.json.package.signPaths` so release packaging signs it before signing the bundle.
 
 Plugin UI copy should be localized by the plugin itself. Put plugin string catalogs under `Plugins/<PluginName>/Resources`, then look them up from the plugin bundle, for example:

scripts/plugins/build-local-plugins.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
+ORIGINAL_ARGS=("$@")
 
 if [[ -z "${PYTHON3:-}" ]]; then
     if [[ -x /usr/bin/python3 ]]; then
@@ -127,26 +128,31 @@ if [[ -z "$SOURCE_DIR" || -z "$OUTPUT_DIR" ]]; then
     exit 1
 fi
 
-SOURCE_DIR="$(cd "$SOURCE_DIR" 2>/dev/null && pwd || true)"
+SOURCE_DIR="$(cd "$SOURCE_DIR" 2>/dev/null && pwd -P || true)"
 if [[ -z "$SOURCE_DIR" || ! -d "$SOURCE_DIR" ]]; then
     echo "Plugin source directory not found. Expected: $SOURCE_DIR" >&2
     exit 1
 fi
 
-REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
-OFFICIAL_PLUGIN_SOURCE_DIR="$(cd "$REPO_ROOT/Plugins" && pwd)"
+REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)"
+OFFICIAL_PLUGIN_SOURCE_DIR="$(cd "$REPO_ROOT/Plugins" && pwd -P)"
 THIRD_PARTY_NOTICE_MANIFEST="$REPO_ROOT/Sources/Resources/ThirdPartyNotices/manifest.json"
 THIRD_PARTY_NOTICE_GENERATOR="$REPO_ROOT/scripts/licenses/generate-third-party-notices.py"
 if [[ -z "$XCODEBUILD_COMMAND" ]]; then
     XCODEBUILD_COMMAND="$REPO_ROOT/scripts/xcodebuild-filtered.sh"
 fi
+cache_args=(--source-dir "$SOURCE_DIR" --output-dir "$OUTPUT_DIR"
+    --configuration "$CONFIGURATION" --destination "$DESTINATION" --xcodebuild "$XCODEBUILD_COMMAND")
+[[ "$UNSIGNED_BUILD" == "1" ]] && cache_args+=(--unsigned-build)
+# Validate before resetting Packages, including when a cache root was misconfigured inside it.
+"$PYTHON3" "$REPO_ROOT/scripts/plugins/with-build-cache.py" --check-paths "${cache_args[@]}"
 OUTPUT_DIR="$(mkdir -p "$OUTPUT_DIR" && cd "$OUTPUT_DIR" && pwd)"
 PACKAGES_DIR="$OUTPUT_DIR/Packages"
-DERIVED_DATA_DIR="$OUTPUT_DIR/DerivedData"
+DERIVED_DATA_DIR="${MACTOOLS_PLUGIN_DERIVED_ROOT:-}"
 CATALOG_PATH="$OUTPUT_DIR/catalog.dev.json"
 
 rm -rf "$PACKAGES_DIR"
-mkdir -p "$PACKAGES_DIR" "$DERIVED_DATA_DIR"
+mkdir -p "$PACKAGES_DIR"
 
 json_value() {
     local file="$1"
@@ -309,6 +315,11 @@ build_bundle_if_needed() {
         return
     fi
 
+    # Prebuilt/aggregate products keep their existing path; only fallback compiles need a cache owner.
+    if [[ "${MACTOOLS_PLUGIN_CACHE_CHECKOUT:-}" != "$REPO_ROOT" ]]; then
+        exec "$PYTHON3" "$REPO_ROOT/scripts/plugins/with-build-cache.py" "${cache_args[@]}" -- "${ORIGINAL_ARGS[@]}"
+    fi
+
     local project
     project_file_for "$root" "$manifest" project
     if [[ -z "$project" ]]; then
@@ -324,8 +335,13 @@ build_bundle_if_needed() {
     fi
 
     local derived_data
-    derived_data="$DERIVED_DATA_DIR/$(basename "$root")"
-    rm -rf "$derived_data"
+    local project_key
+    project_key="$("$PYTHON3" - "$project" "$scheme" <<'PY'
+import hashlib, json, pathlib, sys
+print(hashlib.sha256(json.dumps([str(pathlib.Path(sys.argv[1]).resolve()), sys.argv[2]]).encode()).hexdigest()[:24])
+PY
+)"
+    derived_data="$DERIVED_DATA_DIR/$project_key"
 
     local build_args=(
         -project "$project"
@@ -343,19 +359,29 @@ build_bundle_if_needed() {
             CODE_SIGN_IDENTITY=
         )
     fi
-    build_args+=(build -quiet)
-
-    if ! "$XCODEBUILD_COMMAND" "${build_args[@]}"; then
+    if ! "$XCODEBUILD_COMMAND" "${build_args[@]}" build -quiet; then
         echo "Failed to build plugin bundle '$scheme' for $root." >&2
         return 1
     fi
 
+    local settings_file="$MACTOOLS_PLUGIN_BUILD_REPORT/$project_key-build-settings.json"
+    "$XCODEBUILD_COMMAND" "${build_args[@]}" -showBuildSettings -json > "$settings_file"
     local built_bundle
-    built_bundle="$(find "$derived_data/Build/Products" -name "$bundle_name" -type d -print | sort | head -1)"
-    if [[ -z "$built_bundle" ]]; then
-        echo "Built bundle not found: $bundle_name" >&2
-        return 1
-    fi
+    # Retained products may include obsolete targets or paths. Use the current scheme's declared product.
+    built_bundle="$("$PYTHON3" - "$settings_file" "$bundle_name" "$derived_data" <<'PY'
+import json, pathlib, sys
+settings_file, name, derived = sys.argv[1:]
+settings = json.load(open(settings_file))
+paths = {pathlib.Path(entry['buildSettings']['TARGET_BUILD_DIR']) / name
+         for entry in settings if entry.get('buildSettings', {}).get('FULL_PRODUCT_NAME') == name}
+if len(paths) != 1:
+    sys.exit('Current scheme must declare exactly one product for ' + name)
+product = paths.pop().resolve()
+if pathlib.Path(derived).resolve() not in product.parents or not product.is_dir():
+    sys.exit('Current product is missing or outside managed compiler storage: ' + str(product))
+print(product)
+PY
+)"
 
     printf -v "$output_var" '%s' "$built_bundle"
 }

scripts/plugins/with-build-cache.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Own a reusable compiler workspace until local plugin packaging has finished."""
+import argparse
+from datetime import datetime, timezone
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import shutil
+import signal
+import subprocess
+import sys
+import uuid
+
+REPO = Path(__file__).resolve().parents[2]
+OUTPUT_SETTINGS = ('OBJROOT', 'SYMROOT', 'CONFIGURATION_BUILD_DIR', 'CONFIGURATION_TEMP_DIR',
+                   'PROJECT_TEMP_DIR', 'SHARED_PRECOMPS_DIR', 'CLANG_MODULE_CACHE_PATH',
+                   'SWIFT_MODULE_CACHE_PATH', 'XCODE_XCCONFIG_FILE')
+INPUT_SETTINGS = ('ARCHS', 'EXCLUDED_ARCHS', 'ONLY_ACTIVE_ARCH', 'SDKROOT', 'SWIFT_VERSION',
+                  'TOOLCHAINS', 'SWIFT_EXEC', 'CC', 'CXX', 'LD', 'ENABLE_TESTABILITY',
+                  'SWIFT_OPTIMIZATION_LEVEL', 'GCC_OPTIMIZATION_LEVEL',
+                  'OTHER_SWIFT_FLAGS', 'OTHER_CFLAGS', 'OTHER_CPLUSPLUSFLAGS', 'OTHER_LDFLAGS',
+                  'SWIFT_ACTIVE_COMPILATION_CONDITIONS', 'GCC_PREPROCESSOR_DEFINITIONS',
+                  'DEVELOPMENT_TEAM', 'CODE_SIGN_IDENTITY', 'CODE_SIGNING_ALLOWED',
+                  'CODE_SIGNING_REQUIRED', 'MACTOOLS_PLUGIN_CACHE_VARIANT')
+
+
+def internal_path(value):
+    path = Path(value).expanduser()
+    for candidate in (path.absolute(), path.resolve()):
+        if candidate == Path('/Volumes') or Path('/Volumes') in candidate.parents:
+            raise ValueError('External storage is deferred; choose an internal path.')
+    return path.resolve()
+
+
+def query(*command):
+    return subprocess.check_output(command, text=True).strip()
+
+
+def signal_group(pid, signum):
+    try:
+        os.killpg(pid, signum)
+    except ProcessLookupError:
+        pass
+    except PermissionError:
+        # Teardown can race an empty group on macOS. Never ignore an EPERM with live members.
+        rows = subprocess.check_output(['/bin/ps', '-axo', 'pgid=,stat='], text=True)
+        for line in rows.splitlines():
+            parts = line.split()
+            if len(parts) == 2 and parts[0] == str(pid) and not parts[1].startswith('Z'):
+                raise
+
+
+def storage_paths(output_dir):
+    cache = internal_path(os.environ.get('MACTOOLS_PLUGIN_CACHE_ROOT',
+                                         str(Path.home() / 'Library/Caches/MacTools/PluginBuilds')))
+    output = internal_path(output_dir)
+    if cache == output or cache in output.parents or output in cache.parents:
+        raise ValueError('Plugin packages/reports and compiler cache must have separate roots.')
+    return cache, output
+
+
+def run(args):
+    overrides = [name for name in OUTPUT_SETTINGS if os.environ.get(name)]
+    if overrides:
+        raise ValueError('Compiler output overrides are unmanaged: ' + ', '.join(overrides))
+    cache, output = storage_paths(args.output_dir)
+    executable = shutil.which(args.xcodebuild)
+    if executable is None:
+        raise ValueError('xcodebuild executable not found: ' + args.xcodebuild)
+    identity = dict(checkout=str(REPO), source=str(Path(args.source_dir).resolve()),
+                    configuration=args.configuration, destination=args.destination,
+                    unsigned=args.unsigned_build, host_architecture=platform.machine(),
+                    developer=os.environ.get('DEVELOPER_DIR') or query('xcode-select', '-p'),
+                    xcodebuild=str(Path(executable).resolve()), xcode_version=query(executable, '-version'),
+                    sdk_build=query('xcrun', '--sdk', 'macosx', '--show-sdk-build-version'),
+                    settings={name: os.environ[name] for name in INPUT_SETTINGS if name in os.environ})
+    key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:24]
+    fresh = os.environ.get('MACTOOLS_PLUGIN_FRESH_BUILD') == '1'
+    workspace = cache / (key + ('-fresh-' + uuid.uuid4().hex[:8] if fresh else ''))
+    report = output / 'BuildReports' / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ-') + uuid.uuid4().hex[:8])
+    report.mkdir(parents=True, exist_ok=False)
+    workspace.mkdir(parents=True, exist_ok=True)
+    temporary = workspace / 'Temporary'
+    temporary.mkdir(exist_ok=True)
+    env = dict(os.environ, MACTOOLS_PLUGIN_CACHE_CHECKOUT=str(REPO),
+               MACTOOLS_PLUGIN_DERIVED_ROOT=str(workspace / 'DerivedData'),
+               MACTOOLS_PLUGIN_BUILD_REPORT=str(report), TMPDIR=str(temporary) + '/')
+    command = ['/bin/bash', str(REPO / 'scripts/plugins/build-local-plugins.sh'), *args.arguments]
+    record = dict(identity=identity, cache_key=key, workspace=str(workspace), report=str(report),
+                  command=command, fresh=fresh, state='waiting-for-lock')
+
+    def save():
+        temporary_report = report / 'build.json.tmp'
+        temporary_report.write_text(json.dumps(record, indent=2) + '\n')
+        temporary_report.replace(report / 'build.json')
+
+    save()
+    print('Plugin build report: ' + str(report), flush=True)
+    with (workspace / '.build.lock').open('a+') as lock:
+        def cancel_wait(signum, frame):
+            record.update(state='failed', exit_code=128 + signum)
+            save()
+            raise SystemExit(128 + signum)
+
+        waiting = {sig: signal.signal(sig, cancel_wait) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)}
+        try:
+            try:
+                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                record['lock_contended'] = True
+                save()
+                fcntl.flock(lock, fcntl.LOCK_EX)
+        finally:
+            for sig, handler in waiting.items():
+                signal.signal(sig, handler)
+        record.update(state='running', started_at=datetime.now(timezone.utc).isoformat())
+        save()
+        child = None
+        received = []
+
+        def forward(signum, frame):
+            received.append(signum)
+            if child is not None:
+                signal_group(child.pid, signum)
+
+        previous = {sig: signal.signal(sig, forward) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)}
+        try:
+            with (report / 'build.log').open('wb') as log:
+                child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
+                                         start_new_session=True, pass_fds=(lock.fileno(),))
+                if received:
+                    forward(received[-1], None)
+                code = child.wait()
+        except OSError as error:
+            code = 2
+            record['error'] = str(error)
+        finally:
+            if child is not None:
+                # A failed/cancelled shell must leave no background compiler writers.
+                signal_group(child.pid, signal.SIGKILL)
+                child.wait()
+            for sig, handler in previous.items():
+                signal.signal(sig, handler)
+        code = 128 + received[-1] if received else (128 - code if code < 0 else code)
+        record.update(state='passed' if code == 0 else 'failed', exit_code=code,
+                      finished_at=datetime.now(timezone.utc).isoformat())
+        save()
+    try:
+        print((report / 'build.log').read_text(errors='replace'), end='', flush=True)
+    except BrokenPipeError:
+        pass
+    return code
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--source-dir', required=True)
+    parser.add_argument('--output-dir', required=True)
+    parser.add_argument('--configuration', required=True)
+    parser.add_argument('--destination', default='')
+    parser.add_argument('--xcodebuild', required=True)
+    parser.add_argument('--unsigned-build', action='store_true')
+    parser.add_argument('--check-paths', action='store_true', help=argparse.SUPPRESS)
+    parser.add_argument('arguments', nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+    if args.arguments[:1] == ['--']:
+        args.arguments = args.arguments[1:]
+    try:
+        if args.check_paths:
+            storage_paths(args.output_dir)
+            sys.exit(0)
+        sys.exit(run(args))
+    except (OSError, ValueError, subprocess.CalledProcessError) as error:
+        print('Plugin build cache: ' + str(error), file=sys.stderr)
+        sys.exit(2)