diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46facefe..c4c0aa0f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,6 +103,8 @@ Use `TEST_FILTER=ClassName/testMethod` for one method, or `make test` for all re Unsigned XCTest builds use `build/DerivedDataTests` by default, separate from the signed app used by `make run`. Override `TEST_DERIVED_DATA` when needed, keeping it separate from `DERIVED_DATA` to avoid leaving a test bundle inside the installed app. +Fallback plugin packaging compiles reuse a locked internal compiler cache, while packages and per-run logs remain in the requested output directory. Use `MACTOOLS_PLUGIN_FRESH_BUILD=1 make build-plugin PLUGIN=` for a cold validation build. See [local plugin build storage](docs/plugins/local-native-plugins.md#development-steps) for cache roots and probe isolation; normal Makefile app/test reuse and aggregate-product packaging keep their existing paths. + For copy and catalog changes, run `make validate-localization` to check English resources, complete coverage of all 11 supported languages, and translation format contracts. `make script-tests` includes this audit and fails when a translation is missing. To inspect per-key findings, run `python3 scripts/audit-localization.py --require-complete --json`. Follow the [plugin localization rules](docs/plugins/development-guidelines.md#protocols-and-ownership), and manually check language switching and affected UI layout. Action-symbol changes must keep manifests and runtime definitions consistent; verify them with `make test TEST_FILTER=PluginRuntimeActionSnapshotTests`. | Change | Verification scope | diff --git a/docs/plugins/local-native-plugins.md b/docs/plugins/local-native-plugins.md index 5c85546a..43b41a31 100644 --- a/docs/plugins/local-native-plugins.md +++ b/docs/plugins/local-native-plugins.md @@ -108,6 +108,8 @@ A full Debug sync mirrors the current checkout: packages missing from its local Debug package copies normalize `minHostVersion` to the locally built app version because the host and plugin bundles come from the same checkout. Release packages preserve each source manifest's declared minimum host version. +Fallback compiles in `build-local-plugins.sh` reuse internal compiler workspaces under `~/Library/Caches/MacTools/PluginBuilds`. The cache separates source checkout, project/scheme, Xcode/SDK, host architecture, configuration, destination, signing mode and compiler flags. A lock covers compilation and copying bundles into packages. Each run retains its log and build record under `/BuildReports/`; packaged bundles remain under `/Packages` and do not depend on the compiler cache. Set `MACTOOLS_PLUGIN_FRESH_BUILD=1` for a separate cold workspace, `MACTOOLS_PLUGIN_CACHE_VARIANT` to isolate a probe, or `MACTOOLS_PLUGIN_CACHE_ROOT` for another internal root. Existing compiler output is preserved. Prebuilt packages and `--products-dir` aggregate products keep their existing packaging path. + If the plugin needs extra frameworks, private include paths, bundle resources, helper/tool targets, or target dependencies, add only those differences in `Plugins//project.yml`. If the plugin package contains an extra executable inside the bundle resources, declare it in `plugin.json.package.signPaths` so release packaging signs it before signing the bundle. Plugin UI copy should be localized by the plugin itself. Put plugin string catalogs under `Plugins//Resources`, then look them up from the plugin bundle, for example: diff --git a/scripts/plugins/build-local-plugins.sh b/scripts/plugins/build-local-plugins.sh index 815736cf..339c26c6 100755 --- a/scripts/plugins/build-local-plugins.sh +++ b/scripts/plugins/build-local-plugins.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +ORIGINAL_ARGS=("$@") if [[ -z "${PYTHON3:-}" ]]; then if [[ -x /usr/bin/python3 ]]; then @@ -127,26 +128,31 @@ if [[ -z "$SOURCE_DIR" || -z "$OUTPUT_DIR" ]]; then exit 1 fi -SOURCE_DIR="$(cd "$SOURCE_DIR" 2>/dev/null && pwd || true)" +SOURCE_DIR="$(cd "$SOURCE_DIR" 2>/dev/null && pwd -P || true)" if [[ -z "$SOURCE_DIR" || ! -d "$SOURCE_DIR" ]]; then echo "Plugin source directory not found. Expected: $SOURCE_DIR" >&2 exit 1 fi -REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -OFFICIAL_PLUGIN_SOURCE_DIR="$(cd "$REPO_ROOT/Plugins" && pwd)" +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" +OFFICIAL_PLUGIN_SOURCE_DIR="$(cd "$REPO_ROOT/Plugins" && pwd -P)" THIRD_PARTY_NOTICE_MANIFEST="$REPO_ROOT/Sources/Resources/ThirdPartyNotices/manifest.json" THIRD_PARTY_NOTICE_GENERATOR="$REPO_ROOT/scripts/licenses/generate-third-party-notices.py" if [[ -z "$XCODEBUILD_COMMAND" ]]; then XCODEBUILD_COMMAND="$REPO_ROOT/scripts/xcodebuild-filtered.sh" fi +cache_args=(--source-dir "$SOURCE_DIR" --output-dir "$OUTPUT_DIR" + --configuration "$CONFIGURATION" --destination "$DESTINATION" --xcodebuild "$XCODEBUILD_COMMAND") +[[ "$UNSIGNED_BUILD" == "1" ]] && cache_args+=(--unsigned-build) +# Validate before resetting Packages, including when a cache root was misconfigured inside it. +"$PYTHON3" "$REPO_ROOT/scripts/plugins/with-build-cache.py" --check-paths "${cache_args[@]}" OUTPUT_DIR="$(mkdir -p "$OUTPUT_DIR" && cd "$OUTPUT_DIR" && pwd)" PACKAGES_DIR="$OUTPUT_DIR/Packages" -DERIVED_DATA_DIR="$OUTPUT_DIR/DerivedData" +DERIVED_DATA_DIR="${MACTOOLS_PLUGIN_DERIVED_ROOT:-}" CATALOG_PATH="$OUTPUT_DIR/catalog.dev.json" rm -rf "$PACKAGES_DIR" -mkdir -p "$PACKAGES_DIR" "$DERIVED_DATA_DIR" +mkdir -p "$PACKAGES_DIR" json_value() { local file="$1" @@ -309,6 +315,11 @@ build_bundle_if_needed() { return fi + # Prebuilt/aggregate products keep their existing path; only fallback compiles need a cache owner. + if [[ "${MACTOOLS_PLUGIN_CACHE_CHECKOUT:-}" != "$REPO_ROOT" ]]; then + exec "$PYTHON3" "$REPO_ROOT/scripts/plugins/with-build-cache.py" "${cache_args[@]}" -- "${ORIGINAL_ARGS[@]}" + fi + local project project_file_for "$root" "$manifest" project if [[ -z "$project" ]]; then @@ -324,8 +335,13 @@ build_bundle_if_needed() { fi local derived_data - derived_data="$DERIVED_DATA_DIR/$(basename "$root")" - rm -rf "$derived_data" + local project_key + project_key="$("$PYTHON3" - "$project" "$scheme" <<'PY' +import hashlib, json, pathlib, sys +print(hashlib.sha256(json.dumps([str(pathlib.Path(sys.argv[1]).resolve()), sys.argv[2]]).encode()).hexdigest()[:24]) +PY +)" + derived_data="$DERIVED_DATA_DIR/$project_key" local build_args=( -project "$project" @@ -343,19 +359,29 @@ build_bundle_if_needed() { CODE_SIGN_IDENTITY= ) fi - build_args+=(build -quiet) - - if ! "$XCODEBUILD_COMMAND" "${build_args[@]}"; then + if ! "$XCODEBUILD_COMMAND" "${build_args[@]}" build -quiet; then echo "Failed to build plugin bundle '$scheme' for $root." >&2 return 1 fi + local settings_file="$MACTOOLS_PLUGIN_BUILD_REPORT/$project_key-build-settings.json" + "$XCODEBUILD_COMMAND" "${build_args[@]}" -showBuildSettings -json > "$settings_file" local built_bundle - built_bundle="$(find "$derived_data/Build/Products" -name "$bundle_name" -type d -print | sort | head -1)" - if [[ -z "$built_bundle" ]]; then - echo "Built bundle not found: $bundle_name" >&2 - return 1 - fi + # Retained products may include obsolete targets or paths. Use the current scheme's declared product. + built_bundle="$("$PYTHON3" - "$settings_file" "$bundle_name" "$derived_data" <<'PY' +import json, pathlib, sys +settings_file, name, derived = sys.argv[1:] +settings = json.load(open(settings_file)) +paths = {pathlib.Path(entry['buildSettings']['TARGET_BUILD_DIR']) / name + for entry in settings if entry.get('buildSettings', {}).get('FULL_PRODUCT_NAME') == name} +if len(paths) != 1: + sys.exit('Current scheme must declare exactly one product for ' + name) +product = paths.pop().resolve() +if pathlib.Path(derived).resolve() not in product.parents or not product.is_dir(): + sys.exit('Current product is missing or outside managed compiler storage: ' + str(product)) +print(product) +PY +)" printf -v "$output_var" '%s' "$built_bundle" } diff --git a/scripts/plugins/with-build-cache.py b/scripts/plugins/with-build-cache.py new file mode 100644 index 00000000..460ce22f --- /dev/null +++ b/scripts/plugins/with-build-cache.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Own a reusable compiler workspace until local plugin packaging has finished.""" +import argparse +from datetime import datetime, timezone +import fcntl +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import signal +import subprocess +import sys +import uuid + +REPO = Path(__file__).resolve().parents[2] +OUTPUT_SETTINGS = ('OBJROOT', 'SYMROOT', 'CONFIGURATION_BUILD_DIR', 'CONFIGURATION_TEMP_DIR', + 'PROJECT_TEMP_DIR', 'SHARED_PRECOMPS_DIR', 'CLANG_MODULE_CACHE_PATH', + 'SWIFT_MODULE_CACHE_PATH', 'XCODE_XCCONFIG_FILE') +INPUT_SETTINGS = ('ARCHS', 'EXCLUDED_ARCHS', 'ONLY_ACTIVE_ARCH', 'SDKROOT', 'SWIFT_VERSION', + 'TOOLCHAINS', 'SWIFT_EXEC', 'CC', 'CXX', 'LD', 'ENABLE_TESTABILITY', + 'SWIFT_OPTIMIZATION_LEVEL', 'GCC_OPTIMIZATION_LEVEL', + 'OTHER_SWIFT_FLAGS', 'OTHER_CFLAGS', 'OTHER_CPLUSPLUSFLAGS', 'OTHER_LDFLAGS', + 'SWIFT_ACTIVE_COMPILATION_CONDITIONS', 'GCC_PREPROCESSOR_DEFINITIONS', + 'DEVELOPMENT_TEAM', 'CODE_SIGN_IDENTITY', 'CODE_SIGNING_ALLOWED', + 'CODE_SIGNING_REQUIRED', 'MACTOOLS_PLUGIN_CACHE_VARIANT') + + +def internal_path(value): + path = Path(value).expanduser() + for candidate in (path.absolute(), path.resolve()): + if candidate == Path('/Volumes') or Path('/Volumes') in candidate.parents: + raise ValueError('External storage is deferred; choose an internal path.') + return path.resolve() + + +def query(*command): + return subprocess.check_output(command, text=True).strip() + + +def signal_group(pid, signum): + try: + os.killpg(pid, signum) + except ProcessLookupError: + pass + except PermissionError: + # Teardown can race an empty group on macOS. Never ignore an EPERM with live members. + rows = subprocess.check_output(['/bin/ps', '-axo', 'pgid=,stat='], text=True) + for line in rows.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[0] == str(pid) and not parts[1].startswith('Z'): + raise + + +def storage_paths(output_dir): + cache = internal_path(os.environ.get('MACTOOLS_PLUGIN_CACHE_ROOT', + str(Path.home() / 'Library/Caches/MacTools/PluginBuilds'))) + output = internal_path(output_dir) + if cache == output or cache in output.parents or output in cache.parents: + raise ValueError('Plugin packages/reports and compiler cache must have separate roots.') + return cache, output + + +def run(args): + overrides = [name for name in OUTPUT_SETTINGS if os.environ.get(name)] + if overrides: + raise ValueError('Compiler output overrides are unmanaged: ' + ', '.join(overrides)) + cache, output = storage_paths(args.output_dir) + executable = shutil.which(args.xcodebuild) + if executable is None: + raise ValueError('xcodebuild executable not found: ' + args.xcodebuild) + identity = dict(checkout=str(REPO), source=str(Path(args.source_dir).resolve()), + configuration=args.configuration, destination=args.destination, + unsigned=args.unsigned_build, host_architecture=platform.machine(), + developer=os.environ.get('DEVELOPER_DIR') or query('xcode-select', '-p'), + xcodebuild=str(Path(executable).resolve()), xcode_version=query(executable, '-version'), + sdk_build=query('xcrun', '--sdk', 'macosx', '--show-sdk-build-version'), + settings={name: os.environ[name] for name in INPUT_SETTINGS if name in os.environ}) + key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:24] + fresh = os.environ.get('MACTOOLS_PLUGIN_FRESH_BUILD') == '1' + workspace = cache / (key + ('-fresh-' + uuid.uuid4().hex[:8] if fresh else '')) + report = output / 'BuildReports' / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ-') + uuid.uuid4().hex[:8]) + report.mkdir(parents=True, exist_ok=False) + workspace.mkdir(parents=True, exist_ok=True) + temporary = workspace / 'Temporary' + temporary.mkdir(exist_ok=True) + env = dict(os.environ, MACTOOLS_PLUGIN_CACHE_CHECKOUT=str(REPO), + MACTOOLS_PLUGIN_DERIVED_ROOT=str(workspace / 'DerivedData'), + MACTOOLS_PLUGIN_BUILD_REPORT=str(report), TMPDIR=str(temporary) + '/') + command = ['/bin/bash', str(REPO / 'scripts/plugins/build-local-plugins.sh'), *args.arguments] + record = dict(identity=identity, cache_key=key, workspace=str(workspace), report=str(report), + command=command, fresh=fresh, state='waiting-for-lock') + + def save(): + temporary_report = report / 'build.json.tmp' + temporary_report.write_text(json.dumps(record, indent=2) + '\n') + temporary_report.replace(report / 'build.json') + + save() + print('Plugin build report: ' + str(report), flush=True) + with (workspace / '.build.lock').open('a+') as lock: + def cancel_wait(signum, frame): + record.update(state='failed', exit_code=128 + signum) + save() + raise SystemExit(128 + signum) + + waiting = {sig: signal.signal(sig, cancel_wait) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} + try: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + record['lock_contended'] = True + save() + fcntl.flock(lock, fcntl.LOCK_EX) + finally: + for sig, handler in waiting.items(): + signal.signal(sig, handler) + record.update(state='running', started_at=datetime.now(timezone.utc).isoformat()) + save() + child = None + received = [] + + def forward(signum, frame): + received.append(signum) + if child is not None: + signal_group(child.pid, signum) + + previous = {sig: signal.signal(sig, forward) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} + try: + with (report / 'build.log').open('wb') as log: + child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True, pass_fds=(lock.fileno(),)) + if received: + forward(received[-1], None) + code = child.wait() + except OSError as error: + code = 2 + record['error'] = str(error) + finally: + if child is not None: + # A failed/cancelled shell must leave no background compiler writers. + signal_group(child.pid, signal.SIGKILL) + child.wait() + for sig, handler in previous.items(): + signal.signal(sig, handler) + code = 128 + received[-1] if received else (128 - code if code < 0 else code) + record.update(state='passed' if code == 0 else 'failed', exit_code=code, + finished_at=datetime.now(timezone.utc).isoformat()) + save() + try: + print((report / 'build.log').read_text(errors='replace'), end='', flush=True) + except BrokenPipeError: + pass + return code + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source-dir', required=True) + parser.add_argument('--output-dir', required=True) + parser.add_argument('--configuration', required=True) + parser.add_argument('--destination', default='') + parser.add_argument('--xcodebuild', required=True) + parser.add_argument('--unsigned-build', action='store_true') + parser.add_argument('--check-paths', action='store_true', help=argparse.SUPPRESS) + parser.add_argument('arguments', nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.arguments[:1] == ['--']: + args.arguments = args.arguments[1:] + try: + if args.check_paths: + storage_paths(args.output_dir) + sys.exit(0) + sys.exit(run(args)) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + print('Plugin build cache: ' + str(error), file=sys.stderr) + sys.exit(2) diff --git a/scripts/tests/test_plugin_build_cache.py b/scripts/tests/test_plugin_build_cache.py new file mode 100644 index 00000000..b44b34d0 --- /dev/null +++ b/scripts/tests/test_plugin_build_cache.py @@ -0,0 +1,250 @@ +"""Fallback plugin compiles reuse output without racing the packaging consumer.""" +import json +import os +from pathlib import Path +import signal +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +from test_copy_plugin_manifest import runtime_envelope + +REPO = Path(__file__).resolve().parents[2] +SCRIPT = REPO / 'scripts/plugins/build-local-plugins.sh' + + +class PluginBuildCacheTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.source = self.root / 'source/Example' + self.source.mkdir(parents=True) + (self.source / 'Example.xcodeproj').mkdir() + (self.source / 'plugin.json').write_text(json.dumps(runtime_envelope( + build={'project': 'Example.xcodeproj', 'scheme': 'Example'}))) + self.bin = self.root / 'bin' + self.bin.mkdir() + self.env = {key: value for key, value in os.environ.items() if not key.startswith('MACTOOLS_PLUGIN_')} + self.env.update(PATH=str(self.bin) + os.pathsep + self.env['PATH'], + PYTHON3=sys.executable, DEVELOPER_DIR=str(self.root / 'Xcode/Developer'), + MACTOOLS_PLUGIN_CACHE_ROOT=str(self.root / 'compiler'), CALLS=str(self.root / 'calls')) + self.tool('xcrun', "import os\nprint(os.environ.get('SDK_BUILD','fixture-sdk'))\n") + self.builder = self.tool('xcodebuild', '''import json,os,pathlib,sys +if '-version' in sys.argv: + print('fixture-xcode') + sys.exit(0) +derived=pathlib.Path(sys.argv[sys.argv.index('-derivedDataPath')+1]) +products=derived/'Build/Products'/os.environ.get('PRODUCT_DIRECTORY','')/sys.argv[sys.argv.index('-configuration')+1] +name=os.environ.get('PRODUCT_NAME','Example.bundle') +if '-showBuildSettings' in sys.argv: + print(json.dumps([{'buildSettings':{'TARGET_BUILD_DIR':str(products),'FULL_PRODUCT_NAME':name}}])) + sys.exit(0) +with open(os.environ['CALLS'],'a') as stream: stream.write(str(derived)+'\\n') +bundle=products/name +bundle.mkdir(parents=True,exist_ok=True) +(bundle/'payload').write_text(os.environ.get('BUNDLE_PAYLOAD','built plugin')) +print('Fixture build log',flush=True) +sys.exit(int(os.environ.get('BUILD_EXIT','0'))) +''') + self.tool('ditto', '''import os,pathlib,subprocess,sys,time +if os.environ.get('HOLD_COPY'): + pathlib.Path(os.environ['HOLD_COPY']).touch() + deadline=time.monotonic()+8 + while not pathlib.Path(os.environ['RELEASE_COPY']).exists() and time.monotonic()