diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..c3c22da9e --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,33 @@ +# Repository-wide Cargo defaults for the development build standard. +# +# The mold linker and the parallel `rustc` frontend are the defaults for every +# development, test, lint and typecheck build. Cargo auto-discovers this file, +# so a bare `cargo build` gets them too; there is no opt-in path and no slower +# alternative to choose. +# +# Release and coverage builds are excluded. Both are held out by assigning +# `RUSTFLAGS`, which displaces every `rustflags` table below; see "The build +# standard" in docs/developers-guide.md for the mechanism and the contracts. +# +# No codegen backend is named here, and none may be. A panic compiled by the +# Cranelift backend does not find the unwind handler it should: `catch_unwind` +# fails to catch and a panic on a spawned thread aborts the process, while a +# bare `#[should_panic]` still passes. That makes the backend unusable for a +# profile any test or binary runs on. The guide records the measurement and the +# command; a contract refuses a backend key so that re-adding one has to go +# through that evidence first. + +# Cargo selects a single rustflags source rather than merging them: a matching +# `[target.*]` table replaces `[build] rustflags` outright, and an externally +# set `RUSTFLAGS` replaces both. Every flag that must survive therefore has to +# be repeated in each source, including the `RUSTFLAGS` the Makefile composes +# for the gate targets. tests/build_tools_cargo_config_tests.rs and +# tests/makefile_test_target/rustflags.rs hold the sources equal. +[build] +rustflags = ["-Zthreads=8"] + +# mold ships for Linux only, so the linker flag is gated behind a target `cfg`. +# macOS and Windows fall through to `[build] rustflags` above and use their +# platform default linker. +[target.'cfg(target_os = "linux")'] +rustflags = ["-Zthreads=8", "-Clink-arg=-fuse-ld=mold"] diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 7ec4857bb..f32bcd2d7 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -176,7 +176,13 @@ jobs: run: | $profileHome = [Environment]::GetFolderPath([Environment+SpecialFolder]::UserProfile) $whitaker = Join-Path $profileHome '.local\bin\whitaker.ps1' - $env:RUSTFLAGS = "$env:RUSTFLAGS -D warnings" + # Assigning RUSTFLAGS displaces every `rustflags` table in + # `.cargo/config.toml`, so the value has to carry the standard's + # flags as well as the deny. Name `-Zthreads=8` explicitly: mold is + # Linux-only so its flag is absent here, but the parallel frontend is + # not, and dropping it would lint a differently compiled tree from the + # one the other lanes measure. + $env:RUSTFLAGS = "$env:RUSTFLAGS -D warnings -Zthreads=8" $env:DYLINT_TOML = Get-Content dylint.toml -Raw & $whitaker --all --no-deps --package netsuke-build '--' --all-targets --all-features if ($LASTEXITCODE -ne 0) { @@ -320,6 +326,15 @@ jobs: # been made. shell: pwsh run: | + # The `setup-rust` action exports its `rustflags` input as + # `RUSTFLAGS`, and an externally set `RUSTFLAGS` displaces every + # `rustflags` table in `.cargo/config.toml`. This step invokes Cargo + # directly rather than through the Makefile, so nothing composes the + # standard back in: without this line the binary users get would be + # the one build here not compiled with the parallel frontend. mold is + # Linux-only, so its flag stays absent. Same assignment, and same + # reason, as `Lint (Whitaker)` above. + $env:RUSTFLAGS = "$env:RUSTFLAGS -Zthreads=8" cargo build --locked --bin netsuke if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18643b278..7ba358104 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,11 @@ jobs: tool: nextest@${{ env.NEXTEST_VERSION }} - name: Reset sccache statistics run: sccache --zero-stats + - name: Install the build standard + # The pinned `mold` is the repository's default linker and + # `check-build-tools` gates every build target on it, so this runs + # before the first compile rather than as a convenience. + run: make install-build-tools - name: Show rustc version run: | rustup show @@ -257,6 +262,15 @@ jobs: # `rustflags` input. Pull requests only; on the trunk # `coverage-upload` runs the same set over the same commit. if: github.event_name == 'pull_request' + env: + # One of the two exclusions from the build standard. A build whose + # output is a measurement is a reproducibility claim, so it takes + # neither the parallel frontend nor the linker change. Assigning + # RUSTFLAGS is the whole mechanism: it displaces every `rustflags` + # table in `.cargo/config.toml`. Stated here rather than inherited + # from `setup-rust`, so the exclusion is visible at the step it + # applies to and a contract has something to read. + RUSTFLAGS: -D warnings uses: leynos/shared-actions/.github/actions/generate-coverage@a5765019912a8ab6882b12db049c7cde635f3a85 with: language: rust diff --git a/.github/workflows/coverage-main.yml b/.github/workflows/coverage-main.yml index 6e03d52d4..2aa751fa7 100644 --- a/.github/workflows/coverage-main.yml +++ b/.github/workflows/coverage-main.yml @@ -213,6 +213,15 @@ jobs: - name: Reset sccache statistics run: sccache --zero-stats - name: Test and Measure Coverage + env: + # One of the two exclusions from the build standard. A build whose + # output is a measurement is a reproducibility claim, so it takes + # neither the parallel frontend nor the linker change. Assigning + # RUSTFLAGS is the whole mechanism: it displaces every `rustflags` + # table in `.cargo/config.toml`. Stated here rather than inherited + # from `setup-rust`, so the exclusion is visible at the step it + # applies to and a contract has something to read. + RUSTFLAGS: -D warnings uses: leynos/shared-actions/.github/actions/generate-coverage@a5765019912a8ab6882b12db049c7cde635f3a85 with: language: rust diff --git a/.github/workflows/netsukefile-test.yml b/.github/workflows/netsukefile-test.yml index 8e1f9a60f..87dff8346 100644 --- a/.github/workflows/netsukefile-test.yml +++ b/.github/workflows/netsukefile-test.yml @@ -107,6 +107,11 @@ jobs: uses: seanmiddleditch/gha-setup-ninja@3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6 - name: Reset sccache statistics run: sccache --zero-stats + - name: Install the build standard + # `make build` below is gated on `check-build-tools` and links with the + # pinned `mold`, so it must be present before the first compile rather + # than installed on demand. + run: make install-build-tools - name: Show rustc version run: | rustup show diff --git a/AGENTS.md b/AGENTS.md index 05804921b..7222b5720 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -190,18 +190,28 @@ directive anywhere. CI-pinned version. `--git --include-untracked` selects the tracked and untracked Markdown files Git does not ignore, and `--check` exits `1` when any of them would be reformatted. - - `make lint` executes: + - On Linux, `make lint` executes: ```sh - RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings" \ + RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings -Zthreads=8 -Clink-arg=-fuse-ld=mold" \ RUSTDOCFLAGS="--cfg docsrs -D warnings" cargo doc --workspace --no-deps - RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings" \ + RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings -Zthreads=8 -Clink-arg=-fuse-ld=mold" \ cargo clippy --workspace --all-targets --all-features -- -D warnings + RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings -Zthreads=8 -Clink-arg=-fuse-ld=mold" \ whitaker --all -- --all-targets --all-features yamllint --config-file .yamllint.yml .github/workflows actionlint ``` + The Makefile composes that `RUSTFLAGS` value from one variable. + `-Clink-arg=-fuse-ld=mold` is Linux-only, so every sample in this section + drops it elsewhere — macOS and Windows use their platform linker and the + value ends `-D warnings -Zthreads=8`. Only that one flag is platform-gated; + `-Zthreads=8` and `-D warnings` apply everywhere. The flags are restated in + the Makefile rather than left to `.cargo/config.toml` because an assigned + `RUSTFLAGS` replaces every `rustflags` table in that file; see *Build + standard* below. + linting every target with all features enabled, denying all Clippy warnings, running the Whitaker Dylint suite (see `docs/whitaker-users-guide.md`; install via `cargo install @@ -226,15 +236,18 @@ directive anywhere. otherwise `$HOME/go/bin`; override `GO_BIN` to point at a different directory, or pass `ACTIONLINT=/path/to/actionlint` to name the binary directly, as CI does. - - `make test` executes: + - On Linux, `make test` executes: ```sh - RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings" \ + RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings -Zthreads=8 -Clink-arg=-fuse-ld=mold" \ cargo nextest run --workspace --all-targets --all-features - RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings" \ + RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-D warnings -Zthreads=8 -Clink-arg=-fuse-ld=mold" \ cargo test --workspace --doc --all-features ``` + The platform caveat stated under `make lint` applies unchanged: the linker + flag is Linux-only and the rest of the value is not. + running every unit, integration, and behavioural test through [cargo-nextest](https://nexte.st/), then the doctests separately because nextest cannot execute them. Both passes deny warnings, and `make test` @@ -551,14 +564,40 @@ The following tooling is available in this environment: These practices help maintain a high-quality codebase and facilitate collaboration. -## Fast development builds - -`make dev-build` and `make dev-test` compile with the opt-in Cranelift backend -and the mold linker configured in `tools/dev-fast/config.toml`. Run -`make install-dev-fast` to install the pinned nightly's -`rustc-codegen-cranelift-preview` component and, on Linux, the pinned `mold` -release. `make dev-fast-check` preflights those prerequisites before Cargo is -invoked. Linux `x86_64` and `aarch64` hosts use `mold`; macOS and Windows use -their platform linker instead. The fragment is passed explicitly with -`--config`, so release, coverage, and verification builds are unaffected; never -copy its contents into `.cargo/config.toml`, which Cargo applies to every build. +## Build standard + +The `mold` linker and the parallel `rustc` frontend (`-Zthreads=8`) are the +**defaults** for development, test, lint, and typecheck builds. They are +committed to `.cargo/config.toml`, which Cargo auto-discovers, so a bare +`cargo build` gets them too. The Cranelift codegen backend is deliberately not +part of the standard and a contract refuses one; the developers' guide records +why. + +Run `make install-build-tools` to install the pinned nightly and, on Linux, the +pinned `mold` release. `make check-build-tools` preflights those prerequisites, +and is a prerequisite of `make build`, `make test`, `make lint`, and +`make typecheck`, so a missing tool reports an installation hint before Cargo +runs. Linux hosts use `mold`; macOS and Windows keep their platform linker, +which the `cfg(target_os = "linux")` gate in the configuration expresses. + +Two build shapes are excluded and must stay excluded: + +- **Release and packaging.** The release recipe assigns `RUSTFLAGS` so the + configuration's `rustflags` tables do not apply. A shipped artefact is built + on the platform linker and a single-threaded frontend. +- **Coverage.** A build whose output is a measurement is a reproducibility + claim. The coverage steps assign `RUSTFLAGS` at the step itself and carry + neither `-Zthreads` nor the linker flag. + +Cargo picks a single `rustflags` source rather than merging them: a matching +`[target.*]` table replaces `[build] rustflags`, and an externally set +`RUSTFLAGS` replaces both. Every gate recipe assigns `RUSTFLAGS` to deny +warnings, so the standard's flags are restated in the Makefile and composed +into that value. Changing one source without the other fails a contract test; +do not "simplify" by deleting a restatement. + +There is no separate accelerated target. `make build`, `make test`, +`make lint`, and `make typecheck` are the build targets, and they all run on +the standard. See "The build standard" in +[developers' guide](docs/developers-guide.md) for the full ownership boundary, +the benchmark, and the fallback behaviour. diff --git a/Cargo.lock b/Cargo.lock index 425de8a5f..176681abc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2978,6 +2978,7 @@ dependencies = [ "sha2", "tempfile", "thiserror 1.0.69", + "toml 0.8.23", "tracing", "tracing-subscriber", ] diff --git a/Makefile b/Makefile index 20e41533a..3e872cb7c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help all clean test test-nextest doctest test-workflow-contracts test-windows-msi-release-rank test-release-admission test-coverage-artifact build release lint lint-clippy lint-whitaker lint-python lint-workflow-scripts github-actions-lint doc-coverage doc-coverage-test validate-coverage-artifact fmt check-fmt typecheck typecheck-python markdownlint spelling nixie install-kani kani-check kani-full kani-ir install-verus verus formal-pr install-dev-fast dev-fast-check dev-build dev-test bench-build bench-config-load bench-glob-expansion +.PHONY: help all clean test test-nextest doctest test-workflow-contracts test-windows-msi-release-rank test-release-admission test-coverage-artifact build release lint lint-clippy lint-whitaker lint-python lint-workflow-scripts github-actions-lint doc-coverage doc-coverage-test validate-coverage-artifact fmt check-fmt typecheck typecheck-python markdownlint spelling nixie install-kani kani-check kani-full kani-ir install-verus verus formal-pr install-build-tools check-build-tools bench-build bench-config-load bench-glob-expansion RUST_TOOLCHAIN_FILE ?= rust-toolchain.toml # Export this path before shell probes expand it, so Make does not interpolate @@ -6,7 +6,7 @@ RUST_TOOLCHAIN_FILE ?= rust-toolchain.toml export RUST_TOOLCHAIN_FILE # Threshold and toolchain for the Rustdoc doc-comment coverage gate. The # threshold mirrors the 80% bar stated in AGENTS.md; the toolchain recollects -# the channel from rust-toolchain.toml the same way the dev-fast variables do, +# the channel from rust-toolchain.toml the same way the build-tools variables do, # so overriding either stays independent. DOC_COVERAGE_THRESHOLD ?= 80 DOC_COVERAGE_TOOLCHAIN ?= $(shell awk -F'"' '/^[[:space:]]*channel[[:space:]]*=/ { print $$2; exit }' "$$RUST_TOOLCHAIN_FILE") @@ -45,24 +45,49 @@ KANI_FLAGS ?= KANI_INSTALL_FLAGS ?= KANI_CHECK_FLAGS ?= KANI_VERSION_FILE ?= tools/kani/VERSION -# Opt-in local build acceleration. The Cargo fragment is deliberately kept out -# of an auto-discovered `.cargo/config.toml`, because Cranelift and mold must -# stay opt-in so release, packaging, coverage, and formal-verification paths -# keep the supported LLVM backend and platform linker. The toolchain is not -# pinned separately — dev-fast uses the repository's own nightly. +# The development build standard: the `mold` linker and the parallel `rustc` +# frontend are the defaults for development, test, lint, and typecheck builds. +# `.cargo/config.toml` carries them so a bare `cargo` invocation gets them too; +# release and coverage builds are excluded there and below. The toolchain is not +# pinned separately — the standard uses the repository's own nightly from +# `rust-toolchain.toml`. MOLD_VERSION_FILE ?= tools/mold/VERSION MOLD_SHA256SUMS_FILE ?= tools/mold/SHA256SUMS -DEV_FAST_CONFIG ?= tools/dev-fast/config.toml -# Preserve the raw override before export: otherwise Make expands a caller's -# literal dollar signs while preparing the environment for the recipe shell. -override DEV_FAST_CONFIG := $(value DEV_FAST_CONFIG) -DEV_FAST_PREFIX ?= $(HOME)/.local +BUILD_TOOLS_PREFIX ?= $(HOME)/.local # Exported rather than interpolated into the recipes. Make hands an exported # variable to the child process directly, so a path containing a quote cannot # break the command line the shell parses; a `VAR='$(VAR)'` prefix could. export MOLD_VERSION_FILE MOLD_SHA256SUMS_FILE -export DEV_FAST_CONFIG DEV_FAST_PREFIX -DEV_FAST_TOOLCHAIN = $$(awk -F'"' '/^[[:space:]]*channel[[:space:]]*=/ { print $$2; exit }' "$$RUST_TOOLCHAIN_FILE") +export BUILD_TOOLS_PREFIX + +# Cargo picks a single `rustflags` source rather than merging them, and an +# externally set `RUSTFLAGS` outranks every `rustflags` table in +# `.cargo/config.toml`. Every gate target below sets `RUSTFLAGS` to deny +# warnings, and CI's `setup-rust` exports the same value for a whole job, so +# without restating the flags here the gates would silently fall back to the +# platform linker and a single-threaded frontend while still reporting success. +# tests/makefile_test_target/rustflags.rs holds these variables equal to the +# configuration file; changing one without the other fails that test. +STANDARD_THREADS_FLAG ?= -Zthreads=8 +STANDARD_MOLD_FLAG ?= -Clink-arg=-fuse-ld=mold +# `mold` ships for Linux only. macOS and Windows keep their platform linker, +# matching the `cfg(target_os = "linux")` gate in `.cargo/config.toml`. +BUILD_HOST_OS := $(shell uname -s) +STANDARD_RUSTFLAGS = $(STANDARD_THREADS_FLAG)$(if $(filter Linux,$(BUILD_HOST_OS)), $(STANDARD_MOLD_FLAG)) +# Warnings-as-errors plus the standard, appended to whatever the caller set. +GATE_RUSTFLAGS = RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings $(STANDARD_RUSTFLAGS)" +# A debug build that keeps the caller's warning policy rather than imposing one. +DEBUG_RUSTFLAGS = RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }$(STANDARD_RUSTFLAGS)" +# Release builds take neither the parallel frontend nor `mold`: assigning +# `RUSTFLAGS` at all, even to an empty inherited value, displaces the +# configuration file's `rustflags` tables, which is the whole mechanism. The +# configuration names no codegen backend at all, so nothing else is needed. +RELEASE_RUSTFLAGS = RUSTFLAGS="$${RUSTFLAGS-}" +# Kani denies warnings like the gates, but takes none of the standard: it drives +# `rustc` through `kani-compiler` on its own bundled toolchain, so the parallel +# frontend and `mold` would neither apply nor be honoured there. Inherited flags +# still survive, since `cargo kani` appends this value to its own. +KANI_RUSTFLAGS = RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" # Command name, resolved by the recipe shell from the curated PATH, which # carries `$HOME/.bun/bin` where the global markdownlint install lands. MDLINT ?= markdownlint-cli2 @@ -185,10 +210,13 @@ VERUS_FLAGS ?= VERUS_INSTALL_FLAGS ?= WHITAKER ?= whitaker +# The build-tools install prefix leads: `-fuse-ld=mold` resolves by PATH order, so +# the pinned release must outrank any distribution `mold`. Every build target +# now links with it, so this is global rather than target-specific. # GO_BIN is appended after the three fixed directories so a tool present in # more than one location keeps its current precedence; CI's explicit # `ACTIONLINT=` override still wins over all of them. -export PATH := $(HOME)/.cargo/bin:$(HOME)/.local/bin:$(HOME)/.bun/bin:$(GO_BIN):$(PATH) +export PATH := $(BUILD_TOOLS_PREFIX)/bin:$(HOME)/.cargo/bin:$(HOME)/.local/bin:$(HOME)/.bun/bin:$(GO_BIN):$(PATH) build: target/debug/$(APP) ## Build debug binary release: target/release/$(APP) ## Build release binary @@ -200,11 +228,11 @@ clean: ## Remove build artefacts test: test-nextest doctest ## Run every Rust test with warnings treated as errors -test-nextest: ## Run all non-doctest Rust tests through cargo-nextest - RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" $(CARGO) nextest run --workspace --all-targets --all-features $(NEXTEST_BUILD_JOBS) $(NEXTEST_TEST_JOBS) +test-nextest: check-build-tools ## Run all non-doctest Rust tests through cargo-nextest + $(GATE_RUSTFLAGS) $(CARGO) nextest run --workspace --all-targets --all-features $(NEXTEST_BUILD_JOBS) $(NEXTEST_TEST_JOBS) -doctest: ## Run doctests, which cargo-nextest cannot execute - RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" $(CARGO) test --workspace --doc --all-features $(BUILD_JOBS) +doctest: check-build-tools ## Run doctests, which cargo-nextest cannot execute + $(GATE_RUSTFLAGS) $(CARGO) test --workspace --doc --all-features $(BUILD_JOBS) test-workflow-contracts: ## Validate GitHub Actions workflow contracts $(UV_ENV) $(UV) run --no-project --python $(PYTHON_BASELINE) --with 'pytest>=8' --with 'pyyaml>=6' --with 'hypothesis>=6' --with 'cmd-mox==0.2.0' pytest tests/workflow_contracts -q --doctest-modules @@ -228,8 +256,14 @@ test-coverage-artifact: ## Test hostile LCOV artefact validation scripts/tests/test_validate_coverage_archive.py -c /dev/null --rootdir=. \ -p no:cacheprovider -target/%/$(APP): ## Build binary in debug or release mode - $(CARGO) build $(BUILD_JOBS) $(if $(findstring release,$(@)),--release) --bin $(APP) +# Split rather than a single `target/%/$(APP)` pattern: the two profiles no +# longer share a command line. The debug build takes the standard; the release +# build is one of the two exclusions. +target/debug/$(APP): | check-build-tools ## Build the debug binary + $(DEBUG_RUSTFLAGS) $(CARGO) build $(BUILD_JOBS) --bin $(APP) + +target/release/$(APP): ## Build the release binary on the LLVM backend + $(RELEASE_RUSTFLAGS) $(CARGO) build $(BUILD_JOBS) --release --bin $(APP) lint: lint-clippy lint-whitaker lint-python github-actions-lint ## Run the Rust, Python, and GitHub Actions lint suites with warnings denied @@ -253,15 +287,15 @@ lint-workflow-scripts: ## Load every trusted workflow module under the Python ba "$$module" || { echo "$$module does not load under Python $(PYTHON_BASELINE)" >&2; exit 1; }; \ done -lint-clippy: ## Run rustdoc and Clippy with warnings denied - RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" $(CARGO) doc --workspace --no-deps - RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" $(CARGO) clippy $(CLIPPY_FLAGS) +lint-clippy: check-build-tools ## Run rustdoc and Clippy with warnings denied + $(GATE_RUSTFLAGS) $(CARGO) doc --workspace --no-deps + $(GATE_RUSTFLAGS) $(CARGO) clippy $(CLIPPY_FLAGS) -lint-whitaker: ## Run the Whitaker Dylint suite with warnings denied - DYLINT_TOML="$$(cat dylint.toml)" RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" $(WHITAKER) --all --no-deps --package netsuke-build -- --all-targets --all-features +lint-whitaker: check-build-tools ## Run the Whitaker Dylint suite with warnings denied + DYLINT_TOML="$$(cat dylint.toml)" $(GATE_RUSTFLAGS) $(WHITAKER) --all --no-deps --package netsuke-build -- --all-targets --all-features # Run from the crate directory as well so Whitaker loads the narrow # `test_support::fs` exemption from test_support/dylint.toml. - cd test_support && DYLINT_TOML="$$(cat dylint.toml)" RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" $(WHITAKER) --all --no-deps --package test_support -- --all-targets --all-features + cd test_support && DYLINT_TOML="$$(cat dylint.toml)" $(GATE_RUSTFLAGS) $(WHITAKER) --all --no-deps --package test_support -- --all-targets --all-features # actionlint is resolved in the recipe shell below, never while Make parses the # file: only the recipe shell receives the curated PATH, so a parse-time probe @@ -313,8 +347,8 @@ check-fmt: ## Verify formatting $(RUFF) format --check $(PYTHON_SOURCES) $(MDTABLEFIX) --check $(MDTABLEFIX_SELECT) $(MDTABLEFIX_RULES) -typecheck: typecheck-python ## Typecheck all targets and features - RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" $(CARGO) check --all-targets --all-features $(BUILD_JOBS) +typecheck: check-build-tools typecheck-python ## Typecheck all targets and features + $(GATE_RUSTFLAGS) $(CARGO) check --all-targets --all-features $(BUILD_JOBS) typecheck-python: ## Typecheck the Python sources with ty # `uv tool run` materialises one venv holding ty plus the test-suite @@ -348,7 +382,7 @@ kani-check: ## Check the installed Kani verifier version @$(PROVER_TOOLS) kani check-version --kani-command "$(KANI)" $(KANI_CHECK_FLAGS) || { status=$$?; printf 'prover-tools: target=kani-check failed exit=%s\n' "$$status" >&2; exit "$$status"; } kani-full: ## Run the full Kani verification suite - RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings" $(KANI) $(KANI_FLAGS) + $(KANI_RUSTFLAGS) $(KANI) $(KANI_FLAGS) kani-ir: kani-full ## Run the IR Kani verification suite @@ -367,27 +401,15 @@ verus: ## Run the Verus proof entry point formal-pr: ## Run pull-request formal-verification checks $(MAKE) kani-check -install-dev-fast: ## Install the pinned mold linker and Cranelift backend - @scripts/install-dev-fast.sh - -dev-fast-check: ## Check the mold and Cranelift local build prerequisites - @scripts/dev-fast-check.sh - -# Every dev-fast target needs the install prefix ahead of a distribution mold: -# the check probes PATH for it, and `-fuse-ld=mold` resolves by PATH order. -# Target-specific exports do not reach prerequisites, so `dev-fast-check` is -# listed in its own right as well as being a prerequisite of the others. -DEV_FAST_TARGETS = install-dev-fast dev-fast-check dev-build dev-test bench-build -$(DEV_FAST_TARGETS): export PATH := $(DEV_FAST_PREFIX)/bin:$(PATH) - -dev-build: dev-fast-check ## Build the debug binary with Cranelift and mold - RUSTUP_TOOLCHAIN=$(DEV_FAST_TOOLCHAIN) $(CARGO) $(CARGO_LOCKED) --config "$$DEV_FAST_CONFIG" build $(BUILD_JOBS) --bin $(APP) +install-build-tools: ## Install the pinned mold linker and the pinned toolchain + @scripts/install-build-tools.sh -dev-test: dev-fast-check ## Run the nextest pass with Cranelift and mold - RUSTUP_TOOLCHAIN=$(DEV_FAST_TOOLCHAIN) $(CARGO) --config "$$DEV_FAST_CONFIG" nextest run $(CARGO_LOCKED) --workspace --all-targets --all-features $(NEXTEST_BUILD_JOBS) $(NEXTEST_TEST_JOBS) +check-build-tools: ## Check the mold linker and toolchain prerequisites + @scripts/check-build-tools.sh -bench-build: dev-fast-check ## Time clean and incremental debug builds for both paths - @CARGO="$(CARGO)" scripts/bench-build.sh +bench-build: check-build-tools ## Time clean and incremental debug builds for all three paths + @CARGO="$(CARGO)" STANDARD_THREADS_FLAG="$(STANDARD_THREADS_FLAG)" \ + STANDARD_MOLD_FLAG="$(STANDARD_MOLD_FLAG)" scripts/bench-build.sh bench-config-load: ## Benchmark cached configuration loading without layer copies $(CARGO) bench --bench config_load_cached_merge diff --git a/README.md b/README.md index aee95c3d6..5a731f9ff 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,19 @@ cd netsuke cargo install --path . ``` +Because that install runs inside the checkout, it inherits the repository's +build standard: the parallel `rustc` frontend, and on Linux the `mold` linker. +The pinned nightly comes from `rust-toolchain.toml`, which `rustup` provisions +automatically; on Linux, `mold` must also be reachable through `PATH` or gcc's +own search directories. `make install-build-tools` unpacks the pinned release +into `~/.local/bin` and does not edit any shell profile, and the make targets +add that directory to `PATH` for themselves only, so a direct `cargo install` +needs it added first. On macOS and Windows the linker flag is not set, so no +extra prerequisite applies. Build on another platform, or remove +`.cargo/config.toml` first, to use the platform linker instead. The +[user's guide](docs/users-guide.md#install-netsuke) covers the same ground in +more detail. + ### Your first build Create a new directory and add a file named `Netsukefile`: diff --git a/docs/adr-029-mold-and-parallel-frontend-as-build-defaults.md b/docs/adr-029-mold-and-parallel-frontend-as-build-defaults.md new file mode 100644 index 000000000..5bac896dc --- /dev/null +++ b/docs/adr-029-mold-and-parallel-frontend-as-build-defaults.md @@ -0,0 +1,219 @@ +# Architecture decision record (ADR): Make `mold` and the parallel frontend the default build + +## Status + +Accepted. + +## Date + +2026-09-19 + +## Context and problem statement + +Netsuke's development loop spent most of its wall-clock time in two places that +are not the code: linking, and `rustc`'s single-threaded front-end. Both have +supported, low-risk accelerations for years, and both were reachable only by +opting in — an environment variable exported by whoever remembered, or a local +edit to a file that is git-ignored. The result was that a contributor's build +speed depended on whether they had read a particular section of the developers' +guide, and that continuous integration and local runs compiled under different +flags without either side recording that they did. + +Two candidate accelerations are available to the whole workspace: + +- the `mold` linker (Linux only), which replaces GNU `ld` for the final link; +- the parallel `rustc` front-end, `-Zthreads=N`, which on nightly parallelizes + front-end work that has been serial since the compiler was written. + +A third looks obvious and is not: the Cranelift codegen backend, which promises +a much larger win because it replaces LLVM entirely, and which is unusable here +for a reason that a shallow probe does not reveal. + +Choosing a default also means choosing what the default must _not_ reach. Two +build shapes are not development builds: a release or packaging build, whose +output is shipped, and a coverage build, whose output is a measurement. An +acceleration reaching either would change what is shipped or compared. + +Finally, the mechanism matters as much as the flags. A default that lives in a +target, or in an exported variable, is not a default: it is an opt-in that some +paths take and others miss, and the misses are silent. + +## Decision + +Commit the build standard to `.cargo/config.toml`, the file Cargo +auto-discovers, so that every build in the repository takes it — including a +bare `cargo build`, and including builds that no Make target drives: + +- `-Zthreads=8` in `[build] rustflags`, taking effect on every platform; +- the same flag plus `-Clink-arg=-fuse-ld=mold` in a + `cfg(target_os = "linux")` table. `mold` ships for Linux only, so macOS and + Windows fall through to the `[build]` table and use their platform linker. + +The flags are restated in the Makefile as `STANDARD_THREADS_FLAG` and +`STANDARD_MOLD_FLAG` and composed into the `RUSTFLAGS` each gate builds. Cargo +selects a single `rustflags` source rather than merging them, so an assigned +`RUSTFLAGS` — which every gate sets, to deny warnings — replaces every table in +the configuration file. Restating is therefore required rather than redundant, +and contract tests hold the two sources equal in both directions. + +**Exclusions.** Release and coverage builds are held out by assigning +`RUSTFLAGS` at the point they run, which is the same mechanism that makes the +restatement necessary. Release assigns it because the artefact ships; coverage +assigns it at the step, not on the job, so the exclusion is visible where it +applies and a contract has something to read. Continuous integration's release +lanes are already covered because the toolchain action exports `RUSTFLAGS` for +the whole job. + +**Cranelift is refused.** No `codegen-backend` key may appear in the +configuration, under any profile, and a contract refuses one along three +routes: a direct profile key, a key beneath a profile's `package` table, and +`-Zcodegen-backend=` inside rustflags. + +**The toolchain and the tools are separate.** `rust-toolchain.toml` supplies +the compiler and is provisioned by `rustup` automatically. `mold` is a binary +the contributor installs: `make install-build-tools` fetches the pinned release +and verifies it against `tools/mold/SHA256SUMS`, and `make check-build-tools` +verifies it is present and is the pinned version. Every gate target depends on +the check, so a missing prerequisite reports an installation hint rather than +surfacing later as a linker error. + +## Rationale + +**Why the configuration file, not a target.** A Make target can only accelerate +the builds that go through it. `cargo build`, `cargo test`, an editor's +`cargo check`, and rust-analyzer all bypass the Makefile, and a contributor who +uses them would get the slow path while the gates got the fast one — with +nothing to indicate the difference. Auto-discovery is what makes this a default +rather than an opt-in, and it is why the file must be treated as applying to +everything, including the shapes that must be excluded. + +**Why the flags are restated.** Documented under _Decision_. This is the single +most surprising property of the mechanism: the composition that makes the gates +work is what would silently disable the standard if the flags were not +repeated. Keeping the two sources equal is therefore a testable contract rather +than a convention, and the guide asks explicitly that they not be consolidated. + +**Why Cranelift is excluded, stated narrowly.** A Cranelift-compiled panic does +not find the unwind handler it should. Measured on `nightly-2026-08-23` +(`librustc_codegen_cranelift-1.100.0-nightly.so`) in a crate with no +dependencies, with `[profile.dev] codegen-backend = "cranelift"` and the +standard's flags: + +| Case | Cranelift | LLVM control | +| ------------------------- | -------------------------- | ------------ | +| `#[should_panic]` | passes | passes | +| `catch_unwind` | does not catch; test fails | passes | +| Panic on a spawned thread | aborts the process | passes | + +_Table 1: Panic behaviour under the Cranelift backend and under LLVM._ + +The `#[should_panic]` row is why the refusal has to be justified on this +specific evidence rather than on a general claim that "panics do not unwind". +That case passes, because libtest's outermost handler is the panic's own +handler and nothing between them has to work. `catch_unwind` sits in between +and the unwinder walks past it; a spawned thread has no handler above it, so +the unwinder reaches the end of the stack and the process leaves on SIGABRT with +`failed to initiate panic, error 5` — error 5 being exactly that. + +What was ruled out, each by its own run: not the linker (it aborts with the +platform linker too), not the parallel front-end (LLVM with `-Zthreads=8` +passes), not a compiler-cache wrapper (it aborts with `RUSTC_WRAPPER` unset), +and not a missing flag (`-Cforce-unwind-tables=yes` and an explicit +`-Cpanic=unwind` both still abort). Scoping the backend to `[profile.test]` +alone does not rescue it either: a clean `cargo test` under that setting builds +every crate on LLVM, dependencies included, so the only beneficiary would be +`make build` — producing an artefact that aborts on a panic, differing from the +binary the tests exercise. + +**Why the thread count is a variable.** No tuning sweep was done, and none is +claimed: `8` is the value the standard carries, chosen to exceed the core count +of the machines contributors use so that the front end keeps work in flight +through I/O waits. It is deliberately reached through one variable in two +places, `STANDARD_THREADS_FLAG` and the configuration file, held equal by +contract, so that raising it is a one-line change; a higher or lower value +should be justified by the benchmark rather than by this record. + +## Consequences + +- A bare `cargo build` in a checkout is accelerated, with no opt-in and no + slower alternative to select. There is no `install-dev-fast` or + `dev-fast-check` target any more: `install-build-tools` and + `check-build-tools` name what they do, and the build targets are the build + targets. +- A Linux contributor who has not run `make install-build-tools` gets a + _failure_ at link time, not a silent fallback: gcc is passed `-fuse-ld=mold` + explicitly, and with no `mold` reachable it reports that it cannot find the + linker and stops. The gate targets report this earlier and more clearly, as a + missing prerequisite. +- A `cargo install --path .` inside the checkout inherits the standard, because + it reads the same configuration file. The users' guide and README therefore + document the prerequisite for source installs and name the platform + difference; a registry install builds from packaged source, where neither the + configuration file nor `rust-toolchain.toml` applies, and needs neither. +- The release and coverage exclusions are asserted by contract, not assumed. + Adding anything new to `.cargo/config.toml` reaches those shapes too, so a + setting that is only safe for the development loop belongs in the Makefile's + composed `RUSTFLAGS` instead. +- The standard is currently justified by what it does rather than by a recorded + figure. The one measurement attempt, on 2026-09-17, ran on a shared host + whose load average went from 0.7 to 117 and reversed its verdict twice. No + table is recorded here until a run on an otherwise-idle host produces samples + that agree; the developers' guide states the conditions such a run needs. +- Re-testing Cranelift on a toolchain bump is a deliberate act, and the + developers' guide carries the probe. `CARGO_PROFILE_DEV_CODEGEN_BACKEND` + remains available for a single scoped experiment. + +## Alternatives considered + +- **Export the flags from the Makefile's shell environment instead.** Rejected + because it accelerates only Make-driven builds, leaves `cargo build` and + rust-analyzer slow, and is harder to make a contract read than a committed + file. +- **Keep two targets, one standard and one accelerated.** Rejected as the shape + that let the acceleration go unused: an "opt-in fast target" is a target + nobody runs, and it doubles the CI matrix to prove something the default + could prove on its own. +- **Adopt the Cranelift backend.** Rejected on the evidence above. The + performance case is real and the correctness case is disqualifying: a debug + binary that aborts at 134 where it should exit 101 is a different program + from the one the tests exercise. +- **Scope Cranelift to `make build` only.** Rejected because a profile override + does not confine the backend to the artefacts the developer intends, and the + one artefact it does reach would then behave differently from every test. +- **Let `mold` fall back to the platform linker when absent.** Rejected as the + worst of both worlds: the build would succeed, so nothing would prompt the + contributor to install it, and the numbers they compare against a colleague's + would be measuring different links. +- **Pin `mold` through the toolchain file.** Rejected because `rustup` manages + compilers, not linkers; the version pin and its checksums live in + `tools/mold/` and are verified by the installer. + +## Implementation references + +- The standard itself: [`.cargo/config.toml`](../.cargo/config.toml) +- Composed flags and gate prerequisites: [`Makefile`](../Makefile) +- Capability check and installer: + [`scripts/check-build-tools.sh`](../scripts/check-build-tools.sh) and + [`scripts/install-build-tools.sh`](../scripts/install-build-tools.sh) +- Version and checksum pins: [`tools/mold/`](../tools/mold) +- Configuration contract: + [`tests/build_tools_cargo_config_tests.rs`](../tests/build_tools_cargo_config_tests.rs) +- Makefile and configuration held equal: + [`tests/build_tools_make_target_tests.rs`](../tests/build_tools_make_target_tests.rs) + and + [`tests/makefile_test_target/rustflags.rs`](../tests/makefile_test_target/rustflags.rs), + whose model is composed through real Make and shell expansion in + [`tests/makefile_test_target/rustflags_expansion.rs`](../tests/makefile_test_target/rustflags_expansion.rs) +- Exclusion contracts: + [`tests/workflow_contracts/build_standard_wiring_test.py`](../tests/workflow_contracts/build_standard_wiring_test.py) +- Benchmark: [`scripts/bench-build.sh`](../scripts/bench-build.sh) +- Contributor guidance: + [`developers-guide.md`](developers-guide.md#the-build-standard) + +## Related decisions + +- [ADR-006: Adopt the Polonius borrow checker on a pinned nightly toolchain][adr-006] +- [ADR-025: Local pull-request coverage ratcheting with persistent coverage data owned by `main`][adr-025] + +[adr-006]: adr-006-adopt-polonius-nightly-toolchain.md +[adr-025]: adr-025-main-owned-coverage-publication.md diff --git a/docs/contents.md b/docs/contents.md index 57c160805..515445a9b 100644 --- a/docs/contents.md +++ b/docs/contents.md @@ -173,6 +173,10 @@ operator, user, and contributor references are easier to find. - [ADR-028](adr-028-defer-split-build-dir-harness-trim.md): Deferred trim of the split-build-dir harness test, with the serialized-lane measurements that made the figure unstable and the ten-run gate that reopens the question. +- [ADR-029](adr-029-mold-and-parallel-frontend-as-build-defaults.md): + `mold` and the parallel `rustc` front end as the committed default build, + with the release and coverage exclusions, the Cranelift refusal, and the + toolchain-versus-tools boundary. - [ADR-032](adr-032-windows-reparse-point-same-handle-open.md): Windows final component validation through a same-handle reparse-point open. - [ADR-033](adr-033-record-split-build-cargo-messages.md): Recorded Cargo @@ -215,8 +219,8 @@ operator, user, and contributor references are easier to find. ## Contributor guidance - [developers-guide.md](developers-guide.md): Engineering workflow, quality - gates, Lading release configuration, local build acceleration, testing - strategy, and stdlib resolver-boundary conventions. + gates, Lading release configuration, the build standard, testing strategy, + and stdlib resolver-boundary conventions. - [polonius.md](polonius.md): Polonius migration audit, borrow-centric API evolution log, and principled refusals. - [documentation-style-guide.md](documentation-style-guide.md): Documentation diff --git a/docs/developers-guide.md b/docs/developers-guide.md index ae6a6cb2f..f9643a731 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -553,17 +553,26 @@ that restates it is a build that can silently drop it. A contract test every checkout consumer — plain Cargo invocations, rust-analyzer, Clippy, and Whitaker — without any Cargo configuration. `cargo kani setup` is a separate boundary: Kani 0.67.0 installs and uses its bundled `nightly-2025-11-21` -toolchain rather than the checkout toolchain. The repository has no -`.cargo/config.toml`; carrying the flag was that file's only purpose, and it -was deleted when the pin moved past 2026-08-04. - -Makefile recipes still set `RUSTFLAGS`, but only to deny warnings. Each builds -the value as `RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }-D warnings"`; the -`$${RUSTFLAGS:+$$RUSTFLAGS }` expansion prepends any `RUSTFLAGS` already set by -the caller (for example a CI wrapper), so those flags survive rather than being -silently discarded. `make kani-full` and the binary-build recipe set no -`RUSTFLAGS` at all: Kani compiles third-party crates the workspace lint policy -does not govern, and a plain binary build is not a lint gate. +toolchain rather than the checkout toolchain. `.cargo/config.toml` carried the +Polonius flag until the pin moved past 2026-08-04; that file was deleted then +and has since returned for the build standard alone, so look to *The build +standard* above for what it holds now. + +Makefile recipes set `RUSTFLAGS` through a small set of named variables rather +than spelling a value out, and the variable a recipe composes states its policy. +`GATE_RUSTFLAGS` appends `-D warnings` and the standard's flags, and every +lint or test gate takes it. `DEBUG_RUSTFLAGS` takes the standard but leaves the +caller's warning policy alone, so `make build` is not a gate. `KANI_RUSTFLAGS` +denies warnings but takes none of the standard: Kani drives `rustc` through +`kani-compiler` on its own bundled toolchain, where neither flag applies. +`RELEASE_RUSTFLAGS` assigns an empty inherited value, which is what holds the +config file's `rustflags` tables off a shipped artefact; see *Exclusions* under +*The build standard*. Every one of them builds its value as +`RUSTFLAGS="$${RUSTFLAGS:+$$RUSTFLAGS }…"`, whose `$${RUSTFLAGS:+$$RUSTFLAGS }` +expansion prepends any `RUSTFLAGS` already set by the caller (for example a CI +wrapper), so those flags survive rather than being silently discarded. +`tests/makefile_test_target/rustflags.rs` holds all four to that composition +and to their individual policies. [ADR-006](adr-006-adopt-polonius-nightly-toolchain.md) records the policy decision, and the [polonius migration notes](polonius.md) track every site @@ -1078,8 +1087,8 @@ enforces all five callers. For each one it asserts: The same test carries the two toolchain-level assertions: that the pinned channel is a dated nightly at or after 2026-08-04, the first nightly on which Polonius is the default analysis, and that no build configuration — the -Makefile, a Cargo configuration fragment, a workflow, or a recreated -`.cargo/config.toml` — passes a `-Zpolonius` directive. +Makefile, the committed `.cargo/config.toml`, a workflow, or a helper script — +passes a `-Zpolonius` directive. Run it with: @@ -1655,9 +1664,10 @@ set -o pipefail make test 2>&1 | tee /tmp/netsuke-make-test.log ``` -These gates always use the repository toolchain and the default codegen -backend. For a faster inner loop between gate runs, see -[local build acceleration](#local-build-acceleration). +These gates run on the repository toolchain and on the codegen backend, linker, +and frontend the repository has chosen as its defaults; there is no separate +faster path to switch to. See [the build standard](#the-build-standard) for +what they apply and why release and coverage builds are held out of it. For documentation changes, also run `make fmt`, `make markdownlint`, and `make nixie`. @@ -2351,240 +2361,411 @@ contract. Its fixtures compile the production module paths selected by negative fixture imports `cli::discovery` and must fail with an unresolved module diagnostic. Update the fixtures whenever the build-script slice changes. -## Local build acceleration +## The build standard -Debug builds and tests can optionally use the [`mold`] linker and the Cranelift -`rustc` codegen backend to shorten the local edit-compile-test loop. This is a -developer convenience only. It is opt-in, it is never used for release -artefacts, and it changes nothing about what CI builds. +The [`mold`] linker and the parallel `rustc` frontend (`-Zthreads=8`) are the +**defaults** for development, test, lint, and typecheck builds. They are +committed to `.cargo/config.toml`, which Cargo auto-discovers, so a bare +`cargo build` in the repository gets them as well as every Make target. Release +and coverage builds are excluded, and the exclusions are enforced rather than +assumed; see *Exclusions* below. + +The Cranelift codegen backend is deliberately **not** part of the standard, and +a contract refuses one. See *Why Cranelift is not part of the standard* below +for the evidence. + +The decision, the exclusions it carries, and the evidence behind the Cranelift +refusal are recorded in +[ADR-029](adr-029-mold-and-parallel-frontend-as-build-defaults.md). This +section is the working reference; that record is why the standard takes this +shape. [`mold`]: https://github.com/rui314/mold The canonical commands are: ```bash -make install-dev-fast # install the pinned mold release and Cranelift backend -make dev-fast-check # verify the prerequisites are present -make dev-build # debug binary via Cranelift and mold -make dev-test # the nextest pass via Cranelift and mold +make install-build-tools # install the pinned mold release and the toolchain +make check-build-tools # verify the prerequisites are present +make build # debug binary on the standard +make test # the full gate on the standard ``` -`make dev-build` and `make dev-test` both depend on `make dev-fast-check`, so a -missing tool reports an installation hint before Cargo is invoked rather than -surfacing as an opaque codegen-backend or linker error. +`make build`, `make test-nextest`, `make doctest`, `make lint-clippy`, +`make lint-whitaker`, and `make typecheck` all depend on +`make check-build-tools`, so a missing tool reports an installation hint before +Cargo is invoked rather than surfacing as an opaque codegen-backend or linker +error. There is no separate accelerated target: `make build`, `make test`, +`make lint`, and `make typecheck` are the build targets, and every one of them +runs on the standard. -`DEV_FAST_CONFIG` defaults to `tools/dev-fast/config.toml` and may be -overridden for a local experiment, for example -`make DEV_FAST_CONFIG=tools/dev-fast/config.local.toml dev-build`. The Makefile -passes the selected path explicitly with Cargo's `--config` option; keep the -fragment out of `.cargo/config.toml` so it cannot affect ordinary, release, -coverage, or verification builds. +Every lane in continuous integration whose builds take the linker flag runs +`make install-build-tools` before its first build. The Windows jobs do not: +`mold` is Linux-only, so the `cfg` gate leaves the flag inert there and an +install step would provision nothing. A contract test asserts that command lane +by lane, and names the Windows omission as intended rather than missing. `CARGO_LOCKED` defaults to empty. Set `CARGO_LOCKED=--locked` to enable -repository lockfile verification for `dev-build` and `dev-test`, for example -`make CARGO_LOCKED=--locked dev-test`. +repository lockfile verification. + +### Why the flags are written down twice + +Cargo selects a single `rustflags` source rather than merging them. A matching +`[target.*]` table replaces `[build] rustflags` outright, and an externally set +`RUSTFLAGS` environment variable replaces both. Every gate recipe assigns +`RUSTFLAGS` to deny warnings, and CI's `setup-rust` action exports the same +value for a whole job, so a repository that named the flags only in +`.cargo/config.toml` would link with the platform linker and a single-threaded +frontend during exactly the builds it most wanted accelerated — and report +success while doing it. + +The flags are therefore restated in the Makefile, in `STANDARD_THREADS_FLAG` and +`STANDARD_MOLD_FLAG`, and composed into the `RUSTFLAGS` each gate builds. +`tests/build_tools_make_target_tests.rs` and +`tests/build_tools_cargo_config_tests.rs` hold the sources equal in both +directions: a flag the Makefile passes but the configuration omits fails one +test, and a flag named in `[build] rustflags` but missing from the Linux table +fails another. Do not consolidate them. + +Whether a build actually linked with `mold` is checkable after the fact, not +only inferable from the command line: + +```console +$ strings target/debug/netsuke | grep '^mold ' +mold 2.41.0 (7c4c0addcb833120bf41cc3db7b2652694e0d814; compatible with GNU ld) +``` + +The linker writes its own version into the artefact, so a build that silently +fell back to the platform linker carries no such line. That is worth checking +after any change to how `RUSTFLAGS` is composed, because the fallback is +otherwise completely silent. ### Toolchain contract Two pins fix the linker; the toolchain is not pinned separately. Change the pins together, never individually. -The scripts locate these files relative to their own path, so `make dev-*`, a -direct `scripts/dev-fast-check.sh`, and a run from any working directory all -resolve the same committed pins. Setting `MOLD_VERSION_FILE`, +The scripts locate these files relative to their own path, so the Make targets, +a direct `scripts/check-build-tools.sh`, and a run from any working directory +all resolve the same committed pins. Setting `MOLD_VERSION_FILE`, `MOLD_SHA256SUMS_FILE`, or `RUST_TOOLCHAIN_FILE` overrides the corresponding default; the tests use that to point the scripts at fixtures. Either way a -missing or empty file is reported as `dev-fast: missing version pin: ` +missing or empty file is reported as `build-tools: missing version pin: ` rather than silently becoming an empty version. -- `rust-toolchain.toml` supplies the toolchain. dev-fast deliberately shares - the repository's own dated nightly rather than pinning a second one, keeping - the accelerated loop and the gates on the same toolchain. The - `make install-dev-fast` target adds `rustc-codegen-cranelift-preview` to that - toolchain. +- `rust-toolchain.toml` supplies the toolchain. The build standard deliberately + shares the repository's own dated nightly rather than pinning a second one, + keeping the accelerated loop and the gates on the same toolchain. + `make install-build-tools` installs that toolchain and adds no component to + it: the standard needs none, because it names no codegen backend. - `tools/mold/VERSION` holds the `mold` release tag. - `tools/mold/SHA256SUMS` holds the SHA-256 checksum of each supported `mold` - release artefact. `make install-dev-fast` refuses to install an artefact that - is absent from this file or whose checksum does not match. - -`make install-dev-fast` unpacks `mold` under `~/.local` by default; override -the location with `DEV_FAST_PREFIX`. Every `dev-*` recipe prepends -`$(DEV_FAST_PREFIX)/bin` to `PATH`, so an overridden prefix is the one actually -selected — `-fuse-ld=mold` resolves by `PATH` order, and the Makefile otherwise -puts `~/.local/bin` first unconditionally. Invoking the scripts directly rather -than through `make` means arranging that `PATH` order manually. - -`make dev-fast-check` prints the resolved `mold` path alongside its version, so -an unexpected pick is visible. A version that differs from the pin fails the + release artefact. `make install-build-tools` refuses to install an artefact + that is absent from this file or whose checksum does not match. + +`make install-build-tools` unpacks `mold` under `~/.local` by default; override +the location with `BUILD_TOOLS_PREFIX`. The Makefile prepends +`$(BUILD_TOOLS_PREFIX)/bin` to `PATH`, so an overridden prefix is the one +actually selected — `-fuse-ld=mold` resolves by `PATH` order, and the Makefile +otherwise puts `~/.local/bin` first unconditionally. Invoking the scripts +directly rather than through `make` means arranging that `PATH` order manually. + +`make check-build-tools` prints the resolved `mold` path alongside its version, +so an unexpected pick is visible. A version that differs from the pin fails the check, as does a missing `mold` or one that cannot report its version; run -`make install-dev-fast` to install the pinned release ahead of any distribution -`mold` on `PATH`. An advisory pin is not a pin: tolerating drift would let the -linker actually in use stop matching what the repository claims. - -For screen readers: the following flowchart traces `make install-dev-fast` from -start to exit. It reads the pinned linker version, then branches on the host -platform. On Linux it selects the architecture, downloads the release tarball, -verifies its checksum, unpacks it into the install prefix, and reports the -`PATH` requirement; on other platforms it skips the linker entirely and falls -back to the platform default. Both branches then converge on the toolchain -half, which reads the pinned nightly, fails early if `rustup` is absent, and -otherwise installs the toolchain and the Cranelift backend component before -printing a readiness message. +`make install-build-tools` to install the pinned release ahead of any +distribution `mold` on `PATH`. An advisory pin is not a pin: tolerating drift +would let the linker actually in use stop matching what the repository claims. + +For screen readers: the following flowchart traces `make install-build-tools` +from start to exit. It reads the pinned linker version, then branches on the +host platform. On Linux it selects the architecture, downloads the release +tarball, verifies its checksum, unpacks it into the install prefix, and reports +the `PATH` requirement; on other platforms it skips the linker entirely and +falls back to the platform default. Both branches then converge on the +toolchain half, which reads the pinned nightly, fails early if `rustup` is +absent, and otherwise installs the toolchain before printing a readiness +message. ```mermaid flowchart TD - A["Start install-dev-fast.sh"] --> B["Source dev-fast-common.sh"] + A["Start install-build-tools.sh"] --> B["Source build-tools-common.sh"] B --> C["mold_version"] C --> D{"is_linux"} D -- No --> E["Skip linker installation
Fall back to platform linker"] D -- Yes --> F["mold_arch"] F --> G["Download tarball from MOLD_RELEASE_BASE_URL"] G --> H["verify_mold_archive"] - H --> I["tar extract into DEV_FAST_PREFIX"] - I --> J["Report DEV_FAST_PREFIX/bin PATH requirement"] + H --> I["tar extract into BUILD_TOOLS_PREFIX"] + I --> J["Report BUILD_TOOLS_PREFIX/bin PATH requirement"] - E --> K["cranelift_toolchain"] + E --> K["install_toolchain"] J --> K K --> L{"rustup on PATH?"} L -- No --> M["fail: install rustup"] L -- Yes --> N["rustup toolchain install pinned nightly --profile minimal"] - N --> O["rustup component add rustc-codegen-cranelift-preview"] - O --> P["Print ready; verify with make dev-fast-check"] + N --> P["Print ready; verify with make check-build-tools"] M --> Q["Exit"] P --> Q ``` -**Figure**: `make install-dev-fast` control flow. The `is_linux` branch is what -keeps macOS and Windows on the platform linker while still installing -Cranelift, and `verify_mold_archive` is the point at which an artefact absent +**Figure**: `make install-build-tools` control flow. The `is_linux` branch is +what keeps macOS and Windows on the platform linker while still installing the +toolchain, and `verify_mold_archive` is the point at which an artefact absent from `tools/mold/SHA256SUMS`, or one whose checksum does not match, aborts the installation. The final node only reports the `PATH` requirement for direct -script invocation; the `dev-*` recipes prepend `$(DEV_FAST_PREFIX)/bin` -themselves. +script invocation; the Makefile prepends `$(BUILD_TOOLS_PREFIX)/bin` itself. ### Ownership boundary -The accelerated configuration lives in `tools/dev-fast/config.toml`, which is -deliberately *not* `.cargo/config.toml`. Cargo auto-discovers the latter, so -placing Cranelift and the Linux-only `mold` linker there would silently apply -them to every build in the repository, including release, packaging, coverage, -and formal-verification builds. The fragment is instead passed explicitly with -`cargo --config tools/dev-fast/config.toml` from the `make dev-*` targets, and -must not be sourced from any target that CI invokes. - -No repository-root `.cargo/config.toml` exists any more. It once carried the -Polonius flag, and was deleted when the pinned nightly began enabling the -analysis by default. The rule is about what would belong in that file if it -returned, not about whether it may exist: settings needed everywhere may go -there; settings that are only safe for the accelerated dev loop must not. - -The fragment sets the `codegen-backend` unstable flag, -`codegen-backend = "cranelift"` on the `dev` profile, and a -`cfg(target_os = "linux")`-gated rustflags list carrying -`-Clink-arg=-fuse-ld=mold`. +The configuration lives in `.cargo/config.toml`, the file Cargo auto-discovers. +That placement is the mechanism: the standard applies to every build in the +repository whether or not it went through a Make target, which is what makes it +a default rather than an opt-in. + +The file carries two settings and nothing else: + +- `-Zthreads=8` in `[build] rustflags`; +- the same flag repeated in a `cfg(target_os = "linux")` table, beside + `-Clink-arg=-fuse-ld=mold`. The repetition is required, not redundant: see + *Why the flags are written down twice* above. + +It names no codegen backend, for any profile, and a contract refuses one. That +is a refusal rather than an omission; see *Why Cranelift is not part of the +standard* below. + +Adding anything else to that file applies it to release and coverage builds +too. A setting that is only safe for the development loop does not belong +there; put it in the Makefile's composed `RUSTFLAGS` instead, where a target +chooses whether to take it. + +The file once carried the Polonius flag and was deleted when the pinned nightly +began enabling the analysis by default. It must not carry one now: the analysis +still comes from the pin, and `tests/polonius_toolchain_contract.rs` reads this +file among others to keep the directive from returning. + +### Exclusions + +Two build shapes are excluded from the standard, and each exclusion works by a +different mechanism. + +**Release and packaging.** A `[target.*]` table applies to every profile, so +`make release` assigns `RUSTFLAGS` — to the caller's inherited value, or to +nothing — because *assigning it at all* is what displaces the configuration's +tables. A recipe that left the variable unset would ship an artefact built with +the parallel frontend and `mold`. On CI the release lanes are already covered, +because `setup-rust` exports `RUSTFLAGS` for the whole job. + +**Coverage.** A build whose output is a measurement is a reproducibility claim, +so it takes neither the parallel frontend nor the linker change. Both coverage +steps assign `RUSTFLAGS` at the step itself rather than inheriting it from the +toolchain action, so the exclusion is visible where it applies and a contract +has something to read. `tests/workflow_contracts/build_standard_wiring_test.py` +asserts the assignment and, separately, that no excluded flag appears; the two +fail to different edits. + +### Why Cranelift is not part of the standard + +The Cranelift codegen backend is the obvious third member of this set, and it +is deliberately absent. A Cranelift-compiled panic does not find the unwind +handler it should. The wording matters, because a probe that only checks +whether a panic unwinds at all reads as a pass: what fails is every handler +other than the outermost one. + +Measured on 2026-09-18 on `nightly-2026-08-23`, whose Cranelift is +`librustc_codegen_cranelift-1.100.0-nightly.so`, in a crate with no +dependencies at all, with `[profile.dev] codegen-backend = "cranelift"` and the +standard's `-Zthreads=8` and `mold` flags: + +```rust +/// A panic raised on the main thread, caught by `catch_unwind`. +#[test] +fn main_thread_catch_unwind() { + let caught = std::panic::catch_unwind(|| panic!("boom")); + assert!(caught.is_err(), "catch_unwind should report the panic"); +} + +/// A panic raised on a thread this test spawned. +#[test] +fn spawned_thread_panic_unwinds() { + let handle = std::thread::spawn(|| panic!("boom")); + assert!(handle.join().is_err()); +} + +/// A panic raised on the test's own thread, caught by libtest. +#[test] +#[should_panic(expected = "boom")] +fn should_panic_attribute() { + panic!("boom"); +} +``` + +```sh +cargo test --lib -- --nocapture --test-threads=1 +``` + +Table: panic behaviour under the Cranelift backend and under LLVM. + +| Case | Cranelift | LLVM control | +| ------------------------- | -------------------------- | ------------ | +| `#[should_panic]` | passes | passes | +| `catch_unwind` | does not catch; test fails | passes | +| Panic on a spawned thread | aborts the process | passes | + +The control is the same crate and the same flags with the backend key removed; +it passes all three, so the backend is the cause and neither the linker nor the +parallel frontend is. + +The `#[should_panic]` row is why the reason has to be stated this narrowly. It +passes because libtest's own outermost handler catches the panic, and nothing +between the panic and that handler has to work for it to do so. `catch_unwind` +sits between, and the unwinder walks straight past it. A spawned thread has no +handler above it at all, so the unwinder reaches the end of the stack: + +```text +fatal runtime error: failed to initiate panic, error 5, aborting +``` + +Error 5 is exactly that, the end of the stack with no handler found, and the +process leaves on SIGABRT. The reach is therefore every `catch_unwind` in the +tree, every test that asserts a spawned thread panicked, and a debug binary +that would abort with 134 where it now exits 101 — but not, on this nightly, a +bare `#[should_panic]`. + +What was ruled out, each by its own run: it is not the linker, because it +aborts with the platform linker too; not the parallel frontend, because LLVM +with `-Zthreads=8` passes; not a compiler-cache wrapper, because it aborts with +`RUSTC_WRAPPER` unset; and not a missing flag, because +`-Cforce-unwind-tables=yes` and an explicit `-Cpanic=unwind` both still abort. +It is not the pinned toolchain either: the same crate aborts on the newest +upstream nightly, where the LLVM control passes. + +Scoping it to a profile does not rescue it. +`[profile.test] codegen-backend = "llvm"` makes the suite pass, but reading the +compiler invocations of a clean `cargo test` under that setting shows every +crate built on LLVM, dependencies included. `cargo check` and Clippy generate +no code, so `make typecheck` and `make lint` would gain nothing either. That +leaves `make build` as the only beneficiary — the one artefact that would then +abort on a panic and so behave differently from the binary the tests exercise. + +`tests/build_tools_cargo_config_tests.rs` therefore refuses a `codegen-backend` +key under any profile. Re-test on a toolchain bump with the crate above before +relaxing it; the environment override `CARGO_PROFILE_DEV_CODEGEN_BACKEND` +remains available for a single scoped experiment. ### Composition rules -- **Quality gates.** `make check-fmt`, `make lint`, `make lint-clippy`, - `make test`, and `make typecheck` are unchanged and remain on the - repository's pinned nightly toolchain from `rust-toolchain.toml` with the - default LLVM backend. The `dev-*` targets are not part of `make test`, - `make lint`, `make check-fmt`, or `make all`, mirroring the Kani boundary - described below. Run the ordinary gates before proposing a change; - `make dev-test` is a faster inner-loop proxy, not a substitute. +- **Quality gates.** `make lint`, `make lint-clippy`, `make lint-whitaker`, + `make test`, and `make typecheck` run on the standard, on the repository's + pinned nightly from `rust-toolchain.toml`. They are gated on + `make check-build-tools`, so they stop with an installation hint rather than + a codegen-backend error. `make check-fmt` compiles nothing and is unaffected. - **`RUSTFLAGS`.** `make test-nextest`, `make doctest`, `make typecheck`, and - the rustdoc stage of `make lint` append `-D warnings` to any flags inherited - from the caller. An externally set `RUSTFLAGS` overrides the `[target.*]` - `rustflags` in a Cargo configuration file, so the `dev-*` targets - deliberately do not set it. Exporting `RUSTFLAGS` in the shell silently - disables `mold` for these targets. + the rustdoc and Clippy stages of `make lint` append `-D warnings` *and* the + standard's flags to whatever the caller set. An externally set `RUSTFLAGS` + overrides every `rustflags` table in a Cargo configuration file, which is + precisely why the Makefile restates the flags rather than relying on the + file. Exporting `RUSTFLAGS` in the shell no longer silences the linker for + these targets, because they compose rather than replace; it does still + silence it for a bare `cargo build`. - **`RUSTDOC_FLAGS`.** Make defaults this caller-overridable variable to `--cfg docsrs -D warnings` and exports it as Cargo's supported `RUSTDOCFLAGS` environment variable for `make doctest`, the rustdoc stage of `make lint-clippy`, and `make doc-coverage`. The unsupported `RUSTDOC_FLAGS` name is not exported, so Cargo cannot warn about it. Caller overrides retain their literal contents, including quotes in Rust `--cfg` values. -- **Release and packaging.** `make release` and everything under - `.github/workflows/build-and-package.yml` use the release profile, the LLVM - backend, and the platform linker. Cranelift is applied to the `dev` profile - only, so it cannot reach a shipped artefact even if the fragment were loaded. - `make build` produces a debug binary, but through the default backend and - linker; `make dev-build` is the accelerated counterpart. -- **Coverage.** Coverage is generated through LLVM source-based instrumentation - in `.github/workflows/ci.yml` and `coverage-main.yml`. Cranelift does not - emit that instrumentation. Never combine the `dev-fast` fragment with a - coverage run. +- **Release, packaging, and coverage.** See *Exclusions* above. These are the + two shapes the standard must not reach, and each is held out by a different + mechanism. - **Formal verification.** Kani manages its own supporting nightly toolchain - during `cargo kani setup`. That nightly is unrelated to the repository's - Polonius nightly and must not be conflated with it; verification must run on - Kani's own toolchain and the LLVM backend. The same applies to Verus. -- **Test runner.** `make dev-test` is the accelerated counterpart of - `make test-nextest`, not of `make test`: it runs the same - `cargo nextest run --workspace --all-targets --all-features`, and so is - governed by the same [`.config/nextest.toml`](#nextest-configuration). It - omits the `doctest` pass, because `cargo test --doc` is a separate and - comparatively quick runner; run `make test` before proposing a change. The - acceleration is applied through `RUSTUP_TOOLCHAIN` and `cargo --config`, both - Cargo-level rather than runner-level, which is why they compose with nextest - unchanged. Note the target uses `NEXTEST_BUILD_JOBS`, not `BUILD_JOBS`: - nextest reserves `-j` for test concurrency, so a Cargo-shaped `-j` would - silently become a thread count. It forwards `NEXTEST_TEST_JOBS` as well, so - both worker bounds mean the same thing under `make dev-test` as under - `make test-nextest`; a bound honoured by one and dropped by the other would - make a local run diverge from the gate for no stated reason. - `tests/makefile_test_target.rs` holds the two targets to that agreement. + during `cargo kani setup`, and drives `rustc` through `kani-compiler`. + Reading the compiler invocations a `cargo kani` run produces shows + `-Zthreads` never reaching `kani-compiler`, so the harnesses need no override + and none is added; CI's `kani-smoke` job runs `make kani-ir` on every pull + request, which is where that continues to be checked. Verus drives its own + toolchain the same way. If a proof tool ever does inherit a flag it cannot + take, the remedy is an override scoped to that one target with the reason + recorded, not a change to the shared configuration. +- **Dylint and Whitaker.** `make lint-whitaker` execs `cargo dylint`, which + re-invokes Cargo under Whitaker's own pinned nightly with a driver as + `RUSTC_WORKSPACE_WRAPPER`. That older Cargo reads this configuration without + complaint, and dylint drives `cargo check`, so nothing here reaches code + generation. It would only need revisiting if dylint moved to a + codegen-producing command. +- **Test runner.** The standard is applied at the Cargo level, through + `RUSTFLAGS` and the profile, rather than at the runner level, which is why it + composes with nextest unchanged; `make test-nextest` is governed by the same + [`.config/nextest.toml`](#nextest-configuration) as before. Note the target + uses `NEXTEST_BUILD_JOBS`, not `BUILD_JOBS`: nextest reserves `-j` for test + concurrency, so a Cargo-shaped `-j` would silently become a thread count. - **rust-analyzer.** No rust-analyzer configuration is committed, so the - language server uses the repository toolchain and the default backend. Opting - rust-analyzer into Cranelift is a personal, machine-local choice; it needs a - separate target directory to avoid thrashing the cache shared with - `make test`. -- **Polonius.** The analysis comes from the pinned nightly (ADR-006), and the - `dev-*` targets use that same toolchain, so the fragment needs no - Polonius-specific cooperation and must not add a `-Zpolonius` directive. - Cargo does still pick a single rustflags source rather than merging them, so - anything the fragment's `[target.*]` table must carry has to be named there - in full. + language server picks up `.cargo/config.toml` like any other Cargo caller and + takes the standard. Give it a separate target directory to avoid thrashing + the cache shared with `make test`. +- **Polonius.** The analysis comes from the pinned nightly (ADR-006), so the + configuration needs no Polonius-specific cooperation and must not add a + `-Zpolonius` directive; `tests/polonius_toolchain_contract.rs` reads + `.cargo/config.toml` to keep it out. ### Fallback behaviour - **Non-Linux hosts.** `mold` ships for Linux only, so on macOS and Windows - `make install-dev-fast` skips the linker installation, the - `cfg(target_os = "linux")` gate keeps the link argument inert, and - `make dev-fast-check` prints the fallback to the platform linker explicitly. - Cranelift still applies. -- **Unsupported architecture.** `make install-dev-fast` fails with a clear + `make install-build-tools` skips the linker installation, the + `cfg(target_os = "linux")` gate keeps the link argument inert, the Makefile + omits it from the composed `RUSTFLAGS`, and `make check-build-tools` prints + the fallback to the platform linker explicitly. The parallel frontend still + applies on every platform, being a compiler flag rather than a tool that has + to be installed. +- **Unsupported architecture.** `make install-build-tools` fails with a clear message rather than guessing when `uname -m` is not one of the architectures recorded in `tools/mold/SHA256SUMS`. -- **Missing tools.** `make dev-fast-check` names the absent component — `mold`, - `rustup`, the pinned toolchain, or the Cranelift backend — and points at - `make install-dev-fast`. It exits non-zero, so `make dev-build` and - `make dev-test` stop before Cargo runs. +- **Missing tools.** `make check-build-tools` names the absent component — + `mold`, `rustup`, or the pinned toolchain — and points at + `make install-build-tools`. It exits non-zero, so `make build`, `make test`, + `make lint`, and `make typecheck` stop before Cargo runs. ### Testing the tooling -Six suites cover the tooling's observable behaviour. All are hermetic — no +Eight suites cover the tooling's observable behaviour. All are hermetic — no network, and no real `mold`, `rustup`, or Cargo — so they run as part of `make test` on any Linux host. -- `tests/dev_fast_check_tests.rs`: the capability gate. Which diagnostic each - failure mode emits, exit status, pin resolution, and refusal of a malformed - pin. -- `tests/dev_fast_install_tests.rs`: the installer's happy path and its - refusals, plus the benchmark script's Markdown output. -- `tests/dev_fast_checksum_tests.rs`: property coverage for checksum +- `tests/build_tools_check_tests.rs`: the capability gate. Which diagnostic each + failure mode emits, exit status, and the non-Linux path, where the linker is + skipped and the toolchain half still runs `rustup toolchain install`. +- `tests/build_tools_pin_tests.rs`: pin resolution. Which file each pin is read + from, that boundary whitespace is trimmed, that an explicit override wins, + that the committed pins are the fallback rather than an empty string, and + that a malformed pin is refused rather than rewritten. +- `tests/build_tools_install_tests.rs`: the installer's happy path and its + refusals, `make install-build-tools` forwarding, and the benchmark script's + Markdown output. +- `tests/build_tools_checksum_tests.rs`: property coverage for checksum verification against a model. -- `tests/dev_fast_make_target_tests.rs`: the Make recipes. Toolchain and - fragment selection; that a failed gate reaches zero Cargo invocations - (`dev-build` and `dev-test` stop before Cargo runs); the fragment's contents; - and `install-dev-fast` forwarding. -- `tests/dev_fast_bench_tests.rs`: `make bench-build`. Per-variant target - directories, the clean/incremental cycle, and both variant rows. -- `tests/dev_fast_bench_lock_tests.rs`: the benchmark's exclusion lock. That a - held lock rejects a second run before it mutates anything, that the lock is +- `tests/build_tools_make_target_tests.rs`: the Make recipes. That each gate + composes the standard's flags *and* the warning policy into `RUSTFLAGS`, that + the debug build takes the standard without the warning policy, that the + release build assigns `RUSTFLAGS` and carries neither flag, and that a failed + capability check reaches zero Cargo invocations. +- `tests/build_tools_cargo_config_tests.rs`: the committed `.cargo/config.toml`. + That both `rustflags` sources repeat the shared flags, that no profile names + a codegen backend, that no `rustflags` source carries one either — in the + array form or the space-separated string form Cargo reads identically — and + that Cargo itself resolves the keys — `cargo config get` reports Cargo's own + view, so a key nested under the wrong table shows up as a missing value + rather than parsing cleanly and being ignored. +- `tests/build_tools_bench_tests.rs`: `make bench-build`. Per-variant target + directories, the clean/incremental cycle, all three variant rows, and that + every pass clears both compiler wrappers so a measurement cannot be a cache + read. Its `order_cases` module holds the cases about the order the variants + ran in: that the draw is shuffled rather than walked straight through, that + one seed replays one order, and that a repeat count below two is refused. +- `tests/build_tools_bench_lock_tests.rs`: the benchmark's exclusion lock. That + a held lock rejects a second run before it mutates anything, that the lock is released however a run ends, and that a later run can take it after an aborted one. -The fixtures live in `test_support::dev_fast`: +The fixtures live in `test_support::build_tools`: - `Sandbox` builds `PATH` from nothing — an explicit allowlist of ordinary utilities symlinked into a temporary directory, plus whichever fakes a case @@ -2606,14 +2787,17 @@ The fixtures live in `test_support::dev_fast`: checksum verification, and strip depth. Each release owns its version, so no caller threads a version string around. - `RecordingCargo` is a fake `cargo` that logs the arguments, - `RUSTUP_TOOLCHAIN`, and `PATH` of every invocation, turning a recipe's - command line into a checkable fact. It also records the target directory and - whether that directory already existed, which makes a benchmark's - clean-then-incremental cycle observable: the clean pass sees - `TargetState::Absent` because the harness wiped the directory, and the - incremental pass that follows sees `Present`. Seed a stale target directory - before asserting on that, or the wipe is indistinguishable from doing - nothing. It records the benchmark touch file's timestamp too, compared + `RUSTUP_TOOLCHAIN`, `PATH`, and `RUSTFLAGS` of every invocation, turning a + recipe's command line and environment into checkable facts. It records whether + `RUSTFLAGS` was assigned separately from its value, because unset and empty + are different facts: an empty assignment still displaces the configuration + file's tables, which is exactly what the release exclusion relies on. It also + records the target directory and whether that directory already existed, + which makes a benchmark's clean-then-incremental cycle observable: the clean + pass sees `TargetState::Absent` because the harness wiped the directory, and + the incremental pass that follows sees `Present`. Seed a stale target + directory before asserting on that, or the wipe is indistinguishable from + doing nothing. It records the benchmark touch file's timestamp too, compared against a backdated baseline rather than between passes so the assertion does not depend on filesystem timestamp granularity. - `PinOverrides` selects whether a script run supplies the pin-file variables. @@ -2622,21 +2806,21 @@ The fixtures live in `test_support::dev_fast`: entries are kept apart deliberately: a command-line variable outranks a `?=` default, whereas an environment entry is the only channel for a setting a script reads without the Makefile naming it. -- `test_support::dev_fast::scenario` builds on the fixtures above to assemble - two starting points. `BuildScenario` is a sandbox where `make dev-fast-check` - passes — pinned `mold` on the install prefix, a `rustup` reporting the - Cranelift component, and a `RecordingCargo` installed — and is shared by the - Make-target and benchmark suites. `BuildScenario::run(target)` returns the - single Cargo invocation a target must produce. The scenario is shared by both - suites so each can inspect that invocation without relying on process-global - state. `InstallerScenario` is a sandbox with a published `FakeRelease` and a - usable `rustup`, letting a test concentrate on the linker half of the - installer; the installer and checksum suites share it. The module also exports - `TEST_MOLD_VERSION`, deliberately not a real `mold` version so a test that - accidentally reaches the network fails rather than silently succeeding - against an upstream artefact, and `WRONG_SHA256`. `InstallerFixture` groups - the installer's pin path, checksum path, and release URL, and renders them via - `script_env()`. +- `test_support::build_tools::scenario` builds on the fixtures above to assemble + two starting points. `BuildScenario` is a sandbox where + `make check-build-tools` passes — pinned `mold` on the install prefix, a + `rustup` reporting the pinned toolchain, and a `RecordingCargo` installed — + and is shared by the Make-target and benchmark suites. + `BuildScenario::run(target)` returns the single Cargo invocation a target + must produce. The scenario is shared by both suites so each can inspect that + invocation without relying on process-global state. `InstallerScenario` is a + sandbox with a published `FakeRelease` and a usable `rustup`, letting a test + concentrate on the linker half of the installer; the installer and checksum + suites share it. The module also exports `TEST_MOLD_VERSION`, deliberately + not a real `mold` version so a test that accidentally reaches the network + fails rather than silently succeeding against an upstream artefact, and + `WRONG_SHA256`. `InstallerFixture` groups the installer's pin path, checksum + path, and release URL, and renders them via `script_env()`. A scenario earns its place here once a second suite needs it, and not before; suite-specific conveniences stay with their suite — the installer tests keep @@ -2675,17 +2859,48 @@ the corpus small and the strategy structural. `test_support` is a workspace member, so `make test` (whose nextest command uses `--workspace`), rustdoc, Clippy, and Whitaker visit its unit tests and library code. Keep fixture tests beside the fixture when they exercise a local -invariant; use the `tests/dev_fast_*.rs` integration crates when the assertion -spans the application-facing sandbox or Makefile contract. +invariant; use the `tests/build_tools_*.rs` integration crates when the +assertion spans the application-facing sandbox or Makefile contract. ### Benchmark evidence -`make bench-build` measures both paths with one repeatable command. It builds -the `netsuke` binary from an empty target directory, touches `src/main.rs`, and -rebuilds. Each variant uses its own target directory under `target/bench/`, so -neither warms the other's cache nor disturbs the working `target/` tree. The -timer reads `EPOCHREALTIME`, so this target needs Bash 5.0 or newer; it fails -with a named prerequisite on older shells rather than reporting zeroes. +`make bench-build` measures three build shapes with one repeatable command: the +platform-linker baseline, the repository's `mold` default, and that default +with the parallel frontend added. The linker and the frontend get a row each +because they pay off at different points in a build, and one row for both would +hide which of them is earning its keep. Each variant builds the `netsuke` +binary from an empty target directory, touches `src/main.rs`, and rebuilds. +Each uses its own target directory under `target/bench/`, so none warms +another's cache nor disturbs the working `target/` tree. The timer reads +`EPOCHREALTIME`, so this target needs Bash 5.0 or newer; it fails with a named +prerequisite on older shells rather than reporting zeroes. + +`mold` is Linux-only, so the benchmark drops its row elsewhere. On a non-Linux +host the threaded row keeps the parallel frontend and loses the linker flag, +and its caption reads "Platform linker, parallel frontend" so the table never +names a linker change it did not make. The capability check tolerates a +non-Linux host rather than aborting, which is what makes this path reachable. + +Every measured build runs with `RUSTC_WRAPPER` and `RUSTC_WORKSPACE_WRAPPER` +assigned empty. This is not tidiness. A developer shell commonly exports a +compiler wrapper chaining to `sccache`, and with one in force a variant's first +clean pass fills the cache while every later pass reads it back, so the table +times cache retrieval under variant labels and the row order decides the +winner. The flags are part of the cache key, so the variants warm each other +unevenly and nothing in the output reveals it. Both variables are named because +Cargo honours them independently, and both are *assigned* rather than unset, +because only an assignment displaces an exported value. + +`CARGO_ENCODED_RUSTFLAGS` is *removed* instead — `env -u`, not an empty +assignment. Cargo consults it before `RUSTFLAGS` and uses the first source it +finds, so an inherited value would leave every variant compiling with the same +flags while the table still showed three rows, and an empty encoded list is +still a source that would displace every variant's own `RUSTFLAGS`. Only +removing the variable leaves `RUSTFLAGS` to decide the build. A developer with +that variable exported — `cargo nextest` sets it, as do some wrapper setups — +would otherwise get a table that compares nothing. Removing it is an extension +to `env` rather than POSIX, but it is present in both GNU coreutils and the BSD +`env` the benchmark can reach on macOS. `BENCH_ROOT` and `BENCH_TOUCH_FILE` default to the shared `target/bench` directory and the tracked `src/main.rs`, so two runs in one checkout would @@ -2699,12 +2914,53 @@ run ends, including on interrupt. To benchmark two things at once, override `BENCH_ROOT`, so distinct roots do not contend. If a killed run ever leaves the directory behind, remove it. -Results below were recorded on a 24-core x86_64 Linux host, with both variants -on the repository's then-pinned `nightly-2026-06-25` supplying Cranelift -0.132.0, and `mold` 2.41.0. Regenerate the table verbatim with -`make bench-build`. Absolute figures move with machine load, so the ratio -between the two rows is the durable signal, not the seconds; the run below is -representative of three consecutive runs that agreed to within 0.4 s. +The variants are measured in a shuffled order, redrawn for each of +`BENCH_REPEATS` samples (`2` by default). Separate target directories isolate +build artefacts and nothing else: page-cache warmth and other tenants on a +shared host are not isolated by any directory, and they are where the ordering +bias lives. Drawing a fresh order per sample spreads that bias across the +variants instead of pinning it to whichever ran first, and repeating turns it +into visible spread rather than one number. The script prints `order sample N:` +for each draw and `order measured:` for the run as a whole, because a shuffle +is not reconstructible after the fact — without the record, a table disagreeing +with an earlier one cannot be told apart from a run that measured the variants +in a different order, which is the exact confusion the shuffle exists to +remove. Paste that record with any table recorded here, so the next reader can +tell which it was. + +The draw is an input rather than ambient state. `BENCH_SEED` seeds it, the run +prints `order seed: N` before it measures anything, and passing that value back +replays the same order. An unseeded run draws a seed and prints it, so a table +somebody has already taken can still be replayed. `BENCH_REPEATS` below two is +refused rather than clamped: nought prints an empty table and exits nought, and +one prints a table indistinguishable in shape from a valid one while carrying +exactly the single-sample bias the repeats exist to spread. Neither failure is +visible in the output a reader pastes here, which is why the script names it +instead. + +No table is recorded here yet, and the reason is worth keeping. The figures +this section used to carry were taken before the wrapper defect above was +found, so they timed a mixture of compilation and cache retrieval. The attempt +to replace them on 2026-09-17 ran on a shared host whose load went from 0.7 to +117 during the round: across three runs the same variant's clean build ranged +from 37 s to 154 s, and reversing the row order reversed the verdict twice. A +number produced under those conditions is not a slower or faster reading of the +truth; it is a reading of the host. + +A run worth recording therefore needs all of: a host doing nothing else, the +load average quoted beside the table, and samples that agree. Regenerate with +`make bench-build` and paste the table verbatim. Until then, treat the standard +as justified by what it does rather than by a figure — `mold` and the parallel +frontend cost nothing at runtime and are trivially reversible — and measure a +representative workload before concluding the acceleration is or is not worth +the setup. + +Two limits bound whatever that run reports. The benchmark builds only +`--bin netsuke`, the smallest useful target, so it under-represents what +`make test` sees, where every test binary's link is also on the linker. And the +incremental row rebuilds a single crate and links once, which on any host is a +couple of seconds dominated by that link, so it discriminates far less between +variants than the clean row does. `make bench-glob-expansion` measures `glob_paths("**/*.txt", Some(base))` against its equivalent absolute, unbased pattern. Its deterministic fixture is @@ -2714,23 +2970,6 @@ optimized-away query. Use it when changing glob-base preparation, path rebasing, or separator formatting; compare the two cases on the same machine, not their absolute timings across hosts. -| Variant | Clean build (s) | Incremental build (s) | -| ------------------------------- | --------------- | --------------------- | -| Default (LLVM, platform linker) | 11.6 | 0.8 | -| dev-fast (Cranelift, `mold`) | 10.7 | 0.6 | - -Table: Debug build wall-clock time for the default and accelerated paths. - -Be realistic about the size of this: roughly 8% off a clean build and a quarter -off an incremental one, which on this host is a few hundred milliseconds. Two -things bound it. Both variants now share one nightly, so the comparison -isolates Cranelift and `mold` rather than also capturing a toolchain change — -earlier figures in this document did not, and overstated the gain. And the -benchmark builds only `--bin netsuke`, the smallest useful target, so it -under-represents what `make dev-test` sees, where Cranelift has every test -binary's codegen to save on. Measure the actual workload before concluding the -acceleration is or is not worth the setup. - ## Formal-verification tooling Kani is the repository-supported bounded model checker for local @@ -4144,19 +4383,19 @@ is not obvious from the name: string normalization. Keep this exception in `test_support::fs`; production code remains capability-scoped or uses its dedicated normalizer. - `copy(from, to) -> io::Result` forwards to `std::fs::copy`, returning - the number of bytes copied and propagating its failure. The `dev_fast` + the number of bytes copied and propagating its failure. The `build_tools` release fixtures use it to place a built archive under its versioned name. - `modified(path) -> io::Result` returns the file's modification time. It propagates both the metadata failure and the platform's failure to report a timestamp, so it is `io::Result` rather than an `Option`. The - `dev_fast` staging fixtures use it to assert a file was or was not rebuilt. + `build_tools` staging fixtures use it to assert a file was or was not rebuilt. - `write_with_mtime(path, contents, mtime) -> io::Result<()>` (Unix only) creates or truncates `path`, writes `contents`, and sets the modification time to `mtime`, propagating whichever step fails. The staging fixtures use it to backdate a file so a later build sees it as stale. `write_with_mtime` is the reason `test_support/dylint.toml` carries no -`dev_fast` exemption. Backdating a fixture needs one open file for both the +`build_tools` exemption. Backdating a fixture needs one open file for both the write and the timestamp, which reads like an irreducibly ambient operation that has to happen at the call site. Taking the timestamp as an argument keeps the handle inside this module instead: the caller never sees a `File`, so the @@ -4486,10 +4725,11 @@ the resulting configuration applies the override only when a child command is spawned; callers should configure this through `StdlibConfig` rather than constructing the internal value directly. -The `test_support::dev_fast` sandbox reuses `mockable::Env` only while locating -the host utilities it explicitly links into its hermetic `PATH`. +The `test_support::build_tools` sandbox reuses `mockable::Env` only while +locating the host utilities it explicitly links into its hermetic `PATH`. `real_utility_with_env` is the test seam for that lookup; it is not a general -executable-discovery API and must not be used outside dev-fast test scaffolding. +executable-discovery API and must not be used outside build-tools test +scaffolding. #### Annotating a sanctioned site diff --git a/docs/repository-layout.md b/docs/repository-layout.md index 0eb651bce..0e0a6ddcd 100644 --- a/docs/repository-layout.md +++ b/docs/repository-layout.md @@ -12,6 +12,7 @@ output and some leaf files so the long-lived structure remains visible. ```plaintext . +├── .cargo/ ├── .github/ │ ├── actions/ │ └── workflows/ @@ -44,7 +45,6 @@ output and some leaf files so the long-lived structure remains visible. │ ├── fixtures/ │ └── snapshots/ └── tools/ - ├── dev-fast/ ├── kani/ └── mold/ ``` @@ -63,6 +63,14 @@ output and some leaf files so the long-lived structure remains visible. overview, linked from the localization menu at the top of each README. They follow [the localization glossary](localization-glossary.md) and are exempt from the en-GB-oxendict spelling gate via `typos.local.toml`. +- `.cargo/`: Cargo configuration that Cargo auto-discovers. It holds the + repository's build standard: the `rustflags` every build takes — the parallel + `rustc` frontend, plus the `mold` linker under a Linux-only `cfg` table. It + names no codegen backend, and a contract refuses one. Because it reaches + release and coverage builds too, those two shapes are excluded by assigning + `RUSTFLAGS` at the point they run; a setting that is only safe for the + development loop does not belong here. See + [developers' guide](developers-guide.md). - `.github/actions/`: Reusable GitHub Actions used by workflow definitions. - `.github/workflows/`: Continuous Integration (CI), release, packaging, and repository automation workflows. @@ -123,9 +131,6 @@ output and some leaf files so the long-lived structure remains visible. - `tests/features/`: Cross-platform behavioural feature files. - `tests/features_unix/`: Unix-specific behavioural feature files. - `tests/snapshots/`: Checked-in integration-test snapshots. -- `tools/dev-fast/`: Non-auto-loaded Cargo configuration fragment for the - opt-in Cranelift and `mold` build path. Cargo never discovers this file on - its own; only the `make dev-*` targets pass it through `cargo --config`. - `tools/kani/`: Kani formal-verification harness configuration and related local tooling. - `tools/mold/`: Pinned `mold` linker release version and the SHA-256 checksums diff --git a/docs/users-guide.md b/docs/users-guide.md index ff0bea4f1..bace175f3 100644 --- a/docs/users-guide.md +++ b/docs/users-guide.md @@ -98,6 +98,33 @@ cd netsuke cargo install --path . ``` +That command runs inside the checkout, so it inherits the repository's build +standard: the parallel `rustc` frontend, and on Linux the `mold` linker. A +`cargo install --path .` therefore needs the same two prerequisites as +`make build`: + +- the pinned nightly from `rust-toolchain.toml`, which `rustup` provisions + automatically because the file is present; +- `mold`, on Linux only, reachable through `PATH` or gcc's own search + directories. `make install-build-tools` installs the pinned release; a + distribution `mold` also works for a local install, though the development + gates additionally check the version against `tools/mold/VERSION`. The + installer unpacks into `$(BUILD_TOOLS_PREFIX)/bin`, `~/.local/bin` by + default, and does not edit any shell profile; the make targets add that + directory to `PATH` for the recipes they run, so a shell the make targets do + not drive needs that directory added before installing. + +The linker is named explicitly rather than left to gcc's default, so a Linux +host without `mold` fails the link rather than quietly falling back. The +platform default linker is used instead on macOS and Windows — where the +configuration names no linker — or when `.cargo/config.toml` has been removed +before installing. A registry install — `cargo install netsuke-build` — builds +from packaged source, where neither `.cargo/config.toml` nor +`rust-toolchain.toml` is present, so no linker flag applies and `mold` is not +needed on any platform. The pinned nightly is still required, and with no +`rust-toolchain.toml` to supply it the command selects it explicitly: see the +crates.io commands earlier in this section. + ### Complete Windows setup The MSI does not add its installation directory to `PATH`. Add it to the diff --git a/docs/v0-1-0-migration-guide.md b/docs/v0-1-0-migration-guide.md index c7122e71e..674e4098b 100644 --- a/docs/v0-1-0-migration-guide.md +++ b/docs/v0-1-0-migration-guide.md @@ -31,6 +31,52 @@ rustup toolchain install nightly-2026-08-23 cargo +nightly-2026-08-23 install netsuke-build ``` +## Install the linker the build standard uses + +Builds that run inside a checkout now take a committed build standard, because +`.cargo/config.toml` is auto-discovered rather than opt-in. There is nothing to +enable for the compiler part: the parallel `rustc` front end applies on every +platform, and the pinned nightly above supplies the compiler it needs. + +Linux is the exception. That configuration also passes `-fuse-ld=mold`, and +`mold` is a separate program the compiler must be able to find. On Linux, +install it before the first build: + +```sh +make install-build-tools +export PATH="${BUILD_TOOLS_PREFIX:-$HOME/.local}/bin:$PATH" +``` + +The installer unpacks the pinned release into `$(BUILD_TOOLS_PREFIX)/bin`, +`~/.local/bin` by default, and does not edit any shell profile; the `PATH` +export above is therefore a manual step, and one to keep for any later shell. +The Make targets add that directory for the recipes they run, so a build driven +by `make` needs no export. A distribution `mold` on `PATH` also works for a +local install, though the development gates additionally check the version +against `tools/mold/VERSION`. + +Without it the build fails at link time rather than falling back quietly: the +linker is named explicitly, so gcc reports that it cannot find `mold` and +stops. macOS and Windows name no linker and need no extra prerequisite; so does +any platform once `.cargo/config.toml` is removed. + +Two build shapes are deliberately excluded and keep the platform linker: a +release or packaging build, whose output ships, and a coverage build, whose +output is a measurement. Both assign `RUSTFLAGS` at the point they run, which +displaces every table in the configuration file. + +The opt-in `install-dev-fast` and `dev-fast-check` targets are gone. The +acceleration is the default now, so there is nothing to opt into: +`install-build-tools` installs the pinned linker, and `check-build-tools` +verifies it is present and is the pinned version. Every gate target depends on +that check, so a missing prerequisite is reported as such rather than surfacing +later as a linker error. + +The [users' guide](users-guide.md#install-netsuke) covers the same ground in +more detail, and the +[developers' guide](developers-guide.md#the-build-standard) records the +standard's rationale and its exclusions. + ## Netsuke is a build tool, not a library Netsuke is intended to be used as a command-line build tool. The only surfaces diff --git a/scripts/bench-build.sh b/scripts/bench-build.sh index 269011f4d..310b2a5d8 100755 --- a/scripts/bench-build.sh +++ b/scripts/bench-build.sh @@ -1,21 +1,32 @@ #!/usr/bin/env bash -# Benchmark the default (LLVM + platform linker) debug build against the opt-in -# mold + Cranelift path. +# Benchmark the three debug build shapes: the platform-linker baseline, the +# repository's `mold` default, and that default with the parallel `rustc` +# frontend added. Separating the linker from the frontend is the point: they +# pay off at different points in the build, so one row for both would hide +# which of them is earning its keep. # # Each variant is measured twice: a clean build from an empty target directory, # and an incremental rebuild after touching the binary's entry point. Variants # use separate target directories so neither warms nor invalidates the other's # cache, and neither disturbs the working `target/` tree. Results are printed as # a Markdown table so the developers' guide can be regenerated verbatim. +# +# The variants are selected by environment override rather than by a Cargo +# configuration fragment. `.cargo/config.toml` is the committed default, so the +# baseline is expressed by displacing its `rustflags` entirely, and the two +# accelerated rows differ only by one flag. set -euo pipefail script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=scripts/dev-fast-common.sh -. "$script_dir/dev-fast-common.sh" +# shellcheck source=scripts/build-tools-common.sh +. "$script_dir/build-tools-common.sh" : "${CARGO:=cargo}" -: "${DEV_FAST_CONFIG:?DEV_FAST_CONFIG must be set}" +# Supplied by the Makefile from the same variables the gate targets compose, so +# a flag cannot be benchmarked in a shape the gates do not actually use. +: "${STANDARD_THREADS_FLAG:=-Zthreads=8}" +: "${STANDARD_MOLD_FLAG:=-Clink-arg=-fuse-ld=mold}" # The timer below reads EPOCHREALTIME, which Bash gained in 5.0. Fail here with # a named prerequisite rather than silently reporting every duration as zero. @@ -26,8 +37,39 @@ BENCH_ROOT=${BENCH_ROOT:-target/bench} BENCH_BIN=${BENCH_BIN:-netsuke} BENCH_TOUCH_FILE=${BENCH_TOUCH_FILE:-src/main.rs} BENCH_LOCK_DIR=${BENCH_LOCK_DIR:-$BENCH_ROOT.lock} +# How many times each variant is measured. One sample cannot separate a variant +# from the host, and shared state that no target directory isolates — page-cache +# warmth, other load — is exactly what the ordering bias lives in. +BENCH_REPEATS=${BENCH_REPEATS:-2} +# Refused rather than clamped. A run with nought repeats prints an empty table +# and exits nought, and a run with one prints a table that looks exactly like a +# valid one while carrying the single-sample bias this script exists to remove. +# Both are worse than a named failure, because neither is visible in the output +# a reader pastes into the guide. +# +# The empty string is not among the refusals: `:-` above has already replaced it +# with the default, as it does for every other `BENCH_` variable, so a pattern +# for it here could never match and would read as a guard that holds. +case $BENCH_REPEATS in + *[!0-9]*) fail "BENCH_REPEATS must be a whole number of at least 2; got '$BENCH_REPEATS'" ;; +esac +[ "$BENCH_REPEATS" -ge 2 ] || + fail "BENCH_REPEATS must be a whole number of at least 2; got '$BENCH_REPEATS'" -# Populated as "