diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8a6793..1899954 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -147,6 +147,9 @@ jobs: needs: [test, orchard, artwork, base] env: HARDWARE_TAG: ${{ needs.orchard.outputs.tag }} + # The root image keeps its owners: as root, directly on this runner. + ORCHARD_BUILD_NATIVE: '1' + ORCHARD_BUILD_ENV_ID: ubuntu-24.04-arm-root-image runs-on: ubuntu-24.04-arm permissions: contents: read @@ -211,8 +214,13 @@ jobs: # What orchard supplies, from its bundles as they are: nothing is fetched or built. python -m orchard.inputs --output out/orchard-inputs.json "${kernels[@]}" \ --m1n1 "$tools/quiet/m1n1.bin" --pongo "$tools/quiet/pongo.bin" --tools "$tools/boot" + # A published orchard build's files are linked from the release; a run's own are hosted with it. + source=(--orchard-release "$HARDWARE_TAG") + if [[ -n "${HARDWARE_ARTIFACT_DIR:-}" && -d "${HARDWARE_ARTIFACT_DIR:-}" ]]; then + source=(--orchard-inputs out/orchard-inputs.json) + fi python build.py --base out/base/base.tar.gz --base-receipt out/base/base.tar.gz.json \ - --inputs out/inputs/inputs.json --orchard-inputs out/orchard-inputs.json --orchard .orchard \ + --inputs out/inputs/inputs.json "${source[@]}" --orchard .orchard \ --version "$VERSION" --output out/release python audit.py out/release --orchard .orchard python -m orchard.manifests --verify-local out/release/index.toml diff --git a/audit.py b/audit.py index 70c3d52..9a2bd55 100644 --- a/audit.py +++ b/audit.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 -"""Audit actual root/recovery payloads without extracting links on the host.""" +"""Audit the actual root and platform layers without extracting links on the host.""" import argparse -import gzip import hashlib import json from pathlib import Path, PurePosixPath -import stat import struct import tarfile import tomllib @@ -15,31 +13,6 @@ HARDWARE = None # the orchard snapshot; main() sets it, else orchard_root() finds it -def newc(data): - result = {} - offset = 0 - while True: - header = data[offset:offset + 110] - if len(header) != 110 or header[:6] != b'070701': - raise ValueError('Invalid newc header') - fields = [int(header[i:i + 8], 16) for i in range(6, 110, 8)] - mode, size, namesize = fields[1], fields[6], fields[11] - start = offset + 110 - name = data[start:start + namesize - 1].decode() - if namesize < 1 or data[start + namesize - 1] != 0: - raise ValueError('Invalid newc name') - start = (start + namesize + 3) & ~3 - payload = data[start:start + size] - if len(payload) != size: - raise ValueError('Truncated newc member') - offset = (start + size + 3) & ~3 - if name == 'TRAILER!!!': - return result - if name.startswith('/') or '..' in PurePosixPath(name).parts or name in result: - raise ValueError('Unsafe/duplicate newc member') - result[name] = (payload, mode) - - def check_elf_pages(data): """The real J71b kernel uses 16 KiB pages; the container kernel may not.""" if data[:6] != b'\x7fELF\x02\x01': @@ -76,51 +49,17 @@ def audit_layer(path, platform, row): return files, description -def audit_recovery(image, release, kernel_release, layer): - framing = json.loads(Path(str(image) + '.json').read_text()) - assert sha(image) == framing['image_sha256'] - components, dtbs = {}, 0 - with image.open('rb') as stream: - for row in framing['components']: - stream.seek(row['offset']) - payload = stream.read(row['bytes']) - assert len(payload) == row['bytes'] and hashlib.sha256(payload).hexdigest() == row['sha256'] - dtbs += row['name'] == 'dtb' - components[row['name']] = payload - profile = (HARDWARE or orchard_root()) / 'boot/initramfs/recovery/boot.json' # the snapshot's tested profile - assert components['bootargs'] == json.loads(profile.read_text())['bootargs'].encode(), 'Recovery must preserve the tested standard boot profile' - recovery = newc(gzip.decompress(components['initramfs'])) - config = json.loads(recovery['etc/orchard/release.json'][0]) - assert (config['release'], config['kernel_release']) == (release, kernel_release) - # The recovery runs the kernel the installed entry boots: the same modules and regulatory data. - shared = 0 - for name, data in layer.items(): - if name.startswith(('usr/lib/modules/', 'usr/lib/firmware/')) and not name.endswith(('/vmlinuz', '/initrd')) \ - and '/dtb/' not in name: - assert recovery[name][0] == data, name - shared += 1 - for data, mode in recovery.values(): - if stat.S_ISREG(mode): - check_elf_pages(data) - for name in ('usr/bin/python3', 'usr/bin/orchard-device', 'init'): - assert name in recovery - for name in ('blkid', 'sfdisk', 'mke2fs'): - data, mode = recovery['usr/sbin/' + name] - assert stat.S_ISREG(mode) and mode & 0o111 == 0o111 and data.startswith(b'\x7fELF') - assert recovery['etc/mke2fs.conf'][0] - for name in ('device.py', 'identity.py', 'adt.py', 'storage-fence'): - assert 'usr/libexec/orchard/' + name in recovery - assert not any(name.startswith('root/.ssh') or name.startswith('run/recovery/') for name in recovery) - return {'members': len(recovery), 'dtbs': dtbs, 'shared_with_layer': shared} - - def audit(directory): manifest = tomllib.loads((directory / 'release.toml').read_text()) assert manifest['schema'] == 2 and manifest['distro'] == 'minimal-linux' record = json.loads((directory / 'rootfs.json').read_text()) rootfile = directory / 'rootfs.tar.gz' assert rootfile.stat().st_size == record['bytes'] and sha(rootfile) == record['sha256'] - assert manifest['root']['sha256'] == record['sha256'] + # The checks read the tree the image was made from; orchard.image.root_image made it, and checks it. + for row in (manifest['root'], manifest['root']['map'], manifest['logo'], manifest['pongo'], manifest['m1n1'], + manifest['install_tools']): + path = directory / row['file'] + assert path.stat().st_size == row['bytes'] and sha(path) == row['sha256'], row['file'] entries = {} size = elf_loads = 0 with tarfile.open(rootfile, 'r|gz') as archive: @@ -160,21 +99,17 @@ def audit(directory): assert [row[0] for row in users if 1000 <= int(row[2]) < 60000] == ['tablet'] platforms = {} for name, files in sorted(manifest['platforms'].items()): - for role in ('boot', 'layer'): - path = directory / files[role]['file'] - assert path.stat().st_size == files[role]['bytes'] and sha(path) == files[role]['sha256'] + path = directory / files['layer']['file'] + assert path.stat().st_size == files['layer']['bytes'] and sha(path) == files['layer']['sha256'] layer, description = audit_layer(directory / files['layer']['file'], name, files['layer']) assert sorted(description['boards']) == sorted(files['boards']) # The installer refuses a layer file the root already has. assert not (layer.keys() & {n for n, (data, _, _) in entries.items() if data is not None}) - result = audit_recovery(directory / files['boot']['file'], record['release'], - files['layer']['kernel_release'], layer) - assert result['dtbs'] == len(set(description['boards'].values())) - platforms[name] = {'boards': sorted(description['boards']), 'recovery': result} + platforms[name] = {'boards': sorted(description['boards'])} return {'root_sha256': record['sha256'], 'root_bytes': record['bytes'], 'expanded_bytes': size, 'root_members': len(entries), 'platforms': platforms, 'elf_16k_compatible_load_segments': elf_loads, - 'standard_recovery_boot_profile': True, 'signed_regulatory_database_present': True, + 'signed_regulatory_database_present': True, 'init': 'BusyBox PID 1 + OpenRC', 'libc': 'musl', 'compression': 'gzip', 'systemd': False, 'physical_device_operations_performed': False, 'hardware_validated': False} diff --git a/build.py b/build.py index 637b492..a17226a 100644 --- a/build.py +++ b/build.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""One Alpine/OpenRC release for every platform: the root, per platform a recovery and orchard's layer. +"""One Alpine/OpenRC release for every platform: the root as a block image, its logo, and orchard's files. The root holds no kernel; orchard's platform layer brings kernel, initrd, DTBs and modules at install, and orchard writes the boot entry. @@ -16,6 +16,7 @@ import sys import tarfile import tempfile +import tomllib from snapshot import command as orchard_command, root as orchard_root, sha @@ -34,18 +35,6 @@ def verify_inputs(path): return result, spec -def kernels(resolved, explicit=()): - """{platform: staged kernel bundle} of orchard.inputs' RESOLVED platforms; --kernel-dir overrides one.""" - known = resolved['platforms'] - given = {} - for value in explicit: - name, separator, path = value.partition('=') - if not separator or name not in known: - raise ValueError(f'Use --kernel-dir PLATFORM=DIRECTORY with a platform of: {", ".join(known)}') - given[name] = Path(path) - return {name: given.get(name) or Path(row['kernel']) for name, row in known.items()} - - def overlay(): result = {} for path in (PROJECT / 'system/overlay').rglob('*'): @@ -167,8 +156,11 @@ def receipt(path): return {'file': path.name, 'bytes': path.stat().st_size, 'sha256': sha(path)} -def manifests(output, version, root, platforms, orchard): - """release.toml (orchard's schema 2, every platform) and the index that pins it.""" +def manifests(output, version, root, offered, orchard): + """release.toml (orchard's schema 2, every platform) and the index that pins it. + + ROOT is orchard.image.root_image's table, OFFERED orchard's platforms.toml beside it. + """ document = ('schema = 2\nkind = "orchard.install"\ndistro = "minimal-linux"\n' 'name = "Minimal Alpine"\nversion = ' + json.dumps(version) + '\n' 'boot_method = "checkm8-pongo"\nmin_orchard = "0.2.0"\nprotocol = 1\n') @@ -178,12 +170,14 @@ def manifests(output, version, root, platforms, orchard): document += table('install', {'default_root': '/minimal', 'ready': '/run/orchard-ready'}) document += table('install.account', {'user': 'tablet', 'password': True, 'pin': False, 'passwordless_sudo': False}) - row = receipt(output / 'rootfs.tar.gz') - row.update({key: root[key] for key in ('compression', 'expanded_bytes', 'members')}) - document += table('root', row) + table('pongo', receipt(output / 'pongo.bin')) - for name, files in sorted(platforms.items()): - document += f'\n[platforms.{name}]\nboards = {json.dumps(files["boards"])}\n' - document += table(f'platforms.{name}.boot', files['boot']) + table(f'platforms.{name}.layer', files['layer']) + document += table('root', {key: value for key, value in root.items() if key != 'map'}) + document += table('root.map', root['map']) + document += table('logo', receipt(output / 'logo.bin')) # the Alpine mark, drawn by orchard's m1n1 + document += table('pongo', offered['loaders']['pongo']) + table('m1n1', offered['loaders']['m1n1']) + document += table('install_tools', offered['install_tools']) + for name, layer in sorted(offered['platforms'].items()): + document += f'\n[platforms.{name}]\nboards = {json.dumps(layer["boards"])}\n' + document += table(f'platforms.{name}.layer', {key: value for key, value in layer.items() if key != 'boards'}) path = output / 'release.toml' path.write_text(document) (output / 'index.toml').write_text('schema = 2\nkind = "orchard.index"\ndistro = "minimal-linux"\n' @@ -191,60 +185,39 @@ def manifests(output, version, root, platforms, orchard): return path -def recovery(orchard, output, base, base_receipt, components, platform_name, bundle, boards): - """The platform's RAM recovery, which starts every board of it.""" - dtbs = [bundle / 'dtbs' / (dtb + '.dtb') for dtb in sorted(set(boards.values()))] - image = output / f'recovery-{platform_name}.bin' - command = [*(f'--dtb={path}' for path in dtbs), - '--base', str(base), '--base-receipt', str(base_receipt), - '--kernel', str(bundle / 'Image.gz'), '--modules-dir', str(bundle / 'lib/modules')] - for name in ('loader', 'busybox', 'dropbear', 'dropbearkey', 'wpa_supplicant', 'wpa_cli', 'iw'): - command += ['--' + name.replace('_', '-'), str(components[name])] - command += ['--logo', str(components['logo'])] # the Alpine mark, drawn by orchard's m1n1 - command += ['--root-receipt', str(output / 'rootfs.json'), '--output', str(image)] - orchard_command(orchard, 'image.recovery', *command) - return image - - def main(): parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) - for name in ('base', 'base-receipt', 'inputs', 'orchard-inputs', 'output'): + for name in ('base', 'base-receipt', 'inputs', 'output'): parser.add_argument('--' + name, type=Path, required=True) parser.add_argument('--version', required=True) parser.add_argument('--orchard', type=Path, help='orchard snapshot (default: ORCHARD_CHECKOUT, then .orchard)') - parser.add_argument('--kernel-dir', action='append', default=[], metavar='PLATFORM=DIRECTORY', - help="a platform's staged kernel bundle instead of the one orchard.inputs names") + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument('--orchard-release', metavar='TAG', help="take orchard's files finished from this build") + source.add_argument('--orchard-inputs', type=Path, help="orchard's inputs for this build (orchard.inputs)") args = parser.parse_args() if args.output.exists() or args.output.is_symlink(): parser.error('Choose a new output directory') orchard = orchard_root(args.orchard) components, inputs = verify_inputs(args.inputs) - resolved = json.loads(args.orchard_inputs.read_text()) - try: - bundles = kernels(resolved, args.kernel_dir) - except ValueError as error: - parser.error(str(error)) + resolved = json.loads(args.orchard_inputs.read_text()) if args.orchard_inputs else None + revision = resolved['orchard'] if resolved else None args.output.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix='.minimal-build-', dir=args.output.parent) as temp: output = Path(temp) / 'release' output.mkdir() - root = root_archive(args.base, args.base_receipt, components, output, args.version, orchard) - platforms = {} - for name, bundle in bundles.items(): - layer_receipt = Path(temp) / f'layer-{name}.json' - orchard_command(orchard, 'image.platform', name, '--output', output, '--kernel-dir', bundle, - '--busybox', components['busybox'], '--receipt', layer_receipt, stdout=subprocess.DEVNULL) - layer = json.loads(layer_receipt.read_text()) - image = recovery(orchard, output, args.base, args.base_receipt, components, name, bundle, - resolved['platforms'][name]['boards']) - platforms[name] = {'boards': layer.pop('boards'), 'layer': layer, 'boot': receipt(image)} - shutil.copyfile(components['pongo'], output / 'pongo.bin') - revision = resolved['orchard'] - manifests(output, args.version, root, platforms, revision) + root_archive(args.base, args.base_receipt, components, output, args.version, orchard) + source = ('--release', args.orchard_release) if args.orchard_release else ('--inputs', args.orchard_inputs) + orchard_command(orchard, 'image.platforms_index', output, *source, stdout=subprocess.DEVNULL) + offered = tomllib.loads((output / 'platforms.toml').read_text()) + revision = revision or {'commit': offered['commit'], 'modified': False} + orchard_command(orchard, 'image.root_image', output / 'rootfs.tar.gz', '--output', output) + root = json.loads((output / 'root-image.json').read_text()) + shutil.copyfile(components['logo'], output / 'logo.bin') + manifests(output, args.version, root, offered, revision) public_inputs = {**inputs, 'files': {name: {k: v for k, v in row.items() if k != 'path'} for name, row in inputs['files'].items()}} (output / 'build-receipt.json').write_text(json.dumps({'distro': 'minimal-linux', - 'version': args.version, 'orchard': revision, 'platforms': platforms, + 'version': args.version, 'orchard': revision, 'platforms': offered['platforms'], 'base': json.loads(args.base_receipt.read_text()), 'inputs': public_inputs, 'physical_device_operations_performed': False, 'hardware_validated': False}, indent=2) + '\n') shutil.rmtree(output / '.work', ignore_errors=True) diff --git a/tests/test_cached_kernel.py b/tests/test_cached_kernel.py deleted file mode 100644 index 89f5795..0000000 --- a/tests/test_cached_kernel.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Which kernels a release uses: what orchard.inputs resolved, or an explicit bundle; no builds or devices.""" -import importlib.util -from pathlib import Path -import unittest - -PROJECT = Path(__file__).resolve().parents[1] -spec = importlib.util.spec_from_file_location('minimal_build', PROJECT / 'build.py') -build = importlib.util.module_from_spec(spec) -spec.loader.exec_module(build) - -RESOLVED = {'platforms': {'apple-16k': {'kernel': '/cache/apple-16k', 'boards': {'j71b': 't8010-j71b'}}, - 'apple-4k': {'kernel': '/cache/apple-4k', 'boards': {'n71': 's8000-n71'}}}} - - -class CachedKernelTests(unittest.TestCase): - def test_a_release_covers_every_resolved_platform(self): - self.assertEqual(build.kernels(RESOLVED), {'apple-16k': Path('/cache/apple-16k'), - 'apple-4k': Path('/cache/apple-4k')}) - - def test_an_explicit_bundle_replaces_the_resolved_one(self): - self.assertEqual(build.kernels(RESOLVED, ['apple-16k=/explicit'])['apple-16k'], Path('/explicit')) - - def test_unknown_platforms_are_refused(self): - for explicit in (['apple-64k=/x'], ['/x']): - with self.subTest(explicit=explicit), self.assertRaisesRegex(ValueError, 'apple-16k'): - build.kernels(RESOLVED, explicit) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_publication.py b/tests/test_publication.py index ea0eee0..f3b9d1d 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -16,17 +16,18 @@ def release(source): - """A fake release: one root, Pongo, and a recovery and layer per platform.""" - for name in ('rootfs.tar.gz', 'pongo.bin', 'recovery-apple-16k.bin', 'platform-apple-16k.tar.gz', - 'recovery-apple-4k.bin', 'platform-apple-4k.tar.gz'): + """A fake release: the root image, its logo, and what orchard offers.""" + for name in ('root.blocks.zst', 'root.map.json', 'logo.bin', 'pongo.bin', 'm1n1.bin', 'install-tools.tar.gz', + 'platform-apple-16k.tar.gz', 'platform-apple-4k.tar.gz'): (source / name).write_bytes(name.encode()) - platforms = {} - for name, boards in (('apple-16k', ['j71b']), ('apple-4k', ['n61'])): - layer = build.receipt(source / f'platform-{name}.tar.gz') - layer.update(compression='gzip', expanded_bytes=10, members=2, kernel_release='7.3-' + name) - platforms[name] = {'boards': boards, 'boot': build.receipt(source / f'recovery-{name}.bin'), 'layer': layer} - return build.manifests(source, '0.1.0', {'compression': 'gzip', 'expanded_bytes': 100, 'members': 3}, - platforms, {'commit': 'c' * 40, 'modified': False}) + platforms = {name: {**build.receipt(source / f'platform-{name}.tar.gz'), 'compression': 'gzip', 'expanded_bytes': 10, + 'members': 2, 'kernel_release': '7.3-' + name, 'boards': boards} + for name, boards in (('apple-16k', ['j71b']), ('apple-4k', ['n61']))} + root = {**build.receipt(source / 'root.blocks.zst'), 'compression': 'zstd', 'format': 'blocks', 'expanded_bytes': 100, + 'members': 3, 'map': build.receipt(source / 'root.map.json')} + offered = {'loaders': {'pongo': build.receipt(source / 'pongo.bin'), 'm1n1': build.receipt(source / 'm1n1.bin')}, + 'install_tools': build.receipt(source / 'install-tools.tar.gz'), 'platforms': platforms} + return build.manifests(source, '0.1.0', root, offered, {'commit': 'c' * 40, 'modified': False}) class Publication(unittest.TestCase): @@ -53,8 +54,8 @@ def test_minimal_manifest_is_rewritten_and_rehashed(self): self.assertEqual(receipt['url'], 'https://github.com/OrchardLinux/minimal-linux/releases/download/v0.1.0/release.toml') self.assertEqual(receipt['sha256'], hashlib.sha256((output / 'release.toml').read_bytes()).hexdigest()) hosted = tomllib.loads((output / 'release.toml').read_text()) - for row in (hosted['root'], hosted['pongo'], *(files[role] for files in hosted['platforms'].values() - for role in ('boot', 'layer'))): + for row in (hosted['root'], hosted['root']['map'], hosted['logo'], hosted['pongo'], hosted['m1n1'], + hosted['install_tools'], *(files['layer'] for files in hosted['platforms'].values())): self.assertNotIn('file', row) self.assertIn('/releases/download/v0.1.0/', row['url']) self.assertEqual((output / 'pongo.bin').read_bytes(), b'pongo.bin')