diff --git a/.gitignore b/.gitignore index 6297aff..a2df5fc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ pxt_modules built compile_commands.json -generated-docs/* +generated-docs node_modules .DS_Store .vscode/settings.json \ No newline at end of file diff --git a/icons.ts b/icons.ts index 7629f4c..c36005e 100644 --- a/icons.ts +++ b/icons.ts @@ -1,5 +1,7 @@ namespace ui { // Shared icon bitmaps. + //% packable + //% whenUsed export const microbitLogo = bmp` ............................ ......5555555555555555...... @@ -23,6 +25,8 @@ namespace ui { // SMALL ICONS: //------------- + //% packable + //% whenUsed export const microbit_logo_btn = bmp` . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -42,6 +46,8 @@ namespace ui { . . . . . . . . . . . . . . . . ` + //% packable + //% whenUsed export const microbitLogoWhiteBackground = bmp` 11111111111111111 11115555555551111 @@ -61,6 +67,9 @@ namespace ui { 11111444444441111 ` + // Not packable: an image pack falls back to this when a record cannot be + // read, so it has to stay a plain literal. + //% whenUsed export const MISSING = bmp` . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -80,6 +89,8 @@ namespace ui { . . . . . . . . . . . . . . . . ` + //% packable + //% whenUsed export const green_tick = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 @@ -102,6 +113,8 @@ namespace ui { /// /// HARDWARE-SPECIFIC LANGUAGE TILES /// + //% packable + //% whenUsed export const tile_button_a = bmp` . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -121,6 +134,8 @@ namespace ui { . . . . . . . . . . . . . . . . ` + //% packable + //% whenUsed export const tile_button_b = bmp` . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -140,12 +155,12 @@ namespace ui { . . . . . . . . . . . . . . . . ` - //------------------------ // SENSORS AND ACTUATORS: //------------------------ - + //% packable + //% whenUsed export const pin_0 = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 @@ -165,6 +180,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const pin_1 = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 @@ -184,6 +201,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const pin_2 = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 @@ -203,6 +222,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const compass = bmp` 1 1 1 1 5 5 5 5 5 5 5 1 1 1 1 1 1 1 5 5 5 5 5 5 5 5 5 1 1 1 @@ -222,6 +243,8 @@ namespace ui { 1 1 1 1 5 5 5 5 5 5 5 1 1 1 1 ` + //% packable + //% whenUsed export const car_right_turn = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 c c c 1 1 1 1 1 @@ -241,6 +264,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const car_right_spin = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 @@ -260,6 +285,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const finger_press = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 f 1 1 1 1 1 1 1 1 @@ -279,6 +306,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const magnet = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 6 1 1 1 1 1 1 1 1 1 1 1 1 1 6 1 1 1 6 1 @@ -298,6 +327,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 6 1 1 1 1 1 ` + //% packable + //% whenUsed export const thermometer = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 f 1 1 1 1 1 1 1 1 @@ -317,6 +348,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const led_light_sensor = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 8 8 8 8 5 5 5 8 8 8 8 1 1 1 @@ -336,6 +369,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const microphone = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 @@ -355,6 +390,8 @@ namespace ui { 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ` + //% packable + //% whenUsed export const accelerometer = bmp` 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 8 1 1 1 1 1 1 1 @@ -378,6 +415,8 @@ namespace ui { // Large Icons: //------------- + //% packable + //% whenUsed export const largeEditIcon = bmp` .666666666666666666666666666666. 66666666666666666666666666666666 @@ -414,6 +453,8 @@ namespace ui { .bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb. ` + //% packable + //% whenUsed export const largeSettingsGear = bmp` 66666666666666666666666666666666 66666666666666666666666666666666 @@ -449,6 +490,8 @@ namespace ui { 66666666666666666666666666666666 ` + //% packable + //% whenUsed export const linearGraph1 = bmp` .111111111111111111111111111111. 11111111111111111111111111111111 @@ -485,6 +528,8 @@ namespace ui { .bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb. ` + //% packable + //% whenUsed export const radio_set_group = bmp` .666666666666666666666666666666. 66666666666666666666666666666666 @@ -521,6 +566,8 @@ namespace ui { .666666666666666666666666666666. ` + //% packable + //% whenUsed export const largeDisk = bmp` .666666666666666666666666666666. 66666666666666666666666666666666 diff --git a/img-tools/README.md b/img-tools/README.md new file mode 100644 index 0000000..9b08120 --- /dev/null +++ b/img-tools/README.md @@ -0,0 +1,296 @@ +# Image packing + +`img-tools` compresses an app's `bmp` image literals into a single packed +buffer at build time and generates a keyed lookup for them, `_img.get(key)`. +On a typical icon set, pixel data shrinks by around 45%, and the per-image +const overhead goes away too. + +Images stay authored as ordinary `bmp` literals in your source. You annotate +the ones to pack, map your lookup keys to them in a key table, and run the +generator before building. + +## Is it right for your app? + +Packing trades flash for heap. + +- An unpacked `bmp` literal is drawn directly from flash and uses no heap. +- A packed image is decoded into the heap on its first `_img.get`, and the + decoded bitmap is cached for the lifetime of the program. Nothing is evicted. + +Heap cost therefore scales with the number of distinct images accessed over a +session, not with how many are on screen at once. An app that eventually draws +every packed image holds all of them decoded. The generator reports that worst +case on every run: + +``` +img-gen: ... 17648 B RAM if all decoded +``` + +Packing fits if your app can afford that on top of its normal heap usage. +Verify it on a device with a session that reaches every screen. If the heap is +tight, pack selectively: rarely drawn images cost heap only when they are +drawn, while images drawn in every session always cost their full decoded size. + +## Setup + +**1. Pin the same ui-core version on both dependency channels.** The encoder +runs from your npm devDependency; the decoder ships in your pxt dependency. +`runImageGen` compares the resolved versions (the `version` in each side's +`pxt.json`) and fails on a mismatch. + +```json +// package.json +"devDependencies": { + "@microbit-apps/ui-core": "github:microbit-apps/ui-core#v0.0.9" +} +``` + +In `pxt.json`, ui-core can be a direct or transitive dependency. + +**2. Add a script for the `img-gen` bin.** It runs from the project root. + +```json +// package.json +"scripts": { + "img:gen": "img-gen" +} +``` + +Settings are optional and go in `img.config.json`, beside `pxt.json`. See +[Configuration](#configuration). + +**3. Add the generated file to `pxt.json`.** The generator writes `img.g.ts`, +which defines the `_img` namespace. List it in `files` ahead of any file that +uses `_img` at top level, since file order sets global initialization order. +Commit it: the project does not build without it. + +```json +"files": ["img.g.ts", "assets.ts", "..."] +``` + +## Marking an image to pack + +```ts +namespace icons { + //% packable + //% whenUsed + export const thermometer = bmp` + . 1 1 . + . 1 1 . + . 1 1 . + 1 2 2 1 + ` +} +``` + +- `//% packable` makes the const eligible for packing. The image is identified + by its const name, `thermometer`, or `icons.thermometer` when qualified. +- `//% whenUsed` makes the const tree-shakable, so once nothing references it + the literal is dropped from the binary. Without it the literal ships + regardless, and packing would duplicate the image, so the generator skips it + with a warning. + +The two attributes can also share a line: `//% packable whenUsed`. + +The literal remains the source of truth. Edit it in place and regenerate. + +## The key table + +`img.keys.json`, beside `pxt.json`, maps your lookup keys to images: + +```json +{ + "numeric": { + "16": "thermometer", + "180": "thermometer" + }, + "names": { + "delete": "btn_delete", + "thermometer": "thermometer" + } +} +``` + +- **`numeric`** maps integer keys, such as existing tile or item ids. It + generates a flat table indexed by key, with one entry per integer up to the + largest key, so keep keys small. Several keys can map to the same image. +- **`names`** maps string keys. The generated name index holds exactly the keys + listed here, so list every string key you look up, including identity + mappings like `"thermometer": "thermometer"`. String keys cannot contain + commas. + +Both sections are optional. Each packable image needs at least one key; an +image no key reaches could never be retrieved, so the generator excludes it +with a warning. + +If `img.keys.json` doesn't exist, the generator writes a starter table and uses +it for that run: one identity string key per app image that can be packed. +Images excluded for another reason, such as a missing `//% whenUsed`, are +omitted, so the starter never references an image the next run would reject. +The run reports that it wrote one and asks you to review it. Check it before +relying on it: each string key is the image's own name, which may not be the key +your code passes to `_img.get`; there are no numeric keys; and every packable app +image is listed, including any your app never draws. An existing +table is never overwritten. + +An image can be referenced by its bare const name or by its namespace-qualified +name. See [Using a library's images](#using-a-librarys-images). + +## Running the generator + +```sh +npm run img:gen +``` + +Run it before building, and after any change to a packed image or the key +table. A clean run prints a summary: + +``` +img-gen: 110 images (110 unique, 0 shared), 110 compressed, 0 uncompressed; 17648 B -> 9653 B (7995 B saved, 45.3%); 17648 B RAM if all decoded +img-gen: name index holds 24 key(s), about 285 B +``` + +`--verbose` adds a per-image breakdown: compressed size, and whether each image +was compressed, left uncompressed, or shares another image's record. It also +shows notes that are otherwise hidden. Errors exit with status 1. + +## Retrieving images + +```ts +const img = _img.get(16) // numeric key +const del = _img.get("delete") // string key +``` + +`_img.get(key: string | number, nullIfMissing?: boolean)` returns the pack's +fallback image (`ui.MISSING`) for an unknown key, so callers never need a null +check. With `nullIfMissing`, it returns `undefined` instead: + +```ts +const maybe = _img.get("delete", true) +if (maybe) draw(maybe) +``` + +Because decoded bitmaps are cached, every key that resolves to the same image +returns the same `Bitmap` instance. + +`img.g.ts` also exports `_img.count` and `_img.byIndex(i)` for iterating the +pack, and `_img.indexOfKey` / `_img.indexOfName` when the corresponding key +type is in use. + +## Using a library's images + +A dependency's images are packed only if your key table references them, and +only if the library marks them `//% packable`. Referencing one moves it into +your pack, and the library's literal is tree-shaken out. + +```json +"numeric": { "16": "ui.thermometer" } +``` + +Qualify the name with the declaring namespace. A bare name resolves to your +app's declaration when one exists, so `"thermometer"` would resolve to your own +`thermometer`, not `ui.thermometer`. The generator reports a bare-name +collision between two dependencies, or within your app; one between your app +and a dependency only shows with `--verbose`, since the bare name already +resolves to your own image. + +**For library authors:** consumers' key tables reference your images by const +name, which makes those names part of your public API. To rename a const +without breaking consumers, keep the published name with +`//% packable="thermometer"`. + +## Web-only images + +Images used only by code that isn't compiled for hardware, such as the body of +a `//% shim=TD_NOOP` function, should be `//% whenUsed` without `//% packable`. +They are tree-shaken from the hardware build as-is. + +## Diagnostics + +| Message | Severity | Meaning and fix | +|---|---|---| +| `key table names "x", which no package declares` | error | The key table references an unknown image. Fix the name, or annotate the const `//% packable`. | +| `key table names x, which cannot be packed` | error | The key table references an image that was excluded. The preceding note gives the reason. | +| `resolves to 0.0.8 for build tooling but 0.0.9 for device code` | error | npm and pxt resolve different ui-core versions. Pin both to the same one. | +| `is //% packable but not //% whenUsed` | warning (info for a dependency's image) | Add `//% whenUsed`. | +| `no key in the key table reaches it, so it is left out` | warning | Add a key, or drop `//% packable`. | +| `is still referenced as x, so it stays a literal` | info | Code references the const directly, so it can't be removed. Retrieve it through `_img.get` instead, or leave it unpacked. | +| `"x" matches both a.x and b.x` | info (detail if one is your app's) | A bare `"x"` is ambiguous. The message names the image it resolves to and the qualified name to write for the other. When one of them is yours, a bare name resolves to yours, so the note only shows with `--verbose`. | +| `has the same pixels as x ... costing N B twice` | info (warning if `x` is yours) | A packed image is pixel-identical to a literal that still ships. Remove one if you can. | + +Errors abort the run and are reported together. Warnings and info never abort. + +## Checking a build + +After building, `img-gen --check` scans the compiled output (`built/*.asm`) for +image buffers byte-identical to a packed image, which would mean the image ships +twice. It regenerates nothing. + +```sh +npx img-gen --check +``` + +``` +img-check: 29 image buffer(s) in 1 output(s), none duplicating the pack +``` + +It reports only and never fails. + +## Configuration + +`img.config.json` is optional. Every setting has a default, and unknown settings +are rejected. + +| Setting | Default | Description | +|---|---|---| +| `keysPath` | `img.keys.json` | Key table path, relative to the project root. | +| `outName` | `img.g.ts` | Generated file name. | +| `fallback` | none | Image to keep out of the pack because the app uses it as its missing-image placeholder. | +| `pins` | `true` | `false` skips the ui-core version check. | + +```json +{ + "keysPath": "assets/img.keys.json", + "fallback": "placeholder" +} +``` + +### Calling it from a script + +For anything the config doesn't cover, such as running without writing or +capturing output, call the API directly. It takes the same settings, plus: + +| Option | Default | Description | +|---|---|---| +| `root` | required | Project directory containing `pxt.json`. | +| `verbose` | `false` | Same as `--verbose`. | +| `write` | `true` | `false` runs everything but writing `img.g.ts` or a starter key table. | +| `log` | `console.log` | Receives each line of output. | + +`keysPath` is a path here, resolved as given rather than against `root`. + +```js +import { runImageGen, runImageCheck } from "@microbit-apps/ui-core/img" + +const result = runImageGen({ root, write: false, log: () => {} }) +runImageCheck({ root, pack: result.pack }) +``` + +`runImageGen` throws on errors and otherwise returns what it decided: the pack, +the resolved keys, the diagnostics, and the generated source. + +## Future directions + +**LRU eviction.** Decoded bitmaps are currently cached for the program's +lifetime. An opt-in LRU cache could bound heap use by evicting the least +recently used bitmaps and decoding them again on the next access. + +Eviction only reclaims heap for a bitmap that has no other live references, and +retained-mode widgets keep references to their bitmaps for as long as they +exist. Evicting a bitmap that a widget still holds reclaims nothing, and the +next `_img.get` for that image decodes a second copy, which increases heap use. +pxt has no weak references, so the cache can't tell held bitmaps from free ones. + +LRU therefore suits draw paths that call `_img.get` at draw time and don't +retain the result. It would also drop the guarantee that a key always returns +the same `Bitmap` instance. diff --git a/img-tools/bin/img-gen.mjs b/img-tools/bin/img-gen.mjs new file mode 100755 index 0000000..e41b514 --- /dev/null +++ b/img-tools/bin/img-gen.mjs @@ -0,0 +1,64 @@ +#!/usr/bin/env node +// Image pack generator for apps that need no wrapper. Run from the project +// root. +// +// Configuration is optional. An img.config.json beside pxt.json may set any of: +// keysPath and outName (paths relative to the project root), fallback, and +// pins. Apps that need more call runImageGen from their own script instead. +// +// Flags: +// --verbose per-image breakdown, plus notes that are otherwise hidden +// --check after a build: report packed images that also ship as buffers, +// without regenerating anything + +import { existsSync, readFileSync } from "node:fs" +import { join } from "node:path" +import { runImageGen } from "../gen.mjs" +import { runImageCheck } from "../verify.mjs" + +const CONFIG_KEYS = ["keysPath", "outName", "fallback", "pins"] + +function readConfig(root) { + const path = join(root, "img.config.json") + if (!existsSync(path)) return {} + let config + try { + config = JSON.parse(readFileSync(path, "utf8")) + } catch (e) { + throw new Error(`${path} is not valid JSON: ${e.message}`) + } + if (!config || typeof config !== "object" || Array.isArray(config)) + throw new Error(`${path} must be a JSON object`) + const unknown = Object.keys(config).filter(k => CONFIG_KEYS.indexOf(k) < 0) + if (unknown.length > 0) + throw new Error(`${path}: unknown setting(s) ${unknown.join(", ")}; expected ${CONFIG_KEYS.join(", ")}`) + return config +} + +try { + const root = process.cwd() + const argv = process.argv.slice(2) + const unknownFlags = argv.filter(a => a !== "--verbose" && a !== "--check") + if (unknownFlags.length > 0) throw new Error(`unknown argument(s) ${unknownFlags.join(" ")}`) + const check = argv.indexOf("--check") >= 0 + + const config = readConfig(root) + const options = { + root, + keysPath: config.keysPath ? join(root, config.keysPath) : undefined, + outName: config.outName, + fallback: config.fallback, + pins: config.pins, + verbose: argv.indexOf("--verbose") >= 0, + } + + if (check) { + const { pack } = runImageGen({ ...options, write: false, log: () => {} }) + runImageCheck({ root, pack }) + } else { + runImageGen(options) + } +} catch (e) { + console.error(`img-gen: ${e.message}`) + process.exit(1) +} diff --git a/img-tools/codec.mjs b/img-tools/codec.mjs new file mode 100644 index 0000000..9466382 --- /dev/null +++ b/img-tools/codec.mjs @@ -0,0 +1,166 @@ +// Build-time encoder for compressed image records. The device-side decoder +// is `bitmaps.ofCompressed` in img.ts, and the record format is documented +// there. These two must agree byte for byte, which is why a consumer has to +// pin the same ui-core ref for its build tooling and its device code. +// +// Typical use: +// +// import { encodeImage, encodeRecord } from "@microbit-apps/ui-core/img" +// const f4 = encodeImage(rows) // rows as the pixel editor prints them +// const record = encodeRecord(f4) // smallest method, always a record +// +// `encodeRecord` round-trips every record through `decodeRecord` and throws +// on a mismatch, so a bad stream fails the build rather than the device. + +const MAGIC = 0x87 +const BPP = 4 +const METHOD_UNCOMPRESSED = 0x0000 +const METHOD_SIGNATURE = 0xc000 +const METHOD_RLE = 1 + +/** Decoded size of an F4 buffer: header plus 32-bit-padded columns. */ +function decodedSize(w, h) { + return 8 + (((h * 4 + 31) >> 5) << 2) * w +} + +/** + * Encodes pixel rows into an F4 image buffer, byte-identical to what the pxt + * compiler emits for a `bmp` literal. `rows` is an array of row strings + * ("." is transparent, a hex digit is a color, whitespace is ignored), or a + * single string with embedded newlines. + */ +export function encodeImage(rows) { + rows = (Array.isArray(rows) ? rows : rows.split("\n")) + .map(r => r.replace(/\s+/g, "")) + .filter(r => r.length > 0) + if (rows.length === 0) throw new Error("image has no rows") + const w = Math.max(...rows.map(r => r.length)) + const h = rows.length + const colBytes = ((h * 4 + 31) >> 5) << 2 + const buf = Buffer.alloc(8 + colBytes * w) + buf[0] = MAGIC + buf[1] = BPP + buf.writeUInt16LE(w, 2) + buf.writeUInt16LE(h, 4) + for (let x = 0; x < w; x++) { + for (let y = 0; y < h; y++) { + const ch = rows[y][x] || "." + const v = ch === "." ? 0 : parseInt(ch, 16) + if (Number.isNaN(v)) throw new Error(`bad pixel ${ch}`) + const off = 8 + x * colBytes + (y >> 1) + if (y & 1) buf[off] |= v << 4 + else buf[off] |= v + } + } + return buf +} + +/** Run-length encodes an F4 payload's nibble stream. */ +function rleNibble(payload) { + const nibbles = [] + for (const b of payload) nibbles.push(b & 15, b >> 4) + const out = [] + let i = 0 + while (i < nibbles.length) { + let j = i + while (j < nibbles.length && nibbles[j] === nibbles[i] && j - i < 16) + j++ + out.push((nibbles[i] << 4) | (j - i - 1)) + i = j + } + return Buffer.from(out) +} + +/** + * Encodes an F4 buffer into a record, using whichever method is smallest. + * Always returns a record, and never one larger than the image it holds. + */ +export function encodeRecord(f4) { + if (f4[0] !== MAGIC || f4[1] !== BPP) + throw new Error("not an F4 image buffer (magic/bpp mismatch)") + const payload = f4.slice(8) + const packed = rleNibble(payload) + + const header = Buffer.from(f4.slice(0, 8)) + let record + if (packed.length < payload.length) { + header.writeUInt16LE(METHOD_SIGNATURE | METHOD_RLE, 6) + record = Buffer.concat([header, packed]) + } else { + header.writeUInt16LE(METHOD_UNCOMPRESSED, 6) + record = Buffer.concat([header, payload]) + } + + const check = decodeRecord(record) + const expected = Buffer.from(f4) + expected.writeUInt16LE(0, 6) + if (!check.equals(expected)) throw new Error("record round-trip failed") + return record +} + +/** True when `encodeRecord` left this record uncompressed. */ +export function isUncompressed(record, offset = 0) { + return record.readUInt16LE(offset + 6) === METHOD_UNCOMPRESSED +} + +/** + * Decodes one record back into an F4 buffer, with the padding field zeroed. + * Reference implementation of the device-side `bitmaps.ofCompressed`. + * + * Throws when `src` at `offset` is not a record. + */ +export function decodeRecord(src, offset = 0) { + if (src[offset] !== MAGIC || src[offset + 1] !== BPP) + throw new Error("not a record (magic/bpp mismatch)") + const w = src.readUInt16LE(offset + 2) + const h = src.readUInt16LE(offset + 4) + const size = decodedSize(w, h) + const marker = src.readUInt16LE(offset + 6) + + const out = Buffer.alloc(size) + src.copy(out, 0, offset, offset + 8) + out.writeUInt16LE(0, 6) + + if (marker === METHOD_UNCOMPRESSED) { + src.copy(out, 8, offset + 8, offset + size) + return out + } + if ((marker & 0xff00) !== METHOD_SIGNATURE) + throw new Error(`not a record (marker 0x${marker.toString(16)})`) + if ((marker & 0xff) !== METHOD_RLE) + throw new Error(`unknown method ${marker & 0xff}`) + + let s = offset + 8 + let d = 16 + const dEnd = size * 2 + while (d < dEnd) { + const run = src[s++] + const color = run >> 4 + const count = (run & 15) + 1 + if (color) { + for (let k = 0; k < count; k++) { + out[d >> 1] |= color << ((d & 1) * 4) + d++ + } + } else { + d += count + } + } + return out +} + +/** Byte length of the record starting at `offset`. */ +export function recordLength(src, offset = 0) { + const w = src.readUInt16LE(offset + 2) + const h = src.readUInt16LE(offset + 4) + if (src.readUInt16LE(offset + 6) === METHOD_UNCOMPRESSED) return decodedSize(w, h) + // A compressed payload is walked to find its end. + const size = decodedSize(w, h) + let s = offset + 8 + let d = 16 + while (d < size * 2) { + const run = src[s++] + d += (run & 15) + 1 + } + return s - offset +} diff --git a/img-tools/diagnostics.mjs b/img-tools/diagnostics.mjs new file mode 100644 index 0000000..a197d2f --- /dev/null +++ b/img-tools/diagnostics.mjs @@ -0,0 +1,50 @@ +// Collected diagnostics, ordered by what the app can act on. +// +// error the run cannot produce a correct result and the app can fix it +// warning the app owns the problem but the pack is still correct +// info the cause is in a dependency, so it is reported and not acted on +// detail only shown when the caller asks for verbose output +// +// Errors are collected rather than thrown; call `check()` at the point the run +// is ready to fail, and it reports every problem found so far at once. + +/** Creates a collector. `verbose` decides whether `detail` notes are shown. */ +export function createDiagnostics(verbose) { + const notes = [] + const add = (severity, text) => notes.push({ severity, text }) + return { + notes, + error: text => add("error", text), + warning: text => add("warning", text), + info: text => add("info", text), + detail: text => add("detail", text), + + /** Notes worth showing, given the verbosity. */ + visible() { + return notes.filter(n => verbose || n.severity !== "detail") + }, + + /** + * Writes any notes not already written. Call this before `check()` so + * a failing run still explains itself: the notes saying why an image + * could not be packed are what make the failure actionable. + */ + flush(log) { + for (const n of this.visible()) { + if (n.logged) continue + n.logged = true + log(`img-gen: ${n.severity}: ${n.text}`) + } + }, + + /** Throws one error naming every problem found, if there were any. */ + check() { + const errors = notes.filter(n => n.severity === "error") + if (errors.length === 0) return + const lines = errors.map(e => " - " + e.text).join("\n") + throw new Error( + `${errors.length} problem(s) prevent generating the image pack:\n${lines}`, + ) + }, + } +} diff --git a/img-tools/emit.mjs b/img-tools/emit.mjs new file mode 100644 index 0000000..eeaaf06 --- /dev/null +++ b/img-tools/emit.mjs @@ -0,0 +1,121 @@ +// Generation of the app's img.g.ts. +// +// The file holds the pack, its offset table and the lookups the app's key +// spaces need. Packed images are reached through those lookups; the file +// declares no named const for one. +// +// Only the lookups an app actually uses are emitted: an app with no numeric +// keys gets no numeric table, and one that never looks an image up by string +// gets no name index. + +/** Emits a hex literal, split across lines so the file stays readable. */ +function hexLit(buf, indent) { + const hex = buf.toString("hex") + if (hex.length <= 96) return "hex`" + hex + "`" + const lines = [] + for (let i = 0; i < hex.length; i += 96) lines.push(indent + " " + hex.slice(i, i + 96)) + return "hex`\n" + lines.join("\n") + "\n" + indent + "`" +} + +// Index tables are u8 up to 255 images and u16le past that, with the matching +// accessor emitted alongside. +function indexTable(values, wide) { + const buf = Buffer.alloc(values.length * (wide ? 2 : 1)) + values.forEach((v, i) => (wide ? buf.writeUInt16LE(v, i * 2) : (buf[i] = v))) + return buf +} + +function readIndex(name, expr, wide) { + return wide ? `${name}.getNumber(NumberFormat.UInt16LE, (${expr}) * 2)` : `${name}[${expr}]` +} + +/** + * Emits img.g.ts content. + * + * `pack` is what buildPack returned; `names` maps each addressable string key + * to a pack index; `numeric` maps each numeric key to a pack index. `sources` + * is the list of files the images were discovered in, for the header. + */ +export function emitImgG(pack, names, numeric, sources) { + const count = pack.entries.length + const wide = count > 255 + const L = [] + + L.push("// AUTO-GENERATED by @microbit-apps/ui-core/img -- do not edit.") + L.push("//") + L.push("// Images are declared with //% img=\"name\" in:") + for (const s of sources) L.push("// " + s) + L.push("// and keyed by the app's key table. Edit an image where it is declared,") + L.push("// then regenerate; the pixels here are a build product.") + L.push("namespace _img {") + L.push(" // Compressed image records end to end; bitmaps.ofCompressed reads one.") + L.push(" const BLOB = " + hexLit(pack.blob, " ")) + L.push(" // u16le record offset per image index; identical images share an offset.") + L.push(" const OFFSETS = " + hexLit(pack.offsets, " ")) + L.push(" const PACK = new ui.ImagePack(BLOB, OFFSETS)") + L.push("") + L.push(" /** Number of image indices in the pack. */") + L.push(" export const count = " + count) + L.push("") + L.push(" /** The image at a pack index, decoded on first use and then cached. */") + L.push(" export function byIndex(index: number): Bitmap {") + L.push(" return PACK.get(index)") + L.push(" }") + + const nameKeys = Object.keys(names).sort() + if (nameKeys.length > 0) { + const table = indexTable(nameKeys.map(k => names[k]), wide) + L.push("") + L.push(" // String keys, and the pack index of each.") + L.push(" const KEYS = " + JSON.stringify(nameKeys.join(",")) + ".split(\",\")") + L.push(" const KEY_INDEX = " + hexLit(table, " ")) + L.push("") + L.push(" /** Pack index for a string key, or -1 when no image has it. */") + L.push(" export function indexOfName(name: string): number {") + L.push(" const i = KEYS.indexOf(name)") + L.push(" if (i < 0) return -1") + L.push(" return " + readIndex("KEY_INDEX", "i", wide)) + L.push(" }") + } + + const numKeys = Object.keys(numeric).map(Number) + if (numKeys.length > 0) { + // A flat table indexed by the key itself. Entries are index + 1, so + // 0 means the key names no image. + const size = Math.max(...numKeys) + 1 + const values = new Array(size).fill(0) + for (const k of numKeys) values[k] = numeric[k] + 1 + L.push("") + L.push(" // Numeric key -> pack index + 1; 0 means the key names no image.") + L.push(" const NUM_INDEX = " + hexLit(indexTable(values, wide), " ")) + L.push("") + L.push(" /** Pack index for a numeric key, or -1 when no image has it. */") + L.push(" export function indexOfKey(key: number): number {") + L.push(" if (key < 0 || key >= " + size + ") return -1") + L.push(" return " + readIndex("NUM_INDEX", "key", wide) + " - 1") + L.push(" }") + } + + L.push("") + L.push(" /**") + L.push(" * The image for one of the app's keys. An unknown key gives the pack's") + L.push(" * fallback image, or undefined when `nullIfMissing` is set.") + L.push(" */") + L.push(" export function get(") + L.push(" id: string | number,") + L.push(" nullIfMissing?: boolean,") + L.push(" ): Bitmap | undefined {") + const lookups = [] + if (nameKeys.length > 0) lookups.push('typeof id == "string" ? indexOfName(id)') + if (numKeys.length > 0) lookups.push("indexOfKey(id)") + if (lookups.length === 2) L.push(" const index = " + lookups[0] + " : " + lookups[1]) + else if (nameKeys.length > 0) L.push(" const index = indexOfName(id)") + else if (numKeys.length > 0) L.push(" const index = indexOfKey(id)") + else L.push(" const index = -1") + L.push(" if (index < 0 && nullIfMissing) return undefined") + L.push(" // An out-of-range index is what yields the pack's fallback.") + L.push(" return PACK.get(index)") + L.push(" }") + L.push("}") + return L.join("\n") + "\n" +} diff --git a/img-tools/gen.mjs b/img-tools/gen.mjs new file mode 100644 index 0000000..5843537 --- /dev/null +++ b/img-tools/gen.mjs @@ -0,0 +1,328 @@ +// Image pack generation. +// +// Discovers the images declared with //% packable across the app and the +// packages it compiles against, joins them to the app's key table, packs the +// ones the app can pack, and writes img.g.ts. +// +// An app packs its own images, and a dependency's only when its key table names +// one. An image is left out when packing it would copy it rather than move it: +// when it lacks //% whenUsed, or when code still refers to its const. +// +// A consuming app calls runImageGen from a thin wrapper holding whatever is +// specific to it, the way it wraps runLocGen. + +import { existsSync, readFileSync, writeFileSync } from "node:fs" +import { basename, join } from "node:path" + +import { scanSource, scanLiterals, countReferences } from "./scan.mjs" +import { discoverPackages, isApp } from "./packages.mjs" +import { readKeyTable, referencedNames } from "./keys.mjs" +import { buildPack } from "./pack.mjs" +import { emitImgG } from "./emit.mjs" +import { createDiagnostics } from "./diagnostics.mjs" +import { checkPins } from "./pins.mjs" +import { encodeImage } from "./codec.mjs" + +// Normalize options into a fully-defaulted config. +function resolveConfig(options) { + const root = options.root + if (!root) throw new Error("runImageGen: options.root is required") + const outName = options.outName || "img.g.ts" + return { + root, + outName, + outPath: options.outPath || join(root, outName), + keysPath: options.keysPath || join(root, "img.keys.json"), + // An image to keep out of the pack because the pack falls back to it. + fallback: options.fallback || null, + pins: options.pins !== false, + write: options.write !== false, + verbose: !!options.verbose, + log: options.log || (s => console.log(s)), + } +} + +// Every compiled source in the graph, read once, with what each one declares. +function readGraph(cfg) { + const packages = discoverPackages(cfg.root, cfg.outName) + const sources = [] + const declared = [] + const literals = [] + for (const pkg of packages) { + for (const rel of pkg.files) { + const path = join(pkg.dir, rel) + if (!existsSync(path)) continue + const text = readFileSync(path, "utf8") + sources.push({ pkg, rel, text }) + for (const img of scanSource(text, rel)) declared.push({ ...img, pkg }) + for (const lit of scanLiterals(text, rel)) literals.push({ ...lit, pkg }) + } + } + return { packages, sources, declared, literals } +} + +// The qualified name is an image's identity: unique, because a namespace +// cannot declare the same const twice. +function qualified(img) { + return img.namespace ? img.namespace + "." + img.name : img.name +} + +// Index every declared image twice: by its bare name, where an app declaration +// wins over a dependency's, and by its qualified name, which a key table uses +// to say exactly which image it means when a bare name is ambiguous. +function indexImages(graph, diag, keysName) { + const byBare = new Map() + const byQualified = new Map() + const all = [] + for (const img of graph.declared) { + const q = qualified(img) + const prior = byQualified.get(q) + if (prior) { + const where = `${prior.file}:${prior.line} and ${img.file}:${img.line}` + const text = `${q} is declared twice in ${img.pkg.name}: ${where}` + isApp(img.pkg) ? diag.error(text) : diag.info(text) + continue + } + byQualified.set(q, img) + all.push(img) + + const held = byBare.get(img.name) + if (!held) { + byBare.set(img.name, img) + continue + } + const winner = isApp(img.pkg) ? img : held + const loser = winner === img ? held : img + byBare.set(img.name, winner) + const w = qualified(winner) + const l = qualified(loser) + const text = + `"${img.name}" matches both ${w} and ${l}. A bare "${img.name}" in ${keysName} ` + + `resolves to ${w}; to use ${l} instead, write "${l}".` + // When exactly one side is the app's, a bare name resolves to the image + // the app itself declared, which is what its author expects. + if (isApp(winner.pkg) !== isApp(loser.pkg)) diag.detail(text) + else diag.info(text) + } + return { byBare, byQualified, all } +} + +// A key table entry is an image's own name, or its qualified name when that is +// ambiguous. The exact name wins, so a `packable="a.b"` override still resolves. +function resolveImage(index, name) { + return index.byBare.get(name) || index.byQualified.get(name) || null +} + +// An image whose const is still used somewhere would be copied by packing +// rather than moved, so it stays a literal. A const name is unique within its +// namespace, so its own declaration is the single use expected. +function isReferenced(graph, img) { + let uses = 0 + for (const s of graph.sources) uses += countReferences(s.text, img.constName, img.namespace) + return uses > 1 +} + +// Why an image cannot be packed regardless of the key table, as a note to +// report, or null when nothing prevents it. +function exclusion(cfg, graph, img) { + const where = `${img.pkg.name} ${img.file}:${img.line}` + if (cfg.fallback && (img.name === cfg.fallback || qualified(img) === cfg.fallback)) + return { severity: "info", text: `${qualified(img)} is the pack's fallback image, so it stays a literal` } + if (!img.whenUsed) + return { + severity: isApp(img.pkg) ? "warning" : "info", + text: `${where}: ${qualified(img)} is //% packable but not //% whenUsed, so packing it would copy it rather than move it`, + } + if (isReferenced(graph, img)) + return { severity: "info", text: `${where}: ${qualified(img)} is still referenced as ${img.constName}, so it stays a literal` } + return null +} + +// The images the app can pack, with a note for each one it cannot. +function selectPackable(cfg, graph, index, wanted, diag) { + const packable = [] + for (const img of index.all) { + if (!isApp(img.pkg) && !wanted.has(img)) continue + const why = exclusion(cfg, graph, img) + if (why) { + diag[why.severity](why.text) + continue + } + // A packed image is reached only through a key, so one without a key + // would take up space in the pack with no way to draw it. + if (!wanted.has(img)) { + diag.warning(`${img.pkg.name} ${img.file}:${img.line}: ${qualified(img)} is //% packable but no key in the key table reaches it, so it is left out`) + continue + } + packable.push(img) + } + return packable +} + +// A key table for an app that has none: one string key per image of the app's +// that can be packed, named after the image. Images that could not be packed +// are left out, so the table never names an image the next run rejects. +function starterKeyTable(cfg, graph, index) { + const names = {} + for (const img of index.all) { + if (!isApp(img.pkg) || exclusion(cfg, graph, img)) continue + // A bare name that resolves elsewhere would key the wrong image. + const name = index.byBare.get(img.name) === img ? img.name : qualified(img) + names[name] = name + } + return { numeric: {}, names } +} + +// Reports each packed image whose pixels also ship as a plain literal, with +// the bytes that costs. +function reportDuplicates(graph, packed, diag) { + const annotated = new Set(graph.declared.map(d => `${d.pkg.name}/${d.file}/${d.constName}`)) + const plain = graph.literals.filter( + l => !annotated.has(`${l.pkg.name}/${l.file}/${l.constName}`), + ) + const contentOf = new Map() + for (const lit of plain) { + const key = encodeImage(lit.rows).toString("hex") + if (!contentOf.has(key)) contentOf.set(key, []) + contentOf.get(key).push(lit) + } + + for (const e of packed.entries) { + if (e.shared) continue + const twins = contentOf.get(e.f4.toString("hex")) || [] + for (const twin of twins) { + const where = `${twin.pkg.name} ${twin.file}:${twin.line}` + const text = `packed "${e.name}" has the same pixels as ${twin.constName} at ${where}, costing ${e.rawBytes} B twice` + isApp(twin.pkg) ? diag.warning(text) : diag.info(text) + } + } + + // Identical images between two dependencies, with the app's pack holding + // neither, are reported only when asked for. + for (const [, group] of contentOf) { + const deps = group.filter(l => !isApp(l.pkg)) + if (deps.length > 1 && deps.length === group.length) + diag.detail( + `identical images in dependencies: ` + + deps.map(l => `${l.pkg.name}/${l.constName}`).join(", "), + ) + } +} + +/** + * Generates the app's image pack and img.g.ts. + * + * Returns everything it decided -- the pack, the lookups, the images it left + * out and why, and the emitted source -- so a wrapper or a test can inspect the + * result without reading the file back. + */ +export function runImageGen(options) { + const cfg = resolveConfig(options) + const diag = createDiagnostics(cfg.verbose) + + if (cfg.pins) { + const pin = checkPins(cfg.root, options.pinNames) + if (!pin.ok) diag.error(pin.reason) + else if (pin.reason) diag.detail(`pin check skipped: ${pin.reason}`) + } + + const graph = readGraph(cfg) + const index = indexImages(graph, diag, basename(cfg.keysPath)) + + let keys + if (!existsSync(cfg.keysPath)) { + keys = starterKeyTable(cfg, graph, index) + const count = Object.keys(keys.names).length + if (count > 0) { + if (cfg.write) writeFileSync(cfg.keysPath, JSON.stringify(keys, null, 4) + "\n") + diag.info( + `no key table at ${cfg.keysPath}; ${cfg.write ? "wrote" : "would write"} a starter ` + + `with ${count} string key(s), one per image that can be packed`, + ) + // The starter is a guess at the app's keys, so point out what it + // cannot know. + if (cfg.write) + diag.info( + `review ${cfg.keysPath} before relying on it: each string key is the image's own ` + + `name, which may not be the key your code passes to _img.get; it has no numeric ` + + `keys; and it lists every app image that can be packed, including any your app ` + + `never draws`, + ) + } + } else { + try { + keys = readKeyTable(cfg.keysPath) + } catch (e) { + diag.error(e.message) + diag.flush(cfg.log) + diag.check() + } + } + const wanted = new Set() + for (const name of [...referencedNames(keys)].sort()) { + const img = resolveImage(index, name) + if (img) wanted.add(img) + else diag.error(`key table names "${name}", which no package declares`) + } + + const packable = selectPackable(cfg, graph, index, wanted, diag) + const indexOf = new Map(packable.map((img, i) => [img, i])) + for (const img of wanted) + if (!indexOf.has(img)) + diag.error(`key table names ${qualified(img)}, which cannot be packed (see the note above)`) + diag.flush(cfg.log) + diag.check() + + const pack = buildPack(packable.map(img => ({ name: qualified(img), rows: img.rows }))) + reportDuplicates(graph, pack, diag) + diag.flush(cfg.log) + diag.check() + + // The key table declares every string key the app looks an image up by. + const names = {} + for (const [key, value] of Object.entries(keys.names)) + names[key] = indexOf.get(resolveImage(index, value)) + const numeric = {} + for (const [key, value] of Object.entries(keys.numeric)) + numeric[Number(key)] = indexOf.get(resolveImage(index, value)) + + const files = [...new Set(packable.map(i => `${i.pkg.name}/${i.file}`))] + const source = emitImgG(pack, names, numeric, files) + if (cfg.write) writeFileSync(cfg.outPath, source) + + report(cfg, pack, names, numeric, diag) + return { pack, names, numeric, source, images: packable, diagnostics: diag.notes, config: cfg } +} + +function report(cfg, pack, names, numeric, diag) { + diag.flush(cfg.log) + + const s = pack.stats + const packTotal = s.packBytes + s.offsetBytes + const saved = s.rawBytes - packTotal + const pct = s.rawBytes > 0 ? ((saved / s.rawBytes) * 100).toFixed(1) : "0.0" + cfg.log( + `img-gen: ${s.images} images (${s.unique} unique, ${s.shared} shared), ` + + `${s.compressed} compressed, ${s.uncompressed} uncompressed; ` + + `${s.rawBytes} B -> ${packTotal} B (${saved} B saved, ${pct}%); ` + + `${s.decodedWorstCase} B RAM if all decoded`, + ) + // Approximate flash cost of the emitted name index. + const stringKeys = Object.keys(names) + const indexBytes = stringKeys.join(",").length + stringKeys.length * (pack.entries.length > 255 ? 2 : 1) + cfg.log( + `img-gen: name index holds ${stringKeys.length} key(s), about ${indexBytes} B`, + ) + if (!cfg.verbose) return + + cfg.log(`img-gen: wrote ${cfg.outPath}`) + cfg.log(`img-gen: ${Object.keys(numeric).length} numeric key(s)`) + for (const e of pack.entries) { + if (e.shared) { + cfg.log(` ${e.name}: shares the record of ${e.shared}`) + continue + } + const how = e.uncompressed ? "uncompressed" : "compressed" + cfg.log(` ${e.name}: ${e.rawBytes} B -> ${e.recordBytes} B ${how} @${e.offset}`) + } +} diff --git a/img-tools/index.mjs b/img-tools/index.mjs new file mode 100644 index 0000000..0916c4f --- /dev/null +++ b/img-tools/index.mjs @@ -0,0 +1,20 @@ +// Public image compression tooling API. +// +// The codec works with the F4 buffers that `bmp` literals compile to: an +// 8-byte header followed by 4-bit pixel data, column by column. +// +// A consuming app builds a thin wrapper over runImageGen, supplying its root +// and any app-specific keys, and calls runImageCheck after a build to see what +// the pack duplicates. The codec, scanner, package discovery, key table reader, +// packer, emitter and pin check are exposed for wrappers and tests that need +// the pieces directly. + +export { encodeImage, encodeRecord, decodeRecord, isUncompressed, recordLength } from "./codec.mjs" +export { runImageGen } from "./gen.mjs" +export { runImageCheck, imagesInBuild, findBuildOutputs } from "./verify.mjs" +export { scanSource, scanLiterals, countReferences, stripBlockComments } from "./scan.mjs" +export { discoverPackages } from "./packages.mjs" +export { readKeyTable, emptyKeyTable, referencedNames } from "./keys.mjs" +export { buildPack } from "./pack.mjs" +export { emitImgG } from "./emit.mjs" +export { checkPins } from "./pins.mjs" diff --git a/img-tools/keys.mjs b/img-tools/keys.mjs new file mode 100644 index 0000000..37a8d70 --- /dev/null +++ b/img-tools/keys.mjs @@ -0,0 +1,69 @@ +// The app's key table: the keys it looks images up by, and nothing else. +// +// Shape (both sections optional): +// +// { +// "numeric": { "16": "thermometer", "180": "thermometer" }, +// "names": { "delete": "btn_delete" } +// } +// +// `numeric` maps the app's own small-integer keys to image names, for a key +// space the app already has and cannot renumber. `names` lists every string key +// the app looks an image up by, including one that equals its image's name +// (`"sad": "sad"`); the emitted name index holds exactly the keys listed here. +// +// An image is named by the const that declares it. Where two packages declare +// the same const name, write the namespace too (`"ui.microphone"`) to say which +// one is meant; a bare name resolves to the app's. +// +// The table holds no pixels. Images are declared where they are drawn, with +// `//% packable`, and the two are joined by name. + +import { existsSync, readFileSync } from "node:fs" + +/** An empty key table: the app declares no keys of its own. */ +export function emptyKeyTable() { + return { numeric: {}, names: {} } +} + +/** + * Reads and validates a key table, returning an empty one when the file does + * not exist. Throws on a malformed table, naming the entry at fault. + */ +export function readKeyTable(path) { + if (!existsSync(path)) return emptyKeyTable() + + let raw + try { + raw = JSON.parse(readFileSync(path, "utf8")) + } catch (e) { + throw new Error(`key table ${path} is not valid JSON: ${e.message}`) + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) + throw new Error(`key table ${path} must be a JSON object`) + + const table = emptyKeyTable() + for (const [key, value] of Object.entries(raw.numeric || {})) { + if (!/^\d+$/.test(key)) + throw new Error(`key table ${path}: numeric key "${key}" is not a non-negative integer`) + if (typeof value !== "string" || value.length === 0) + throw new Error(`key table ${path}: numeric key ${key} must name an image`) + table.numeric[Number(key)] = value + } + for (const [key, value] of Object.entries(raw.names || {})) { + if (key.indexOf(",") >= 0) + throw new Error(`key table ${path}: string key "${key}" may not contain a comma`) + if (typeof value !== "string" || value.length === 0) + throw new Error(`key table ${path}: string key "${key}" must name an image`) + table.names[key] = value + } + return table +} + +/** Every image name the table refers to, deduplicated. */ +export function referencedNames(table) { + const names = new Set() + for (const n of Object.values(table.numeric)) names.add(n) + for (const n of Object.values(table.names)) names.add(n) + return names +} diff --git a/img-tools/pack.mjs b/img-tools/pack.mjs new file mode 100644 index 0000000..0778ba6 --- /dev/null +++ b/img-tools/pack.mjs @@ -0,0 +1,102 @@ +// Pack assembly: records laid end to end, with a u16le offset per image index. +// +// Images with identical pixels share one record and one offset. +// +// Every record is decoded back out of the assembled blob and compared against +// the image it came from, so a mispacked blob fails the build rather than the +// device. + +import { encodeImage, encodeRecord, decodeRecord, isUncompressed, recordLength } from "./codec.mjs" + +const OFFSET_LIMIT = 0x10000 + +/** + * Builds a pack from `images`, an array of { name, rows } in index order. + * + * Returns { blob, offsets, entries, indexByName, stats }. `entries` carries one + * record per index, with `shared` naming the earlier image it reuses, if any. + * `stats` totals the counts and byte sizes for reporting. + */ +export function buildPack(images) { + const chunks = [] + const entries = [] + const indexByName = new Map() + const offsetByContent = new Map() + let off = 0 + + images.forEach((img, index) => { + if (indexByName.has(img.name)) + throw new Error(`image "${img.name}" is declared more than once`) + if (img.name.indexOf(",") >= 0) + throw new Error(`image name "${img.name}" may not contain a comma`) + + const f4 = encodeImage(img.rows) + const key = f4.toString("hex") + const seen = offsetByContent.get(key) + if (seen !== undefined) { + entries.push({ ...img, index, offset: seen.offset, f4, uncompressed: seen.uncompressed, + recordBytes: 0, rawBytes: f4.length, shared: seen.name }) + indexByName.set(img.name, index) + return + } + + let record + try { + record = encodeRecord(f4) + } catch (e) { + throw new Error(`encoding "${img.name}" failed: ${e.message}`) + } + const uncompressed = isUncompressed(record) + offsetByContent.set(key, { offset: off, uncompressed, name: img.name }) + entries.push({ ...img, index, offset: off, f4, uncompressed, + recordBytes: record.length, rawBytes: f4.length, shared: null }) + indexByName.set(img.name, index) + chunks.push(record) + off += record.length + }) + + if (off >= OFFSET_LIMIT) + throw new Error(`pack is ${off} bytes, past the ${OFFSET_LIMIT}-byte range of a u16 offset table`) + + const blob = Buffer.concat(chunks) + const offsets = Buffer.alloc(2 * entries.length) + entries.forEach((e, i) => offsets.writeUInt16LE(e.offset, 2 * i)) + + verifyPack(blob, entries) + + const unique = entries.filter(e => !e.shared) + const stats = { + images: entries.length, + unique: unique.length, + shared: entries.length - unique.length, + uncompressed: unique.filter(e => e.uncompressed).length, + compressed: unique.filter(e => !e.uncompressed).length, + rawBytes: unique.reduce((n, e) => n + e.rawBytes, 0), + packBytes: blob.length, + offsetBytes: offsets.length, + // RAM cost once every image is decoded; ui.ImagePack never evicts. + decodedWorstCase: unique.reduce((n, e) => n + e.f4.length, 0), + } + return { blob, offsets, entries, indexByName, stats } +} + +/** Decodes every record out of the assembled blob and compares it to its source. */ +function verifyPack(blob, entries) { + for (const e of entries) { + let back + try { + back = decodeRecord(blob, e.offset) + } catch (err) { + throw new Error(`"${e.name}" does not decode at offset ${e.offset}: ${err.message}`) + } + const expected = Buffer.from(e.f4) + expected.writeUInt16LE(0, 6) + if (!back.equals(expected)) + throw new Error(`"${e.name}" decodes to different pixels at offset ${e.offset}`) + if (!e.shared) { + const len = recordLength(blob, e.offset) + if (len !== e.recordBytes) + throw new Error(`"${e.name}" record is ${e.recordBytes} bytes but walks as ${len}`) + } + } +} diff --git a/img-tools/packages.mjs b/img-tools/packages.mjs new file mode 100644 index 0000000..175983b --- /dev/null +++ b/img-tools/packages.mjs @@ -0,0 +1,49 @@ +// Package discovery: the app and every package it compiles against. +// +// Dependencies resolve under pxt_modules and are visited in transitive order, +// a package before anything that depends on it, with the app last. Each is +// { name, dir, files }, where `files` is the package's compiled sources -- +// testFiles are excluded, since an image declared in one never ships. + +import { existsSync, readFileSync } from "node:fs" +import { join } from "node:path" + +function readPxt(dir) { + const path = join(dir, "pxt.json") + if (!existsSync(path)) return null + return JSON.parse(readFileSync(path, "utf8")) +} + +function compiledFiles(pxt, skip) { + return (pxt.files || []).filter(f => f.endsWith(".ts") && f !== skip) +} + +/** + * The packages an app at `root` compiles against, dependencies first and the + * app last under the name "app". `skip` names a file to leave out of every + * package's file list, for the generated file itself. + */ +export function discoverPackages(root, skip) { + const modulesDir = join(root, "pxt_modules") + const seen = new Set() + const packages = [] + const visit = name => { + if (seen.has(name)) return + seen.add(name) + const dir = join(modulesDir, name) + const pxt = readPxt(dir) + if (!pxt) return + for (const dep of Object.keys(pxt.dependencies || {})) visit(dep) + packages.push({ name, dir, files: compiledFiles(pxt, skip) }) + } + const appPxt = readPxt(root) + if (!appPxt) throw new Error(`no pxt.json at ${root}`) + for (const dep of Object.keys(appPxt.dependencies || {})) visit(dep) + packages.push({ name: "app", dir: root, files: compiledFiles(appPxt, skip) }) + return packages +} + +/** True for the package holding the app's own sources. */ +export function isApp(pkg) { + return pkg.name === "app" +} diff --git a/img-tools/pins.mjs b/img-tools/pins.mjs new file mode 100644 index 0000000..bb62bca --- /dev/null +++ b/img-tools/pins.mjs @@ -0,0 +1,57 @@ +// Pin check for the package shipping both the build tooling and the device +// runtime. +// +// The encoder runs from the npm install and the decoder ships from the pxt +// module, so a build is only sound when both resolve to the same version of +// the package. Resolved versions are compared, not the specs that produced +// them: a linked development setup legitimately carries different specs that +// resolve to one checkout. + +import { existsSync, readFileSync } from "node:fs" +import { dirname, join } from "node:path" + +function versionOf(dir) { + const path = join(dir, "pxt.json") + if (!existsSync(path)) return null + return JSON.parse(readFileSync(path, "utf8")).version || null +} + +// npm may hoist an install to any node_modules above the app. +function findNpmDir(root, npmName) { + let dir = root + for (;;) { + const candidate = join(dir, "node_modules", npmName) + if (existsSync(candidate)) return candidate + const parent = dirname(dir) + if (parent === dir) return null + dir = parent + } +} + +/** + * Compares the versions an app at `root` resolves for the shared package. + * + * Returns { ok, reason, npm, pxt }. `ok` is false only when both sides were + * found and disagree; when either is missing there is nothing to compare and + * `reason` says which. + */ +export function checkPins(root, options = {}) { + const pxtName = options.pxtName || "ui-core" + const npmName = options.npmName || "@microbit-apps/ui-core" + + const pxtDir = join(root, "pxt_modules", pxtName) + const npmDir = findNpmDir(root, npmName) + const pxt = existsSync(pxtDir) ? versionOf(pxtDir) : null + const npm = npmDir ? versionOf(npmDir) : null + + if (!pxt) return { ok: true, reason: `no pxt module ${pxtName}`, npm, pxt } + if (!npm) return { ok: true, reason: `no npm install of ${npmName}`, npm, pxt } + if (npm !== pxt) + return { + ok: false, + reason: `${npmName} resolves to ${npm} for build tooling but ${pxt} for device code`, + npm, + pxt, + } + return { ok: true, reason: null, npm, pxt } +} diff --git a/img-tools/scan.mjs b/img-tools/scan.mjs new file mode 100644 index 0000000..c7a6190 --- /dev/null +++ b/img-tools/scan.mjs @@ -0,0 +1,180 @@ +// Source scanning for packable images. +// +// A packable image is an `export const = bmp` literal carrying a +// `//% packable` attribute in the comment block directly above it. The image +// is named by its const, so a key table entry and a declaration never repeat +// the same string. +// +// `//% packable="name"` overrides that name. It exists so a package can keep a +// name stable while renaming the const behind it -- a library cannot edit the +// key tables of the apps that use it. An app aliasing its own key to an image +// belongs in its key table instead. +// +// Images inside a /* */ block are ignored, matching what the compiler sees. +// +// The same source is also read two other ways: for every `bmp` literal a file +// declares, annotated or not, and for whether an image's const is referenced +// by name anywhere, which decides whether packing it would move it or copy it. + +// One `//%` attribute block followed by an exported bmp literal. +const DECL = + /((?:[ \t]*\/\/%[^\n]*\n)+)[ \t]*export[ \t]+const[ \t]+(\w+)[ \t]*=[ \t]*bmp`([^`]*)`/g + +// Any exported bmp literal, with or without attributes. +const LITERAL = /export[ \t]+const[ \t]+(\w+)[ \t]*=[ \t]*bmp`([^`]*)`/g + +// The namespace a declaration sits in, for reports that name the const. +const NAMESPACE = /\bnamespace[ \t]+([\w.]+)/g + +/** + * Blanks out comments, keeping every newline so line numbers still match the + * file on disk. A "/*" or "//" inside a string or template is left alone. + * + * `opts.lineComments` also blanks `//` comments, which discovery needs to keep + * because `//%` attributes live in them. `opts.strings` blanks the text inside + * quotes, which reference counting needs: a name in string data is not a use + * of the const that shares it. Interpolations in a template are kept, since + * those hold real code. + */ +function blankComments(src, opts) { + const lineComments = !!(opts && opts.lineComments) + const strings = !!(opts && opts.strings) + let out = "" + let i = 0 + const blank = text => { + for (const ch of text) out += ch === "\n" ? "\n" : " " + } + while (i < src.length) { + const c = src[i] + const d = src[i + 1] + if (c === "/" && d === "*") { + const end = src.indexOf("*/", i + 2) + const stop = end < 0 ? src.length : end + 2 + blank(src.slice(i, stop)) + i = stop + } else if (c === "/" && d === "/") { + const end = src.indexOf("\n", i) + const stop = end < 0 ? src.length : end + if (lineComments) blank(src.slice(i, stop)) + else out += src.slice(i, stop) + i = stop + } else if (c === '"' || c === "'" || c === "`") { + let j = i + 1 + while (j < src.length && src[j] !== c) j += src[j] === "\\" ? 2 : 1 + const stop = Math.min(j + 1, src.length) + const body = src.slice(i + 1, stop - 1) + if (!strings) out += src.slice(i, stop) + else { + out += c + // Keep ${...} in a template: it is code, not text. + for (const part of body.split(/(\$\{[^}]*\})/g)) { + if (c === "`" && part.startsWith("${")) out += part + else blank(part) + } + out += stop - 1 > i ? c : "" + } + i = stop + } else { + out += c + i++ + } + } + return out +} + +/** Source with block comments blanked, leaving `//%` attributes readable. */ +export function stripBlockComments(src) { + return blankComments(src, {}) +} + +/** + * Reads `//% packable` from an attribute block: null when absent, otherwise + * the name it carries, or "" for the bare form, which names by the const. + */ +function packableAttr(attrs) { + // pxt allows several attributes on one `//%` line, so `packable` may + // follow another one: `//% whenUsed packable`. + const m = /\/\/%(?:[ \t]*|[^\n]*?[ \t])packable\b(\s*=\s*("([^"]*)"|'([^']*)'|(\S+)))?/.exec(attrs) + if (!m) return null + if (!m[1]) return "" + const raw = m[3] !== undefined ? m[3] : m[4] !== undefined ? m[4] : m[5] + return (raw || "").trim() +} + +/** The innermost namespace opened before `index`, or null. */ +function namespaceAt(src, index) { + let ns = null + let m + NAMESPACE.lastIndex = 0 + while ((m = NAMESPACE.exec(src)) && m.index < index) ns = m[1] + return ns +} + +/** + * Annotated images in one source file, in declaration order. Each is + * { name, constName, namespace, rows, whenUsed, file, line } -- `name` is the + * const's own name unless `packable=` overrode it, and `rows` the pixel rows + * as written. + */ +export function scanSource(src, file) { + const clean = stripBlockComments(src) + const found = [] + let m + DECL.lastIndex = 0 + while ((m = DECL.exec(clean))) { + const declared = packableAttr(m[1]) + if (declared === null) continue + found.push({ + name: declared || m[2], + constName: m[2], + namespace: namespaceAt(clean, m.index), + rows: m[3], + whenUsed: /\/\/%[^\n]*\bwhenUsed\b/.test(m[1]), + file, + line: clean.slice(0, m.index).split("\n").length, + }) + } + return found +} + +/** + * Every exported `bmp` literal in one source file, annotated or not, as + * { constName, rows, file, line }. Used to compare a pack against the images + * that still ship as literals. + */ +export function scanLiterals(src, file) { + const clean = stripBlockComments(src) + const found = [] + let m + LITERAL.lastIndex = 0 + while ((m = LITERAL.exec(clean))) + found.push({ + constName: m[1], + rows: m[2], + file, + line: clean.slice(0, m.index).split("\n").length, + }) + return found +} + +/** + * Counts references to a const named `name` declared in `namespace ns`, + * ignoring comments and the text inside quotes. + * + * A reference is `ns.name` from anywhere, or a bare `name` in a file that opens + * that namespace -- so `music.playTone()` is not a use of `icondb.music`. The + * const's own declaration is one such bare use, which is why a caller compares + * the total against 1 rather than 0. Pass no `ns` to count bare uses anywhere. + */ +export function countReferences(src, name, ns) { + const code = blankComments(src, { lineComments: true, strings: true }) + const escaped = ns ? ns.replace(/\./g, "\\.") : null + let total = 0 + if (ns) { + const qualified = code.match(new RegExp("\\b" + escaped + "\\." + name + "\\b", "g")) + total += qualified ? qualified.length : 0 + if (!new RegExp("\\bnamespace\\s+" + escaped + "\\b").test(code)) return total + } + const bare = code.match(new RegExp("(? { + const root = app() + try { + const r = imgGen(root) + assert.equal(r.code, 0, r.err) + assert.match(r.out, /wrote a starter with 1 string key/) + assert.match(r.out, /img-gen: 1 images/) + assert.ok(existsSync(join(root, "img.g.ts"))) + assert.ok(existsSync(join(root, "img.keys.json"))) + } finally { + removeProject(root) + } +}) + +test("img.config.json sets the key table path and output name", () => { + const root = app({ + "img.config.json": { keysPath: "config/keys.json", outName: "images.g.ts" }, + "config/keys.json": { names: { save: "disk" } }, + }) + try { + const r = imgGen(root) + assert.equal(r.code, 0, r.err) + assert.ok(existsSync(join(root, "images.g.ts"))) + assert.ok(!existsSync(join(root, "img.g.ts"))) + assert.match(readFileSync(join(root, "images.g.ts"), "utf8"), /const KEYS = "save"/) + } finally { + removeProject(root) + } +}) + +test("an unknown config setting is an error, not ignored", () => { + const root = app({ "img.config.json": { nameKeys: "all" } }) + try { + const r = imgGen(root) + assert.equal(r.code, 1) + assert.match(r.err, /unknown setting\(s\) nameKeys; expected keysPath, outName, fallback, pins/) + } finally { + removeProject(root) + } +}) + +test("a malformed config file is an error", () => { + const root = app({ "img.config.json": "{ nope" }) + try { + const r = imgGen(root) + assert.equal(r.code, 1) + assert.match(r.err, /img\.config\.json is not valid JSON/) + } finally { + removeProject(root) + } +}) + +test("an unknown argument is an error", () => { + const root = app() + try { + const r = imgGen(root, "--nope") + assert.equal(r.code, 1) + assert.match(r.err, /unknown argument\(s\) --nope/) + } finally { + removeProject(root) + } +}) + +test("a failing run exits 1 with the problems and no stack trace", () => { + const root = app({ "img.keys.json": { names: { a: "no_such_image" } } }) + try { + const r = imgGen(root) + assert.equal(r.code, 1) + assert.match(r.err, /img-gen: 1 problem\(s\)/) + assert.match(r.err, /no_such_image/) + assert.doesNotMatch(r.err, /\n\s+at /) + } finally { + removeProject(root) + } +}) + +test("--verbose adds the per-image breakdown", () => { + const root = app({ "img.keys.json": { names: { disk: "disk" } } }) + try { + const r = imgGen(root, "--verbose") + assert.equal(r.code, 0, r.err) + assert.match(r.out, /icons\.disk: 40 B -> \d+ B compressed @0/) + } finally { + removeProject(root) + } +}) + +test("--check reports duplicates in the last build without regenerating", () => { + const f4 = encodeImage(ROWS) + const asm = `_hexlit1:\n .word pxt::buffer_vt\n .word ${f4.length}\n .hex ${f4.toString("hex")}\n` + const root = app({ "img.keys.json": { names: { disk: "disk" } }, "built/out.asm": asm }) + try { + const r = imgGen(root, "--check") + assert.equal(r.code, 0, r.err) + assert.match(r.out, /img-check: 1 packed image\(s\) also ship as buffers/) + assert.ok(!existsSync(join(root, "img.g.ts"))) + } finally { + removeProject(root) + } +}) diff --git a/img-tools/test/codec.test.mjs b/img-tools/test/codec.test.mjs new file mode 100644 index 0000000..705ae7e --- /dev/null +++ b/img-tools/test/codec.test.mjs @@ -0,0 +1,121 @@ +import { test } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { fileURLToPath } from "node:url" +import { dirname, join } from "node:path" + +import { encodeImage, encodeRecord, decodeRecord, isUncompressed, recordLength } from "../codec.mjs" +import { scanLiterals } from "../scan.mjs" +import { seeded } from "./helpers.mjs" + +const uiCore = join(dirname(fileURLToPath(import.meta.url)), "..", "..") +const hex = s => Buffer.from(s.replace(/\s+/g, ""), "hex") + +// The image the compiler would emit, with padding zeroed, which is what a +// record decodes back to. +const zeroPadding = f4 => { + const b = Buffer.from(f4) + b.writeUInt16LE(0, 6) + return b +} + +test("encodeImage lays out pixels as the compiler does", () => { + // Two columns of two rows: each column is padded to 4 bytes, and within a + // column the lower row takes the low nibble. + assert.deepEqual(encodeImage(["1 2", "3 4"]), hex("87 04 0200 0200 0000 31000000 42000000")) +}) + +test("encodeImage pads each column to 32 bits", () => { + // Nine rows need 36 bits, so each column takes 8 bytes. + const rows = [".", ".", ".", ".", ".", ".", ".", ".", "f"] + const f4 = encodeImage(rows) + assert.equal(f4.length, 8 + 8) + assert.equal(f4[8 + 4], 0x0f) +}) + +test("encodeImage accepts rows as a string, ignores whitespace, and pads ragged rows", () => { + assert.deepEqual(encodeImage("1 2\n3 4\n"), encodeImage(["12", "34"])) + assert.deepEqual(encodeImage(["12", "3"]), encodeImage(["12", "3."])) +}) + +test("encodeImage rejects an empty image and a bad pixel", () => { + assert.throws(() => encodeImage([]), /no rows/) + assert.throws(() => encodeImage(["1g"]), /bad pixel g/) +}) + +test("encodeRecord compresses when it helps, marking the record", () => { + const f4 = encodeImage(Array(16).fill(". . . . . . . . . . . . . . . .")) + const record = encodeRecord(f4) + assert.ok(record.length < f4.length) + assert.equal(record.readUInt16LE(6), 0xc001) + assert.equal(isUncompressed(record), false) +}) + +test("encodeRecord leaves an image that does not compress uncompressed and unchanged", () => { + const row = y => Array.from({ length: 16 }, (_, x) => ((x + y) % 2 ? "1" : "2")).join("") + const f4 = encodeImage(Array.from({ length: 16 }, (_, y) => row(y))) + const record = encodeRecord(f4) + assert.equal(isUncompressed(record), true) + assert.deepEqual(record, f4) +}) + +test("encodeRecord rejects a buffer that is not an image", () => { + assert.throws(() => encodeRecord(Buffer.from([0x00, 0x04, 1, 0, 1, 0, 0, 0, 0])), /not an F4/) +}) + +test("records round-trip for many sizes and kinds of pixels", () => { + const rand = seeded(1) + const sizes = [[1, 1], [1, 9], [9, 1], [7, 7], [16, 16], [16, 17], [33, 5], [5, 33]] + for (const [w, h] of sizes) { + for (const kind of ["noise", "runs", "blank"]) { + let color = 0 + const rows = Array.from({ length: h }, () => + Array.from({ length: w }, () => { + if (kind === "blank") return "." + if (kind === "noise" || rand() < 0.1) color = Math.floor(rand() * 16) + return color.toString(16) + }).join(""), + ) + const f4 = encodeImage(rows) + const record = encodeRecord(f4) + assert.ok(record.length <= f4.length, `${w}x${h} ${kind}: record grew`) + assert.deepEqual(decodeRecord(record), zeroPadding(f4), `${w}x${h} ${kind}`) + assert.equal(recordLength(record), record.length, `${w}x${h} ${kind}: length`) + } + } +}) + +test("decodeRecord and recordLength read a record at an offset", () => { + const f4 = encodeImage(Array(8).fill("11112222")) + const record = encodeRecord(f4) + const blob = Buffer.concat([Buffer.from([0xff, 0xff, 0xff]), record]) + assert.deepEqual(decodeRecord(blob, 3), zeroPadding(f4)) + assert.equal(recordLength(blob, 3), record.length) +}) + +test("decodeRecord rejects what it cannot read", () => { + const record = encodeRecord(encodeImage(Array(8).fill("11111111"))) + const withMarker = m => { + const b = Buffer.from(record) + b.writeUInt16LE(m, 6) + return b + } + assert.throws(() => decodeRecord(withMarker(0xa000)), /not a record/) + assert.throws(() => decodeRecord(withMarker(0xc002)), /unknown method 2/) + assert.throws(() => decodeRecord(Buffer.from([0x12, 0x04, 1, 0, 1, 0, 0, 0])), /magic/) +}) + +// ui-core's device tests carry records made by this encoder. If the encoder +// or the art changes, those vectors are stale and the device tests stop +// meaning anything, so this checks them against what the encoder makes now. +test("the device test vectors match what the encoder produces", () => { + const testTs = readFileSync(join(uiCore, "test.ts"), "utf8") + const vector = name => hex(new RegExp(`const ${name} = hex${"`"}([0-9a-f]*)${"`"}`).exec(testTs)[1]) + + const icons = readFileSync(join(uiCore, "icons.ts"), "utf8") + const missing = scanLiterals(icons, "icons.ts").find(l => l.constName === "MISSING") + assert.deepEqual(encodeRecord(encodeImage(missing.rows)), vector("rleRecord")) + + const checker = /const checker = bmp`([^`]*)`/.exec(testTs)[1] + assert.deepEqual(encodeRecord(encodeImage(checker)), vector("uncompressedRecord")) +}) diff --git a/img-tools/test/gen.test.mjs b/img-tools/test/gen.test.mjs new file mode 100644 index 0000000..0c6ad4d --- /dev/null +++ b/img-tools/test/gen.test.mjs @@ -0,0 +1,276 @@ +import { test } from "node:test" +import assert from "node:assert/strict" +import { existsSync, readFileSync } from "node:fs" +import { join } from "node:path" + +import { runImageGen } from "../gen.mjs" +import { bmpConst, ns, makeProject, removeProject, captureLog } from "./helpers.mjs" + +const P = ["packable", "whenUsed"] + +// An app with two dependencies, exercising every way an image can be packed +// or left out. +function project(keys, extra = {}) { + return makeProject({ + "pxt.json": { + name: "app", + dependencies: { lib: "*", other: "*" }, + files: ["assets.ts", "draw.ts", "img.g.ts"], + testFiles: ["test.ts"], + }, + "assets.ts": ns( + "icondb", + bmpConst("own", ["1 2"], P), + bmpConst("referenced", ["3 4"], P), + bmpConst("loose", ["5 6"], ["packable"]), + bmpConst("twin_of_lib", ["7 7"], P), + bmpConst("no_key", ["d d"], P), + bmpConst("renamedSinceRelease", ["a a"], ['packable="stable_name"', "whenUsed"]), + bmpConst("microphone", ["b b"], P), + ), + "draw.ts": ns("app", " export function draw() { return icondb.referenced }"), + "test.ts": ns("t", bmpConst("in_a_test_file", ["9"], P)), + "pxt_modules/lib/pxt.json": { name: "lib", files: ["icons.ts"] }, + "pxt_modules/lib/icons.ts": ns( + "lib", + bmpConst("from_lib", ["8 8"], P), + bmpConst("lib_unwanted", ["a b"], P), + bmpConst("microphone", ["c c"], P), + bmpConst("shared_icon", ["e e"], P), + bmpConst("plainTwin", ["7 7"]), + bmpConst("sharedWithOther", ["c d"]), + ), + "pxt_modules/other/pxt.json": { name: "other", files: ["icons.ts"] }, + "pxt_modules/other/icons.ts": ns( + "other", + bmpConst("sharedWithLib", ["c d"]), + bmpConst("shared_icon", ["f f"], P), + ), + ...(keys ? { "img.keys.json": keys } : {}), + ...extra, + }) +} + +const KEYS = { + numeric: { 3: "own" }, + names: { + own: "own", + twin: "twin_of_lib", + borrowed: "from_lib", + stable: "stable_name", + mic_app: "microphone", + mic_lib: "lib.microphone", + }, +} + +function run(keys, options = {}, extra) { + const root = project(keys, extra) + const log = captureLog() + try { + const result = runImageGen({ root, write: false, log, ...options }) + return { result, log, error: null } + } catch (error) { + return { result: null, log, error } + } finally { + removeProject(root) + } +} + +const notes = (result, severity) => result.diagnostics.filter(n => n.severity === severity).map(n => n.text) +const packed = result => result.pack.entries.map(e => e.name).sort() + +test("packs the app's images and the dependency images its key table names", () => { + const { result, error } = run(KEYS) + assert.equal(error, null) + assert.deepEqual(packed(result), [ + "icondb.microphone", + "icondb.own", + "icondb.stable_name", + "icondb.twin_of_lib", + "lib.from_lib", + "lib.microphone", + ]) +}) + +test("keys resolve to the right images", () => { + const { result } = run(KEYS) + const at = i => result.pack.entries[i].name + assert.equal(at(result.numeric[3]), "icondb.own") + assert.equal(at(result.names.stable), "icondb.stable_name") + assert.equal(at(result.names.borrowed), "lib.from_lib") + assert.equal(at(result.names.mic_app), "icondb.microphone") + assert.equal(at(result.names.mic_lib), "lib.microphone") +}) + +test("each image left out is reported once, for its own reason", () => { + const { result } = run(KEYS) + const warnings = notes(result, "warning") + const infos = notes(result, "info") + assert.equal(warnings.length, 2) + assert.match(warnings.find(t => t.includes("loose")), /not \/\/% whenUsed/) + assert.match(warnings.find(t => t.includes("no_key")), /no key in the key table reaches it/) + assert.match(infos.find(t => t.includes("icondb.referenced")), /still referenced as referenced/) + assert.match(infos.find(t => t.includes("twin_of_lib")), /same pixels as plainTwin.*costing 16 B twice/) +}) + +test("images in testFiles are never discovered", () => { + const { result } = run(KEYS) + assert.ok(!result.diagnostics.some(n => n.text.includes("in_a_test_file"))) + assert.ok(!packed(result).some(n => n.includes("in_a_test_file"))) +}) + +test("a duplicate between two dependencies only shows when verbose", () => { + const quiet = run(KEYS) + assert.doesNotMatch(quiet.log.text(), /identical images in dependencies/) + const verbose = run(KEYS, { verbose: true }) + assert.match(verbose.log.text(), /identical images in dependencies: lib\/sharedWithOther, other\/sharedWithLib/) +}) + +test("a key naming an image nothing declares fails the run", () => { + const { error, log } = run({ names: { a: "no_such_image" } }) + assert.match(error.message, /1 problem\(s\)/) + assert.match(error.message, /"no_such_image", which no package declares/) + assert.match(log.text(), /no_such_image/) +}) + +test("a key naming an image that cannot be packed fails, after saying why", () => { + const { error, log } = run({ names: { a: "loose" } }) + assert.match(error.message, /icondb\.loose, which cannot be packed/) + assert.match(log.text(), /warning: .*icondb\.loose is \/\/% packable but not \/\/% whenUsed/) +}) + +test("every problem is reported at once", () => { + const { error } = run({ names: { a: "nope1", b: "nope2" } }) + assert.match(error.message, /2 problem\(s\)/) + assert.match(error.message, /nope1/) + assert.match(error.message, /nope2/) +}) + +test("mismatched ui-core versions fail the run", () => { + const { error } = run(KEYS, {}, { + "node_modules/@microbit-apps/ui-core/pxt.json": { version: "0.0.8" }, + "pxt_modules/ui-core/pxt.json": { version: "0.0.9" }, + }) + assert.match(error.message, /0\.0\.8 for build tooling but 0\.0\.9 for device code/) +}) + +test("the same image declared twice in the app fails the run", () => { + const { error } = run(KEYS, {}, { "draw.ts": ns("icondb", bmpConst("own", ["f"], P)) }) + assert.match(error.message, /icondb\.own is declared twice in app/) +}) + +test("the fallback image stays out of the pack", () => { + const { result } = run(KEYS, { fallback: "no_key" }) + assert.ok(!packed(result).includes("icondb.no_key")) + assert.match(notes(result, "info").join("\n"), /icondb\.no_key is the pack's fallback image/) + assert.ok(!notes(result, "warning").some(t => t.includes("no_key"))) +}) + +test("the generated file is written only when asked, and matches the result", () => { + const root = project(KEYS) + try { + runImageGen({ root, write: false, log: () => {} }) + assert.equal(existsSync(join(root, "img.g.ts")), false) + const result = runImageGen({ root, log: () => {} }) + assert.equal(readFileSync(join(root, "img.g.ts"), "utf8"), result.source) + assert.match(result.source, /^namespace _img \{$/m) + assert.match(result.source, /\/\/ {3}app\/assets\.ts/) + assert.match(result.source, /\/\/ {3}lib\/icons\.ts/) + } finally { + removeProject(root) + } +}) + +test("a run prints a one-line summary", () => { + const { log } = run(KEYS) + assert.match(log.text(), /img-gen: 6 images \(6 unique, 0 shared\)/) + assert.match(log.text(), /RAM if all decoded/) +}) + +test("a missing key table is scaffolded with every image that can be packed", () => { + const root = project(null) + try { + const log = captureLog() + const result = runImageGen({ root, log }) + const written = JSON.parse(readFileSync(join(root, "img.keys.json"), "utf8")) + assert.deepEqual(written.numeric, {}) + assert.deepEqual(Object.keys(written.names).sort(), ["microphone", "no_key", "own", "stable_name", "twin_of_lib"]) + for (const [k, v] of Object.entries(written.names)) assert.equal(k, v) + assert.match(log.text(), /no key table .*; wrote a starter with 5 string key\(s\)/) + assert.match(log.text(), /info: review .*img\.keys\.json before relying on it/) + assert.ok(!result.diagnostics.some(n => n.text.includes("no key in the key table"))) + assert.deepEqual(packed(result), [ + "icondb.microphone", + "icondb.no_key", + "icondb.own", + "icondb.stable_name", + "icondb.twin_of_lib", + ]) + + // The next run reads the file it wrote, and nothing it keyed is rejected. + const again = runImageGen({ root, log: () => {} }) + assert.deepEqual(packed(again), packed(result)) + } finally { + removeProject(root) + } +}) + +test("the starter leaves out images that cannot be packed, and dependencies' images", () => { + const root = project(null) + try { + runImageGen({ root, log: () => {} }) + const names = Object.keys(JSON.parse(readFileSync(join(root, "img.keys.json"), "utf8")).names) + for (const absent of ["loose", "referenced", "from_lib", "lib.microphone", "in_a_test_file"]) + assert.ok(!names.includes(absent), absent) + } finally { + removeProject(root) + } +}) + +test("the starter is used but not written when writing is off", () => { + const root = project(null) + try { + const log = captureLog() + const result = runImageGen({ root, write: false, log }) + assert.equal(existsSync(join(root, "img.keys.json")), false) + assert.match(log.text(), /would write a starter/) + assert.doesNotMatch(log.text(), /before relying on it/) + assert.equal(result.pack.entries.length, 5) + } finally { + removeProject(root) + } +}) + +test("an existing key table is never overwritten", () => { + const root = project(KEYS) + try { + const before = readFileSync(join(root, "img.keys.json"), "utf8") + runImageGen({ root, log: () => {} }) + assert.equal(readFileSync(join(root, "img.keys.json"), "utf8"), before) + } finally { + removeProject(root) + } +}) + +test("a name the app shares with a dependency is only reported when verbose", () => { + const quiet = run(KEYS) + assert.doesNotMatch(quiet.log.text(), /"microphone" matches both/) + const detail = notes(quiet.result, "detail").find(t => t.includes('"microphone"')) + assert.equal( + detail, + '"microphone" matches both icondb.microphone and lib.microphone. A bare "microphone" in ' + + 'img.keys.json resolves to icondb.microphone; to use lib.microphone instead, write "lib.microphone".', + ) + assert.match(run(KEYS, { verbose: true }).log.text(), /detail: "microphone" matches both/) +}) + +test("a name two dependencies share is reported, saying what to write", () => { + const { result, log } = run(KEYS) + const info = notes(result, "info").find(t => t.includes('"shared_icon"')) + assert.equal( + info, + '"shared_icon" matches both lib.shared_icon and other.shared_icon. A bare "shared_icon" in ' + + 'img.keys.json resolves to lib.shared_icon; to use other.shared_icon instead, write "other.shared_icon".', + ) + assert.match(log.text(), /info: "shared_icon" matches both/) +}) diff --git a/img-tools/test/helpers.mjs b/img-tools/test/helpers.mjs new file mode 100644 index 0000000..5e55df0 --- /dev/null +++ b/img-tools/test/helpers.mjs @@ -0,0 +1,56 @@ +// Shared fixtures for the img-tools tests. + +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { dirname, join } from "node:path" + +export const BT = "`" + +/** A `bmp` literal const, with any `//%` attribute lines given. */ +export function bmpConst(constName, rows, attrs = []) { + const lines = attrs.map(a => " //% " + a) + const body = rows.map(r => " " + r).join("\n") + lines.push(` export const ${constName} = bmp${BT}\n${body}\n ${BT}`) + return lines.join("\n") +} + +/** A namespace wrapping the given declarations. */ +export function ns(name, ...decls) { + return `namespace ${name} {\n${decls.join("\n\n")}\n}\n` +} + +/** + * Writes a project tree into a fresh temporary directory and returns its root. + * `files` maps relative paths to contents; objects are written as JSON. + */ +export function makeProject(files) { + const root = mkdtempSync(join(tmpdir(), "img-tools-test-")) + for (const [rel, content] of Object.entries(files)) { + const path = join(root, rel) + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, typeof content === "string" ? content : JSON.stringify(content, null, 4)) + } + return root +} + +export function removeProject(root) { + rmSync(root, { recursive: true, force: true }) +} + +/** A log function that collects lines, for asserting on what a run printed. */ +export function captureLog() { + const lines = [] + const log = s => lines.push(s) + log.lines = lines + log.text = () => lines.join("\n") + return log +} + +/** A deterministic pseudo-random generator, so failures reproduce. */ +export function seeded(seed) { + let s = seed >>> 0 + return () => { + s = (s * 1664525 + 1013904223) >>> 0 + return s / 0x100000000 + } +} diff --git a/img-tools/test/keys.test.mjs b/img-tools/test/keys.test.mjs new file mode 100644 index 0000000..769e370 --- /dev/null +++ b/img-tools/test/keys.test.mjs @@ -0,0 +1,47 @@ +import { test } from "node:test" +import assert from "node:assert/strict" +import { join } from "node:path" + +import { readKeyTable, emptyKeyTable, referencedNames } from "../keys.mjs" +import { makeProject, removeProject } from "./helpers.mjs" + +function withTable(content, fn) { + const root = makeProject({ "img.keys.json": content }) + try { + return fn(join(root, "img.keys.json")) + } finally { + removeProject(root) + } +} + +test("a missing key table is an empty one", () => { + assert.deepEqual(readKeyTable("/nonexistent/img.keys.json"), emptyKeyTable()) +}) + +test("a key table reads numeric and string keys", () => { + withTable({ numeric: { 16: "thermometer", 180: "thermometer" }, names: { delete: "btn_delete" } }, path => { + const table = readKeyTable(path) + assert.deepEqual(table.numeric, { 16: "thermometer", 180: "thermometer" }) + assert.deepEqual(table.names, { delete: "btn_delete" }) + assert.deepEqual([...referencedNames(table)].sort(), ["btn_delete", "thermometer"]) + }) +}) + +test("either section may be left out", () => { + withTable({ names: { a: "b" } }, path => assert.deepEqual(readKeyTable(path).numeric, {})) + withTable({ numeric: { 1: "b" } }, path => assert.deepEqual(readKeyTable(path).names, {})) +}) + +test("a malformed key table is rejected with the entry at fault", () => { + const cases = [ + ["{ not json", /not valid JSON/], + [[], /must be a JSON object/], + [{ numeric: { "-1": "x" } }, /numeric key "-1"/], + [{ numeric: { two: "x" } }, /numeric key "two"/], + [{ numeric: { 3: "" } }, /numeric key 3 must name an image/], + [{ names: { "a,b": "x" } }, /may not contain a comma/], + [{ names: { a: 7 } }, /string key "a" must name an image/], + ] + for (const [content, message] of cases) + withTable(content, path => assert.throws(() => readKeyTable(path), message, JSON.stringify(content))) +}) diff --git a/img-tools/test/pack-emit.test.mjs b/img-tools/test/pack-emit.test.mjs new file mode 100644 index 0000000..d11c02a --- /dev/null +++ b/img-tools/test/pack-emit.test.mjs @@ -0,0 +1,89 @@ +import { test } from "node:test" +import assert from "node:assert/strict" + +import { buildPack } from "../pack.mjs" +import { emitImgG } from "../emit.mjs" +import { encodeImage, decodeRecord } from "../codec.mjs" + +const solid = c => Array(8).fill(c.repeat(8)) + +test("a pack decodes every image back to its pixels", () => { + const images = [ + { name: "a", rows: solid("1") }, + { name: "b", rows: ["12", "34"] }, + { name: "c", rows: solid("2") }, + ] + const pack = buildPack(images) + assert.equal(pack.entries.length, 3) + images.forEach((img, i) => { + const expected = encodeImage(img.rows) + expected.writeUInt16LE(0, 6) + const offset = pack.offsets.readUInt16LE(i * 2) + assert.deepEqual(decodeRecord(pack.blob, offset), expected, img.name) + }) +}) + +test("identical images share one record", () => { + const pack = buildPack([ + { name: "a", rows: solid("1") }, + { name: "b", rows: solid("2") }, + { name: "c", rows: solid("1") }, + ]) + assert.equal(pack.offsets.readUInt16LE(0), pack.offsets.readUInt16LE(4)) + assert.equal(pack.entries[2].shared, "a") + assert.equal(pack.stats.unique, 2) + assert.equal(pack.stats.shared, 1) +}) + +test("pack stats count bytes once per unique image", () => { + const pack = buildPack([ + { name: "a", rows: solid("1") }, + { name: "b", rows: solid("1") }, + ]) + const raw = encodeImage(solid("1")).length + assert.equal(pack.stats.rawBytes, raw) + assert.equal(pack.stats.decodedWorstCase, raw) + assert.equal(pack.stats.packBytes, pack.blob.length) + assert.equal(pack.stats.offsetBytes, 4) +}) + +test("an image name must be unique and free of commas", () => { + assert.throws(() => buildPack([{ name: "a", rows: ["1"] }, { name: "a", rows: ["2"] }]), /more than once/) + assert.throws(() => buildPack([{ name: "a,b", rows: ["1"] }]), /comma/) +}) + +test("an empty pack is valid", () => { + const pack = buildPack([]) + assert.equal(pack.blob.length, 0) + assert.match(emitImgG(pack, {}, {}, []), /export const count = 0/) +}) + +test("only the lookups an app uses are emitted", () => { + const pack = buildPack([{ name: "a", rows: ["1"] }]) + const none = emitImgG(pack, {}, {}, []) + assert.doesNotMatch(none, /KEYS|NUM_INDEX/) + + const namesOnly = emitImgG(pack, { a: 0 }, {}, []) + assert.match(namesOnly, /const KEYS = "a"\.split/) + assert.doesNotMatch(namesOnly, /NUM_INDEX/) + + const numericOnly = emitImgG(pack, {}, { 3: 0 }, []) + assert.match(numericOnly, /NUM_INDEX/) + assert.doesNotMatch(numericOnly, /KEYS/) +}) + +test("the numeric table holds index + 1 per key, 0 for keys with no image", () => { + const pack = buildPack([{ name: "a", rows: ["1"] }, { name: "b", rows: ["2"] }]) + const src = emitImgG(pack, {}, { 1: 0, 3: 1 }, []) + const table = /const NUM_INDEX = hex`([0-9a-f]*)`/.exec(src)[1] + assert.equal(table, "00010002") + assert.match(src, /key >= 4\) return -1/) +}) + +test("index tables widen past 255 images", () => { + const images = Array.from({ length: 300 }, (_, i) => ({ name: "i" + i, rows: ["1"] })) + const pack = buildPack(images) + const src = emitImgG(pack, { i299: 299 }, { 0: 299 }, []) + assert.match(src, /KEY_INDEX\.getNumber\(NumberFormat\.UInt16LE/) + assert.match(src, /NUM_INDEX\.getNumber\(NumberFormat\.UInt16LE/) +}) diff --git a/img-tools/test/packages-pins.test.mjs b/img-tools/test/packages-pins.test.mjs new file mode 100644 index 0000000..ab623d0 --- /dev/null +++ b/img-tools/test/packages-pins.test.mjs @@ -0,0 +1,116 @@ +import { test } from "node:test" +import assert from "node:assert/strict" +import { mkdirSync, symlinkSync } from "node:fs" +import { join } from "node:path" + +import { discoverPackages } from "../packages.mjs" +import { checkPins } from "../pins.mjs" +import { makeProject, removeProject } from "./helpers.mjs" + +test("packages come dependencies first, each once, with the app last", () => { + const root = makeProject({ + "pxt.json": { name: "app", dependencies: { top: "*", base: "*" }, files: ["app.ts"] }, + "pxt_modules/top/pxt.json": { name: "top", dependencies: { base: "*" }, files: ["top.ts"] }, + "pxt_modules/base/pxt.json": { name: "base", files: ["base.ts"] }, + }) + try { + assert.deepEqual(discoverPackages(root).map(p => p.name), ["base", "top", "app"]) + } finally { + removeProject(root) + } +}) + +test("a package's files are its compiled sources only", () => { + const root = makeProject({ + "pxt.json": { name: "app", files: ["a.ts", "img.g.ts", "README.md"], testFiles: ["test.ts"] }, + }) + try { + const [app] = discoverPackages(root, "img.g.ts") + assert.deepEqual(app.files, ["a.ts"]) + } finally { + removeProject(root) + } +}) + +test("a dependency with nothing under pxt_modules is skipped", () => { + const root = makeProject({ "pxt.json": { name: "app", dependencies: { core: "*" }, files: [] } }) + try { + assert.deepEqual(discoverPackages(root).map(p => p.name), ["app"]) + } finally { + removeProject(root) + } +}) + +test("a project without pxt.json is an error", () => { + const root = makeProject({}) + try { + assert.throws(() => discoverPackages(root), /no pxt.json/) + } finally { + removeProject(root) + } +}) + +const pinned = (npm, pxt) => { + const files = {} + if (npm) files["node_modules/@microbit-apps/ui-core/pxt.json"] = { version: npm } + if (pxt) files["pxt_modules/ui-core/pxt.json"] = { version: pxt } + return makeProject(files) +} + +test("matching versions pass the pin check", () => { + const root = pinned("0.0.9", "0.0.9") + try { + assert.deepEqual(checkPins(root), { ok: true, reason: null, npm: "0.0.9", pxt: "0.0.9" }) + } finally { + removeProject(root) + } +}) + +test("different versions fail the pin check, naming both", () => { + const root = pinned("0.0.8", "0.0.9") + try { + const r = checkPins(root) + assert.equal(r.ok, false) + assert.match(r.reason, /0\.0\.8 for build tooling but 0\.0\.9 for device code/) + } finally { + removeProject(root) + } +}) + +test("a missing side leaves nothing to compare", () => { + for (const [npm, pxt, why] of [[null, "0.0.9", /no npm install/], ["0.0.9", null, /no pxt module/]]) { + const root = pinned(npm, pxt) + try { + const r = checkPins(root) + assert.equal(r.ok, true) + assert.match(r.reason, why) + } finally { + removeProject(root) + } + } +}) + +test("a linked setup passes, since both sides are one checkout", () => { + const root = makeProject({ "checkout/pxt.json": { version: "0.0.9" } }) + try { + mkdirSync(join(root, "node_modules/@microbit-apps"), { recursive: true }) + mkdirSync(join(root, "pxt_modules"), { recursive: true }) + symlinkSync(join(root, "checkout"), join(root, "node_modules/@microbit-apps/ui-core")) + symlinkSync(join(root, "checkout"), join(root, "pxt_modules/ui-core")) + assert.equal(checkPins(root).ok, true) + } finally { + removeProject(root) + } +}) + +test("an npm install hoisted above the project is found", () => { + const outer = makeProject({ + "node_modules/@microbit-apps/ui-core/pxt.json": { version: "0.0.8" }, + "app/pxt_modules/ui-core/pxt.json": { version: "0.0.9" }, + }) + try { + assert.equal(checkPins(join(outer, "app")).ok, false) + } finally { + removeProject(outer) + } +}) diff --git a/img-tools/test/scan.test.mjs b/img-tools/test/scan.test.mjs new file mode 100644 index 0000000..546b104 --- /dev/null +++ b/img-tools/test/scan.test.mjs @@ -0,0 +1,93 @@ +import { test } from "node:test" +import assert from "node:assert/strict" + +import { scanSource, scanLiterals, countReferences, stripBlockComments } from "../scan.mjs" +import { BT, bmpConst, ns } from "./helpers.mjs" + +const names = src => scanSource(src, "a.ts").map(i => i.name) + +test("a packable image is named by its const", () => { + const [img] = scanSource(ns("icons", bmpConst("disk", ["1"], ["packable", "whenUsed"])), "a.ts") + assert.equal(img.name, "disk") + assert.equal(img.constName, "disk") + assert.equal(img.namespace, "icons") + assert.equal(img.whenUsed, true) + assert.equal(img.file, "a.ts") + assert.equal(img.line, 2) +}) + +test("packable= overrides the name, quoted or not", () => { + assert.deepEqual(names(ns("a", bmpConst("x", ["1"], ['packable="stable"']))), ["stable"]) + assert.deepEqual(names(ns("a", bmpConst("x", ["1"], ["packable=bare"]))), ["bare"]) +}) + +test("packable is found wherever it sits on a //% line", () => { + for (const attrs of [["whenUsed packable"], ["packable whenUsed"], ["whenUsed", "packable"]]) { + const [img] = scanSource(ns("a", bmpConst("x", ["1"], attrs)), "a.ts") + assert.ok(img, attrs.join(" / ")) + assert.equal(img.whenUsed, true, attrs.join(" / ")) + } +}) + +test("images without packable are not discovered", () => { + assert.deepEqual(names(ns("a", bmpConst("x", ["1"], ["whenUsed"]))), []) + assert.deepEqual(names(ns("a", bmpConst("x", ["1"]))), []) + assert.deepEqual(names(ns("a", bmpConst("x", ["1"], ["notpackable"]))), []) +}) + +test("whenUsed is optional and reported", () => { + const [img] = scanSource(ns("a", bmpConst("x", ["1"], ["packable"])), "a.ts") + assert.equal(img.whenUsed, false) +}) + +test("an image inside a block comment is ignored", () => { + const src = ns("a", " /*\n" + bmpConst("hidden", ["1"], ["packable"]) + "\n */", bmpConst("shown", ["2"], ["packable"])) + assert.deepEqual(names(src), ["shown"]) +}) + +test("a comment opener inside a string does not hide what follows", () => { + const src = ns("a", ' const s = "a /* b"', bmpConst("after", ["1"], ["packable"])) + assert.deepEqual(names(src), ["after"]) +}) + +test("blanking comments keeps line numbers", () => { + const src = "a\n/* one\ntwo */\nb\n" + assert.equal(stripBlockComments(src).split("\n").length, src.split("\n").length) + const [img] = scanSource("/*\n\n*/\n" + ns("a", bmpConst("x", ["1"], ["packable"])), "a.ts") + assert.equal(img.line, 5) +}) + +test("scanLiterals finds every bmp literal, annotated or not", () => { + const src = ns("a", bmpConst("one", ["1"], ["packable"]), bmpConst("two", ["2"])) + assert.deepEqual(scanLiterals(src, "a.ts").map(l => l.constName), ["one", "two"]) +}) + +test("countReferences counts the declaration as one use", () => { + const src = ns("icons", bmpConst("disk", ["1"], ["packable"])) + assert.equal(countReferences(src, "disk", "icons"), 1) +}) + +test("countReferences finds qualified uses anywhere and bare uses only inside the namespace", () => { + const other = ns("app", " function f() { return icons.disk }") + assert.equal(countReferences(other, "disk", "icons"), 1) + const bareElsewhere = ns("app", " function f() { return disk }") + assert.equal(countReferences(bareElsewhere, "disk", "icons"), 0) + const bareInside = ns("icons", " function f() { return disk }") + assert.equal(countReferences(bareInside, "disk", "icons"), 1) +}) + +test("countReferences ignores a different namespace's member of the same name", () => { + const src = ns("app", " function f() { music.playTone(1, 2) }") + assert.equal(countReferences(src, "music", "icons"), 0) +}) + +test("countReferences ignores comments and string data but not interpolated code", () => { + const src = ns( + "app", + " // icons.disk", + ' const a = "icons.disk"', + " const b = " + BT + "icons.disk" + BT, + " const c = " + BT + "${icons.disk}" + BT, + ) + assert.equal(countReferences(src, "disk", "icons"), 1) +}) diff --git a/img-tools/test/verify.test.mjs b/img-tools/test/verify.test.mjs new file mode 100644 index 0000000..6dc0ce6 --- /dev/null +++ b/img-tools/test/verify.test.mjs @@ -0,0 +1,59 @@ +import { test } from "node:test" +import assert from "node:assert/strict" + +import { imagesInBuild, findBuildOutputs, runImageCheck } from "../verify.mjs" +import { buildPack } from "../pack.mjs" +import { encodeImage } from "../codec.mjs" +import { makeProject, removeProject, captureLog } from "./helpers.mjs" + +// A buffer literal the way the compiler writes one into its assembly output. +const literal = buf => + `_hexlit1:\n .word pxt::buffer_vt\n .word ${buf.length}\n .hex ${buf.toString("hex")}\n` + +const icon = encodeImage(Array(16).fill("1111222233334444")) + +test("imagesInBuild finds image buffers and skips everything else", () => { + const notImage = Buffer.from("hello world") + const wrongLength = Buffer.concat([icon, Buffer.alloc(4)]) + const wrongDepth = Buffer.from(icon) + wrongDepth[1] = 8 + const asm = [icon, notImage, wrongLength, wrongDepth].map(literal).join("\n") + const found = imagesInBuild(asm) + assert.equal(found.length, 1) + assert.deepEqual(found[0], icon) +}) + +test("findBuildOutputs lists the assembly files in built/", () => { + const root = makeProject({ "built/a.asm": "", "built/b.asm": "", "built/binary.hex": "" }) + try { + assert.deepEqual(findBuildOutputs(root).map(p => p.split("/").pop()).sort(), ["a.asm", "b.asm"]) + } finally { + removeProject(root) + } +}) + +test("runImageCheck reports a packed image that also ships as a buffer", () => { + const root = makeProject({ "built/out.asm": literal(icon) }) + try { + const log = captureLog() + const pack = buildPack([{ name: "icon", rows: Array(16).fill("1111222233334444") }]) + const r = runImageCheck({ root, pack, log }) + assert.deepEqual(r.duplicates.map(d => d.name), ["icon"]) + assert.match(log.text(), /1 packed image\(s\) also ship as buffers, 136 B/) + } finally { + removeProject(root) + } +}) + +test("runImageCheck reports nothing when the pack is not duplicated", () => { + const root = makeProject({ "built/out.asm": literal(icon) }) + try { + const log = captureLog() + const pack = buildPack([{ name: "other", rows: ["1 2", "3 4"] }]) + const r = runImageCheck({ root, pack, log }) + assert.equal(r.duplicates.length, 0) + assert.match(log.text(), /1 image buffer\(s\) in 1 output\(s\), none duplicating the pack/) + } finally { + removeProject(root) + } +}) diff --git a/img-tools/verify.mjs b/img-tools/verify.mjs new file mode 100644 index 0000000..c91abed --- /dev/null +++ b/img-tools/verify.mjs @@ -0,0 +1,90 @@ +// Post-build check: what the pack holds against what the build actually shipped. +// +// Run it after a build. It reads the image buffers out of the compiled output +// and reports any whose pixels a packed image already carries, which is flash +// spent twice. Nothing here fails a build. + +import { existsSync, readFileSync, readdirSync } from "node:fs" +import { join } from "node:path" + +import { decodeRecord } from "./codec.mjs" + +// A buffer literal in the compiled output: its declared length and its bytes. +const BUFFER = /\.word\s+pxt::buffer_vt\s+\.word\s+(\d+)\s+\.hex\s+([0-9a-f]+)/g + +const MAGIC = 0x87 + +function decodedSize(w, h) { + return 8 + (((h * 4 + 31) >> 5) << 2) * w +} + +/** + * The F4 images in one compiled output. A buffer counts as an image on the + * same terms the runtime uses: the magic byte, a supported depth, and a length + * matching exactly what its width and height imply. + */ +export function imagesInBuild(asm) { + const found = [] + let m + BUFFER.lastIndex = 0 + while ((m = BUFFER.exec(asm))) { + const buf = Buffer.from(m[2], "hex") + if (buf.length < 9 || buf[0] !== MAGIC) continue + if (buf[1] !== 1 && buf[1] !== 4) continue + if (buf.length !== decodedSize(buf.readUInt16LE(2), buf.readUInt16LE(4))) continue + found.push(buf) + } + return found +} + +/** The compiled outputs under a project's built/ directory. */ +export function findBuildOutputs(root) { + const dir = join(root, "built") + if (!existsSync(dir)) return [] + return readdirSync(dir) + .filter(f => f.endsWith(".asm")) + .map(f => join(dir, f)) +} + +/** + * Compares a pack against the images in the build at `root`. + * + * `pack` is what buildPack returned. Returns { outputs, images, duplicates }, + * where each duplicate names a packed image whose pixels also ship as a + * separate buffer, with the bytes that costs. + */ +export function runImageCheck(options) { + const root = options.root + if (!root) throw new Error("runImageCheck: options.root is required") + const pack = options.pack + if (!pack) throw new Error("runImageCheck: options.pack is required") + const log = options.log || (s => console.log(s)) + + const outputs = options.outputs || findBuildOutputs(root) + const packed = new Map() + for (const e of pack.entries) { + if (e.shared) continue + packed.set(decodeRecord(pack.blob, e.offset).toString("hex"), e) + } + + const duplicates = [] + let images = 0 + for (const path of outputs) { + for (const buf of imagesInBuild(readFileSync(path, "utf8"))) { + images++ + // A packed image decodes with its padding zeroed, which is how + // every compiler-emitted image is written too. + const hit = packed.get(buf.toString("hex")) + if (hit) duplicates.push({ name: hit.name, bytes: buf.length, output: path }) + } + } + + if (duplicates.length === 0) + log(`img-check: ${images} image buffer(s) in ${outputs.length} output(s), none duplicating the pack`) + else { + const bytes = duplicates.reduce((n, d) => n + d.bytes, 0) + log(`img-check: ${duplicates.length} packed image(s) also ship as buffers, ${bytes} B:`) + for (const d of duplicates) log(` ${d.name}: ${d.bytes} B`) + } + return { outputs, images, duplicates } +} diff --git a/img.ts b/img.ts new file mode 100644 index 0000000..2d4618c --- /dev/null +++ b/img.ts @@ -0,0 +1,164 @@ +// Compressed image records. +// +// A record is a pxt F4 image header followed by a method-specific payload. +// The header's `padding` field carries the method: +// +// 0x0000 uncompressed -- the payload is raw F4 pixel data +// 0xC0nn compressed by method nn (1, nibble RLE, is the only one defined) +// +// pxt writes `padding` as 0 and never reads it, so a plain `bmp` literal is +// already a valid uncompressed record and can be handed to `ofCompressed` +// unchanged. A buffer whose marker is neither 0 nor 0xC0-prefixed is +// rejected rather than read as an image. +// +// Nibble RLE: one byte per run, the high nibble the color and the low nibble +// the run length minus 1 (runs of 1 to 16), over the payload's nibble stream +// in storage order -- low nibble first, per-column padding included. A run of +// color 0 only advances the cursor, since a fresh buffer starts zeroed. +// +// Records are produced at build time by img-tools. +namespace bitmaps { + const IMG_MAGIC = 0x87 + const IMG_BPP = 4 + const METHOD_UNCOMPRESSED = 0x0000 + const METHOD_SIGNATURE = 0xc000 + const METHOD_SIGNATURE_MASK = 0xff00 + const METHOD_RLE = 1 + + /** + * Size of the F4 buffer a record decodes to: the 8-byte header plus w + * columns of h nibbles, each column padded to a 32-bit boundary. + */ + function decodedSize(w: number, h: number): number { + return 8 + (((h * 4 + 31) >> 5) << 2) * w + } + + /** + * Decodes one image record into a new Bitmap. + * + * Unlike `ofBuffer`, which wraps a flash buffer in place, this always + * allocates: a record has to be expanded into RAM. Returns `null` when + * `src` at `offset` is not a record this runtime understands. + * + * Records may be packed end to end, so `offset` selects one. + */ + export function ofCompressed(src: Buffer, offset = 0): Bitmap { + if (!src || offset < 0 || offset + 8 > src.length) return null + if (src[offset] != IMG_MAGIC || src[offset + 1] != IMG_BPP) return null + + const w = src.getNumber(NumberFormat.UInt16LE, offset + 2) + const h = src.getNumber(NumberFormat.UInt16LE, offset + 4) + if (w <= 0 || h <= 0) return null + const size = decodedSize(w, h) + + const marker = src.getNumber(NumberFormat.UInt16LE, offset + 6) + if (marker == METHOD_UNCOMPRESSED) { + // An uncompressed record is byte-for-byte an ordinary image. + if (offset + size > src.length) return null + return bitmaps.ofBuffer(src.slice(offset, size)) + } + if ((marker & METHOD_SIGNATURE_MASK) != METHOD_SIGNATURE) return null + if ((marker & 0xff) != METHOD_RLE) return null + + const out = Buffer.create(size) + for (let i = 0; i < 8; ++i) out[i] = src[offset + i] + // Keep pxt's zero-padding invariant, so no method value reaches a + // buffer pxt inspects. + out[6] = 0 + out[7] = 0 + + let s = offset + 8 + let d = 16 // nibble cursor; the 8 header bytes are 16 nibbles + const dEnd = size * 2 + while (d < dEnd) { + if (s >= src.length) return null + const run = src[s++] + const color = run >> 4 + const count = (run & 15) + 1 + if (color) { + for (let k = 0; k < count; ++k) { + out[d >> 1] |= color << ((d & 1) * 4) + ++d + } + } else { + d += count + } + } + return bitmaps.ofBuffer(out) + } +} + +namespace ui { + /** + * Lazily decoded access to a pack of image records. + * + * A pack is one buffer of records laid end to end plus a u16le offset + * table, one entry per image index. Several indices may share an offset, + * which is how identical images are kept once. + * + * An image is decoded on first use and kept for the life of the program. + * Nothing is evicted, so a pack's fully decoded size is its worst-case + * RAM cost. + * + * A record the runtime cannot read yields the fallback image rather than + * null, so callers never have to null-check an image. The fallback is + * `ui.MISSING` unless one is supplied, and it must not itself be packed -- + * were it a record, a decode failure would have nothing to fall back to. + */ + export class ImagePack { + private blob_: Buffer + private offsets_: Buffer + private fallback_: Bitmap + private cache_: Bitmap[] + private count_: number + + constructor(blob: Buffer, offsets: Buffer, fallback?: Bitmap) { + this.blob_ = blob + this.offsets_ = offsets + // Normalized so the field is a Bitmap or null, never undefined. + this.fallback_ = fallback ? fallback : null + this.cache_ = [] + this.count_ = offsets.length >> 1 + } + + /** Number of image indices in the pack. */ + public get length(): number { + return this.count_ + } + + private offsetAt(index: number): number { + return this.offsets_.getNumber(NumberFormat.UInt16LE, index * 2) + } + + private fallbackImage(): Bitmap { + return this.fallback_ ? this.fallback_ : MISSING + } + + /** + * The image at `index`, decoding it on first use. Out-of-range + * indices and unreadable records both yield the fallback. + */ + public get(index: number): Bitmap { + if (index < 0 || index >= this.count_) return this.fallbackImage() + + let img = this.cache_[index] + if (img) return img + + const offset = this.offsetAt(index) + // The same record may be shared by several indices; decode it + // once so they hand back one object rather than equal copies. + for (let i = 0; i < this.count_; ++i) { + const cached = this.cache_[i] + if (cached && this.offsetAt(i) == offset) { + img = cached + break + } + } + if (!img) img = bitmaps.ofCompressed(this.blob_, offset) + if (!img) img = this.fallbackImage() + + this.cache_[index] = img + return img + } + } +} diff --git a/loc-tools/emit.mjs b/loc-tools/emit.mjs index 1b3f2ef..1300d2c 100644 --- a/loc-tools/emit.mjs +++ b/loc-tools/emit.mjs @@ -23,9 +23,15 @@ export const GENERIC_DEFAULT_LOC_G = `// Generated placeholder for a per-languag // Emit loc.g.ts content for one language. `table` is the id-to-string catalog // (emitted only when non-empty); `charsets` maps field names to their resolved -// strings (each emitted as a `_loc` member assignment). Both live inside one -// reopened `namespace _loc` block. -export function emitLocG(lang, table, charsets) { +// strings (each emitted as a `_loc` member assignment). `font`, when given, is +// the name of a `bitmaps` font the language renders in, assigned to +// `_loc.defaultFont` so `ui.locFont()` returns it instead of the built-in +// fallback. All live inside one reopened `namespace _loc` block. +// +// The font is emitted as a reference (`bitmaps.font12`), not as glyph data: +// the font is already in the image, and a second copy would cost flash and +// could drift from the one the coverage check validated against. +export function emitLocG(lang, table, charsets, font) { const keys = Object.keys(table).sort() const fields = Object.keys(charsets).sort() const lines = [] @@ -36,6 +42,7 @@ export function emitLocG(lang, table, charsets) { lines.push("// this file after the run. Do not commit this generated content.") lines.push("// lang: " + lang) lines.push("namespace _loc {") + if (font) lines.push(" defaultFont = bitmaps." + font) if (keys.length > 0) { const entries = keys.map(k => " " + JSON.stringify(k) + ": " + JSON.stringify(table[k])) lines.push(" table = {") diff --git a/loc-tools/gen.mjs b/loc-tools/gen.mjs index 956313b..c60568c 100644 --- a/loc-tools/gen.mjs +++ b/loc-tools/gen.mjs @@ -44,6 +44,11 @@ function resolveConfig(options) { srcLang, fonts, font: options.font || "font8", + // Per-language build-font override: a { lang: fontName } map or a + // lang => fontName function. Falls back to `font` for any language it + // does not name. The resolved font drives both glyph validation and + // the `_loc.defaultFont` assignment, so the two cannot disagree. + fontForLang: options.fontForLang || null, layers: discoverLayers(root), langs: options.langs || null, reservedLangNames: options.reservedLangNames || [], @@ -87,6 +92,18 @@ function buildAndCopy(cfg, lang) { return { dst, size: statSync(dst).size } } +// The font `ui.locFont()` falls back to when `_loc.defaultFont` is unset. A +// language rendering in this font needs no assignment emitted. +const RUNTIME_FALLBACK_FONT = "font8" + +// Resolve the build font for one language. +function resolveFont(cfg, lang) { + const f = cfg.fontForLang + if (!f) return cfg.font + const name = typeof f === "function" ? f(lang) : f[lang] + return name || cfg.font +} + // Validate a set of source strings rendered in the small-font context against // that font's coverage; drop and report those the font cannot render. function dropUnrenderable(merged, keys, coverage, lang, tag, only) { @@ -179,9 +196,15 @@ function processLanguage(cfg, lang) { // Glyph validation: drop entries whose translation contains code points the // build font cannot render. Code points below 33 are always acceptable. // Small-font-context keys are exempt: validated above. - const cov = cfg.fonts.coverage(cfg.font) + const font = resolveFont(cfg, lang) + const cov = cfg.fonts.coverage(font) + // An empty coverage set means the glyph table could not be read -- for + // example a placeholder whose real data ships in a .jres -- not a font + // with no glyphs. Validating against it would drop every translation. + if (cov.size === 0) + throw new Error(`font ${font}: no glyph coverage found; cannot validate ${lang}. Check that its glyph table is readable from the text source.`) const smallKeys = cfg.smallStrings || new Set() - const glyphDropped = dropUnrenderable(merged, smallKeys, cov, lang, cfg.font, false) + const glyphDropped = dropUnrenderable(merged, smallKeys, cov, lang, font, false) // Length warning (never drops): translated text much longer than source. let lengthWarnings = 0 @@ -221,7 +244,7 @@ function processLanguage(cfg, lang) { return report } - writeFileSync(cfg.locGPath, emitLocG(lang, merged, charsets)) + writeFileSync(cfg.locGPath, emitLocG(lang, merged, charsets, font === RUNTIME_FALLBACK_FONT ? null : font)) const built = buildAndCopy(cfg, lang) console.log( diff --git a/package-lock.json b/package-lock.json index e1c871d..9677ba7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,14 @@ { - "name": "ui-core", + "name": "@microbit-apps/ui-core", "lockfileVersion": 3, "requires": true, "packages": { "": { + "name": "@microbit-apps/ui-core", + "bin": { + "loc-gen": "loc-tools/bin/loc-gen.mjs", + "loc-strings": "loc-tools/bin/loc-strings.mjs" + }, "devDependencies": { "prettier": "^3.8.3" } diff --git a/package.json b/package.json index 8f24d65..d012087 100644 --- a/package.json +++ b/package.json @@ -2,15 +2,18 @@ "name": "@microbit-apps/ui-core", "type": "module", "exports": { - "./loc": "./loc-tools/index.mjs" + "./loc": "./loc-tools/index.mjs", + "./img": "./img-tools/index.mjs" }, "bin": { "loc-strings": "./loc-tools/bin/loc-strings.mjs", - "loc-gen": "./loc-tools/bin/loc-gen.mjs" + "loc-gen": "./loc-tools/bin/loc-gen.mjs", + "img-gen": "./img-tools/bin/img-gen.mjs" }, "scripts": { "format": "prettier --write \"**/*.ts\"", - "loc:strings": "node loc-tools/bin/loc-strings.mjs" + "loc:strings": "node loc-tools/bin/loc-strings.mjs", + "test:img": "node --test \"img-tools/test/*.test.mjs\"" }, "devDependencies": { "prettier": "^3.8.3" diff --git a/pxt.json b/pxt.json index c1e412c..0533b12 100644 --- a/pxt.json +++ b/pxt.json @@ -10,6 +10,7 @@ "files": [ "ns.ts", "loc.ts", + "img.ts", "geometry.ts", "viewport.ts", "draw-surface.ts", diff --git a/test.ts b/test.ts index 7bf33dc..cdaed18 100644 --- a/test.ts +++ b/test.ts @@ -521,10 +521,7 @@ namespace ui.core.tests { locc("switch", "on") == "activee", "locc plain-string fallback", ) - control.assert( - locc("switch", "off") == "off", - "locc source fallback", - ) + control.assert(locc("switch", "off") == "off", "locc source fallback") // locf: interpolation with reordered placeholders in translation. _loc.table = { @@ -565,12 +562,143 @@ namespace ui.core.tests { _loc.defaultFont = undefined } + /** + * Smoke harness for compressed image records (img.ts). + * + * Vectors come from img-tools encodeRecord, one per method: ui.MISSING + * compresses, and the checkerboard does not, so it stays uncompressed. No + * real icon measured so far fails to compress, so without a deliberately + * incompressible fixture the uncompressed path would ship untested. + */ + export function runImgTest(): void { + const rleRecord = hex`87041000100001c00f0f0f02290521052105200020032000200520012001200120052002210220052002210220052001200120012005200020032000200521052105290f0f0f02` + const uncompressedRecord = hex`87041000100000001212121212121212212121212121212112121212121212122121212121212121121212121212121221212121212121211212121212121212212121212121212112121212121212122121212121212121121212121212121221212121212121211212121212121212212121212121212112121212121212122121212121212121` + const atOffset = hex`ffffff87041000100001c00f0f0f02290521052105200020032000200520012001200120052002210220052002210220052001200120012005200020032000200521052105290f0f0f02` + const badMarker = hex`87041000100000a00f0f0f02290521052105200020032000200520012001200120052002210220052002210220052001200120012005200020032000200521052105290f0f0f02` + const checker = bmp` + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + ` + + samePixels(bitmaps.ofCompressed(rleRecord), ui.MISSING, "rle") + samePixels(bitmaps.ofCompressed(uncompressedRecord), checker, "uncompressed") + samePixels( + bitmaps.ofCompressed(atOffset, 3), + ui.MISSING, + "rle at offset", + ) + + // The zero-padding invariant is checked in img-tools, not here: + // nothing in the device API exposes a Bitmap's bytes. + + // A buffer that is not a record is rejected rather than read as an + // enormous image; the caller gets null, as from ofBuffer. + control.assert( + bitmaps.ofCompressed(badMarker) === null, + "unknown method rejected", + ) + control.assert( + bitmaps.ofCompressed(hex`00010203`) === null, + "non-record rejected", + ) + } + + function samePixels(actual: Bitmap, expected: Bitmap, what: string): void { + control.assert(!!actual, what + ": decoded") + control.assert(actual.width == expected.width, what + ": width") + control.assert(actual.height == expected.height, what + ": height") + let bad = 0 + for (let x = 0; x < expected.width; ++x) + for (let y = 0; y < expected.height; ++y) + if (actual.getPixel(x, y) != expected.getPixel(x, y)) ++bad + control.assert(bad == 0, what + ": pixels") + } + + /** + * Smoke harness for ui.ImagePack (img.ts). + * + * The fixture pack holds two records, with index 0 and index 2 sharing + * one offset, which is how the generator stores an image used twice. + */ + export function runImagePackTest(): void { + const pack = new ui.ImagePack( + hex`87041000100001c00f0f0f02290521052105200020032000200520012001200120052002210220052002210220052001200120012005200020032000200521052105290f0f0f0287041000100000001212121212121212212121212121212112121212121212122121212121212121121212121212121221212121212121211212121212121212212121212121212112121212121212122121212121212121121212121212121221212121212121211212121212121212212121212121212112121212121212122121212121212121`, + hex`000047000000`, + ) + const checker = bmp` + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 + 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 + ` + + samePixels(pack.get(0), ui.MISSING, "pack rle entry") + samePixels(pack.get(1), checker, "pack uncompressed entry") + control.assert(pack.length == 3, "pack length") + + // Decoded once and kept: the same index hands back one object, and + // so do two indices that share a record. + control.assert(pack.get(0) === pack.get(0), "cached by index") + control.assert( + pack.get(0) === pack.get(2), + "shared record decoded once", + ) + + // Out of range yields the fallback rather than null. + control.assert(pack.get(99) === ui.MISSING, "out of range falls back") + control.assert(pack.get(-1) === ui.MISSING, "negative falls back") + + // An unreadable record yields the fallback, not null. + const broken = new ui.ImagePack( + hex`87041000100000a00f0f0f02290521052105200020032000200520012001200120052002210220052002210220052001200120012005200020032000200521052105290f0f0f02`, + hex`0000`, + ) + control.assert(broken.get(0) === ui.MISSING, "bad record falls back") + + // A pack may carry its own fallback instead of ui.MISSING. + const custom = bmp`5` + const withFallback = new ui.ImagePack( + hex`87041000100000a00f0f0f02290521052105200020032000200520012001200120052002210220052002210220052001200120012005200020032000200521052105290f0f0f02`, + hex`0000`, + custom, + ) + control.assert(withFallback.get(0) === custom, "custom fallback") + } + runGeometrySmokeTest() runViewportSmokeTest(2) runAssetResolverSmokeTest() runRuntimeSmokeTest() runLayoutSmokeTest() runLocTest() + runImgTest() + runImagePackTest() control.__log(1, "All tests passed!") }