From 3a1df6e08d8e06b519044d32c10ca0466bac9d40 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 03:07:24 +0000 Subject: [PATCH] perf(build): write the database in one transaction Every INSERT was an fsynced autocommit, on every PR and release: 43 s to 23 s on the full corpus. Also `git add -f` on the weekly URL cache, which is gitignored and so has never been committed. Part of AUREO-1470 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013X2d8vEWjRnkZydpXo94JF --- .github/workflows/url-health.yml | 7 ++++- CLAUDE.md | 10 +++++++ scripts/build-sqlite.ts | 47 ++++++++++++++++++++++---------- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/.github/workflows/url-health.yml b/.github/workflows/url-health.yml index 68c759291..276d1ff60 100644 --- a/.github/workflows/url-health.yml +++ b/.github/workflows/url-health.yml @@ -249,7 +249,12 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add .github/url-cache.json + # The cache is in .gitignore so a local `pnpm validate-urls` never + # stages it; this job is the one writer meant to commit it, and + # without -f git refuses an ignored path and the step fails under + # `set -e`. The guard above has already confirmed it is the only + # file that changed. + git add -f .github/url-cache.json git diff --staged --quiet && exit 0 git -c core.hooksPath=/dev/null commit -m "chore: update URL cache" git -c core.hooksPath=/dev/null push || echo "Push failed - may need manual intervention" diff --git a/CLAUDE.md b/CLAUDE.md index ef2e60a72..7d6807f30 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,6 +130,16 @@ cohort working: (`apps/studio/src-tauri/src/data/mod.rs`), and only after the builds that cannot read the new schema are considered abandonable. Additive changes never bump it. +- **`pnpm build` writes every row inside one transaction with + `synchronous = OFF`**, after `schema.sql` has switched the file to WAL. + Every INSERT used to be its own fsynced autocommit, which was 20 of the + build's 43 seconds on every PR and release. SQLite refuses the WAL + switch mid-transaction and the WAL flag is part of the shipped file + header, so the schema runs first; better-sqlite3's defensive mode + silently answers `journal_mode = OFF` with `delete`, so that pragma is + not an option; and a failed build closes the handle and deletes the + file rather than relying on a rollback, so a partial database never + looks complete. ## Project Structure diff --git a/scripts/build-sqlite.ts b/scripts/build-sqlite.ts index acb571788..3647deb19 100644 --- a/scripts/build-sqlite.ts +++ b/scripts/build-sqlite.ts @@ -207,12 +207,37 @@ export function buildDatabase(options: BuildOptions = {}): void { const db = new Database(outputFile); let totalProducts: number; try { - totalProducts = populateDatabase(db, dataDir, version); - } finally { - // A missing id throws partway through; the handle still has to go, or - // the next build cannot unlink the file it left behind. + // A fresh file written by one process, and read by nobody until it is + // done, has nothing an fsync protects: a failed build deletes it below. + // Not stored in the file, so there is nothing to restore before VACUUM. + db.pragma("synchronous = OFF"); + + // Apply schema. It stays outside the transaction because it switches + // the journal to WAL, which SQLite refuses mid-transaction, and the + // switch is kept: WAL is written into the file header, and every + // catalog.sqlite shipped so far has carried it. (journal_mode = OFF is + // not an option either way: better-sqlite3 opens in SQLite's defensive + // mode, which answers that pragma with DELETE and changes nothing.) + db.exec(fs.readFileSync(SCHEMA_FILE, "utf-8")); + + // One transaction for every row. As autocommits, each INSERT was its own + // synced transaction and the full corpus took 43 s to build; this takes + // about 23 s. + totalProducts = db.transaction(() => populateDatabase(db, dataDir, version))(); + + // Optimize database. VACUUM refuses to run inside a transaction. + db.exec("VACUUM"); + db.exec("ANALYZE"); + } catch (err) { + // A missing id throws partway through. The handle has to go, or the + // next build cannot unlink the file it left behind, and so does the + // file: a catalog.sqlite with its schema and part of its rows reads as + // a built one to whoever opens it next. db.close(); + fs.rmSync(outputFile, { force: true }); + throw err; } + db.close(); // Get file size const stats = fs.statSync(outputFile); @@ -226,14 +251,12 @@ export function buildDatabase(options: BuildOptions = {}): void { } /** - * Fill an open, empty database from the collections under `dataDir`. - * Returns the number of product entries written. + * Fill an open database that already carries the schema from the + * collections under `dataDir`. Returns the number of product entries + * written. Runs inside the one transaction `buildDatabase` opens, so + * nothing here may VACUUM, commit or change the journal mode. */ function populateDatabase(db: Database.Database, dataDir: string, version: string): number { - // Apply schema - const schema = fs.readFileSync(SCHEMA_FILE, "utf-8"); - db.exec(schema); - // Insert approved locales const insertLocale = db.prepare(` INSERT INTO locales (code, name, native_name, enabled) @@ -1546,10 +1569,6 @@ function populateDatabase(db: Database.Database, dataDir: string, version: strin updateMeta.run("updated_at", new Date().toISOString()); updateMeta.run("schema_version", CATALOG_SCHEMA_VERSION); - // Optimize database - db.exec("VACUUM"); - db.exec("ANALYZE"); - return softwareCount + contentCount + hardwareCount + accessoryCount; }