From 80f508b7c6cc748c07218a4923e4dd275fb87136 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 16:15:30 +0000 Subject: [PATCH] Strip USFM \ref markers before converting USFM to USX3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BibleMultiConverter 0.0.9 doesn't know \ref, so it kept both sides of the pipe ("Psalm 91:11–12|PSA 91:11-12") and added a space where the marker was ("( 1 Enoch 1:9|1:9)"). convertUsfmToUsx3() now copies the USFM files to a temporary directory and replaces \ref TEXT|TARGET\ref* (and \+ref) with TEXT before running the converter, so every fetcher that converts USFM to USX3 gets the fix. The BSB marker file now records the preprocessing version so existing BSB downloads are converted again. Fixes #144 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019LsURyVgHxWYTXpbFoYSPV --- packages/helloao-cli/bsb.spec.ts | 1 + packages/helloao-cli/bsb.ts | 5 +- packages/helloao-cli/conversion.spec.ts | 90 +++++++++++++++++++++ packages/helloao-cli/conversion.ts | 101 +++++++++++++++++++++++- 4 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 packages/helloao-cli/conversion.spec.ts diff --git a/packages/helloao-cli/bsb.spec.ts b/packages/helloao-cli/bsb.spec.ts index 1c45653..290d7dd 100644 --- a/packages/helloao-cli/bsb.spec.ts +++ b/packages/helloao-cli/bsb.spec.ts @@ -16,6 +16,7 @@ jest.mock('./conversion.js', () => ({ findBibleMultiConverterJar: jest.fn(async () => '/fake/converter.jar'), promptForBibleMultiConverter: jest.fn(async () => null), convertUsfmToUsx3: jest.fn(), + USFM_PREPROCESSING_VERSION: 1, })); const encoder = new TextEncoder(); diff --git a/packages/helloao-cli/bsb.ts b/packages/helloao-cli/bsb.ts index bd78cec..c21fd98 100644 --- a/packages/helloao-cli/bsb.ts +++ b/packages/helloao-cli/bsb.ts @@ -16,6 +16,7 @@ import { log } from '@helloao/tools'; import { InputTranslationMetadata } from '@helloao/tools/generation/common-types.js'; import { convertUsfmToUsx3, + USFM_PREPROCESSING_VERSION, findBibleMultiConverterJar, printConversionInstructions, promptForBibleMultiConverter, @@ -147,7 +148,9 @@ export async function fetchBsbTranslation( const content = new Uint8Array(await response.arrayBuffer()); const sha256 = createHash('sha256').update(content).digest('hex'); const marker = `Source: ${url}\nSHA-256: ${sha256}\nFormat: ${ - convertToUsx3 ? 'USX3' : 'USFM' + convertToUsx3 + ? `USX3\nPreprocessing: ${USFM_PREPROCESSING_VERSION}` + : 'USFM' }\n`; if ( diff --git a/packages/helloao-cli/conversion.spec.ts b/packages/helloao-cli/conversion.spec.ts new file mode 100644 index 0000000..4aa075d --- /dev/null +++ b/packages/helloao-cli/conversion.spec.ts @@ -0,0 +1,90 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { preprocessUsfm, preprocessUsfmDirectory } from './conversion.js'; + +describe('preprocessUsfm()', () => { + it('should replace \\ref with its display text', () => { + expect( + preprocessUsfm( + '\\f + \\fr 4:6 \\ft See \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\\f*' + ) + ).toBe('\\f + \\fr 4:6 \\ft See Psalm 91:11–12\\f*'); + }); + + it('should not add a space after an opening parenthesis', () => { + expect(preprocessUsfm('\\ft (\\ref 1 Enoch 1:9|1:9\\ref*)')).toBe( + '\\ft (1 Enoch 1:9)' + ); + }); + + it('should support \\ref without a target', () => { + expect(preprocessUsfm('See \\ref Genesis 1:1\\ref*.')).toBe( + 'See Genesis 1:1.' + ); + }); + + it('should support attribute targets and nested \\+ref', () => { + expect( + preprocessUsfm( + 'A \\ref John 3:16|loc="JHN 3:16"\\ref* and \\+ref Mark 1:1|MRK 1:1\\+ref*' + ) + ).toBe('A John 3:16 and Mark 1:1'); + }); + + it('should handle multiple references on one line', () => { + expect( + preprocessUsfm( + '\\ref Isaiah 40:3|ISA 40:3\\ref*; \\ref Malachi 3:1|MAL 3:1\\ref*' + ) + ).toBe('Isaiah 40:3; Malachi 3:1'); + }); + + it('should leave other markers unchanged', () => { + const usfm = + '\\v 1 In the beginning \\f + \\fr 1:1 \\ft text|with pipe\\f*\n\\rem \\refx'; + expect(preprocessUsfm(usfm)).toBe(usfm); + }); +}); + +describe('preprocessUsfmDirectory()', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), 'conversion-spec-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('should write preprocessed USFM files to the output directory without changing the input', async () => { + const input = path.join(dir, 'in'); + const output = path.join(dir, 'out'); + await mkdir(input); + const original = + '\\id MAT\n\\v 6 \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\n'; + await writeFile(path.join(input, 'MAT.usfm'), original); + await writeFile(path.join(input, 'GEN.usfm'), '\\id GEN\n'); + await writeFile( + path.join(input, 'notes.json'), + '{"a":"\\\\ref x|y\\\\ref*"}' + ); + + const changed = await preprocessUsfmDirectory(input, output); + + expect(changed).toBe(1); + expect(await readFile(path.join(output, 'MAT.usfm'), 'utf-8')).toBe( + '\\id MAT\n\\v 6 Psalm 91:11–12\n' + ); + expect(await readFile(path.join(output, 'GEN.usfm'), 'utf-8')).toBe( + '\\id GEN\n' + ); + expect(await readFile(path.join(output, 'notes.json'), 'utf-8')).toBe( + '{"a":"\\\\ref x|y\\\\ref*"}' + ); + expect(await readFile(path.join(input, 'MAT.usfm'), 'utf-8')).toBe( + original + ); + }); +}); diff --git a/packages/helloao-cli/conversion.ts b/packages/helloao-cli/conversion.ts index b01fb8a..f7342ea 100644 --- a/packages/helloao-cli/conversion.ts +++ b/packages/helloao-cli/conversion.ts @@ -1,5 +1,14 @@ -import { mkdir, readdir } from 'node:fs/promises'; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from 'node:fs/promises'; import { existsSync } from 'fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import { execFile as execFileCallback } from 'child_process'; import { promisify } from 'util'; import { confirm, input } from '@inquirer/prompts'; @@ -84,7 +93,77 @@ export async function promptForBibleMultiConverter(): Promise { } /** - * Converts USFM directory to USX3 using BibleMultiConverter + * The version of the USFM preprocessing that convertUsfmToUsx3() does. + * Increase this when preprocessUsfm() changes so that fetchers that skip unchanged sources + * (e.g. fetchBsbTranslation()) convert them again. + */ +export const USFM_PREPROCESSING_VERSION = 1; + +/** + * Matches a USFM 3 reference (`\\ref TEXT|TARGET\\ref*`), including the nested form (`\\+ref ...\\+ref*`). + * Group 1 is the "+" of a nested marker, group 2 is the display text, and group 3 is the optional link target + * (e.g. `PSA 91:11-12` or `loc="PSA 91:11-12"`). + */ +const USFM_REF_REGEX = + /\\(\+?)ref\s((?:(?!\\\+?ref\*)[^|])*?)(?:\|((?:(?!\\\+?ref\*)[\s\S])*?))?\\\1ref\*/g; + +/** + * The extensions of the files that are treated as USFM by preprocessUsfmDirectory(). + */ +const USFM_EXTENSIONS = new Set(['.usfm', '.sfm']); + +/** + * Rewrites the USFM markup that BibleMultiConverter doesn't understand so that the converted USX3 contains + * only the text that readers should see. + * + * - `\\ref TEXT|TARGET\\ref*` is replaced by `TEXT`. BibleMultiConverter doesn't know the `\\ref` marker, so it drops + * the markers and keeps the text on both sides of the pipe (e.g. "Psalm 91:11–12|PSA 91:11-12"), and it adds a space + * where the opening marker was (e.g. "( 1 Enoch 1:9|1:9)"). See https://github.com/HelloAOLab/bible-api/issues/144. + * + * @param usfm The USFM content. + * @returns The preprocessed USFM content. + */ +export function preprocessUsfm(usfm: string): string { + return usfm.replace(USFM_REF_REGEX, (_match, _plus, text: string) => text); +} + +/** + * Copies the files in the given directory to the given output directory and runs preprocessUsfm() on every USFM file. + * Other files are copied unchanged. + * @param inputDir The directory that contains the USFM files. + * @param outputDir The directory that the preprocessed files should be written to. + * @returns The number of USFM files that were changed. + */ +export async function preprocessUsfmDirectory( + inputDir: string, + outputDir: string +): Promise { + await mkdir(outputDir, { recursive: true }); + let changed = 0; + for (const entry of await readdir(inputDir, { withFileTypes: true })) { + if (!entry.isFile()) { + continue; + } + const inputPath = path.join(inputDir, entry.name); + const outputPath = path.join(outputDir, entry.name); + const content = await readFile(inputPath); + if (USFM_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) { + const text = content.toString('utf-8'); + const processed = preprocessUsfm(text); + if (processed !== text) { + changed += 1; + await writeFile(outputPath, processed, 'utf-8'); + continue; + } + } + await writeFile(outputPath, content); + } + return changed; +} + +/** + * Converts USFM directory to USX3 using BibleMultiConverter. + * The USFM files are first preprocessed with preprocessUsfm() in a temporary directory, so the input directory is not modified. */ export async function convertUsfmToUsx3( inputDir: string, @@ -92,9 +171,21 @@ export async function convertUsfmToUsx3( jarPath: string ): Promise { const logger = log.getLogger(); + let preprocessedDir: string | null = null; try { logger.log(`Converting USFM files from ${inputDir} to USX3 format...`); + preprocessedDir = await mkdtemp( + path.join(tmpdir(), 'bible-api-usfm-preprocessed-') + ); + const changed = await preprocessUsfmDirectory( + inputDir, + preprocessedDir + ); + if (changed > 0) { + logger.log(`Preprocessed ${changed} USFM files before conversion.`); + } + // Handle overwrite logic for output directory // if (overwrite && existsSync(outputDir)) { // logger.log(`Overwriting existing USX3 directory: ${outputDir}`); @@ -110,7 +201,7 @@ export async function convertUsfmToUsx3( jarPath, 'ParatextConverter', 'USFM', - inputDir, + preprocessedDir, 'USX3', outputDir, '*.usx', @@ -139,6 +230,10 @@ export async function convertUsfmToUsx3( } catch (error: any) { logger.error('Conversion failed:', error.message); return false; + } finally { + if (preprocessedDir) { + await rm(preprocessedDir, { recursive: true, force: true }); + } } }