diff --git a/packages/helloao-cli/bsb.spec.ts b/packages/helloao-cli/bsb.spec.ts index 1c45653..290d7dd 100644 --- a/packages/helloao-cli/bsb.spec.ts +++ b/packages/helloao-cli/bsb.spec.ts @@ -16,6 +16,7 @@ jest.mock('./conversion.js', () => ({ findBibleMultiConverterJar: jest.fn(async () => '/fake/converter.jar'), promptForBibleMultiConverter: jest.fn(async () => null), convertUsfmToUsx3: jest.fn(), + USFM_PREPROCESSING_VERSION: 1, })); const encoder = new TextEncoder(); diff --git a/packages/helloao-cli/bsb.ts b/packages/helloao-cli/bsb.ts index bd78cec..c21fd98 100644 --- a/packages/helloao-cli/bsb.ts +++ b/packages/helloao-cli/bsb.ts @@ -16,6 +16,7 @@ import { log } from '@helloao/tools'; import { InputTranslationMetadata } from '@helloao/tools/generation/common-types.js'; import { convertUsfmToUsx3, + USFM_PREPROCESSING_VERSION, findBibleMultiConverterJar, printConversionInstructions, promptForBibleMultiConverter, @@ -147,7 +148,9 @@ export async function fetchBsbTranslation( const content = new Uint8Array(await response.arrayBuffer()); const sha256 = createHash('sha256').update(content).digest('hex'); const marker = `Source: ${url}\nSHA-256: ${sha256}\nFormat: ${ - convertToUsx3 ? 'USX3' : 'USFM' + convertToUsx3 + ? `USX3\nPreprocessing: ${USFM_PREPROCESSING_VERSION}` + : 'USFM' }\n`; if ( diff --git a/packages/helloao-cli/conversion.spec.ts b/packages/helloao-cli/conversion.spec.ts new file mode 100644 index 0000000..4aa075d --- /dev/null +++ b/packages/helloao-cli/conversion.spec.ts @@ -0,0 +1,90 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { preprocessUsfm, preprocessUsfmDirectory } from './conversion.js'; + +describe('preprocessUsfm()', () => { + it('should replace \\ref with its display text', () => { + expect( + preprocessUsfm( + '\\f + \\fr 4:6 \\ft See \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\\f*' + ) + ).toBe('\\f + \\fr 4:6 \\ft See Psalm 91:11–12\\f*'); + }); + + it('should not add a space after an opening parenthesis', () => { + expect(preprocessUsfm('\\ft (\\ref 1 Enoch 1:9|1:9\\ref*)')).toBe( + '\\ft (1 Enoch 1:9)' + ); + }); + + it('should support \\ref without a target', () => { + expect(preprocessUsfm('See \\ref Genesis 1:1\\ref*.')).toBe( + 'See Genesis 1:1.' + ); + }); + + it('should support attribute targets and nested \\+ref', () => { + expect( + preprocessUsfm( + 'A \\ref John 3:16|loc="JHN 3:16"\\ref* and \\+ref Mark 1:1|MRK 1:1\\+ref*' + ) + ).toBe('A John 3:16 and Mark 1:1'); + }); + + it('should handle multiple references on one line', () => { + expect( + preprocessUsfm( + '\\ref Isaiah 40:3|ISA 40:3\\ref*; \\ref Malachi 3:1|MAL 3:1\\ref*' + ) + ).toBe('Isaiah 40:3; Malachi 3:1'); + }); + + it('should leave other markers unchanged', () => { + const usfm = + '\\v 1 In the beginning \\f + \\fr 1:1 \\ft text|with pipe\\f*\n\\rem \\refx'; + expect(preprocessUsfm(usfm)).toBe(usfm); + }); +}); + +describe('preprocessUsfmDirectory()', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), 'conversion-spec-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('should write preprocessed USFM files to the output directory without changing the input', async () => { + const input = path.join(dir, 'in'); + const output = path.join(dir, 'out'); + await mkdir(input); + const original = + '\\id MAT\n\\v 6 \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\n'; + await writeFile(path.join(input, 'MAT.usfm'), original); + await writeFile(path.join(input, 'GEN.usfm'), '\\id GEN\n'); + await writeFile( + path.join(input, 'notes.json'), + '{"a":"\\\\ref x|y\\\\ref*"}' + ); + + const changed = await preprocessUsfmDirectory(input, output); + + expect(changed).toBe(1); + expect(await readFile(path.join(output, 'MAT.usfm'), 'utf-8')).toBe( + '\\id MAT\n\\v 6 Psalm 91:11–12\n' + ); + expect(await readFile(path.join(output, 'GEN.usfm'), 'utf-8')).toBe( + '\\id GEN\n' + ); + expect(await readFile(path.join(output, 'notes.json'), 'utf-8')).toBe( + '{"a":"\\\\ref x|y\\\\ref*"}' + ); + expect(await readFile(path.join(input, 'MAT.usfm'), 'utf-8')).toBe( + original + ); + }); +}); diff --git a/packages/helloao-cli/conversion.ts b/packages/helloao-cli/conversion.ts index b01fb8a..f7342ea 100644 --- a/packages/helloao-cli/conversion.ts +++ b/packages/helloao-cli/conversion.ts @@ -1,5 +1,14 @@ -import { mkdir, readdir } from 'node:fs/promises'; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from 'node:fs/promises'; import { existsSync } from 'fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import { execFile as execFileCallback } from 'child_process'; import { promisify } from 'util'; import { confirm, input } from '@inquirer/prompts'; @@ -84,7 +93,77 @@ export async function promptForBibleMultiConverter(): Promise { } /** - * Converts USFM directory to USX3 using BibleMultiConverter + * The version of the USFM preprocessing that convertUsfmToUsx3() does. + * Increase this when preprocessUsfm() changes so that fetchers that skip unchanged sources + * (e.g. fetchBsbTranslation()) convert them again. + */ +export const USFM_PREPROCESSING_VERSION = 1; + +/** + * Matches a USFM 3 reference (`\\ref TEXT|TARGET\\ref*`), including the nested form (`\\+ref ...\\+ref*`). + * Group 1 is the "+" of a nested marker, group 2 is the display text, and group 3 is the optional link target + * (e.g. `PSA 91:11-12` or `loc="PSA 91:11-12"`). + */ +const USFM_REF_REGEX = + /\\(\+?)ref\s((?:(?!\\\+?ref\*)[^|])*?)(?:\|((?:(?!\\\+?ref\*)[\s\S])*?))?\\\1ref\*/g; + +/** + * The extensions of the files that are treated as USFM by preprocessUsfmDirectory(). + */ +const USFM_EXTENSIONS = new Set(['.usfm', '.sfm']); + +/** + * Rewrites the USFM markup that BibleMultiConverter doesn't understand so that the converted USX3 contains + * only the text that readers should see. + * + * - `\\ref TEXT|TARGET\\ref*` is replaced by `TEXT`. BibleMultiConverter doesn't know the `\\ref` marker, so it drops + * the markers and keeps the text on both sides of the pipe (e.g. "Psalm 91:11–12|PSA 91:11-12"), and it adds a space + * where the opening marker was (e.g. "( 1 Enoch 1:9|1:9)"). See https://github.com/HelloAOLab/bible-api/issues/144. + * + * @param usfm The USFM content. + * @returns The preprocessed USFM content. + */ +export function preprocessUsfm(usfm: string): string { + return usfm.replace(USFM_REF_REGEX, (_match, _plus, text: string) => text); +} + +/** + * Copies the files in the given directory to the given output directory and runs preprocessUsfm() on every USFM file. + * Other files are copied unchanged. + * @param inputDir The directory that contains the USFM files. + * @param outputDir The directory that the preprocessed files should be written to. + * @returns The number of USFM files that were changed. + */ +export async function preprocessUsfmDirectory( + inputDir: string, + outputDir: string +): Promise { + await mkdir(outputDir, { recursive: true }); + let changed = 0; + for (const entry of await readdir(inputDir, { withFileTypes: true })) { + if (!entry.isFile()) { + continue; + } + const inputPath = path.join(inputDir, entry.name); + const outputPath = path.join(outputDir, entry.name); + const content = await readFile(inputPath); + if (USFM_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) { + const text = content.toString('utf-8'); + const processed = preprocessUsfm(text); + if (processed !== text) { + changed += 1; + await writeFile(outputPath, processed, 'utf-8'); + continue; + } + } + await writeFile(outputPath, content); + } + return changed; +} + +/** + * Converts USFM directory to USX3 using BibleMultiConverter. + * The USFM files are first preprocessed with preprocessUsfm() in a temporary directory, so the input directory is not modified. */ export async function convertUsfmToUsx3( inputDir: string, @@ -92,9 +171,21 @@ export async function convertUsfmToUsx3( jarPath: string ): Promise { const logger = log.getLogger(); + let preprocessedDir: string | null = null; try { logger.log(`Converting USFM files from ${inputDir} to USX3 format...`); + preprocessedDir = await mkdtemp( + path.join(tmpdir(), 'bible-api-usfm-preprocessed-') + ); + const changed = await preprocessUsfmDirectory( + inputDir, + preprocessedDir + ); + if (changed > 0) { + logger.log(`Preprocessed ${changed} USFM files before conversion.`); + } + // Handle overwrite logic for output directory // if (overwrite && existsSync(outputDir)) { // logger.log(`Overwriting existing USX3 directory: ${outputDir}`); @@ -110,7 +201,7 @@ export async function convertUsfmToUsx3( jarPath, 'ParatextConverter', 'USFM', - inputDir, + preprocessedDir, 'USX3', outputDir, '*.usx', @@ -139,6 +230,10 @@ export async function convertUsfmToUsx3( } catch (error: any) { logger.error('Conversion failed:', error.message); return false; + } finally { + if (preprocessedDir) { + await rm(preprocessedDir, { recursive: true, force: true }); + } } }