Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/helloao-cli/bsb.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ jest.mock('./conversion.js', () => ({
findBibleMultiConverterJar: jest.fn(async () => '/fake/converter.jar'),
promptForBibleMultiConverter: jest.fn(async () => null),
convertUsfmToUsx3: jest.fn(),
USFM_PREPROCESSING_VERSION: 1,
}));

const encoder = new TextEncoder();
Expand Down
5 changes: 4 additions & 1 deletion packages/helloao-cli/bsb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { log } from '@helloao/tools';
import { InputTranslationMetadata } from '@helloao/tools/generation/common-types.js';
import {
convertUsfmToUsx3,
USFM_PREPROCESSING_VERSION,
findBibleMultiConverterJar,
printConversionInstructions,
promptForBibleMultiConverter,
Expand Down Expand Up @@ -147,7 +148,9 @@ export async function fetchBsbTranslation(
const content = new Uint8Array(await response.arrayBuffer());
const sha256 = createHash('sha256').update(content).digest('hex');
const marker = `Source: ${url}\nSHA-256: ${sha256}\nFormat: ${
convertToUsx3 ? 'USX3' : 'USFM'
convertToUsx3
? `USX3\nPreprocessing: ${USFM_PREPROCESSING_VERSION}`
: 'USFM'
}\n`;

if (
Expand Down
90 changes: 90 additions & 0 deletions packages/helloao-cli/conversion.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { preprocessUsfm, preprocessUsfmDirectory } from './conversion.js';

describe('preprocessUsfm()', () => {
it('should replace \\ref with its display text', () => {
expect(
preprocessUsfm(
'\\f + \\fr 4:6 \\ft See \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\\f*'
)
).toBe('\\f + \\fr 4:6 \\ft See Psalm 91:11–12\\f*');
});

it('should not add a space after an opening parenthesis', () => {
expect(preprocessUsfm('\\ft (\\ref 1 Enoch 1:9|1:9\\ref*)')).toBe(
'\\ft (1 Enoch 1:9)'
);
});

it('should support \\ref without a target', () => {
expect(preprocessUsfm('See \\ref Genesis 1:1\\ref*.')).toBe(
'See Genesis 1:1.'
);
});

it('should support attribute targets and nested \\+ref', () => {
expect(
preprocessUsfm(
'A \\ref John 3:16|loc="JHN 3:16"\\ref* and \\+ref Mark 1:1|MRK 1:1\\+ref*'
)
).toBe('A John 3:16 and Mark 1:1');
});

it('should handle multiple references on one line', () => {
expect(
preprocessUsfm(
'\\ref Isaiah 40:3|ISA 40:3\\ref*; \\ref Malachi 3:1|MAL 3:1\\ref*'
)
).toBe('Isaiah 40:3; Malachi 3:1');
});

it('should leave other markers unchanged', () => {
const usfm =
'\\v 1 In the beginning \\f + \\fr 1:1 \\ft text|with pipe\\f*\n\\rem \\refx';
expect(preprocessUsfm(usfm)).toBe(usfm);
});
});

describe('preprocessUsfmDirectory()', () => {
let dir: string;

beforeEach(async () => {
dir = await mkdtemp(path.join(tmpdir(), 'conversion-spec-'));
});

afterEach(async () => {
await rm(dir, { recursive: true, force: true });
});

it('should write preprocessed USFM files to the output directory without changing the input', async () => {
const input = path.join(dir, 'in');
const output = path.join(dir, 'out');
await mkdir(input);
const original =
'\\id MAT\n\\v 6 \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\n';
await writeFile(path.join(input, 'MAT.usfm'), original);
await writeFile(path.join(input, 'GEN.usfm'), '\\id GEN\n');
await writeFile(
path.join(input, 'notes.json'),
'{"a":"\\\\ref x|y\\\\ref*"}'
);

const changed = await preprocessUsfmDirectory(input, output);

expect(changed).toBe(1);
expect(await readFile(path.join(output, 'MAT.usfm'), 'utf-8')).toBe(
'\\id MAT\n\\v 6 Psalm 91:11–12\n'
);
expect(await readFile(path.join(output, 'GEN.usfm'), 'utf-8')).toBe(
'\\id GEN\n'
);
expect(await readFile(path.join(output, 'notes.json'), 'utf-8')).toBe(
'{"a":"\\\\ref x|y\\\\ref*"}'
);
expect(await readFile(path.join(input, 'MAT.usfm'), 'utf-8')).toBe(
original
);
});
});
101 changes: 98 additions & 3 deletions packages/helloao-cli/conversion.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
import { mkdir, readdir } from 'node:fs/promises';
import {
mkdir,
mkdtemp,
readdir,
readFile,
rm,
writeFile,
} from 'node:fs/promises';
import { existsSync } from 'fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { execFile as execFileCallback } from 'child_process';
import { promisify } from 'util';
import { confirm, input } from '@inquirer/prompts';
Expand Down Expand Up @@ -84,17 +93,99 @@ export async function promptForBibleMultiConverter(): Promise<string | null> {
}

/**
* Converts USFM directory to USX3 using BibleMultiConverter
* The version of the USFM preprocessing that convertUsfmToUsx3() does.
* Increase this when preprocessUsfm() changes so that fetchers that skip unchanged sources
* (e.g. fetchBsbTranslation()) convert them again.
*/
export const USFM_PREPROCESSING_VERSION = 1;

/**
* Matches a USFM 3 reference (`\\ref TEXT|TARGET\\ref*`), including the nested form (`\\+ref ...\\+ref*`).
* Group 1 is the "+" of a nested marker, group 2 is the display text, and group 3 is the optional link target
* (e.g. `PSA 91:11-12` or `loc="PSA 91:11-12"`).
*/
const USFM_REF_REGEX =
/\\(\+?)ref\s((?:(?!\\\+?ref\*)[^|])*?)(?:\|((?:(?!\\\+?ref\*)[\s\S])*?))?\\\1ref\*/g;

/**
* The extensions of the files that are treated as USFM by preprocessUsfmDirectory().
*/
const USFM_EXTENSIONS = new Set(['.usfm', '.sfm']);

/**
* Rewrites the USFM markup that BibleMultiConverter doesn't understand so that the converted USX3 contains
* only the text that readers should see.
*
* - `\\ref TEXT|TARGET\\ref*` is replaced by `TEXT`. BibleMultiConverter doesn't know the `\\ref` marker, so it drops
* the markers and keeps the text on both sides of the pipe (e.g. "Psalm 91:11–12|PSA 91:11-12"), and it adds a space
* where the opening marker was (e.g. "( 1 Enoch 1:9|1:9)"). See https://github.com/HelloAOLab/bible-api/issues/144.
*
* @param usfm The USFM content.
* @returns The preprocessed USFM content.
*/
export function preprocessUsfm(usfm: string): string {
return usfm.replace(USFM_REF_REGEX, (_match, _plus, text: string) => text);
}

/**
* Copies the files in the given directory to the given output directory and runs preprocessUsfm() on every USFM file.
* Other files are copied unchanged.
* @param inputDir The directory that contains the USFM files.
* @param outputDir The directory that the preprocessed files should be written to.
* @returns The number of USFM files that were changed.
*/
export async function preprocessUsfmDirectory(
inputDir: string,
outputDir: string
): Promise<number> {
await mkdir(outputDir, { recursive: true });
let changed = 0;
for (const entry of await readdir(inputDir, { withFileTypes: true })) {
if (!entry.isFile()) {
continue;
}
const inputPath = path.join(inputDir, entry.name);
const outputPath = path.join(outputDir, entry.name);
const content = await readFile(inputPath);
if (USFM_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) {
const text = content.toString('utf-8');
const processed = preprocessUsfm(text);
if (processed !== text) {
changed += 1;
await writeFile(outputPath, processed, 'utf-8');
continue;
}
}
await writeFile(outputPath, content);
}
return changed;
}

/**
* Converts USFM directory to USX3 using BibleMultiConverter.
* The USFM files are first preprocessed with preprocessUsfm() in a temporary directory, so the input directory is not modified.
*/
export async function convertUsfmToUsx3(
inputDir: string,
outputDir: string,
jarPath: string
): Promise<boolean> {
const logger = log.getLogger();
let preprocessedDir: string | null = null;
try {
logger.log(`Converting USFM files from ${inputDir} to USX3 format...`);

preprocessedDir = await mkdtemp(
path.join(tmpdir(), 'bible-api-usfm-preprocessed-')
);
const changed = await preprocessUsfmDirectory(
inputDir,
preprocessedDir
);
if (changed > 0) {
logger.log(`Preprocessed ${changed} USFM files before conversion.`);
}

// Handle overwrite logic for output directory
// if (overwrite && existsSync(outputDir)) {
// logger.log(`Overwriting existing USX3 directory: ${outputDir}`);
Expand All @@ -110,7 +201,7 @@ export async function convertUsfmToUsx3(
jarPath,
'ParatextConverter',
'USFM',
inputDir,
preprocessedDir,
'USX3',
outputDir,
'*.usx',
Expand Down Expand Up @@ -139,6 +230,10 @@ export async function convertUsfmToUsx3(
} catch (error: any) {
logger.error('Conversion failed:', error.message);
return false;
} finally {
if (preprocessedDir) {
await rm(preprocessedDir, { recursive: true, force: true });
}
}
}

Expand Down
Loading