diff --git a/CHANGELOG.md b/CHANGELOG.md index 465051d..84a213f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,9 @@ This is the log of changes for the Free Use Bible API. - Added the `source-all-translations` command, which downloads translations from eBible, [fetch.bible](https://fetch.bible), and pinned OSIS sources into one directory, downloading only new and changed translations. - Each translation is downloaded from only one source, preferring eBible. - `--conversion-concurrency ` limits concurrent USFM-to-USX3 conversions to reduce memory usage. + - Can read the database, the sources, and their directories from a JSON config (`--config `, or `./source.config.json` when no directory is given), like `sync`. See `source.config.example.json` and the [CLI README](./packages/helloao-cli/README.md#sourcing-translations-from-a-config). + - The config can set the path to `BibleMultiConverter.jar` (`bibleMultiConverterPath`). +- A `--bible-multi-converter-path` (or `bibleMultiConverterPath`) that doesn't exist is now an error, instead of silently falling back to another copy of the jar or asking for one. - Added `source-fetch-bible-translations` and `list-fetch-bible-translations` commands. - `import-translations` now accepts multiple directories. - Added `fetch-osis-translations` and `convert-osis-to-usfm` commands for OSIS sources. diff --git a/packages/helloao-cli/README.md b/packages/helloao-cli/README.md index e2b39b6..6253729 100644 --- a/packages/helloao-cli/README.md +++ b/packages/helloao-cli/README.md @@ -54,9 +54,10 @@ Commands: source-fetch-bible-translations [options] [translations...] Finds translation sources from fetch.bible and downloads them. Translations that are already provided by eBible are skipped. - source-all-translations [options] [translations...] + source-all-translations [options] [dir] [translations...] Finds translation sources from eBible and fetch.bible, and downloads them (along with the pinned OSIS translations and the Berean Standard Bible from bereanbible.com) into the given directory so that they can be imported with import-translations. Translations available from eBible are downloaded from eBible, and the rest are downloaded from fetch.bible. Only new and changed translations are downloaded. + With --config (or when is omitted, ./source.config.json), the database, the sources, and the directories they are downloaded to are read from the config, and every argument is a translation ID. See the README for its format. list-fetch-bible-translations [search] List available fetch.bible translations. Optionally filter by search term. list-ebible-translations [search] List available eBible translations. Optionally filter by search term. refresh-ebible-licenses [options] [translations...] Detects the licenses of the eBible translations that are already in the database and stores them. @@ -83,7 +84,7 @@ Commands: > **eBible licenses.** `source-translations` detects the license of each eBible translation (from the Creative Commons link on its ebible.org details page, falling back to a "public domain" copyright notice) and stores it in `metadata.json`. To add licenses to a database that was imported before license detection existed, run `helloao refresh-ebible-licenses --db --dir `. It updates the `license` column of each eBible translation in place, and `--dir` also writes the licenses into the matching `metadata.json` files so that a later re-import doesn't clear them. Translations whose license can't be detected are left unchanged. -> **Sourcing translations.** `source-all-translations ` is the one command for keeping a directory of translations up to date. It reconciles the translation lists from [eBible](https://ebible.org) and [fetch.bible](https://fetch.bible) so that each translation is downloaded from only one source: translations that eBible has are downloaded from eBible (like `source-translations`), and translations that only fetch.bible has (or that eBible lists but has no USFM download for) are downloaded from the [fetch.bible collection](https://github.com/gracious-tech/fetch_collection). Downloads are tracked in the database (`--db`), so later runs only download new and changed translations. Changes are detected from the git history of the fetch.bible collection: each run looks up the collection's latest commit with the GitHub API (3 requests) and compares the git tree SHA of each translation's directory with the one that was downloaded, and files are downloaded from that exact commit. Set `GITHUB_TOKEN` to raise the GitHub API rate limit; if the API is unavailable, changes are detected from each translation's `meta.json` instead. Use `--collection-repo` and `--collection-ref` to download from a fork or another branch. USFM files are converted to USX3 with [BibleMultiConverter](https://github.com/schierlm/BibleMultiConverter) (use `--no-convert-to-usx3` to keep the USFM files). Up to 10 conversions run at once; each one is a separate Java process, so lower this with `--conversion-concurrency ` to reduce memory usage (the same option works with `source-translations` and `source-fetch-bible-translations`). Every translation gets a `metadata.json` (including its license), so the directory can be imported straight away with `import-translations `. To keep the fetch.bible translations in their own directory, add `--fetch-bible-dir `; `` then only holds the eBible translations, and both are imported with `import-translations `. When a translation that was downloaded from fetch.bible later becomes available from eBible, its copy in the fetch.bible directory is removed so that it isn't imported twice. Translations listed in `/ignorelist.txt` (or the fetch.bible directory's `ignorelist.txt`) (by eBible ID, fetch.bible ID, or translation ID) are never downloaded. To only download from fetch.bible, use `source-fetch-bible-translations `, which skips translations that the database (or `--ebible-dir`) shows were already downloaded from eBible. Translations that came from fetch.bible have a `fetch-bible.txt` file in their directory. The pinned OSIS translations (the ones `fetch-osis-translations` downloads, such as `jpn_kou`) are fetched as well, after their SHA-256 hashes are checked, and are never downloaded from eBible or fetch.bible. They have an `osis.txt` file in their directory that records their source, so they are only downloaded again when their source or metadata changes (or with `--overwrite`). Put them in their own directory with `--osis-dir `, or skip them with `--no-osis`. The Berean Standard Bible is downloaded from bereanbible.com (like `fetch-bsb`) instead of eBible, and is only replaced when the download changes. Put it in its own directory with `--bsb-dir `, or use `--no-bsb` to source it from eBible like the other translations. +> **Sourcing translations.** `source-all-translations ` is the one command for keeping a directory of translations up to date. It reconciles the translation lists from [eBible](https://ebible.org) and [fetch.bible](https://fetch.bible) so that each translation is downloaded from only one source: translations that eBible has are downloaded from eBible (like `source-translations`), and translations that only fetch.bible has (or that eBible lists but has no USFM download for) are downloaded from the [fetch.bible collection](https://github.com/gracious-tech/fetch_collection). Downloads are tracked in the database (`--db`), so later runs only download new and changed translations. Changes are detected from the git history of the fetch.bible collection: each run looks up the collection's latest commit with the GitHub API (3 requests) and compares the git tree SHA of each translation's directory with the one that was downloaded, and files are downloaded from that exact commit. Set `GITHUB_TOKEN` to raise the GitHub API rate limit; if the API is unavailable, changes are detected from each translation's `meta.json` instead. Use `--collection-repo` and `--collection-ref` to download from a fork or another branch. USFM files are converted to USX3 with [BibleMultiConverter](https://github.com/schierlm/BibleMultiConverter) (use `--no-convert-to-usx3` to keep the USFM files). Up to 10 conversions run at once; each one is a separate Java process, so lower this with `--conversion-concurrency ` to reduce memory usage (the same option works with `source-translations` and `source-fetch-bible-translations`). Every translation gets a `metadata.json` (including its license), so the directory can be imported straight away with `import-translations `. To keep the fetch.bible translations in their own directory, add `--fetch-bible-dir `; `` then only holds the eBible translations, and both are imported with `import-translations `. When a translation that was downloaded from fetch.bible later becomes available from eBible, its copy in the fetch.bible directory is removed so that it isn't imported twice. Translations listed in `/ignorelist.txt` (or the fetch.bible directory's `ignorelist.txt`) (by eBible ID, fetch.bible ID, or translation ID) are never downloaded. To only download from fetch.bible, use `source-fetch-bible-translations `, which skips translations that the database (or `--ebible-dir`) shows were already downloaded from eBible. Translations that came from fetch.bible have a `fetch-bible.txt` file in their directory. The pinned OSIS translations (the ones `fetch-osis-translations` downloads, such as `jpn_kou`) are fetched as well, after their SHA-256 hashes are checked, and are never downloaded from eBible or fetch.bible. They have an `osis.txt` file in their directory that records their source, so they are only downloaded again when their source or metadata changes (or with `--overwrite`). Put them in their own directory with `--osis-dir `, or skip them with `--no-osis`. The Berean Standard Bible is downloaded from bereanbible.com (like `fetch-bsb`) instead of eBible, and is only replaced when the download changes. Put it in its own directory with `--bsb-dir `, or use `--no-bsb` to source it from eBible like the other translations. The database and the sources can also be read from a config file instead; see [Sourcing translations from a config](#sourcing-translations-from-a-config). > **SWORD commentaries.** `import-commentary`, `import-commentaries`, and `sync` can import commentaries from [SWORD](https://crosswire.org/sword/) modules, like the ones from the [CrossWire module repository](https://crosswire.org/sword/modules/ModDisp.jsp?modType=Commentaries). Put the module's `.zip` file in the commentary's directory, or extract it there (so that the directory contains `mods.d/` and `modules/`). The commentary's metadata comes from the module's `.conf` file: its ID is the module name in lowercase (e.g. `clarke`), and its name is the module's `Description`. To change any of these, add a `metadata.json` with just the values to override (e.g. `{ "id": "adam-clarke" }`). The `zCom`, `zCom4`, `RawCom`, and `RawCom4` drivers are supported, with the KJV versification and OSIS, ThML, GBF, or plain text markup. Entries are converted to plain text, with one line per paragraph; notes are left out. Book and chapter headings become the book and chapter introductions, and a comment that covers several verses (a linked entry) is only listed under its first verse. Some commentaries (like Adam Clarke's) start each comment with a "Verse N" heading: the heading is removed, and any text before the "Verse 1" heading becomes the chapter introduction. @@ -161,11 +162,49 @@ Only the items imported by the sync are uploaded, so translations that were impo The sync state is stored in the `SyncSource`, `SyncUpload`, `SyncPendingDelete`, and `SyncDestination` tables of the database. The first sync of an existing database imports every source and uploads every item once. +### Sourcing translations from a config + +Like `sync`, `source-all-translations` can read its settings from a JSON config instead of the command line. The config sets the database that tracks downloads and lists the sources to download translations from, with the directory each one is downloaded to: + +```json +{ + "db": "./free-use-bible-api.db", + "ignorelist": ["amh_amh"], + "convertToUsx3": true, + "bibleMultiConverterPath": "./tools/BibleMultiConverter.jar", + "conversionConcurrency": 4, + "sources": [ + { "type": "ebible", "path": "./sources/ebible" }, + { + "type": "fetch-bible", + "path": "./sources/fetch-bible", + "repo": "gracious-tech/fetch_collection", + "ref": "main" + }, + { "type": "osis", "path": "./sources/osis" }, + { "type": "bsb", "path": "./sources/bsb" } + ] +} +``` + +``` +helloao source-all-translations --config ./source.config.json [translations...] +helloao source-all-translations # reads ./source.config.json +``` + +Paths in the config are relative to the config file, and `--db` overrides `db`. See `source.config.example.json` in the repository for a starting point. + +- **Sources.** `ebible` is required. `fetch-bible` downloads the translations that eBible doesn't have from the fetch.bible collection (`manifestUrl`, `repo`, and `ref` are optional and only allowed here). `osis` downloads the pinned OSIS translations, and `bsb` downloads the Berean Standard Bible from bereanbible.com. Each type can be listed once, and a source that is left out isn't downloaded from (without `bsb`, the BSB is downloaded from eBible). Sources can share a directory. +- **Ignorelist.** `ignorelist` lists translations that are never downloaded (by eBible ID, fetch.bible ID, or translation ID), in addition to the `ignorelist.txt` file of each source directory. +- **Conversion.** `convertToUsx3` and `conversionConcurrency` set the USFM to USX3 conversion. `bibleMultiConverterPath` is the path to `BibleMultiConverter.jar`, relative to the config file; without it, the jar is searched for in the usual places (like `./tools/BibleMultiConverter.jar`), and you're asked for its location if it isn't found. If the jar isn't at the given path, the command stops with an error instead of searching. `--no-convert-to-usx3`, `--bible-multi-converter-path`, and `--conversion-concurrency` override them. +- **Command line.** With a config, every argument is a translation ID to download, and `--overwrite` and `--no-database` still work. The options that choose sources and directories (`--fetch-bible-dir`, `--osis-dir`, `--no-osis`, `--bsb-dir`, `--no-bsb`, `--manifest-url`, `--collection-repo`, and `--collection-ref`) can't be combined with a config. +- **Importing.** To import the downloaded translations with `sync`, add each source's directory to the sync config as a `translations` source. + The `@helloao/cli` package can also be used as a library. The library exports a variety of actions, utilities, and supporting classes designed to assist with generating and managing a Free Use Bible API. -There are 7 main exports: +There are 8 main exports: - `actions` - This export contains function versions of the CLI commands. They make it easy to call a CLI command from a script. - `db` - This export contains functions that make working with a database easier. It supports operations like importing translations into a database, inserting chapters, verses, etc. and getting an updated database instance from a path. @@ -174,6 +213,7 @@ There are 7 main exports: - `uploads` - This export contains functions that make it easy to upload an API to a destination like S3, the local filesystem, or a zip archive. - `s3` - This export contains a class that can upload files to S3. - `sync` - This export contains the function version of the `sync` command (`sync()` and `syncWithConfig()`) and the functions that find, import, and upload the sources in a sync config. +- `sourceConfig` - This export contains `sourceAllTranslationsFromConfig()`, which runs `source-all-translations` from a source config, and `loadSourceConfig()`. Here are some common operations that you might want to perform: diff --git a/packages/helloao-cli/actions.ts b/packages/helloao-cli/actions.ts index cc3e19b..16a7192 100644 --- a/packages/helloao-cli/actions.ts +++ b/packages/helloao-cli/actions.ts @@ -267,6 +267,12 @@ export interface SourceAllTranslationsOptions extends Omit< SourceFetchBibleTranslationsOptions, 'ebibleDir' | 'includeEBible' | 'ebibleCoverage' | 'catalog' > { + /** + * Whether translations that eBible doesn't provide should be downloaded from fetch.bible. + * Defaults to true. + */ + includeFetchBible?: boolean; + /** * The directory that translations from fetch.bible should be downloaded to. * Defaults to the same directory as the eBible translations. @@ -1912,7 +1918,9 @@ export async function sourceTranslations( source: EBibleSource; }> = []; // Try to find BibleMultiConverter.jar - let jarPath = await findBibleMultiConverterJar(bibleMultiConverterPath); + let jarPath = convertToUsx3 + ? await findBibleMultiConverterJar(bibleMultiConverterPath) + : null; if (!jarPath && convertToUsx3) { jarPath = await promptForBibleMultiConverter(); @@ -2553,6 +2561,7 @@ export async function sourceAllTranslations( const logger = log.getLogger(); const { convertToUsx3 = true, + includeFetchBible = true, fetchBibleDir: _, osisDir: __, includeOsis = true, @@ -2567,7 +2576,7 @@ export async function sourceAllTranslations( // The directories that translations are sourced into, without duplicates const sourceDirs = [ outputDir, - fetchBibleDir, + ...(includeFetchBible ? [fetchBibleDir] : []), ...(includeOsis ? [osisDir] : []), ...(includeBsb ? [bsbDir] : []), ].filter( @@ -2596,10 +2605,16 @@ export async function sourceAllTranslations( } } - logger.log('Fetching eBible and fetch.bible metadata...'); + logger.log( + includeFetchBible + ? 'Fetching eBible and fetch.bible metadata...' + : 'Fetching eBible metadata...' + ); const [ebibleSources, catalog] = await Promise.all([ fetchEBibleMetadata(), - fetchFetchBibleCatalog(options), + includeFetchBible + ? fetchFetchBibleCatalog(options) + : Promise.resolve([] as FetchBibleCatalogEntry[]), ]); // The ignorelists of every directory apply to every source @@ -2645,12 +2660,14 @@ export async function sourceAllTranslations( logger.log(''); logger.log('TRANSLATION SOURCES:'); logger.log(` eBible: ${ebibleAvailable.length} translations`); - logger.log( - ` fetch.bible: ${catalog.length} translations (${catalog.length - fetchBibleOnly.length} are also available from eBible)` - ); - logger.log( - ` Translations that will be sourced from fetch.bible: ${fetchBibleOnly.length}` - ); + if (includeFetchBible) { + logger.log( + ` fetch.bible: ${catalog.length} translations (${catalog.length - fetchBibleOnly.length} are also available from eBible)` + ); + logger.log( + ` Translations that will be sourced from fetch.bible: ${fetchBibleOnly.length}` + ); + } if (includeOsis) { logger.log(` Pinned OSIS: ${osisSources.length} translations`); } @@ -2674,7 +2691,9 @@ export async function sourceAllTranslations( ); if (ebibleUnavailable.size > 0) { logger.log( - `${ebibleUnavailable.size} translations could not be downloaded from eBible. They will be sourced from fetch.bible if available.` + includeFetchBible + ? `${ebibleUnavailable.size} translations could not be downloaded from eBible. They will be sourced from fetch.bible if available.` + : `${ebibleUnavailable.size} translations could not be downloaded from eBible.` ); } const ebibleCoverage = createEBibleCoverage( @@ -2684,21 +2703,28 @@ export async function sourceAllTranslations( ) ); - logger.log(''); - logger.log('=== Sourcing translations from fetch.bible ==='); - const fetchBibleResult = await sourceFetchBibleTranslations( - fetchBibleDir, - translations, - { - ...sourceOptions, - convertToUsx3, - bibleMultiConverterPath, - catalog, - ebibleCoverage, - ebibleDir: outputDir, - ignorelist: otherSourcesIgnorelist, - } - ); + let fetchBibleResult: SourceTranslationsResult = { + downloaded: [], + failed: [], + unavailable: [], + }; + if (includeFetchBible) { + logger.log(''); + logger.log('=== Sourcing translations from fetch.bible ==='); + fetchBibleResult = await sourceFetchBibleTranslations( + fetchBibleDir, + translations, + { + ...sourceOptions, + convertToUsx3, + bibleMultiConverterPath, + catalog, + ebibleCoverage, + ebibleDir: outputDir, + ignorelist: otherSourcesIgnorelist, + } + ); + } const osisResult: SourceTranslationsResult = { downloaded: [], @@ -2769,9 +2795,11 @@ export async function sourceAllTranslations( logger.log(''); logger.log('SOURCING SUMMARY:'); logger.log(` Downloaded from eBible: ${ebibleResult.downloaded.length}`); - logger.log( - ` Downloaded from fetch.bible: ${fetchBibleResult.downloaded.length}` - ); + if (includeFetchBible) { + logger.log( + ` Downloaded from fetch.bible: ${fetchBibleResult.downloaded.length}` + ); + } if (includeOsis) { logger.log( ` Downloaded from pinned OSIS sources: ${osisResult.downloaded.length}` diff --git a/packages/helloao-cli/cli.ts b/packages/helloao-cli/cli.ts index f859f23..0b49e1c 100644 --- a/packages/helloao-cli/cli.ts +++ b/packages/helloao-cli/cli.ts @@ -47,6 +47,7 @@ import { } from './actions.js'; import { getPrismaDb } from './db.js'; import { sync } from './sync.js'; +import { sourceAllTranslationsFromConfig } from './source-config.js'; import { BSB_USFM_ZIP_URL, fetchBsbTranslation } from './bsb.js'; import { FETCH_BIBLE_COLLECTION_REF, @@ -1212,9 +1213,14 @@ async function start() { }); program - .command('source-all-translations [translations...]') + .command('source-all-translations [dir] [translations...]') .description( - 'Finds translation sources from eBible and fetch.bible, and downloads them (along with the pinned OSIS translations and the Berean Standard Bible from bereanbible.com) into the given directory so that they can be imported with import-translations.\nTranslations available from eBible are downloaded from eBible, and the rest are downloaded from fetch.bible. Only new and changed translations are downloaded.' + 'Finds translation sources from eBible and fetch.bible, and downloads them (along with the pinned OSIS translations and the Berean Standard Bible from bereanbible.com) into the given directory so that they can be imported with import-translations.\nTranslations available from eBible are downloaded from eBible, and the rest are downloaded from fetch.bible. Only new and changed translations are downloaded.\n' + + 'With --config (or when is omitted, ./source.config.json), the database, the sources, and the directories they are downloaded to are read from the config, and every argument is a translation ID. See the README for its format.' + ) + .option( + '--config ', + 'Read the database, sources, and directories from this source config.' ) .option( '--no-convert-to-usx3', @@ -1266,24 +1272,73 @@ async function start() { 'The branch (or other git ref) of the fetch.bible collection to download from', FETCH_BIBLE_COLLECTION_REF ) - .action(async (dir, translations, options) => { - await sourceAllTranslations(dir, translations, { - ...program.opts(), - convertToUsx3: options.convertToUsx3, - bibleMultiConverterPath: options.bibleMultiConverterPath, - conversionConcurrency: options.conversionConcurrency, - useDatabase: options.database !== false, - overwrite: options.overwrite, - fetchBibleDir: options.fetchBibleDir, - osisDir: options.osisDir, - includeOsis: options.osis !== false, - bsbDir: options.bsbDir, - includeBsb: options.bsb !== false, - manifestUrl: options.manifestUrl, - repo: options.collectionRepo, - ref: options.collectionRef, - }); - }); + .action( + async ( + dir: string | undefined, + translations: string[], + options: any, + command: Command + ) => { + const sharedOptions = { + ...program.opts(), + convertToUsx3: options.convertToUsx3, + bibleMultiConverterPath: options.bibleMultiConverterPath, + conversionConcurrency: options.conversionConcurrency, + useDatabase: options.database !== false, + overwrite: options.overwrite, + }; + + if (options.config || dir === undefined) { + // The sources and their directories come from the config, + // so the options that set them can't be used. + const conflicting = [ + 'fetchBibleDir', + 'osis', + 'osisDir', + 'bsb', + 'bsbDir', + 'manifestUrl', + 'collectionRepo', + 'collectionRef', + ].filter( + (key) => command.getOptionValueSource(key) === 'cli' + ); + if (conflicting.length > 0) { + command.error( + `error: ${conflicting.map((key) => command.options.find((o) => o.attributeName() === key)!.long).join(', ')} can't be used with a source config. Set the sources in the config instead.` + ); + } + + const configPath = options.config ?? './source.config.json'; + const ids = [ + ...(options.config && dir !== undefined ? [dir] : []), + ...translations, + ]; + await sourceAllTranslationsFromConfig(configPath, ids, { + ...sharedOptions, + // Only override the config when the option was given + convertToUsx3: + command.getOptionValueSource('convertToUsx3') === + 'cli' + ? options.convertToUsx3 + : undefined, + }); + return; + } + + await sourceAllTranslations(dir, translations, { + ...sharedOptions, + fetchBibleDir: options.fetchBibleDir, + osisDir: options.osisDir, + includeOsis: options.osis !== false, + bsbDir: options.bsbDir, + includeBsb: options.bsb !== false, + manifestUrl: options.manifestUrl, + repo: options.collectionRepo, + ref: options.collectionRef, + }); + } + ); program .command('list-fetch-bible-translations [search]') diff --git a/packages/helloao-cli/conversion.ts b/packages/helloao-cli/conversion.ts index b01fb8a..ea6309f 100644 --- a/packages/helloao-cli/conversion.ts +++ b/packages/helloao-cli/conversion.ts @@ -1,5 +1,6 @@ import { mkdir, readdir } from 'node:fs/promises'; import { existsSync } from 'fs'; +import path from 'path'; import { execFile as execFileCallback } from 'child_process'; import { promisify } from 'util'; import { confirm, input } from '@inquirer/prompts'; @@ -8,14 +9,21 @@ import { log } from '@helloao/tools'; const execFile = promisify(execFileCallback); /** - * Finds BibleMultiConverter.jar automatically in common locations + * Finds BibleMultiConverter.jar automatically in common locations. + * @param providedPath The path to BibleMultiConverter.jar. If given, it is used instead of searching, + * and an error is thrown when it doesn't exist (so that a typo isn't silently replaced by another copy). */ export async function findBibleMultiConverterJar( providedPath?: string ): Promise { const logger = log.getLogger(); //Use provided path - if (providedPath && existsSync(providedPath)) { + if (providedPath) { + if (!existsSync(providedPath)) { + throw new Error( + `BibleMultiConverter.jar was not found at the given path: ${path.resolve(providedPath)}` + ); + } return providedPath; } diff --git a/packages/helloao-cli/index.ts b/packages/helloao-cli/index.ts index 0c06e7b..1bc6891 100644 --- a/packages/helloao-cli/index.ts +++ b/packages/helloao-cli/index.ts @@ -5,5 +5,6 @@ import * as actions from './actions.js'; import * as files from './files.js'; import * as s3 from './s3.js'; import * as sync from './sync.js'; +import * as sourceConfig from './source-config.js'; -export { db, downloads, uploads, actions, files, s3, sync }; +export { db, downloads, uploads, actions, files, s3, sync, sourceConfig }; diff --git a/packages/helloao-cli/source-config.spec.ts b/packages/helloao-cli/source-config.spec.ts new file mode 100644 index 0000000..fc7199c --- /dev/null +++ b/packages/helloao-cli/source-config.spec.ts @@ -0,0 +1,182 @@ +import path from 'node:path'; +import { tmpdir } from 'node:os'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { + getSourceAllTranslationsOptions, + loadSourceConfig, +} from './source-config'; +import { findBibleMultiConverterJar } from './conversion'; + +describe('source config', () => { + let dir: string; + let configPath: string; + + beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'helloao-source-config-')); + configPath = path.resolve(dir, 'source.config.json'); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + function writeConfig(config: any) { + writeFileSync(configPath, JSON.stringify(config)); + } + + it('should resolve the paths relative to the config file', async () => { + writeConfig({ + db: './api.db', + ignorelist: ['amh_amh'], + conversionConcurrency: 2, + bibleMultiConverterPath: './tools/BibleMultiConverter.jar', + sources: [ + { type: 'ebible', path: './sources/ebible' }, + { + type: 'fetch-bible', + path: './sources/fetch-bible', + repo: 'someone/fetch_collection', + ref: 'dev', + }, + { type: 'osis', path: './sources/osis' }, + { type: 'bsb', path: './sources/bsb' }, + ], + }); + + const config = await loadSourceConfig(configPath); + const { outputDir, options } = getSourceAllTranslationsOptions(config); + + expect(outputDir).toBe(path.resolve(dir, 'sources/ebible')); + expect(options).toMatchObject({ + db: path.resolve(dir, 'api.db'), + ignorelist: ['amh_amh'], + conversionConcurrency: 2, + bibleMultiConverterPath: path.resolve( + dir, + 'tools/BibleMultiConverter.jar' + ), + includeFetchBible: true, + fetchBibleDir: path.resolve(dir, 'sources/fetch-bible'), + repo: 'someone/fetch_collection', + ref: 'dev', + includeOsis: true, + osisDir: path.resolve(dir, 'sources/osis'), + includeBsb: true, + bsbDir: path.resolve(dir, 'sources/bsb'), + }); + }); + + it('should not download from sources that are left out', async () => { + writeConfig({ sources: [{ type: 'ebible', path: 'ebible' }] }); + + const { options } = getSourceAllTranslationsOptions( + await loadSourceConfig(configPath) + ); + + expect(options.includeFetchBible).toBe(false); + expect(options.includeOsis).toBe(false); + expect(options.includeBsb).toBe(false); + expect(options.db).toBeUndefined(); + }); + + it('should let the given options override the config', async () => { + writeConfig({ + db: './api.db', + ignorelist: ['a'], + conversionConcurrency: 2, + convertToUsx3: false, + sources: [{ type: 'ebible', path: 'ebible' }], + }); + + const { options } = getSourceAllTranslationsOptions( + await loadSourceConfig(configPath), + { + db: '/other.db', + ignorelist: ['b'], + conversionConcurrency: undefined, + convertToUsx3: true, + overwrite: true, + } + ); + + expect(options.db).toBe('/other.db'); + expect(options.ignorelist).toEqual(['a', 'b']); + expect(options.conversionConcurrency).toBe(2); + expect(options.convertToUsx3).toBe(true); + expect(options.overwrite).toBe(true); + }); + + it('should use the BibleMultiConverter path from the config', async () => { + const jarPath = path.resolve(dir, 'tools/BibleMultiConverter.jar'); + mkdirSync(path.dirname(jarPath)); + writeFileSync(jarPath, ''); + writeConfig({ + bibleMultiConverterPath: 'tools/BibleMultiConverter.jar', + sources: [{ type: 'ebible', path: 'ebible' }], + }); + + const { options } = getSourceAllTranslationsOptions( + await loadSourceConfig(configPath) + ); + expect(options.bibleMultiConverterPath).toBe(jarPath); + await expect( + findBibleMultiConverterJar(options.bibleMultiConverterPath) + ).resolves.toBe(jarPath); + + // The command line path overrides the config + const { options: overridden } = getSourceAllTranslationsOptions( + await loadSourceConfig(configPath), + { bibleMultiConverterPath: '/other/BibleMultiConverter.jar' } + ); + expect(overridden.bibleMultiConverterPath).toBe( + '/other/BibleMultiConverter.jar' + ); + }); + + it('should fail when the given BibleMultiConverter path does not exist', async () => { + await expect( + findBibleMultiConverterJar(path.resolve(dir, 'missing.jar')) + ).rejects.toThrow(/was not found at the given path/); + }); + + it('should require exactly one ebible source', async () => { + writeConfig({ sources: [{ type: 'bsb', path: 'bsb' }] }); + await expect(loadSourceConfig(configPath)).rejects.toThrow( + /ebible source is required/ + ); + + writeConfig({ + sources: [ + { type: 'ebible', path: 'a' }, + { type: 'ebible', path: 'b' }, + ], + }); + await expect(loadSourceConfig(configPath)).rejects.toThrow( + /only be one ebible source/ + ); + }); + + it('should reject invalid sources', async () => { + writeConfig({ sources: [{ type: 'ebible' }] }); + await expect(loadSourceConfig(configPath)).rejects.toThrow( + /Invalid source config/ + ); + + writeConfig({ + sources: [{ type: 'ebible', path: 'ebible', ref: 'main' }], + }); + await expect(loadSourceConfig(configPath)).rejects.toThrow( + /only allowed for fetch-bible sources/ + ); + + writeConfig({ + sources: [ + { type: 'ebible', path: 'ebible' }, + { type: 'wordproject', path: 'wp' }, + ], + }); + await expect(loadSourceConfig(configPath)).rejects.toThrow( + /Invalid source config/ + ); + }); +}); diff --git a/packages/helloao-cli/source-config.ts b/packages/helloao-cli/source-config.ts new file mode 100644 index 0000000..d24afe8 --- /dev/null +++ b/packages/helloao-cli/source-config.ts @@ -0,0 +1,244 @@ +import path from 'node:path'; +import { readFile } from 'node:fs/promises'; +import { z } from 'zod'; +import { + sourceAllTranslations, + SourceAllTranslationsOptions, + SourceTranslationsResult, +} from './actions.js'; + +/** + * The types of sources that source-all-translations can download translations from. + */ +export type SourceConfigSourceType = 'ebible' | 'fetch-bible' | 'osis' | 'bsb'; + +/** + * A source of translations in the source config. + */ +export interface SourceConfigSource { + /** + * The type of source. + * - `ebible`: the translations from eBible. Required. + * - `fetch-bible`: the translations from the fetch.bible collection that eBible doesn't provide. + * - `osis`: the pinned OSIS translations (see fetch-osis-translations). + * - `bsb`: the Berean Standard Bible from bereanbible.com (see fetch-bsb). + */ + type: SourceConfigSourceType; + + /** + * The directory that the translations should be downloaded to, relative to the config file. + */ + path: string; + + /** + * The URL of the fetch.bible manifest. Only for `fetch-bible` sources. + */ + manifestUrl?: string; + + /** + * The GitHub repository (owner/name) of the fetch.bible collection. Only for `fetch-bible` sources. + */ + repo?: string; + + /** + * The branch (or other git ref) of the fetch.bible collection. Only for `fetch-bible` sources. + */ + ref?: string; +} + +/** + * The configuration for the source-all-translations command. + */ +export interface SourceConfig { + /** + * The path to the database that tracks downloads, relative to the config file. + */ + db?: string; + + /** + * The IDs (eBible ID, fetch.bible ID, or translation ID) of translations that should never be downloaded. + * Applied in addition to the ignorelist.txt file of each source directory. + */ + ignorelist?: string[]; + + /** + * Whether USFM files should be converted to USX3. Defaults to true. + */ + convertToUsx3?: boolean; + + /** + * The path to BibleMultiConverter.jar, relative to the config file. + */ + bibleMultiConverterPath?: string; + + /** + * The maximum number of BibleMultiConverter processes to run at once. + */ + conversionConcurrency?: number; + + /** + * The sources that translations should be downloaded from, and the directories they are downloaded to. + * Sources that are left out are not downloaded from (except for eBible, which is required). + */ + sources: SourceConfigSource[]; +} + +/** + * A source config whose paths can be resolved. + */ +export interface LoadedSourceConfig extends SourceConfig { + /** + * The directory that the paths in the config are relative to. + */ + baseDir: string; +} + +const fetchBibleOnlyFields = ['manifestUrl', 'repo', 'ref'] as const; + +const sourceConfigSchema = z.object({ + db: z.string().optional(), + ignorelist: z.array(z.string()).optional(), + convertToUsx3: z.boolean().optional(), + bibleMultiConverterPath: z.string().optional(), + conversionConcurrency: z.number().int().positive().optional(), + sources: z + .array( + z + .object({ + type: z.enum(['ebible', 'fetch-bible', 'osis', 'bsb']), + path: z.string(), + manifestUrl: z.url().optional(), + repo: z + .string() + .regex(/^[^/\s]+\/[^/\s]+$/, 'Must be owner/name.') + .optional(), + ref: z.string().optional(), + }) + .superRefine((source, ctx) => { + if (source.type === 'fetch-bible') { + return; + } + for (const field of fetchBibleOnlyFields) { + if (source[field] !== undefined) { + ctx.addIssue({ + code: 'custom', + message: `${field} is only allowed for fetch-bible sources.`, + path: [field], + }); + } + } + }) + ) + .superRefine((sources, ctx) => { + const seen = new Set(); + sources.forEach((source, index) => { + if (seen.has(source.type)) { + ctx.addIssue({ + code: 'custom', + message: `There can only be one ${source.type} source.`, + path: [index, 'type'], + }); + } + seen.add(source.type); + }); + if (!seen.has('ebible')) { + ctx.addIssue({ + code: 'custom', + message: 'An ebible source is required.', + }); + } + }), +}); + +/** + * Loads and validates the source config from the given JSON file. + * @param configPath The path to the config file. + */ +export async function loadSourceConfig( + configPath: string +): Promise { + const fullPath = path.resolve(configPath); + const json = JSON.parse(await readFile(fullPath, 'utf-8')); + const result = sourceConfigSchema.safeParse(json); + if (!result.success) { + throw new Error( + `Invalid source config ${fullPath}:\n${z.prettifyError(result.error)}` + ); + } + + return { + ...result.data, + baseDir: path.dirname(fullPath), + }; +} + +/** + * Converts the given source config into the output directory and options for sourceAllTranslations(). + * @param config The source config. + * @param options The options that override the config (e.g. from the command line). + */ +export function getSourceAllTranslationsOptions( + config: LoadedSourceConfig, + options: SourceAllTranslationsOptions = {} +): { outputDir: string; options: SourceAllTranslationsOptions } { + const resolve = (p: string) => path.resolve(config.baseDir, p); + const find = (type: SourceConfigSourceType) => + config.sources.find((source) => source.type === type); + + const ebible = find('ebible')!; + const fetchBible = find('fetch-bible'); + const osis = find('osis'); + const bsb = find('bsb'); + + return { + outputDir: resolve(ebible.path), + options: { + convertToUsx3: config.convertToUsx3, + bibleMultiConverterPath: config.bibleMultiConverterPath + ? resolve(config.bibleMultiConverterPath) + : undefined, + conversionConcurrency: config.conversionConcurrency, + ...withoutUndefined(options), + db: options.db ?? (config.db ? resolve(config.db) : undefined), + ignorelist: [ + ...(config.ignorelist ?? []), + ...(options.ignorelist ?? []), + ], + includeFetchBible: !!fetchBible, + fetchBibleDir: fetchBible ? resolve(fetchBible.path) : undefined, + manifestUrl: fetchBible?.manifestUrl ?? options.manifestUrl, + repo: fetchBible?.repo ?? options.repo, + ref: fetchBible?.ref ?? options.ref, + includeOsis: !!osis, + osisDir: osis ? resolve(osis.path) : undefined, + includeBsb: !!bsb, + bsbDir: bsb ? resolve(bsb.path) : undefined, + }, + }; +} + +/** + * Downloads the translations from the sources in the source config at the given path. + * @param configPath The path to the source config file. + * @param translations The translations to download. If omitted, every translation is downloaded. + * @param options The options that override the config (e.g. from the command line). + */ +export async function sourceAllTranslationsFromConfig( + configPath: string, + translations?: string[], + options: SourceAllTranslationsOptions = {} +): Promise { + const config = await loadSourceConfig(configPath); + const resolved = getSourceAllTranslationsOptions(config, options); + return await sourceAllTranslations( + resolved.outputDir, + translations, + resolved.options + ); +} + +function withoutUndefined(value: T): Partial { + return Object.fromEntries( + Object.entries(value).filter(([, v]) => v !== undefined) + ) as Partial; +} diff --git a/source.config.example.json b/source.config.example.json new file mode 100644 index 0000000..001f45d --- /dev/null +++ b/source.config.example.json @@ -0,0 +1,11 @@ +{ + "db": "./free-use-bible-api.db", + "ignorelist": [], + "bibleMultiConverterPath": "./tools/BibleMultiConverter.jar", + "sources": [ + { "type": "ebible", "path": "./sources/ebible" }, + { "type": "fetch-bible", "path": "./sources/fetch-bible" }, + { "type": "osis", "path": "./sources/osis" }, + { "type": "bsb", "path": "./sources/bsb" } + ] +}