From 372b72eb78ff9f0988e996201b737f55c12d9e92 Mon Sep 17 00:00:00 2001 From: Golo Roden Date: Mon, 19 Oct 2026 11:31:29 +0200 Subject: [PATCH 1/7] feat: Wait for the test container up to a configurable startup timeout, replace one that does not become reachable, and remove one whose start fails. The container now starts EventSourcingDB the way all client SDKs do. Each attempt gets the startup timeout, 30 seconds by default and configurable with withStartupTimeout, from creating the container until a ping of /api/v1/ping answers 200, reading the mapped port anew on every poll. A running container that does not become reachable in time is replaced, up to 3 containers. A container that exits fails at once with the last 20 lines of its output, and so does a Docker error, with that error as cause. Every attempt carries the label io.eventsourcingdb.container-start, by which its containers are removed once it fails, also while Docker is still creating them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QykegdHe7vhHMBN3kWxgfH --- README.md | 7 + src/Container.test.ts | 48 ++++- src/Container.ts | 43 +++-- src/container/ContainerExitedError.ts | 11 ++ src/container/attachOutput.ts | 42 +++++ src/container/createArchive.ts | 69 +++++++ src/container/defaultStartupTimeout.ts | 9 + src/container/startContainer.test.ts | 252 +++++++++++++++++++++++++ src/container/startContainer.ts | 220 +++++++++++++++++++++ src/container/waitUntilReachable.ts | 95 ++++++++++ 10 files changed, 777 insertions(+), 19 deletions(-) create mode 100644 src/container/ContainerExitedError.ts create mode 100644 src/container/attachOutput.ts create mode 100644 src/container/createArchive.ts create mode 100644 src/container/defaultStartupTimeout.ts create mode 100644 src/container/startContainer.test.ts create mode 100644 src/container/startContainer.ts create mode 100644 src/container/waitUntilReachable.ts diff --git a/README.md b/README.md index 4f2308e..72bde17 100644 --- a/README.md +++ b/README.md @@ -700,6 +700,13 @@ const container = new Container() .withApiToken('secret'); ``` +By default, the `start` function gives the test container 30 seconds to become reachable, from creating it until the database answers a ping, without the time it takes to pull the image. A container that is running but does not become reachable in time is replaced by a new one, up to 3 containers in total. If the container exits before it becomes reachable, or if Docker fails, the `start` function fails at once. Every container whose start fails is removed at once. To change the 30 seconds, call the `withStartupTimeout` function with the time in milliseconds: + +```typescript +const container = new Container() + .withStartupTimeout(60_000); +``` + If you want to sign events, call the `withSigningKey` function. This generates a new signing and verification key pair inside the container: ```typescript diff --git a/src/Container.test.ts b/src/Container.test.ts index 18333fa..b09cd76 100644 --- a/src/Container.test.ts +++ b/src/Container.test.ts @@ -1,10 +1,12 @@ import assert from 'node:assert/strict'; import { suite, test } from 'node:test'; import { Container } from './Container.js'; +import { defaultStartupTimeout } from './container/defaultStartupTimeout.js'; import { getImageTag } from './getImageTag.js'; +import { integrationTimeout } from './testing/integrationTimeout.js'; -suite('Container', { timeout: 30_000 }, () => { - test('starts with a custom port.', async (): Promise => { +suite('Container', () => { + test('starts with a custom port.', integrationTimeout, async (): Promise => { const imageTag = getImageTag(); const container = new Container().withImageTag(imageTag).withPort(4000); @@ -20,4 +22,46 @@ suite('Container', { timeout: 30_000 }, () => { await container.stop(); } }); + + suite('startup timeout', () => { + test('is 30 seconds by default.', (): void => { + assert.equal(defaultStartupTimeout, 30_000); + }); + + test('accepts a startup timeout from 1 millisecond up to the longest timer of Node.js.', (): void => { + assert.ok(new Container().withStartupTimeout(1)); + assert.ok(new Container().withStartupTimeout(2_147_483_647)); + }); + + test('throws an error if the startup timeout is out of range.', (): void => { + for (const invalidStartupTimeout of [ + 0, + -1, + 2_147_483_648, + Number.NaN, + Number.POSITIVE_INFINITY, + ]) { + assert.throws((): Container => new Container().withStartupTimeout(invalidStartupTimeout), { + name: 'Error', + message: `Startup timeout must be between 1 and 2147483647 milliseconds, got '${invalidStartupTimeout}'.`, + }); + } + }); + + test( + 'gives each of 3 containers the configured time to become reachable.', + integrationTimeout, + async (): Promise => { + const imageTag = getImageTag(); + const container = new Container().withImageTag(imageTag).withStartupTimeout(1); + + await assert.rejects(container.start(), { + name: 'Error', + message: + 'Failed to start the container, the database did not become reachable within 0.001s in any of 3 containers', + }); + assert.equal(container.isRunning(), false); + }, + ); + }); }); diff --git a/src/Container.ts b/src/Container.ts index 889915c..4a7ff7d 100644 --- a/src/Container.ts +++ b/src/Container.ts @@ -1,13 +1,9 @@ import crypto from 'node:crypto'; -import type { Content, StartedTestContainer } from 'testcontainers'; -import { GenericContainer, Wait } from 'testcontainers'; import { Client } from './Client.js'; - -interface ContentToCopy { - content: Content; - target: string; - mode?: number; -} +import type { ContentToCopy } from './container/createArchive.js'; +import { defaultStartupTimeout, maximumStartupTimeout } from './container/defaultStartupTimeout.js'; +import type { StartedContainer } from './container/startContainer.js'; +import { startContainer } from './container/startContainer.js'; class Container { readonly #imageName = 'thenativeweb/eventsourcingdb'; @@ -15,7 +11,8 @@ class Container { #internalPort = 3000; #apiToken = 'secret'; #signingKey: crypto.KeyObject | undefined; - #container: StartedTestContainer | undefined; + #startupTimeout = defaultStartupTimeout; + #container: StartedContainer | undefined; public withImageTag(tag: string): this { this.#imageTag = tag; @@ -38,6 +35,17 @@ class Container { return this; } + public withStartupTimeout(timeout: number): this { + if (!(timeout >= 1 && timeout <= maximumStartupTimeout)) { + throw new Error( + `Startup timeout must be between 1 and ${maximumStartupTimeout} milliseconds, got '${timeout}'.`, + ); + } + + this.#startupTimeout = timeout; + return this; + } + public async start(): Promise { const command = [ 'run', @@ -63,12 +71,13 @@ class Container { }); } - this.#container = await new GenericContainer(`${this.#imageName}:${this.#imageTag}`) - .withExposedPorts(this.#internalPort) - .withCommand(command) - .withCopyContentToContainer(contents) - .withWaitStrategy(Wait.forHttp('/api/v1/ping', this.#internalPort).withStartupTimeout(10_000)) - .start(); + this.#container = await startContainer({ + image: `${this.#imageName}:${this.#imageTag}`, + port: this.#internalPort, + command, + contents, + startupTimeout: this.#startupTimeout, + }); } public getHost(): string { @@ -76,7 +85,7 @@ class Container { throw new Error('Container must be running.'); } - return this.#container.getHost(); + return this.#container.host; } public getMappedPort(): number { @@ -84,7 +93,7 @@ class Container { throw new Error('Container must be running.'); } - return this.#container.getMappedPort(this.#internalPort); + return this.#container.mappedPort; } public getBaseUrl(): URL { diff --git a/src/container/ContainerExitedError.ts b/src/container/ContainerExitedError.ts new file mode 100644 index 0000000..78156c2 --- /dev/null +++ b/src/container/ContainerExitedError.ts @@ -0,0 +1,11 @@ +// The container exited before the database in it became reachable, so another +// container would fail the same way. +class ContainerExitedError extends Error { + public constructor(logTail: string) { + super( + `Failed to start the container, it exited before the database became reachable: ${logTail}`, + ); + } +} + +export { ContainerExitedError }; diff --git a/src/container/attachOutput.ts b/src/container/attachOutput.ts new file mode 100644 index 0000000..6bff523 --- /dev/null +++ b/src/container/attachOutput.ts @@ -0,0 +1,42 @@ +import type { ContainerRuntimeClient } from 'testcontainers'; + +type DockerContainer = Awaited>; + +interface ContainerOutput { + // Waits until the output has ended, and returns its last lines. + readTail: (lineCount: number) => Promise; + detach: () => void; +} + +// Follows the output of a container, stdout and stderr, from before its start +// on. The log of Docker may lag behind the exit of a container, while an +// attached stream ends only once all output has arrived. +const attachOutput = async ( + client: ContainerRuntimeClient, + container: DockerContainer, +): Promise => { + const stream = await client.container.attach(container); + const ended = new Promise(resolve => { + stream.once('end', resolve); + stream.once('close', resolve); + }); + + let output = ''; + stream.on('data', (chunk: string): void => { + output += chunk; + }); + + return { + readTail: async (lineCount: number): Promise => { + await ended; + + return output.trimEnd().split('\n').slice(-lineCount).join('\n'); + }, + detach: (): void => { + stream.destroy(); + }, + }; +}; + +export type { ContainerOutput, DockerContainer }; +export { attachOutput }; diff --git a/src/container/createArchive.ts b/src/container/createArchive.ts new file mode 100644 index 0000000..8ddec0d --- /dev/null +++ b/src/container/createArchive.ts @@ -0,0 +1,69 @@ +interface ContentToCopy { + content: string | Buffer; + target: string; + mode?: number; +} + +interface Field { + offset: number; + length: number; +} + +// A tar archive consists of blocks of 512 bytes. Each file has a header block, +// followed by its content, padded to whole blocks, and two empty blocks end +// the archive. The fields of a header that are not set stay zero. +const blockSize = 512; +const endBlockCount = 2; +const nameField: Field = { offset: 0, length: 100 }; +const modeField: Field = { offset: 100, length: 8 }; +const sizeField: Field = { offset: 124, length: 12 }; +const modificationTimeField: Field = { offset: 136, length: 12 }; +const checksumField: Field = { offset: 148, length: 8 }; +const typeField: Field = { offset: 156, length: 1 }; +const magicField: Field = { offset: 257, length: 8 }; +const regularFile = '0'; +const ustarMagic = 'ustar\u000000'; +const octal = 8; +const defaultMode = 0o644; +const millisecondsPerSecond = 1000; +const leadingSlashes = /^\/+/u; + +const writeField = (header: Buffer, { offset, length }: Field, value: string): void => { + header.write(value, offset, length, 'utf8'); +}; + +// A number takes up all but the last byte of its field, in octal, and the +// last byte ends it. +const writeNumber = (header: Buffer, field: Field, value: number): void => { + writeField(header, field, `${value.toString(octal).padStart(field.length - 1, '0')}\u0000`); +}; + +const createEntry = ({ content, target, mode = defaultMode }: ContentToCopy): Buffer[] => { + const data = Buffer.from(content); + const header = Buffer.alloc(blockSize); + + writeField(header, nameField, target.replace(leadingSlashes, '')); + writeNumber(header, modeField, mode); + writeNumber(header, sizeField, data.length); + writeNumber(header, modificationTimeField, Math.floor(Date.now() / millisecondsPerSecond)); + writeField(header, typeField, regularFile); + writeField(header, magicField, ustarMagic); + + // The checksum adds up the bytes of the header, counting its own field as + // spaces, and takes up all but the last two bytes of its field. + writeField(header, checksumField, ' '.repeat(checksumField.length)); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + writeNumber(header, { ...checksumField, length: checksumField.length - 1 }, checksum); + + const padding = Buffer.alloc((blockSize - (data.length % blockSize)) % blockSize); + + return [header, data, padding]; +}; + +// Packs the contents into a tar archive, which is what Docker expects for +// copying files into a container. +const createArchive = (contents: ContentToCopy[]): Buffer => + Buffer.concat([...contents.flatMap(createEntry), Buffer.alloc(endBlockCount * blockSize)]); + +export type { ContentToCopy }; +export { createArchive }; diff --git a/src/container/defaultStartupTimeout.ts b/src/container/defaultStartupTimeout.ts new file mode 100644 index 0000000..b5c869b --- /dev/null +++ b/src/container/defaultStartupTimeout.ts @@ -0,0 +1,9 @@ +// Each attempt to start a container may take this many milliseconds, from +// creating the container until the database answers a ping, unless the +// container is configured otherwise. +const defaultStartupTimeout = 30_000; + +// Node.js runs a timer for at most this many milliseconds. +const maximumStartupTimeout = 2_147_483_647; + +export { defaultStartupTimeout, maximumStartupTimeout }; diff --git a/src/container/startContainer.test.ts b/src/container/startContainer.test.ts new file mode 100644 index 0000000..6ded51a --- /dev/null +++ b/src/container/startContainer.test.ts @@ -0,0 +1,252 @@ +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { IncomingMessage } from 'node:http'; +import { suite, test } from 'node:test'; +import { getContainerRuntimeClient } from 'testcontainers'; +import { getImageTag } from '../getImageTag.js'; +import { integrationTimeout } from '../testing/integrationTimeout.js'; +import { startContainer } from './startContainer.js'; + +const startLabel = 'io.eventsourcingdb.container-start'; +const testLabel = 'io.eventsourcingdb.container-start-test'; +const exitedPrefix = + 'Failed to start the container, it exited before the database became reachable: '; + +const getCommand = (port: number): string[] => [ + 'run', + '--api-token', + 'secret', + '--data-directory-temporary', + '--http-enabled', + '--https-enabled=false', + '--http-port', + String(port), +]; + +// Each test labels its containers, so that it finds them, and only them, while +// other tests use the same Docker. +const listContainers = async (testId: string): Promise => { + const client = await getContainerRuntimeClient(); + + return await client.container.dockerode.listContainers({ + all: true, + filters: { label: [`${testLabel}=${testId}`] }, + }); +}; + +// Watches Docker for the containers that it creates for a test, and returns a +// function that stops watching and returns the start label of each of them. +// Docker keeps only the latest events, so the test watches while it runs, +// instead of asking for its events afterwards. +const watchCreatedContainers = async (testId: string): Promise<() => string[]> => { + const client = await getContainerRuntimeClient(); + const events = await client.container.dockerode.getEvents({ + filters: { type: ['container'], event: ['create'], label: [`${testLabel}=${testId}`] }, + }); + if (!(events instanceof IncomingMessage)) { + throw new Error('Failed to watch Docker.'); + } + + // Lets the test end even if it fails before it stops watching. + events.socket.unref(); + + const chunks: Buffer[] = []; + events.on('data', (chunk: Buffer): void => { + chunks.push(chunk); + }); + + return (): string[] => { + events.destroy(); + + return Buffer.concat(chunks) + .toString('utf8') + .split('\n') + .filter(line => line !== '') + .map(line => JSON.parse(line).Actor.Attributes[startLabel]); + }; +}; + +suite('startContainer', () => { + const image = `thenativeweb/eventsourcingdb:${getImageTag()}`; + + test( + 'replaces a running container whose database does not become reachable, up to 3 containers, and removes each of them.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const stopWatching = await watchCreatedContainers(testId); + + // The database listens on another port than the one that is mapped. + await assert.rejects( + startContainer({ + image, + port: 3000, + command: getCommand(3001), + contents: [], + startupTimeout: 1500, + labels: { [testLabel]: testId }, + }), + { + message: + 'Failed to start the container, the database did not become reachable within 1.5s in any of 3 containers', + }, + ); + + const startIds = stopWatching(); + assert.equal(startIds.length, 3); + assert.equal(new Set(startIds).size, 3); + assert.deepEqual(await listContainers(testId), []); + }, + ); + + test( + 'removes a container whose creation outlasts the startup timeout.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const stopWatching = await watchCreatedContainers(testId); + + await assert.rejects( + startContainer({ + image, + port: 3000, + command: getCommand(3000), + contents: [], + startupTimeout: 1, + labels: { [testLabel]: testId }, + }), + { + message: + 'Failed to start the container, the database did not become reachable within 0.001s in any of 3 containers', + }, + ); + + assert.equal(stopWatching().length, 3); + assert.deepEqual(await listContainers(testId), []); + }, + ); + + test( + 'fails at once with the end of the log if the container exits, and removes it.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const stopWatching = await watchCreatedContainers(testId); + + await assert.rejects( + startContainer({ + image, + port: 3000, + command: [...getCommand(3000), '--no-such-flag'], + contents: [], + startupTimeout: 20_000, + labels: { [testLabel]: testId }, + }), + (error: unknown): boolean => { + assert.ok(error instanceof Error); + assert.ok(error.message.startsWith(exitedPrefix)); + + // The database prints its usage, which ends with this flag, and more + // than 20 lines in all. Docker passes on stdout and stderr in the + // order in which they arrive, so the last line may come from either. + const logLines = error.message.slice(exitedPrefix.length).split('\n'); + assert.equal(logLines.length, 20); + assert.ok(logLines.some(line => line.includes('--with-ui'))); + return true; + }, + ); + + assert.equal(stopWatching().length, 1); + assert.deepEqual(await listContainers(testId), []); + }, + ); + + test( + 'fails at once with the error of Docker if the image does not exist.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const stopWatching = await watchCreatedContainers(testId); + + await assert.rejects( + startContainer({ + image: `thenativeweb/eventsourcingdb:non-existent-${testId}`, + port: 3000, + command: getCommand(3000), + contents: [], + startupTimeout: 20_000, + labels: { [testLabel]: testId }, + }), + (error: unknown): boolean => { + assert.ok(error instanceof Error); + assert.ok(error.cause instanceof Error); + assert.equal(error.message, `Failed to start the container: ${error.cause.message}`); + return true; + }, + ); + + assert.equal(stopWatching().length, 0); + assert.deepEqual(await listContainers(testId), []); + }, + ); + + test( + 'fails at once with the error of Docker if Docker fails after creating the container, and removes it.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const stopWatching = await watchCreatedContainers(testId); + + // Docker cannot copy a file into another file. + await assert.rejects( + startContainer({ + image, + port: 3000, + command: getCommand(3000), + contents: [{ content: 'content', target: '/bin/busybox/file' }], + startupTimeout: 20_000, + labels: { [testLabel]: testId }, + }), + (error: unknown): boolean => { + assert.ok(error instanceof Error); + assert.ok(error.cause instanceof Error); + assert.equal(error.message, `Failed to start the container: ${error.cause.message}`); + return true; + }, + ); + + assert.equal(stopWatching().length, 1); + assert.deepEqual(await listContainers(testId), []); + }, + ); + + test( + 'starts a container whose database becomes reachable.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const stopWatching = await watchCreatedContainers(testId); + + const { host, mappedPort, stop } = await startContainer({ + image, + port: 3000, + command: getCommand(3000), + contents: [], + startupTimeout: 20_000, + labels: { [testLabel]: testId }, + }); + + try { + const response = await fetch(`http://${host}:${mappedPort}/api/v1/ping`); + assert.equal(response.status, 200); + await response.body?.cancel(); + + assert.equal(stopWatching().length, 1); + } finally { + await stop(); + } + + assert.deepEqual(await listContainers(testId), []); + }, + ); +}); diff --git a/src/container/startContainer.ts b/src/container/startContainer.ts new file mode 100644 index 0000000..0d5b487 --- /dev/null +++ b/src/container/startContainer.ts @@ -0,0 +1,220 @@ +import { randomUUID } from 'node:crypto'; +import { Readable } from 'node:stream'; +import type { ContainerRuntimeClient } from 'testcontainers'; +import { + getContainerRuntimeClient, + getReaper, + ImageName, + LABEL_TESTCONTAINERS_SESSION_ID, +} from 'testcontainers'; +import { waitUnlessAborted } from '../http/waitUnlessAborted.js'; +import { isRecord } from '../types/isRecord.js'; +import { attachOutput } from './attachOutput.js'; +import { ContainerExitedError } from './ContainerExitedError.js'; +import type { ContentToCopy } from './createArchive.js'; +import { createArchive } from './createArchive.js'; +import { waitUntilReachable } from './waitUntilReachable.js'; + +interface StartContainerOptions { + image: string; + port: number; + command: string[]; + contents: ContentToCopy[]; + startupTimeout: number; + labels?: Record; +} + +interface StartedContainer { + host: string; + mappedPort: number; + stop: () => Promise; +} + +interface Attempt { + client: ContainerRuntimeClient; + image: ImageName; + labels: Record; + options: StartContainerOptions; + signal: AbortSignal; +} + +const maximumAttempts = 3; +const millisecondsPerSecond = 1000; +const startLabel = 'io.eventsourcingdb.container-start'; +const statusNotFound = 404; +const statusConflict = 409; + +// A container that is gone, or that Docker is removing already, needs no +// removing. +const isGone = (error: unknown): boolean => + isRecord(error) && (error.statusCode === statusNotFound || error.statusCode === statusConflict); + +// Removes the containers of an attempt by its label, whatever state they are +// in, so that this also finds a container whose creation was still pending. +const removeContainers = async (client: ContainerRuntimeClient, startId: string): Promise => { + const { dockerode } = client.container; + const containers = await dockerode.listContainers({ + all: true, + filters: { label: [`${startLabel}=${startId}`] }, + }); + + await Promise.all( + containers.map(async ({ Id: id }): Promise => { + try { + await dockerode.getContainer(id).remove({ force: true, v: true }); + } catch (error) { + if (!isGone(error)) { + throw error; + } + } + }), + ); +}; + +// Creates the container, copies the files into it, starts it, and waits until +// its database is reachable, while it follows the output of the container. +// Once the signal aborts, it does not take the next step, so that it does not +// start a container whose time has run out. +const createAndStart = async ({ + client, + image, + labels, + options, + signal, +}: Attempt): Promise => { + const exposedPort = `${options.port}/tcp`; + const container = await client.container.create({ + Image: image.string, + Cmd: options.command, + Labels: labels, + ExposedPorts: { [exposedPort]: {} }, + HostConfig: { PortBindings: { [exposedPort]: [{ HostPort: '0' }] } }, + }); + + if (options.contents.length > 0) { + signal.throwIfAborted(); + await client.container.putArchive( + container, + Readable.from(createArchive(options.contents)), + '/', + ); + } + + signal.throwIfAborted(); + const output = await attachOutput(client, container); + + try { + signal.throwIfAborted(); + await client.container.start(container); + + const mappedPort = await waitUntilReachable({ + client, + container, + output, + port: options.port, + signal, + }); + + return { + host: client.info.containerRuntime.host, + mappedPort, + stop: async (): Promise => { + await client.container.stop(container, { timeout: 0 }); + await client.container.remove(container, { removeVolumes: true }); + }, + }; + } finally { + output.detach(); + } +}; + +// Starts a container and waits until its database is reachable, for the +// startup timeout at most, from creating the container on. If that fails, it +// removes the container at once. If the time ran out while Docker was still +// creating or starting the container, it then waits for that to end, and +// removes what was created meanwhile. It returns undefined if the database did +// not become reachable in time, so that another container can take its place. +const attemptToStart = async ( + client: ContainerRuntimeClient, + image: ImageName, + sessionId: string, + options: StartContainerOptions, +): Promise => { + const startId = randomUUID(); + const deadline = new AbortController(); + const timer = setTimeout(() => { + deadline.abort(); + }, options.startupTimeout); + + const starting = createAndStart({ + client, + image, + labels: { + ...options.labels, + [LABEL_TESTCONTAINERS_SESSION_ID]: sessionId, + [startLabel]: startId, + }, + options, + signal: deadline.signal, + }); + + try { + return await waitUnlessAborted(starting, deadline.signal); + } catch (error) { + await removeContainers(client, startId); + + // The attempt has failed already, so how the start ends no longer + // matters, only that nothing of it is left. + await Promise.allSettled([starting]); + await removeContainers(client, startId); + + if (deadline.signal.aborted && error === deadline.signal.reason) { + return undefined; + } + + throw error; + } finally { + clearTimeout(timer); + } +}; + +// Starts a container with EventSourcingDB and waits until the database is +// reachable. A container that is running, but whose database does not become +// reachable within the startup timeout, gives way to another one, up to 3 +// containers in total. If Docker fails, or if the container exits, starting +// fails at once. Ryuk removes the containers once the process ends, as it does +// for the other containers of testcontainers. +const startContainer = async (options: StartContainerOptions): Promise => { + try { + const client = await getContainerRuntimeClient(); + const image = ImageName.fromString(options.image); + + // Neither pulling the image nor starting Ryuk counts towards the startup + // timeout. + await client.image.pull(image); + const { sessionId } = await getReaper(client); + + for (let attempt = 1; attempt <= maximumAttempts; attempt++) { + // biome-ignore lint/performance/noAwaitInLoops: Each attempt waits for the one before. + const startedContainer = await attemptToStart(client, image, sessionId, options); + + if (startedContainer !== undefined) { + return startedContainer; + } + } + } catch (error) { + if (error instanceof ContainerExitedError) { + throw error; + } + + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to start the container: ${message}`, { cause: error }); + } + + throw new Error( + `Failed to start the container, the database did not become reachable within ${options.startupTimeout / millisecondsPerSecond}s in any of ${maximumAttempts} containers`, + ); +}; + +export type { StartContainerOptions, StartedContainer }; +export { startContainer }; diff --git a/src/container/waitUntilReachable.ts b/src/container/waitUntilReachable.ts new file mode 100644 index 0000000..eaf8aea --- /dev/null +++ b/src/container/waitUntilReachable.ts @@ -0,0 +1,95 @@ +import { setTimeout as sleep } from 'node:timers/promises'; +import type { ContainerRuntimeClient } from 'testcontainers'; +import type { ContainerOutput, DockerContainer } from './attachOutput.js'; +import { ContainerExitedError } from './ContainerExitedError.js'; + +type ContainerInspection = Awaited>; + +interface Readiness { + client: ContainerRuntimeClient; + container: DockerContainer; + output: ContainerOutput; + port: number; + signal: AbortSignal; +} + +const pollInterval = 100; +const pingTimeout = 1000; +const statusOk = 200; +const logTailLength = 20; +const exitedStates = new Set(['exited', 'dead']); + +// Until Docker has published the port, it has no binding. +const readMappedPort = (inspection: ContainerInspection, port: number): number | undefined => { + const hostPort = inspection.NetworkSettings.Ports[`${port}/tcp`]?.[0]?.HostPort; + + if (hostPort === undefined || hostPort === '') { + return undefined; + } + + return Number(hostPort); +}; + +const isReachable = async (url: URL, signal: AbortSignal): Promise => { + try { + const response = await fetch(url, { + signal: AbortSignal.any([signal, AbortSignal.timeout(pingTimeout)]), + }); + await response.body?.cancel(); + + return response.status === statusOk; + } catch { + // While the database starts, a ping fails, for example since nothing + // listens on the port yet, so this is no error, but a reason to ask again. + return false; + } +}; + +// Returns the mapped port once the database answers on it, and undefined +// until then. +const poll = async ({ + client, + container, + output, + port, + signal, +}: Readiness): Promise => { + const inspection = await client.container.inspect(container); + + if (exitedStates.has(inspection.State.Status)) { + throw new ContainerExitedError(await output.readTail(logTailLength)); + } + + const mappedPort = readMappedPort(inspection, port); + if (mappedPort === undefined) { + return undefined; + } + + const url = new URL(`http://${client.info.containerRuntime.host}:${mappedPort}/api/v1/ping`); + if (!(await isReachable(url, signal))) { + return undefined; + } + + return mappedPort; +}; + +// Waits until the database in the container answers a ping with the status +// 200, and returns the mapped port. It reads the mapped port anew on every +// poll, since Docker sometimes publishes the port late. If the container exits +// meanwhile, it fails with the last 20 lines of the output of the container. +// Once the signal aborts, it fails with the reason of the signal. +const waitUntilReachable = async (readiness: Readiness): Promise => { + for (;;) { + readiness.signal.throwIfAborted(); + + // biome-ignore lint/performance/noAwaitInLoops: Each poll waits for the one before. + const mappedPort = await poll(readiness); + if (mappedPort !== undefined) { + return mappedPort; + } + + await sleep(pollInterval); + } +}; + +export { waitUntilReachable }; From 12a3d87501d70f3944928cb382f2c752c09f8519 Mon Sep 17 00:00:00 2001 From: Golo Roden Date: Mon, 19 Oct 2026 11:49:13 +0200 Subject: [PATCH 2/7] Start the test container through GenericContainer of testcontainers again. testcontainers creates and starts the container again and copies the signing key into it, so the own create and start, the tar writer, and the attached output are gone. Each attempt still carries its own io.eventsourcingdb.container-start label, now set through withLabels, and follows the container by that label from its creation on: it reads the mapped port on every poll, pings /api/v1/ping until it answers 200, which testcontainers waits for through its wait strategy, and fails at once with the last 20 lines of the log, read through testcontainers, if the container exits. If testcontainers gives up on the port of a running container, after 10 seconds or since it cannot find the port on the host, the attempt counts as one whose database did not become reachable, and another container takes its place. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QykegdHe7vhHMBN3kWxgfH --- src/Container.ts | 10 +- src/container/attachOutput.ts | 42 ------ src/container/containersOfAttempt.ts | 49 ++++++ src/container/createArchive.ts | 69 --------- src/container/isReachable.test.ts | 58 +++++++ src/container/isReachable.ts | 21 +++ src/container/startContainer.test.ts | 78 +++++++++- src/container/startContainer.ts | 216 ++++++++++----------------- src/container/waitUntilReachable.ts | 78 ++++------ 9 files changed, 319 insertions(+), 302 deletions(-) delete mode 100644 src/container/attachOutput.ts create mode 100644 src/container/containersOfAttempt.ts delete mode 100644 src/container/createArchive.ts create mode 100644 src/container/isReachable.test.ts create mode 100644 src/container/isReachable.ts diff --git a/src/Container.ts b/src/Container.ts index 4a7ff7d..c02fcf6 100644 --- a/src/Container.ts +++ b/src/Container.ts @@ -1,8 +1,8 @@ import crypto from 'node:crypto'; +import type { StartedTestContainer } from 'testcontainers'; import { Client } from './Client.js'; -import type { ContentToCopy } from './container/createArchive.js'; import { defaultStartupTimeout, maximumStartupTimeout } from './container/defaultStartupTimeout.js'; -import type { StartedContainer } from './container/startContainer.js'; +import type { ContentToCopy } from './container/startContainer.js'; import { startContainer } from './container/startContainer.js'; class Container { @@ -12,7 +12,7 @@ class Container { #apiToken = 'secret'; #signingKey: crypto.KeyObject | undefined; #startupTimeout = defaultStartupTimeout; - #container: StartedContainer | undefined; + #container: StartedTestContainer | undefined; public withImageTag(tag: string): this { this.#imageTag = tag; @@ -85,7 +85,7 @@ class Container { throw new Error('Container must be running.'); } - return this.#container.host; + return this.#container.getHost(); } public getMappedPort(): number { @@ -93,7 +93,7 @@ class Container { throw new Error('Container must be running.'); } - return this.#container.mappedPort; + return this.#container.getMappedPort(this.#internalPort); } public getBaseUrl(): URL { diff --git a/src/container/attachOutput.ts b/src/container/attachOutput.ts deleted file mode 100644 index 6bff523..0000000 --- a/src/container/attachOutput.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { ContainerRuntimeClient } from 'testcontainers'; - -type DockerContainer = Awaited>; - -interface ContainerOutput { - // Waits until the output has ended, and returns its last lines. - readTail: (lineCount: number) => Promise; - detach: () => void; -} - -// Follows the output of a container, stdout and stderr, from before its start -// on. The log of Docker may lag behind the exit of a container, while an -// attached stream ends only once all output has arrived. -const attachOutput = async ( - client: ContainerRuntimeClient, - container: DockerContainer, -): Promise => { - const stream = await client.container.attach(container); - const ended = new Promise(resolve => { - stream.once('end', resolve); - stream.once('close', resolve); - }); - - let output = ''; - stream.on('data', (chunk: string): void => { - output += chunk; - }); - - return { - readTail: async (lineCount: number): Promise => { - await ended; - - return output.trimEnd().split('\n').slice(-lineCount).join('\n'); - }, - detach: (): void => { - stream.destroy(); - }, - }; -}; - -export type { ContainerOutput, DockerContainer }; -export { attachOutput }; diff --git a/src/container/containersOfAttempt.ts b/src/container/containersOfAttempt.ts new file mode 100644 index 0000000..c425496 --- /dev/null +++ b/src/container/containersOfAttempt.ts @@ -0,0 +1,49 @@ +import type { ContainerRuntimeClient } from 'testcontainers'; +import { isRecord } from '../types/isRecord.js'; + +type ContainerInfo = Awaited< + ReturnType +>[number]; + +// Every attempt to start a container labels its container with an id of its +// own, by which it finds the container, whatever state it is in, even before +// testcontainers hands it over. +const startLabel = 'io.eventsourcingdb.container-start'; +const statusNotFound = 404; +const statusConflict = 409; + +// A container that is gone, or that Docker is removing already, needs no +// removing. +const isGone = (error: unknown): boolean => + isRecord(error) && (error.statusCode === statusNotFound || error.statusCode === statusConflict); + +const listContainersOfAttempt = async ( + client: ContainerRuntimeClient, + startId: string, +): Promise => + await client.container.dockerode.listContainers({ + all: true, + filters: { label: [`${startLabel}=${startId}`] }, + }); + +const removeContainersOfAttempt = async ( + client: ContainerRuntimeClient, + startId: string, +): Promise => { + const containers = await listContainersOfAttempt(client, startId); + + await Promise.all( + containers.map(async ({ Id: id }): Promise => { + try { + await client.container.dockerode.getContainer(id).remove({ force: true, v: true }); + } catch (error) { + if (!isGone(error)) { + throw error; + } + } + }), + ); +}; + +export type { ContainerInfo }; +export { listContainersOfAttempt, removeContainersOfAttempt, startLabel }; diff --git a/src/container/createArchive.ts b/src/container/createArchive.ts deleted file mode 100644 index 8ddec0d..0000000 --- a/src/container/createArchive.ts +++ /dev/null @@ -1,69 +0,0 @@ -interface ContentToCopy { - content: string | Buffer; - target: string; - mode?: number; -} - -interface Field { - offset: number; - length: number; -} - -// A tar archive consists of blocks of 512 bytes. Each file has a header block, -// followed by its content, padded to whole blocks, and two empty blocks end -// the archive. The fields of a header that are not set stay zero. -const blockSize = 512; -const endBlockCount = 2; -const nameField: Field = { offset: 0, length: 100 }; -const modeField: Field = { offset: 100, length: 8 }; -const sizeField: Field = { offset: 124, length: 12 }; -const modificationTimeField: Field = { offset: 136, length: 12 }; -const checksumField: Field = { offset: 148, length: 8 }; -const typeField: Field = { offset: 156, length: 1 }; -const magicField: Field = { offset: 257, length: 8 }; -const regularFile = '0'; -const ustarMagic = 'ustar\u000000'; -const octal = 8; -const defaultMode = 0o644; -const millisecondsPerSecond = 1000; -const leadingSlashes = /^\/+/u; - -const writeField = (header: Buffer, { offset, length }: Field, value: string): void => { - header.write(value, offset, length, 'utf8'); -}; - -// A number takes up all but the last byte of its field, in octal, and the -// last byte ends it. -const writeNumber = (header: Buffer, field: Field, value: number): void => { - writeField(header, field, `${value.toString(octal).padStart(field.length - 1, '0')}\u0000`); -}; - -const createEntry = ({ content, target, mode = defaultMode }: ContentToCopy): Buffer[] => { - const data = Buffer.from(content); - const header = Buffer.alloc(blockSize); - - writeField(header, nameField, target.replace(leadingSlashes, '')); - writeNumber(header, modeField, mode); - writeNumber(header, sizeField, data.length); - writeNumber(header, modificationTimeField, Math.floor(Date.now() / millisecondsPerSecond)); - writeField(header, typeField, regularFile); - writeField(header, magicField, ustarMagic); - - // The checksum adds up the bytes of the header, counting its own field as - // spaces, and takes up all but the last two bytes of its field. - writeField(header, checksumField, ' '.repeat(checksumField.length)); - const checksum = header.reduce((sum, byte) => sum + byte, 0); - writeNumber(header, { ...checksumField, length: checksumField.length - 1 }, checksum); - - const padding = Buffer.alloc((blockSize - (data.length % blockSize)) % blockSize); - - return [header, data, padding]; -}; - -// Packs the contents into a tar archive, which is what Docker expects for -// copying files into a container. -const createArchive = (contents: ContentToCopy[]): Buffer => - Buffer.concat([...contents.flatMap(createEntry), Buffer.alloc(endBlockCount * blockSize)]); - -export type { ContentToCopy }; -export { createArchive }; diff --git a/src/container/isReachable.test.ts b/src/container/isReachable.test.ts new file mode 100644 index 0000000..8691af0 --- /dev/null +++ b/src/container/isReachable.test.ts @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict'; +import type { ServerResponse } from 'node:http'; +import { afterEach, beforeEach, suite, test } from 'node:test'; +import type { FakeRequest } from '../testing/FakeServer.js'; +import { FakeServer } from '../testing/FakeServer.js'; +import { isReachable } from './isReachable.js'; + +suite('isReachable', () => { + let server: FakeServer; + + beforeEach(async () => { + server = new FakeServer(); + await server.start(); + }); + + afterEach(async () => { + await server.stop(); + }); + + const answerWith = (status: number): void => { + server.respond = (_request: FakeRequest, response: ServerResponse): void => { + response.writeHead(status); + response.end(); + }; + }; + + test('pings the database.', async (): Promise => { + answerWith(200); + + await isReachable(server.url, new AbortController().signal); + + assert.equal(server.requests.length, 1); + assert.equal(server.requests[0]?.method, 'GET'); + assert.equal(server.requests[0]?.path, '/api/v1/ping'); + }); + + test('accepts the status 200.', async (): Promise => { + answerWith(200); + + assert.equal(await isReachable(server.url, new AbortController().signal), true); + }); + + test('accepts no other status, not even another one of success.', async (): Promise => { + for (const status of [201, 202, 204, 299, 404, 500, 503]) { + answerWith(status); + + // biome-ignore lint/performance/noAwaitInLoops: Each status is checked on its own. + assert.equal(await isReachable(server.url, new AbortController().signal), false, `${status}`); + } + }); + + test('does not accept a server that cannot be reached.', async (): Promise => { + assert.equal( + await isReachable(new URL('http://localhost:1/'), new AbortController().signal), + false, + ); + }); +}); diff --git a/src/container/isReachable.ts b/src/container/isReachable.ts new file mode 100644 index 0000000..69dd2f4 --- /dev/null +++ b/src/container/isReachable.ts @@ -0,0 +1,21 @@ +const pingTimeout = 1000; +const statusOk = 200; + +// Pings the database. Only the status 200 counts, not any other status of +// success. +const isReachable = async (url: URL, signal: AbortSignal): Promise => { + try { + const response = await fetch(new URL('/api/v1/ping', url), { + signal: AbortSignal.any([signal, AbortSignal.timeout(pingTimeout)]), + }); + await response.body?.cancel(); + + return response.status === statusOk; + } catch { + // While the database starts, a ping fails, for example since nothing + // listens on the port yet, so this is no error, but a reason to ask again. + return false; + } +}; + +export { isReachable }; diff --git a/src/container/startContainer.test.ts b/src/container/startContainer.test.ts index 6ded51a..509dfd2 100644 --- a/src/container/startContainer.test.ts +++ b/src/container/startContainer.test.ts @@ -2,7 +2,8 @@ import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; import { IncomingMessage } from 'node:http'; import { suite, test } from 'node:test'; -import { getContainerRuntimeClient } from 'testcontainers'; +import type { StartedTestContainer } from 'testcontainers'; +import { getContainerRuntimeClient, getReaper } from 'testcontainers'; import { getImageTag } from '../getImageTag.js'; import { integrationTimeout } from '../testing/integrationTimeout.js'; import { startContainer } from './startContainer.js'; @@ -147,10 +148,11 @@ suite('startContainer', () => { assert.ok(error.message.startsWith(exitedPrefix)); // The database prints its usage, which ends with this flag, and more - // than 20 lines in all. Docker passes on stdout and stderr in the - // order in which they arrive, so the last line may come from either. + // than 20 lines in all. Docker interleaves stdout and stderr in the + // order in which they arrive, and its log may still lack the last + // line, so the test does not rely on which line comes last. const logLines = error.message.slice(exitedPrefix.length).split('\n'); - assert.equal(logLines.length, 20); + assert.ok(logLines.length <= 20); assert.ok(logLines.some(line => line.includes('--with-ui'))); return true; }, @@ -220,6 +222,66 @@ suite('startContainer', () => { }, ); + test( + 'replaces a running container whose port testcontainers cannot find, and removes it.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const client = await getContainerRuntimeClient(); + const { containerRuntime } = client.info; + const { hostIps } = containerRuntime; + + // Ryuk starts with the first container of the process, so it starts + // before the lookup fails, and only the first container of the test is + // affected. + await getReaper(client); + + // testcontainers looks up the port of a container on the host once per + // start, by the addresses of the host. Without any, the lookup of the + // first container fails, as when Docker publishes the port only for + // another address. + let lookupCount = 0; + Object.defineProperty(containerRuntime, 'hostIps', { + configurable: true, + get: (): typeof hostIps => { + lookupCount += 1; + return lookupCount === 1 ? [] : hostIps; + }, + }); + + const stopWatching = await watchCreatedContainers(testId); + + let container: StartedTestContainer; + try { + container = await startContainer({ + image, + port: 3000, + command: getCommand(3000), + contents: [], + startupTimeout: 20_000, + labels: { [testLabel]: testId }, + }); + } finally { + Object.defineProperty(containerRuntime, 'hostIps', { + configurable: true, + writable: true, + value: hostIps, + }); + } + + try { + const startIds = stopWatching(); + assert.equal(startIds.length, 2); + assert.equal(new Set(startIds).size, 2); + assert.equal((await listContainers(testId)).length, 1); + } finally { + await container.stop(); + } + + assert.deepEqual(await listContainers(testId), []); + }, + ); + test( 'starts a container whose database becomes reachable.', integrationTimeout, @@ -227,7 +289,7 @@ suite('startContainer', () => { const testId = randomUUID(); const stopWatching = await watchCreatedContainers(testId); - const { host, mappedPort, stop } = await startContainer({ + const container = await startContainer({ image, port: 3000, command: getCommand(3000), @@ -237,13 +299,15 @@ suite('startContainer', () => { }); try { - const response = await fetch(`http://${host}:${mappedPort}/api/v1/ping`); + const response = await fetch( + `http://${container.getHost()}:${container.getMappedPort(3000)}/api/v1/ping`, + ); assert.equal(response.status, 200); await response.body?.cancel(); assert.equal(stopWatching().length, 1); } finally { - await stop(); + await container.stop(); } assert.deepEqual(await listContainers(testId), []); diff --git a/src/container/startContainer.ts b/src/container/startContainer.ts index 0d5b487..ddfce34 100644 --- a/src/container/startContainer.ts +++ b/src/container/startContainer.ts @@ -1,20 +1,22 @@ import { randomUUID } from 'node:crypto'; -import { Readable } from 'node:stream'; -import type { ContainerRuntimeClient } from 'testcontainers'; -import { - getContainerRuntimeClient, - getReaper, - ImageName, - LABEL_TESTCONTAINERS_SESSION_ID, +import type { + ContainerRuntimeClient, + Content, + StartedTestContainer, + WaitStrategy, } from 'testcontainers'; +import { GenericContainer, getContainerRuntimeClient, getReaper, ImageName } from 'testcontainers'; import { waitUnlessAborted } from '../http/waitUnlessAborted.js'; -import { isRecord } from '../types/isRecord.js'; -import { attachOutput } from './attachOutput.js'; import { ContainerExitedError } from './ContainerExitedError.js'; -import type { ContentToCopy } from './createArchive.js'; -import { createArchive } from './createArchive.js'; +import { removeContainersOfAttempt, startLabel } from './containersOfAttempt.js'; import { waitUntilReachable } from './waitUntilReachable.js'; +interface ContentToCopy { + content: Content; + target: string; + mode?: number; +} + interface StartContainerOptions { image: string; port: number; @@ -24,156 +26,106 @@ interface StartContainerOptions { labels?: Record; } -interface StartedContainer { - host: string; - mappedPort: number; - stop: () => Promise; -} - -interface Attempt { - client: ContainerRuntimeClient; - image: ImageName; - labels: Record; - options: StartContainerOptions; - signal: AbortSignal; -} - const maximumAttempts = 3; const millisecondsPerSecond = 1000; -const startLabel = 'io.eventsourcingdb.container-start'; -const statusNotFound = 404; -const statusConflict = 409; - -// A container that is gone, or that Docker is removing already, needs no -// removing. -const isGone = (error: unknown): boolean => - isRecord(error) && (error.statusCode === statusNotFound || error.statusCode === statusConflict); - -// Removes the containers of an attempt by its label, whatever state they are -// in, so that this also finds a container whose creation was still pending. -const removeContainers = async (client: ContainerRuntimeClient, startId: string): Promise => { - const { dockerode } = client.container; - const containers = await dockerode.listContainers({ - all: true, - filters: { label: [`${startLabel}=${startId}`] }, - }); - await Promise.all( - containers.map(async ({ Id: id }): Promise => { - try { - await dockerode.getContainer(id).remove({ force: true, v: true }); - } catch (error) { - if (!isGone(error)) { - throw error; - } - } - }), - ); -}; +// testcontainers waits for the port of a container for 10 seconds at most, and +// then looks up the port on the host once. If either fails, the container is +// running, but it cannot be reached. +const portErrors = [ + /^Timed out after \d+ms while waiting for container ports to be bound to the host$/u, + /^No host port found for host IP$/u, +]; + +const isPortError = (error: unknown): boolean => + error instanceof Error && portErrors.some(portError => portError.test(error.message)); + +// Lets testcontainers wait for the readiness of the database, which the +// attempt follows from the creation of the container on. +class ReadinessWait implements WaitStrategy { + readonly #readiness: Promise; + readonly #startupTimeout: number; + + public constructor(readiness: Promise, startupTimeout: number) { + this.#readiness = readiness; + this.#startupTimeout = startupTimeout; + } -// Creates the container, copies the files into it, starts it, and waits until -// its database is reachable, while it follows the output of the container. -// Once the signal aborts, it does not take the next step, so that it does not -// start a container whose time has run out. -const createAndStart = async ({ - client, - image, - labels, - options, - signal, -}: Attempt): Promise => { - const exposedPort = `${options.port}/tcp`; - const container = await client.container.create({ - Image: image.string, - Cmd: options.command, - Labels: labels, - ExposedPorts: { [exposedPort]: {} }, - HostConfig: { PortBindings: { [exposedPort]: [{ HostPort: '0' }] } }, - }); + public async waitUntilReady(): Promise { + await this.#readiness; + } - if (options.contents.length > 0) { - signal.throwIfAborted(); - await client.container.putArchive( - container, - Readable.from(createArchive(options.contents)), - '/', - ); + // testcontainers sets the timeout of a wait strategy only if the container + // has a startup timeout of its own, which this one does not, since the + // attempt has its own clock. + public withStartupTimeout(): this { + return this; } - signal.throwIfAborted(); - const output = await attachOutput(client, container); + // biome-ignore lint/nursery/useThisInClassMethods: The interface of testcontainers asks for this method. + public isStartupTimeoutSet(): boolean { + return true; + } - try { - signal.throwIfAborted(); - await client.container.start(container); - - const mappedPort = await waitUntilReachable({ - client, - container, - output, - port: options.port, - signal, - }); - - return { - host: client.info.containerRuntime.host, - mappedPort, - stop: async (): Promise => { - await client.container.stop(container, { timeout: 0 }); - await client.container.remove(container, { removeVolumes: true }); - }, - }; - } finally { - output.detach(); + public getStartupTimeout(): number { + return this.#startupTimeout; } -}; +} // Starts a container and waits until its database is reachable, for the // startup timeout at most, from creating the container on. If that fails, it // removes the container at once. If the time ran out while Docker was still // creating or starting the container, it then waits for that to end, and // removes what was created meanwhile. It returns undefined if the database did -// not become reachable in time, so that another container can take its place. +// not become reachable, so that another container can take its place. const attemptToStart = async ( client: ContainerRuntimeClient, - image: ImageName, - sessionId: string, options: StartContainerOptions, -): Promise => { +): Promise => { const startId = randomUUID(); const deadline = new AbortController(); const timer = setTimeout(() => { deadline.abort(); }, options.startupTimeout); + const attemptEnded = new AbortController(); - const starting = createAndStart({ + const readiness = waitUntilReachable({ client, - image, - labels: { - ...options.labels, - [LABEL_TESTCONTAINERS_SESSION_ID]: sessionId, - [startLabel]: startId, - }, - options, - signal: deadline.signal, + startId, + port: options.port, + signal: AbortSignal.any([deadline.signal, attemptEnded.signal]), }); + const starting = new GenericContainer(options.image) + .withExposedPorts(options.port) + .withCommand(options.command) + .withCopyContentToContainer(options.contents) + .withLabels({ ...options.labels, [startLabel]: startId }) + .withWaitStrategy(new ReadinessWait(readiness, options.startupTimeout)) + .start(); try { - return await waitUnlessAborted(starting, deadline.signal); + const [container] = await waitUnlessAborted( + Promise.all([starting, readiness]), + deadline.signal, + ); + + return container; } catch (error) { - await removeContainers(client, startId); + attemptEnded.abort(); + await removeContainersOfAttempt(client, startId); // The attempt has failed already, so how the start ends no longer // matters, only that nothing of it is left. - await Promise.allSettled([starting]); - await removeContainers(client, startId); + await Promise.allSettled([starting, readiness]); + await removeContainersOfAttempt(client, startId); - if (deadline.signal.aborted && error === deadline.signal.reason) { + if ((deadline.signal.aborted && error === deadline.signal.reason) || isPortError(error)) { return undefined; } throw error; } finally { + attemptEnded.abort(); clearTimeout(timer); } }; @@ -182,24 +134,22 @@ const attemptToStart = async ( // reachable. A container that is running, but whose database does not become // reachable within the startup timeout, gives way to another one, up to 3 // containers in total. If Docker fails, or if the container exits, starting -// fails at once. Ryuk removes the containers once the process ends, as it does -// for the other containers of testcontainers. -const startContainer = async (options: StartContainerOptions): Promise => { +// fails at once. +const startContainer = async (options: StartContainerOptions): Promise => { try { const client = await getContainerRuntimeClient(); - const image = ImageName.fromString(options.image); // Neither pulling the image nor starting Ryuk counts towards the startup // timeout. - await client.image.pull(image); - const { sessionId } = await getReaper(client); + await client.image.pull(ImageName.fromString(options.image)); + await getReaper(client); for (let attempt = 1; attempt <= maximumAttempts; attempt++) { // biome-ignore lint/performance/noAwaitInLoops: Each attempt waits for the one before. - const startedContainer = await attemptToStart(client, image, sessionId, options); + const container = await attemptToStart(client, options); - if (startedContainer !== undefined) { - return startedContainer; + if (container !== undefined) { + return container; } } } catch (error) { @@ -216,5 +166,5 @@ const startContainer = async (options: StartContainerOptions): Promise>; +import type { ContainerInfo } from './containersOfAttempt.js'; +import { listContainersOfAttempt } from './containersOfAttempt.js'; +import { isReachable } from './isReachable.js'; interface Readiness { client: ContainerRuntimeClient; - container: DockerContainer; - output: ContainerOutput; + startId: string; port: number; signal: AbortSignal; } const pollInterval = 100; -const pingTimeout = 1000; -const statusOk = 200; const logTailLength = 20; const exitedStates = new Set(['exited', 'dead']); -// Until Docker has published the port, it has no binding. -const readMappedPort = (inspection: ContainerInspection, port: number): number | undefined => { - const hostPort = inspection.NetworkSettings.Ports[`${port}/tcp`]?.[0]?.HostPort; +const readLogTail = async ( + client: ContainerRuntimeClient, + id: string, + signal: AbortSignal, +): Promise => { + const logs = await client.container.logs(client.container.getById(id), { tail: logTailLength }); + const chunks: string[] = await logs.toArray({ signal }); - if (hostPort === undefined || hostPort === '') { - return undefined; - } - - return Number(hostPort); + return chunks.join('').trimEnd(); }; -const isReachable = async (url: URL, signal: AbortSignal): Promise => { - try { - const response = await fetch(url, { - signal: AbortSignal.any([signal, AbortSignal.timeout(pingTimeout)]), - }); - await response.body?.cancel(); - - return response.status === statusOk; - } catch { - // While the database starts, a ping fails, for example since nothing - // listens on the port yet, so this is no error, but a reason to ask again. - return false; - } -}; +// Until Docker has published the port, it has no public port. +const readMappedPort = (container: ContainerInfo, port: number): number | undefined => + container.Ports.find( + ({ PrivatePort, PublicPort, Type }) => PrivatePort === port && Type === 'tcp' && PublicPort > 0, + )?.PublicPort; // Returns the mapped port once the database answers on it, and undefined // until then. -const poll = async ({ - client, - container, - output, - port, - signal, -}: Readiness): Promise => { - const inspection = await client.container.inspect(container); +const poll = async ({ client, startId, port, signal }: Readiness): Promise => { + const [container] = await listContainersOfAttempt(client, startId); + if (container === undefined) { + return undefined; + } - if (exitedStates.has(inspection.State.Status)) { - throw new ContainerExitedError(await output.readTail(logTailLength)); + if (exitedStates.has(container.State)) { + throw new ContainerExitedError(await readLogTail(client, container.Id, signal)); } - const mappedPort = readMappedPort(inspection, port); + const mappedPort = readMappedPort(container, port); if (mappedPort === undefined) { return undefined; } - const url = new URL(`http://${client.info.containerRuntime.host}:${mappedPort}/api/v1/ping`); + const url = new URL(`http://${client.info.containerRuntime.host}:${mappedPort}`); if (!(await isReachable(url, signal))) { return undefined; } @@ -73,11 +58,12 @@ const poll = async ({ return mappedPort; }; -// Waits until the database in the container answers a ping with the status -// 200, and returns the mapped port. It reads the mapped port anew on every -// poll, since Docker sometimes publishes the port late. If the container exits -// meanwhile, it fails with the last 20 lines of the output of the container. -// Once the signal aborts, it fails with the reason of the signal. +// Waits from the creation of the container of an attempt on until its database +// answers a ping with the status 200. It looks for the container by the label +// of the attempt, and reads the mapped port anew on every poll, since Docker +// sometimes publishes the port late. If the container exits meanwhile, it fails +// with the last 20 lines of its log. Once the signal aborts, it fails with the +// reason of the signal. const waitUntilReachable = async (readiness: Readiness): Promise => { for (;;) { readiness.signal.throwIfAborted(); From cc6d423240458bde673bf41000ba0a1ee8dfb531 Mon Sep 17 00:00:00 2001 From: Golo Roden Date: Mon, 19 Oct 2026 12:16:49 +0200 Subject: [PATCH 3/7] Keep the outcome of an attempt if removing its container fails, and bound the wait for a pending start. A failed removal no longer replaces the error of the attempt, nor stops the remaining attempts. It is reported on a line of its own after the message. The wait for a start that is still pending after the time ran out lasts 10 seconds at most, after which the containers of the attempt are removed by their label nonetheless. An empty log tail reads as "no log available", since testcontainers does not pass on errors of Docker while reading the log. The integration tests get 180 seconds, so that 3 attempts of 30 seconds fit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QykegdHe7vhHMBN3kWxgfH --- src/container/ContainerExitedError.ts | 6 ++- src/container/isPortError.ts | 12 +++++ src/container/startContainer.ts | 73 +++++++++++++++++++-------- src/testing/integrationTimeout.ts | 5 +- 4 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 src/container/isPortError.ts diff --git a/src/container/ContainerExitedError.ts b/src/container/ContainerExitedError.ts index 78156c2..379a3d8 100644 --- a/src/container/ContainerExitedError.ts +++ b/src/container/ContainerExitedError.ts @@ -1,9 +1,11 @@ // The container exited before the database in it became reachable, so another -// container would fail the same way. +// container would fail the same way. testcontainers reads the log without +// passing on errors of Docker, so an empty log may mean that it is not +// available. class ContainerExitedError extends Error { public constructor(logTail: string) { super( - `Failed to start the container, it exited before the database became reachable: ${logTail}`, + `Failed to start the container, it exited before the database became reachable: ${logTail === '' ? 'no log available' : logTail}`, ); } } diff --git a/src/container/isPortError.ts b/src/container/isPortError.ts new file mode 100644 index 0000000..0e6d80d --- /dev/null +++ b/src/container/isPortError.ts @@ -0,0 +1,12 @@ +// testcontainers waits for the port of a container for 10 seconds at most, and +// then looks up the port on the host once. If either fails, the container is +// running, but it cannot be reached. +const portErrors = [ + /^Timed out after \d+ms while waiting for container ports to be bound to the host$/u, + /^No host port found for host IP$/u, +]; + +const isPortError = (error: unknown): boolean => + error instanceof Error && portErrors.some(portError => portError.test(error.message)); + +export { isPortError }; diff --git a/src/container/startContainer.ts b/src/container/startContainer.ts index ddfce34..58cf3da 100644 --- a/src/container/startContainer.ts +++ b/src/container/startContainer.ts @@ -9,6 +9,7 @@ import { GenericContainer, getContainerRuntimeClient, getReaper, ImageName } fro import { waitUnlessAborted } from '../http/waitUnlessAborted.js'; import { ContainerExitedError } from './ContainerExitedError.js'; import { removeContainersOfAttempt, startLabel } from './containersOfAttempt.js'; +import { isPortError } from './isPortError.js'; import { waitUntilReachable } from './waitUntilReachable.js'; interface ContentToCopy { @@ -28,17 +29,7 @@ interface StartContainerOptions { const maximumAttempts = 3; const millisecondsPerSecond = 1000; - -// testcontainers waits for the port of a container for 10 seconds at most, and -// then looks up the port on the host once. If either fails, the container is -// running, but it cannot be reached. -const portErrors = [ - /^Timed out after \d+ms while waiting for container ports to be bound to the host$/u, - /^No host port found for host IP$/u, -]; - -const isPortError = (error: unknown): boolean => - error instanceof Error && portErrors.some(portError => portError.test(error.message)); +const settleTimeout = 10_000; // Lets testcontainers wait for the readiness of the database, which the // attempt follows from the creation of the container on. @@ -72,15 +63,34 @@ class ReadinessWait implements WaitStrategy { } } +const getMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +// Waits for the promises to settle, but for the given time at most. +const waitForSettling = async (promises: Promise[], timeout: number): Promise => { + let timer: ReturnType | undefined; + + await Promise.race([ + Promise.allSettled(promises), + new Promise(resolve => { + timer = setTimeout(resolve, timeout); + }), + ]); + clearTimeout(timer); +}; + // Starts a container and waits until its database is reachable, for the // startup timeout at most, from creating the container on. If that fails, it // removes the container at once. If the time ran out while Docker was still -// creating or starting the container, it then waits for that to end, and -// removes what was created meanwhile. It returns undefined if the database did -// not become reachable, so that another container can take its place. +// creating or starting the container, it then waits for that to end, for a +// while at most, and removes what was created meanwhile. A failed removal does +// not change the outcome of the attempt, but is collected. It returns +// undefined if the database did not become reachable, so that another +// container can take its place. const attemptToStart = async ( client: ContainerRuntimeClient, options: StartContainerOptions, + removalErrors: unknown[], ): Promise => { const startId = randomUUID(); const deadline = new AbortController(); @@ -112,12 +122,20 @@ const attemptToStart = async ( return container; } catch (error) { attemptEnded.abort(); - await removeContainersOfAttempt(client, startId); + const removeContainers = async (): Promise => { + try { + await removeContainersOfAttempt(client, startId); + } catch (removalError) { + removalErrors.push(removalError); + } + }; + + await removeContainers(); // The attempt has failed already, so how the start ends no longer // matters, only that nothing of it is left. - await Promise.allSettled([starting, readiness]); - await removeContainersOfAttempt(client, startId); + await waitForSettling([starting, readiness], settleTimeout); + await removeContainers(); if ((deadline.signal.aborted && error === deadline.signal.reason) || isPortError(error)) { return undefined; @@ -136,6 +154,18 @@ const attemptToStart = async ( // containers in total. If Docker fails, or if the container exits, starting // fails at once. const startContainer = async (options: StartContainerOptions): Promise => { + const removalErrors: unknown[] = []; + const createError = (message: string, errorOptions?: ErrorOptions): Error => + new Error( + [ + message, + ...removalErrors.map( + removalError => `Failed to remove a container: ${getMessage(removalError)}`, + ), + ].join('\n'), + errorOptions, + ); + try { const client = await getContainerRuntimeClient(); @@ -146,7 +176,7 @@ const startContainer = async (options: StartContainerOptions): Promise Date: Mon, 19 Oct 2026 12:16:49 +0200 Subject: [PATCH 4/7] Test the removal of a container created late, the pull outside the clock, the wording of the port errors, and the default timeout. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QykegdHe7vhHMBN3kWxgfH --- src/Container.test.ts | 19 ++++- src/container/isPortError.test.ts | 65 +++++++++++++++ src/container/startContainer.test.ts | 119 ++++++++++++++++++++------- src/testing/failPortLookups.ts | 40 +++++++++ 4 files changed, 212 insertions(+), 31 deletions(-) create mode 100644 src/container/isPortError.test.ts create mode 100644 src/testing/failPortLookups.ts diff --git a/src/Container.test.ts b/src/Container.test.ts index b09cd76..aa8c582 100644 --- a/src/Container.test.ts +++ b/src/Container.test.ts @@ -1,8 +1,8 @@ import assert from 'node:assert/strict'; import { suite, test } from 'node:test'; import { Container } from './Container.js'; -import { defaultStartupTimeout } from './container/defaultStartupTimeout.js'; import { getImageTag } from './getImageTag.js'; +import { failPortLookups } from './testing/failPortLookups.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('Container', () => { @@ -24,8 +24,21 @@ suite('Container', () => { }); suite('startup timeout', () => { - test('is 30 seconds by default.', (): void => { - assert.equal(defaultStartupTimeout, 30_000); + test('is 30 seconds by default.', integrationTimeout, async (): Promise => { + const imageTag = getImageTag(); + const container = new Container().withImageTag(imageTag); + + // Each attempt fails as soon as the port cannot be found, so that the + // default can show up in the error without waiting for it. + const endFailing = await failPortLookups(Number.POSITIVE_INFINITY); + try { + await assert.rejects(container.start(), { + message: + 'Failed to start the container, the database did not become reachable within 30s in any of 3 containers', + }); + } finally { + endFailing(); + } }); test('accepts a startup timeout from 1 millisecond up to the longest timer of Node.js.', (): void => { diff --git a/src/container/isPortError.test.ts b/src/container/isPortError.test.ts new file mode 100644 index 0000000..0c824a8 --- /dev/null +++ b/src/container/isPortError.test.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { suite, test } from 'node:test'; +import { BoundPorts, getContainerRuntimeClient, ImageName } from 'testcontainers'; +import { inspectContainerUntilPortsExposed } from 'testcontainers/build/generic-container/inspect-container-util-ports-exposed.js'; +import { getImageTag } from '../getImageTag.js'; +import { integrationTimeout } from '../testing/integrationTimeout.js'; +import { isPortError } from './isPortError.js'; + +// The errors come from testcontainers itself, so that an update of it that +// words them differently fails these tests. +suite('isPortError', () => { + test('recognizes the error of testcontainers if it cannot find the port on the host.', (): void => { + assert.throws( + (): BoundPorts => + BoundPorts.fromInspectResult([], { + name: 'container', + hostname: 'container', + ports: { '3000/tcp': [{ hostIp: '0.0.0.0', hostPort: 50_000 }] }, + healthCheckStatus: 'none', + networkSettings: {}, + state: { status: 'running', running: true, startedAt: new Date(), finishedAt: undefined }, + labels: {}, + }), + (error: unknown): boolean => isPortError(error), + ); + }); + + test( + 'recognizes the error of testcontainers if the port is not published in time.', + integrationTimeout, + async (): Promise => { + // A container that has not started yet has no host port bound to its + // port. + const client = await getContainerRuntimeClient(); + const image = ImageName.fromString(`thenativeweb/eventsourcingdb:${getImageTag()}`); + await client.image.pull(image); + + const container = await client.container.create({ + Image: image.string, + Labels: { 'io.eventsourcingdb.container-start-test': randomUUID() }, + ExposedPorts: { '3000/tcp': {} }, + HostConfig: { PortBindings: { '3000/tcp': [{ HostPort: '0' }] } }, + }); + + try { + await assert.rejects( + inspectContainerUntilPortsExposed( + async () => await client.container.inspect(container), + container.id, + 1, + ), + (error: unknown): boolean => isPortError(error), + ); + } finally { + await client.container.remove(container, { removeVolumes: true }); + } + }, + ); + + test('does not recognize other errors.', (): void => { + assert.equal(isPortError(new Error('No such container: container')), false); + assert.equal(isPortError('No host port found for host IP'), false); + }); +}); diff --git a/src/container/startContainer.test.ts b/src/container/startContainer.test.ts index 509dfd2..aecb45d 100644 --- a/src/container/startContainer.test.ts +++ b/src/container/startContainer.test.ts @@ -2,9 +2,11 @@ import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; import { IncomingMessage } from 'node:http'; import { suite, test } from 'node:test'; +import { setTimeout as sleep } from 'node:timers/promises'; import type { StartedTestContainer } from 'testcontainers'; -import { getContainerRuntimeClient, getReaper } from 'testcontainers'; +import { getContainerRuntimeClient } from 'testcontainers'; import { getImageTag } from '../getImageTag.js'; +import { failPortLookups } from '../testing/failPortLookups.js'; import { integrationTimeout } from '../testing/integrationTimeout.js'; import { startContainer } from './startContainer.js'; @@ -227,28 +229,7 @@ suite('startContainer', () => { integrationTimeout, async (): Promise => { const testId = randomUUID(); - const client = await getContainerRuntimeClient(); - const { containerRuntime } = client.info; - const { hostIps } = containerRuntime; - - // Ryuk starts with the first container of the process, so it starts - // before the lookup fails, and only the first container of the test is - // affected. - await getReaper(client); - - // testcontainers looks up the port of a container on the host once per - // start, by the addresses of the host. Without any, the lookup of the - // first container fails, as when Docker publishes the port only for - // another address. - let lookupCount = 0; - Object.defineProperty(containerRuntime, 'hostIps', { - configurable: true, - get: (): typeof hostIps => { - lookupCount += 1; - return lookupCount === 1 ? [] : hostIps; - }, - }); - + const endFailing = await failPortLookups(1); const stopWatching = await watchCreatedContainers(testId); let container: StartedTestContainer; @@ -262,11 +243,7 @@ suite('startContainer', () => { labels: { [testLabel]: testId }, }); } finally { - Object.defineProperty(containerRuntime, 'hostIps', { - configurable: true, - writable: true, - value: hostIps, - }); + endFailing(); } try { @@ -282,6 +259,92 @@ suite('startContainer', () => { }, ); + test( + 'removes a container that testcontainers creates after the startup timeout, and gives up on.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + + // The time runs out before Docker has created the container, and the + // lookup of the port fails, so testcontainers gives up on the container + // without removing it. + const endFailing = await failPortLookups(Number.POSITIVE_INFINITY); + const stopWatching = await watchCreatedContainers(testId); + + try { + await assert.rejects( + startContainer({ + image, + port: 3000, + command: getCommand(3000), + contents: [], + startupTimeout: 1, + labels: { [testLabel]: testId }, + }), + { + message: + 'Failed to start the container, the database did not become reachable within 0.001s in any of 3 containers', + }, + ); + } finally { + endFailing(); + } + + assert.equal(stopWatching().length, 3); + assert.deepEqual(await listContainers(testId), []); + }, + ); + + test( + 'does not count pulling the image towards the startup timeout.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const client = await getContainerRuntimeClient(); + const { image: imageClient } = client; + const { pull } = imageClient; + let pullCount = 0; + + // The first pull takes longer than the startup timeout, the others are + // as fast as Docker. + imageClient.pull = async (...parameters: Parameters): Promise => { + pullCount += 1; + if (pullCount === 1) { + await sleep(2000); + } + await pull.apply(imageClient, parameters); + }; + + const stopWatching = await watchCreatedContainers(testId); + + let container: StartedTestContainer; + try { + container = await startContainer({ + image, + port: 3000, + command: getCommand(3000), + contents: [], + startupTimeout: 1500, + labels: { [testLabel]: testId }, + }); + } finally { + imageClient.pull = pull; + } + + try { + assert.ok(pullCount >= 1); + + // Had the pull counted, the first attempt would have created its + // container late, and another attempt one more. + assert.equal(stopWatching().length, 1); + } finally { + await container.stop(); + } + + assert.deepEqual(await listContainers(testId), []); + }, + ); + test( 'starts a container whose database becomes reachable.', integrationTimeout, diff --git a/src/testing/failPortLookups.ts b/src/testing/failPortLookups.ts new file mode 100644 index 0000000..52bc643 --- /dev/null +++ b/src/testing/failPortLookups.ts @@ -0,0 +1,40 @@ +import { getContainerRuntimeClient, getReaper } from 'testcontainers'; + +// testcontainers looks up the port of a container on the host once per start, +// by the addresses of the host. Without any, the lookup fails with the error +// that testcontainers gives when Docker publishes the port only for another +// address. Lets the given number of lookups fail that way, and returns a +// function that ends that. +const failPortLookups = async (count: number): Promise<() => void> => { + const client = await getContainerRuntimeClient(); + + // Ryuk starts with the first container of the process, so it starts before + // the lookups fail, and does not count among them. + await getReaper(client); + + const { containerRuntime } = client.info; + const { hostIps } = containerRuntime; + let lookupCount = 0; + + Object.defineProperty(containerRuntime, 'hostIps', { + configurable: true, + get: (): typeof hostIps => { + lookupCount += 1; + if (lookupCount <= count) { + return []; + } + return hostIps; + }, + }); + + return (): void => { + Object.defineProperty(containerRuntime, 'hostIps', { + configurable: true, + enumerable: true, + writable: true, + value: hostIps, + }); + }; +}; + +export { failPortLookups }; From c9785b507734e8b533e8185cdd9e5f7d4d020abc Mon Sep 17 00:00:00 2001 From: Golo Roden Date: Mon, 19 Oct 2026 12:41:51 +0200 Subject: [PATCH 5/7] Remove a container that Docker creates only after the wait for a failed attempt has ended. Once the wait for a pending start has run out, the attempt goes on without waiting further, and removes the container of the attempt as soon as the start ends. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QykegdHe7vhHMBN3kWxgfH --- src/container/startContainer.ts | 37 ++++++++++++++++++++++--------- src/testing/integrationTimeout.ts | 5 ++--- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/container/startContainer.ts b/src/container/startContainer.ts index 58cf3da..1b72a8b 100644 --- a/src/container/startContainer.ts +++ b/src/container/startContainer.ts @@ -25,11 +25,12 @@ interface StartContainerOptions { contents: ContentToCopy[]; startupTimeout: number; labels?: Record; + settleTimeout?: number; } const maximumAttempts = 3; const millisecondsPerSecond = 1000; -const settleTimeout = 10_000; +const defaultSettleTimeout = 10_000; // Lets testcontainers wait for the readiness of the database, which the // attempt follows from the creation of the container on. @@ -66,25 +67,32 @@ class ReadinessWait implements WaitStrategy { const getMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); -// Waits for the promises to settle, but for the given time at most. -const waitForSettling = async (promises: Promise[], timeout: number): Promise => { +// Waits for the promises to settle, but for the given time at most, and +// returns whether they have settled. +const waitForSettling = async (promises: Promise[], timeout: number): Promise => { let timer: ReturnType | undefined; - await Promise.race([ - Promise.allSettled(promises), - new Promise(resolve => { - timer = setTimeout(resolve, timeout); + const hasSettled = await Promise.race([ + Promise.allSettled(promises).then(() => true), + new Promise(resolve => { + timer = setTimeout(() => { + resolve(false); + }, timeout); }), ]); clearTimeout(timer); + + return hasSettled; }; // Starts a container and waits until its database is reachable, for the // startup timeout at most, from creating the container on. If that fails, it // removes the container at once. If the time ran out while Docker was still // creating or starting the container, it then waits for that to end, for a -// while at most, and removes what was created meanwhile. A failed removal does -// not change the outcome of the attempt, but is collected. It returns +// while at most, and removes what was created meanwhile. If that takes longer, +// it removes the container once Docker has created it, without waiting for +// that. A failed removal does not change the outcome of the attempt, but is +// collected. It returns // undefined if the database did not become reachable, so that another // container can take its place. const attemptToStart = async ( @@ -134,9 +142,18 @@ const attemptToStart = async ( // The attempt has failed already, so how the start ends no longer // matters, only that nothing of it is left. - await waitForSettling([starting, readiness], settleTimeout); + const hasSettled = await waitForSettling( + [starting, readiness], + options.settleTimeout ?? defaultSettleTimeout, + ); await removeContainers(); + if (!hasSettled) { + // The caller does not wait for a start that Docker does not finish, and + // the removal collects its own errors. + Promise.allSettled([starting]).then(removeContainers); + } + if ((deadline.signal.aborted && error === deadline.signal.reason) || isPortError(error)) { return undefined; } diff --git a/src/testing/integrationTimeout.ts b/src/testing/integrationTimeout.ts index 890c800..e40735c 100644 --- a/src/testing/integrationTimeout.ts +++ b/src/testing/integrationTimeout.ts @@ -3,8 +3,7 @@ // budget for a whole suite shrinks with every start and cancels the last tests // of a suite. So every start and stop, and every test, has a budget of its // own, which node:test keeps apart, since the budget of a test does not count -// its hooks. A start may take 3 attempts of 30 seconds each, plus pulling the -// image and removing the containers of failed attempts. -const integrationTimeout = { timeout: 180_000 }; +// its hooks. +const integrationTimeout = { timeout: 60_000 }; export { integrationTimeout }; From 5d2359cd016e72bc6d8375d4c8ed4a1ada23d95a Mon Sep 17 00:00:00 2001 From: Golo Roden Date: Mon, 19 Oct 2026 12:41:51 +0200 Subject: [PATCH 6/7] Test the late removal, the failed removal, and a missing log, and give 180 seconds only to the hooks that start containers. The other hooks and the tests keep 60 seconds. The pull test waits 6 seconds against a startup timeout of 5 seconds, and the tests label their containers with a label of their own SDK. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QykegdHe7vhHMBN3kWxgfH --- src/Event.test.ts | 3 +- src/apiVersion.test.ts | 3 +- src/container/isPortError.test.ts | 2 +- src/container/startContainer.test.ts | 171 ++++++++++++++++++++++++++- src/observeEvents.test.ts | 3 +- src/ping.test.ts | 3 +- src/readEventType.test.ts | 3 +- src/readEventTypes.test.ts | 3 +- src/readEvents.test.ts | 3 +- src/readSubjects.test.ts | 3 +- src/registerEventSchema.test.ts | 3 +- src/runEventQlQuery.test.ts | 3 +- src/testing/containerStartTimeout.ts | 6 + src/verifyApiToken.test.ts | 3 +- src/writeEvents.test.ts | 3 +- 15 files changed, 198 insertions(+), 17 deletions(-) create mode 100644 src/testing/containerStartTimeout.ts diff --git a/src/Event.test.ts b/src/Event.test.ts index af7032b..6d840ec 100644 --- a/src/Event.test.ts +++ b/src/Event.test.ts @@ -8,6 +8,7 @@ import type { EventCandidate } from './EventCandidate.js'; import { getImageTag } from './getImageTag.js'; import { HashVerificationFailedError } from './HashVerificationFailedError.js'; import { SignatureVerificationFailedError } from './SignatureVerificationFailedError.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; // Returns an event as written, as read, and as observed. Its data contains @@ -48,7 +49,7 @@ suite('Event', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/apiVersion.test.ts b/src/apiVersion.test.ts index ccdc699..67149ce 100644 --- a/src/apiVersion.test.ts +++ b/src/apiVersion.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, suite, test } from 'node:test'; import { Container } from './Container.js'; import type { ApiVersion } from './detectApiVersion.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { getApiVersionOf } from './testing/getApiVersionOf.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; @@ -29,7 +30,7 @@ suite('apiVersion', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/container/isPortError.test.ts b/src/container/isPortError.test.ts index 0c824a8..8fda917 100644 --- a/src/container/isPortError.test.ts +++ b/src/container/isPortError.test.ts @@ -38,7 +38,7 @@ suite('isPortError', () => { const container = await client.container.create({ Image: image.string, - Labels: { 'io.eventsourcingdb.container-start-test': randomUUID() }, + Labels: { 'io.eventsourcingdb.client-javascript-test': randomUUID() }, ExposedPorts: { '3000/tcp': {} }, HostConfig: { PortBindings: { '3000/tcp': [{ HostPort: '0' }] } }, }); diff --git a/src/container/startContainer.test.ts b/src/container/startContainer.test.ts index aecb45d..1df6d72 100644 --- a/src/container/startContainer.test.ts +++ b/src/container/startContainer.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; import { IncomingMessage } from 'node:http'; +import { Readable } from 'node:stream'; import { suite, test } from 'node:test'; import { setTimeout as sleep } from 'node:timers/promises'; import type { StartedTestContainer } from 'testcontainers'; @@ -8,10 +9,11 @@ import { getContainerRuntimeClient } from 'testcontainers'; import { getImageTag } from '../getImageTag.js'; import { failPortLookups } from '../testing/failPortLookups.js'; import { integrationTimeout } from '../testing/integrationTimeout.js'; +import type { ContainerInfo } from './containersOfAttempt.js'; import { startContainer } from './startContainer.js'; const startLabel = 'io.eventsourcingdb.container-start'; -const testLabel = 'io.eventsourcingdb.container-start-test'; +const testLabel = 'io.eventsourcingdb.client-javascript-test'; const exitedPrefix = 'Failed to start the container, it exited before the database became reachable: '; @@ -28,7 +30,7 @@ const getCommand = (port: number): string[] => [ // Each test labels its containers, so that it finds them, and only them, while // other tests use the same Docker. -const listContainers = async (testId: string): Promise => { +const listContainers = async (testId: string): Promise => { const client = await getContainerRuntimeClient(); return await client.container.dockerode.listContainers({ @@ -310,7 +312,7 @@ suite('startContainer', () => { imageClient.pull = async (...parameters: Parameters): Promise => { pullCount += 1; if (pullCount === 1) { - await sleep(2000); + await sleep(6000); } await pull.apply(imageClient, parameters); }; @@ -324,7 +326,7 @@ suite('startContainer', () => { port: 3000, command: getCommand(3000), contents: [], - startupTimeout: 1500, + startupTimeout: 5000, labels: { [testLabel]: testId }, }); } finally { @@ -345,6 +347,167 @@ suite('startContainer', () => { }, ); + test( + 'removes a container that Docker creates only after the wait for it has ended, without waiting for it.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const client = await getContainerRuntimeClient(); + const { container: containerClient } = client; + const { create } = containerClient; + const creations: Promise[] = []; + + // Docker takes longer to create a container than the attempt waits for + // it, and testcontainers gives up on it without removing it, since the + // lookup of its port fails. + containerClient.create = async ( + ...parameters: Parameters + ): ReturnType => { + const creation = sleep(4000).then( + async (): ReturnType => await create.apply(containerClient, parameters), + ); + creations.push(creation); + return await creation; + }; + const endFailing = await failPortLookups(Number.POSITIVE_INFINITY); + + try { + const startedAt = Date.now(); + await assert.rejects( + startContainer({ + image, + port: 3000, + command: getCommand(3000), + contents: [], + startupTimeout: 100, + labels: { [testLabel]: testId }, + settleTimeout: 300, + }), + { + message: + 'Failed to start the container, the database did not become reachable within 0.1s in any of 3 containers', + }, + ); + + // The attempts do not wait for Docker beyond the settle timeout. + assert.ok(Date.now() - startedAt < 4000); + + await Promise.allSettled(creations); + assert.equal(creations.length, 3); + + let containers = await listContainers(testId); + for (let poll = 0; containers.length > 0 && poll < 50; poll += 1) { + // biome-ignore lint/performance/noAwaitInLoops: Each poll waits for the one before. + await sleep(100); + containers = await listContainers(testId); + } + assert.deepEqual(containers, []); + } finally { + endFailing(); + containerClient.create = create; + } + }, + ); + + test( + 'reports a failed removal after the error of the start, and goes on with the next attempt.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const client = await getContainerRuntimeClient(); + const { dockerode } = client.container; + const { getContainer } = dockerode; + + dockerode.getContainer = ( + ...parameters: Parameters + ): ReturnType => + Object.assign(getContainer.apply(dockerode, parameters), { + remove: async (): Promise => { + await Promise.resolve(); + throw new Error('removal failed'); + }, + }); + const endFailing = await failPortLookups(Number.POSITIVE_INFINITY); + const stopWatching = await watchCreatedContainers(testId); + + try { + await assert.rejects( + startContainer({ + image, + port: 3000, + command: getCommand(3000), + contents: [], + startupTimeout: 20_000, + labels: { [testLabel]: testId }, + }), + (error: unknown): boolean => { + assert.ok(error instanceof Error); + + const [message, ...removalFailures] = error.message.split('\n'); + assert.equal( + message, + 'Failed to start the container, the database did not become reachable within 20s in any of 3 containers', + ); + assert.ok(removalFailures.length > 0); + for (const removalFailure of removalFailures) { + assert.equal(removalFailure, 'Failed to remove a container: removal failed'); + } + return true; + }, + ); + + assert.equal(stopWatching().length, 3); + } finally { + endFailing(); + dockerode.getContainer = getContainer; + + for (const { Id: id } of await listContainers(testId)) { + // biome-ignore lint/performance/noAwaitInLoops: The test removes its containers one after another. + await dockerode.getContainer(id).remove({ force: true, v: true }); + } + } + + assert.deepEqual(await listContainers(testId), []); + }, + ); + + test( + 'says so if the log of a container that exits is not available.', + integrationTimeout, + async (): Promise => { + const testId = randomUUID(); + const client = await getContainerRuntimeClient(); + const { container: containerClient } = client; + const { logs } = containerClient; + + // testcontainers ends the log without an error if Docker fails to read + // it. + containerClient.logs = async (): Promise => + await Promise.resolve(Readable.from([])); + + try { + await assert.rejects( + startContainer({ + image, + port: 3000, + command: [...getCommand(3000), '--no-such-flag'], + contents: [], + startupTimeout: 20_000, + labels: { [testLabel]: testId }, + }), + { + message: + 'Failed to start the container, it exited before the database became reachable: no log available', + }, + ); + } finally { + containerClient.logs = logs; + } + + assert.deepEqual(await listContainers(testId), []); + }, + ); + test( 'starts a container whose database becomes reachable.', integrationTimeout, diff --git a/src/observeEvents.test.ts b/src/observeEvents.test.ts index a97156b..0bb4cef 100644 --- a/src/observeEvents.test.ts +++ b/src/observeEvents.test.ts @@ -4,6 +4,7 @@ import { Container } from './Container.js'; import type { Event } from './Event.js'; import type { EventCandidate } from './EventCandidate.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('observeEvents', () => { @@ -13,7 +14,7 @@ suite('observeEvents', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/ping.test.ts b/src/ping.test.ts index f0e09e8..8a6bae8 100644 --- a/src/ping.test.ts +++ b/src/ping.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, suite, test } from 'node:test'; import { Client } from './Client.js'; import { Container } from './Container.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('ping', () => { @@ -12,7 +13,7 @@ suite('ping', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/readEventType.test.ts b/src/readEventType.test.ts index 196cd09..df27765 100644 --- a/src/readEventType.test.ts +++ b/src/readEventType.test.ts @@ -5,6 +5,7 @@ import { DbApiError } from './DbApiError.js'; import { ErrorCode } from './ErrorCode.js'; import type { EventCandidate } from './EventCandidate.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { getApiVersionOf } from './testing/getApiVersionOf.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; @@ -15,7 +16,7 @@ suite('readEventType', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/readEventTypes.test.ts b/src/readEventTypes.test.ts index 7966044..52f2fd5 100644 --- a/src/readEventTypes.test.ts +++ b/src/readEventTypes.test.ts @@ -4,6 +4,7 @@ import { Container } from './Container.js'; import type { EventCandidate } from './EventCandidate.js'; import type { EventType } from './EventType.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('readEventTypes', () => { @@ -13,7 +14,7 @@ suite('readEventTypes', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/readEvents.test.ts b/src/readEvents.test.ts index 44f59f3..abfaabf 100644 --- a/src/readEvents.test.ts +++ b/src/readEvents.test.ts @@ -4,6 +4,7 @@ import { Container } from './Container.js'; import type { Event } from './Event.js'; import type { EventCandidate } from './EventCandidate.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('readEvents', () => { @@ -13,7 +14,7 @@ suite('readEvents', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/readSubjects.test.ts b/src/readSubjects.test.ts index 91606b0..fd06bad 100644 --- a/src/readSubjects.test.ts +++ b/src/readSubjects.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, suite, test } from 'node:test'; import { Container } from './Container.js'; import type { EventCandidate } from './EventCandidate.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('readSubjects', () => { @@ -12,7 +13,7 @@ suite('readSubjects', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/registerEventSchema.test.ts b/src/registerEventSchema.test.ts index 1c6ba2b..af1ff91 100644 --- a/src/registerEventSchema.test.ts +++ b/src/registerEventSchema.test.ts @@ -4,6 +4,7 @@ import { Container } from './Container.js'; import { DbApiError } from './DbApiError.js'; import { ErrorCode } from './ErrorCode.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('registerEventSchema', () => { @@ -13,7 +14,7 @@ suite('registerEventSchema', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/runEventQlQuery.test.ts b/src/runEventQlQuery.test.ts index f5a97a1..f9dd307 100644 --- a/src/runEventQlQuery.test.ts +++ b/src/runEventQlQuery.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, suite, test } from 'node:test'; import { Container } from './Container.js'; import type { EventCandidate } from './EventCandidate.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; import { isRecord } from './types/isRecord.js'; @@ -13,7 +14,7 @@ suite('runEventQlQuery', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/testing/containerStartTimeout.ts b/src/testing/containerStartTimeout.ts new file mode 100644 index 0000000..716d9fe --- /dev/null +++ b/src/testing/containerStartTimeout.ts @@ -0,0 +1,6 @@ +// A hook that starts a container may need 3 attempts of 30 seconds each, plus +// pulling the image and removing the containers of failed attempts, so it has +// a budget larger than the one of a test. +const containerStartTimeout = { timeout: 180_000 }; + +export { containerStartTimeout }; diff --git a/src/verifyApiToken.test.ts b/src/verifyApiToken.test.ts index 27bf3f9..077be1c 100644 --- a/src/verifyApiToken.test.ts +++ b/src/verifyApiToken.test.ts @@ -4,6 +4,7 @@ import { Client } from './Client.js'; import { Container } from './Container.js'; import { DbApiError } from './DbApiError.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('verifyApiToken', () => { @@ -13,7 +14,7 @@ suite('verifyApiToken', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); diff --git a/src/writeEvents.test.ts b/src/writeEvents.test.ts index b4deb53..291f286 100644 --- a/src/writeEvents.test.ts +++ b/src/writeEvents.test.ts @@ -10,6 +10,7 @@ import { isEventQlQueryTrue } from './isEventQlQueryTrue.js'; import { isSubjectOnEventId } from './isSubjectOnEventId.js'; import { isSubjectPopulated } from './isSubjectPopulated.js'; import { isSubjectPristine } from './isSubjectPristine.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; // Both APIs report a precondition that does not hold with the same code. @@ -23,7 +24,7 @@ suite('writeEvents', () => { const imageTag = getImageTag(); container = new Container().withImageTag(imageTag); await container.start(); - }, integrationTimeout); + }, containerStartTimeout); afterEach(async () => { await container.stop(); From ef7a26919704e7964aae8375f48bc7679a7d2d66 Mon Sep 17 00:00:00 2001 From: Golo Roden Date: Mon, 19 Oct 2026 12:55:37 +0200 Subject: [PATCH 7/7] Give the tests that start a container themselves 180 seconds as well. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QykegdHe7vhHMBN3kWxgfH --- src/Container.test.ts | 3 ++- src/Event.test.ts | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Container.test.ts b/src/Container.test.ts index aa8c582..435cc2e 100644 --- a/src/Container.test.ts +++ b/src/Container.test.ts @@ -2,11 +2,12 @@ import assert from 'node:assert/strict'; import { suite, test } from 'node:test'; import { Container } from './Container.js'; import { getImageTag } from './getImageTag.js'; +import { containerStartTimeout } from './testing/containerStartTimeout.js'; import { failPortLookups } from './testing/failPortLookups.js'; import { integrationTimeout } from './testing/integrationTimeout.js'; suite('Container', () => { - test('starts with a custom port.', integrationTimeout, async (): Promise => { + test('starts with a custom port.', containerStartTimeout, async (): Promise => { const imageTag = getImageTag(); const container = new Container().withImageTag(imageTag).withPort(4000); diff --git a/src/Event.test.ts b/src/Event.test.ts index 6d840ec..b2f0175 100644 --- a/src/Event.test.ts +++ b/src/Event.test.ts @@ -163,7 +163,7 @@ suite('Event', () => { test( 'throws an error if the hash verification fails.', - integrationTimeout, + containerStartTimeout, async (): Promise => { await container.stop(); const imageTag = getImageTag(); @@ -202,7 +202,7 @@ suite('Event', () => { test( 'throws an error if the signature verification fails.', - integrationTimeout, + containerStartTimeout, async (): Promise => { await container.stop(); const imageTag = getImageTag(); @@ -238,7 +238,7 @@ suite('Event', () => { }, ); - test('verifies the signature.', integrationTimeout, async (): Promise => { + test('verifies the signature.', containerStartTimeout, async (): Promise => { await container.stop(); const imageTag = getImageTag(); container = new Container().withImageTag(imageTag).withSigningKey(); @@ -272,7 +272,7 @@ suite('Event', () => { test( 'verifies the signature of an event as written, read, and observed, whatever its data and trace context.', - integrationTimeout, + containerStartTimeout, async (): Promise => { await container.stop(); const imageTag = getImageTag();