feat: Wait for the test container up to a configurable startup timeout, replace one that does not become reachable, and remove one whose start fails.

README.md

@@ -700,6 +700,13 @@ const container = new Container()
   .withApiToken('secret');
 ```
 
+By default, the `start` function gives the test container 30 seconds to become reachable, from creating it until the database answers a ping, without the time it takes to pull the image. A container that is running but does not become reachable in time is replaced by a new one, up to 3 containers in total. If the container exits before it becomes reachable, or if Docker fails, the `start` function fails at once. Every container whose start fails is removed at once. To change the 30 seconds, call the `withStartupTimeout` function with the time in milliseconds:
+
+```typescript
+const container = new Container()
+  .withStartupTimeout(60_000);
+```
+
 If you want to sign events, call the `withSigningKey` function. This generates a new signing and verification key pair inside the container:
 
 ```typescript

src/Container.test.ts

@@ -2,9 +2,12 @@ import assert from 'node:assert/strict';
 import { suite, test } from 'node:test';
 import { Container } from './Container.js';
 import { getImageTag } from './getImageTag.js';
+import { containerStartTimeout } from './testing/containerStartTimeout.js';
+import { failPortLookups } from './testing/failPortLookups.js';
+import { integrationTimeout } from './testing/integrationTimeout.js';
 
-suite('Container', { timeout: 30_000 }, () => {
-	test('starts with a custom port.', async (): Promise<void> => {
+suite('Container', () => {
+	test('starts with a custom port.', containerStartTimeout, async (): Promise<void> => {
 		const imageTag = getImageTag();
 		const container = new Container().withImageTag(imageTag).withPort(4000);
 
@@ -20,4 +23,59 @@ suite('Container', { timeout: 30_000 }, () => {
 			await container.stop();
 		}
 	});
+
+	suite('startup timeout', () => {
+		test('is 30 seconds by default.', integrationTimeout, async (): Promise<void> => {
+			const imageTag = getImageTag();
+			const container = new Container().withImageTag(imageTag);
+
+			// Each attempt fails as soon as the port cannot be found, so that the
+			// default can show up in the error without waiting for it.
+			const endFailing = await failPortLookups(Number.POSITIVE_INFINITY);
+			try {
+				await assert.rejects(container.start(), {
+					message:
+						'Failed to start the container, the database did not become reachable within 30s in any of 3 containers',
+				});
+			} finally {
+				endFailing();
+			}
+		});
+
+		test('accepts a startup timeout from 1 millisecond up to the longest timer of Node.js.', (): void => {
+			assert.ok(new Container().withStartupTimeout(1));
+			assert.ok(new Container().withStartupTimeout(2_147_483_647));
+		});
+
+		test('throws an error if the startup timeout is out of range.', (): void => {
+			for (const invalidStartupTimeout of [
+				0,
+				-1,
+				2_147_483_648,
+				Number.NaN,
+				Number.POSITIVE_INFINITY,
+			]) {
+				assert.throws((): Container => new Container().withStartupTimeout(invalidStartupTimeout), {
+					name: 'Error',
+					message: `Startup timeout must be between 1 and 2147483647 milliseconds, got '${invalidStartupTimeout}'.`,
+				});
+			}
+		});
+
+		test(
+			'gives each of 3 containers the configured time to become reachable.',
+			integrationTimeout,
+			async (): Promise<void> => {
+				const imageTag = getImageTag();
+				const container = new Container().withImageTag(imageTag).withStartupTimeout(1);
+
+				await assert.rejects(container.start(), {
+					name: 'Error',
+					message:
+						'Failed to start the container, the database did not become reachable within 0.001s in any of 3 containers',
+				});
+				assert.equal(container.isRunning(), false);
+			},
+		);
+	});
 });

src/Container.ts

@@ -1,20 +1,17 @@
 import crypto from 'node:crypto';
-import type { Content, StartedTestContainer } from 'testcontainers';
-import { GenericContainer, Wait } from 'testcontainers';
+import type { StartedTestContainer } from 'testcontainers';
 import { Client } from './Client.js';
-
-interface ContentToCopy {
-	content: Content;
-	target: string;
-	mode?: number;
-}
+import { defaultStartupTimeout, maximumStartupTimeout } from './container/defaultStartupTimeout.js';
+import type { ContentToCopy } from './container/startContainer.js';
+import { startContainer } from './container/startContainer.js';
 
 class Container {
 	readonly #imageName = 'thenativeweb/eventsourcingdb';
 	#imageTag = 'latest';
 	#internalPort = 3000;
 	#apiToken = 'secret';
 	#signingKey: crypto.KeyObject | undefined;
+	#startupTimeout = defaultStartupTimeout;
 	#container: StartedTestContainer | undefined;
 
 	public withImageTag(tag: string): this {
@@ -38,6 +35,17 @@ class Container {
 		return this;
 	}
 
+	public withStartupTimeout(timeout: number): this {
+		if (!(timeout >= 1 && timeout <= maximumStartupTimeout)) {
+			throw new Error(
+				`Startup timeout must be between 1 and ${maximumStartupTimeout} milliseconds, got '${timeout}'.`,
+			);
+		}
+
+		this.#startupTimeout = timeout;
+		return this;
+	}
+
 	public async start(): Promise<void> {
 		const command = [
 			'run',
@@ -63,12 +71,13 @@ class Container {
 			});
 		}
 
-		this.#container = await new GenericContainer(`${this.#imageName}:${this.#imageTag}`)
-			.withExposedPorts(this.#internalPort)
-			.withCommand(command)
-			.withCopyContentToContainer(contents)
-			.withWaitStrategy(Wait.forHttp('/api/v1/ping', this.#internalPort).withStartupTimeout(10_000))
-			.start();
+		this.#container = await startContainer({
+			image: `${this.#imageName}:${this.#imageTag}`,
+			port: this.#internalPort,
+			command,
+			contents,
+			startupTimeout: this.#startupTimeout,
+		});
 	}
 
 	public getHost(): string {

src/Event.test.ts

@@ -8,6 +8,7 @@ import type { EventCandidate } from './EventCandidate.js';
 import { getImageTag } from './getImageTag.js';
 import { HashVerificationFailedError } from './HashVerificationFailedError.js';
 import { SignatureVerificationFailedError } from './SignatureVerificationFailedError.js';
+import { containerStartTimeout } from './testing/containerStartTimeout.js';
 import { integrationTimeout } from './testing/integrationTimeout.js';
 
 // Returns an event as written, as read, and as observed. Its data contains
@@ -48,7 +49,7 @@ suite('Event', () => {
 		const imageTag = getImageTag();
 		container = new Container().withImageTag(imageTag);
 		await container.start();
-	}, integrationTimeout);
+	}, containerStartTimeout);
 
 	afterEach(async () => {
 		await container.stop();
@@ -162,7 +163,7 @@ suite('Event', () => {
 
 		test(
 			'throws an error if the hash verification fails.',
-			integrationTimeout,
+			containerStartTimeout,
 			async (): Promise<void> => {
 				await container.stop();
 				const imageTag = getImageTag();
@@ -201,7 +202,7 @@ suite('Event', () => {
 
 		test(
 			'throws an error if the signature verification fails.',
-			integrationTimeout,
+			containerStartTimeout,
 			async (): Promise<void> => {
 				await container.stop();
 				const imageTag = getImageTag();
@@ -237,7 +238,7 @@ suite('Event', () => {
 			},
 		);
 
-		test('verifies the signature.', integrationTimeout, async (): Promise<void> => {
+		test('verifies the signature.', containerStartTimeout, async (): Promise<void> => {
 			await container.stop();
 			const imageTag = getImageTag();
 			container = new Container().withImageTag(imageTag).withSigningKey();
@@ -271,7 +272,7 @@ suite('Event', () => {
 
 		test(
 			'verifies the signature of an event as written, read, and observed, whatever its data and trace context.',
-			integrationTimeout,
+			containerStartTimeout,
 			async (): Promise<void> => {
 				await container.stop();
 				const imageTag = getImageTag();

src/apiVersion.test.ts

@@ -3,6 +3,7 @@ import { afterEach, beforeEach, suite, test } from 'node:test';
 import { Container } from './Container.js';
 import type { ApiVersion } from './detectApiVersion.js';
 import { getImageTag } from './getImageTag.js';
+import { containerStartTimeout } from './testing/containerStartTimeout.js';
 import { getApiVersionOf } from './testing/getApiVersionOf.js';
 import { integrationTimeout } from './testing/integrationTimeout.js';
 
@@ -29,7 +30,7 @@ suite('apiVersion', () => {
 		const imageTag = getImageTag();
 		container = new Container().withImageTag(imageTag);
 		await container.start();
-	}, integrationTimeout);
+	}, containerStartTimeout);
 
 	afterEach(async () => {
 		await container.stop();

src/container/ContainerExitedError.ts

@@ -0,0 +1,13 @@
+// The container exited before the database in it became reachable, so another
+// container would fail the same way. testcontainers reads the log without
+// passing on errors of Docker, so an empty log may mean that it is not
+// available.
+class ContainerExitedError extends Error {
+	public constructor(logTail: string) {
+		super(
+			`Failed to start the container, it exited before the database became reachable: ${logTail === '' ? 'no log available' : logTail}`,
+		);
+	}
+}
+
+export { ContainerExitedError };

src/container/containersOfAttempt.ts

@@ -0,0 +1,49 @@
+import type { ContainerRuntimeClient } from 'testcontainers';
+import { isRecord } from '../types/isRecord.js';
+
+type ContainerInfo = Awaited<
+	ReturnType<ContainerRuntimeClient['container']['dockerode']['listContainers']>
+>[number];
+
+// Every attempt to start a container labels its container with an id of its
+// own, by which it finds the container, whatever state it is in, even before
+// testcontainers hands it over.
+const startLabel = 'io.eventsourcingdb.container-start';
+const statusNotFound = 404;
+const statusConflict = 409;
+
+// A container that is gone, or that Docker is removing already, needs no
+// removing.
+const isGone = (error: unknown): boolean =>
+	isRecord(error) && (error.statusCode === statusNotFound || error.statusCode === statusConflict);
+
+const listContainersOfAttempt = async (
+	client: ContainerRuntimeClient,
+	startId: string,
+): Promise<ContainerInfo[]> =>
+	await client.container.dockerode.listContainers({
+		all: true,
+		filters: { label: [`${startLabel}=${startId}`] },
+	});
+
+const removeContainersOfAttempt = async (
+	client: ContainerRuntimeClient,
+	startId: string,
+): Promise<void> => {
+	const containers = await listContainersOfAttempt(client, startId);
+
+	await Promise.all(
+		containers.map(async ({ Id: id }): Promise<void> => {
+			try {
+				await client.container.dockerode.getContainer(id).remove({ force: true, v: true });
+			} catch (error) {
+				if (!isGone(error)) {
+					throw error;
+				}
+			}
+		}),
+	);
+};
+
+export type { ContainerInfo };
+export { listContainersOfAttempt, removeContainersOfAttempt, startLabel };

src/container/defaultStartupTimeout.ts

@@ -0,0 +1,9 @@
+// Each attempt to start a container may take this many milliseconds, from
+// creating the container until the database answers a ping, unless the
+// container is configured otherwise.
+const defaultStartupTimeout = 30_000;
+
+// Node.js runs a timer for at most this many milliseconds.
+const maximumStartupTimeout = 2_147_483_647;
+
+export { defaultStartupTimeout, maximumStartupTimeout };

src/container/isPortError.test.ts

@@ -0,0 +1,65 @@
+import assert from 'node:assert/strict';
+import { randomUUID } from 'node:crypto';
+import { suite, test } from 'node:test';
+import { BoundPorts, getContainerRuntimeClient, ImageName } from 'testcontainers';
+import { inspectContainerUntilPortsExposed } from 'testcontainers/build/generic-container/inspect-container-util-ports-exposed.js';
+import { getImageTag } from '../getImageTag.js';
+import { integrationTimeout } from '../testing/integrationTimeout.js';
+import { isPortError } from './isPortError.js';
+
+// The errors come from testcontainers itself, so that an update of it that
+// words them differently fails these tests.
+suite('isPortError', () => {
+	test('recognizes the error of testcontainers if it cannot find the port on the host.', (): void => {
+		assert.throws(
+			(): BoundPorts =>
+				BoundPorts.fromInspectResult([], {
+					name: 'container',
+					hostname: 'container',
+					ports: { '3000/tcp': [{ hostIp: '0.0.0.0', hostPort: 50_000 }] },
+					healthCheckStatus: 'none',
+					networkSettings: {},
+					state: { status: 'running', running: true, startedAt: new Date(), finishedAt: undefined },
+					labels: {},
+				}),
+			(error: unknown): boolean => isPortError(error),
+		);
+	});
+
+	test(
+		'recognizes the error of testcontainers if the port is not published in time.',
+		integrationTimeout,
+		async (): Promise<void> => {
+			// A container that has not started yet has no host port bound to its
+			// port.
+			const client = await getContainerRuntimeClient();
+			const image = ImageName.fromString(`thenativeweb/eventsourcingdb:${getImageTag()}`);
+			await client.image.pull(image);
+
+			const container = await client.container.create({
+				Image: image.string,
+				Labels: { 'io.eventsourcingdb.client-javascript-test': randomUUID() },
+				ExposedPorts: { '3000/tcp': {} },
+				HostConfig: { PortBindings: { '3000/tcp': [{ HostPort: '0' }] } },
+			});
+
+			try {
+				await assert.rejects(
+					inspectContainerUntilPortsExposed(
+						async () => await client.container.inspect(container),
+						container.id,
+						1,
+					),
+					(error: unknown): boolean => isPortError(error),
+				);
+			} finally {
+				await client.container.remove(container, { removeVolumes: true });
+			}
+		},
+	);
+
+	test('does not recognize other errors.', (): void => {
+		assert.equal(isPortError(new Error('No such container: container')), false);
+		assert.equal(isPortError('No host port found for host IP'), false);
+	});
+});

src/container/isPortError.ts

@@ -0,0 +1,12 @@
+// testcontainers waits for the port of a container for 10 seconds at most, and
+// then looks up the port on the host once. If either fails, the container is
+// running, but it cannot be reached.
+const portErrors = [
+	/^Timed out after \d+ms while waiting for container ports to be bound to the host$/u,
+	/^No host port found for host IP$/u,
+];
+
+const isPortError = (error: unknown): boolean =>
+	error instanceof Error && portErrors.some(portError => portError.test(error.message));
+
+export { isPortError };