From 070017c8f89b52a31abe37b943cf17afcf5b138c Mon Sep 17 00:00:00 2001 From: Marty Pitt Date: Wed, 23 Sep 2026 00:20:22 +0100 Subject: [PATCH 1/2] feat(apigateway): give each REST API a stable id across stack restarts importRestApi minted a fresh id on every start, so anything that had recorded an id (an Orbital registry selection, an env variable pasted into a config file) broke each time the stack restarted. Create the API first with LocalStack's _custom_id_ tag, defaulting the id to the declared name in lower case, then import the spec into it with putRestApi. An explicit apiId can be given. Also documents that response schemas must be named models: API Gateway drops inline schemas on import, and the export then has operations with no response schema. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011c7t8dqC37eQzTt9o44x9v --- NEBULA-AGENT.md | 7 ++++--- docs/pages/providers/apigateway.mdx | 10 ++++++++-- .../nebula/apigateway/ApiGatewayDsl.kt | 18 +++++++++++++----- .../nebula/apigateway/ApiGatewayExecutor.kt | 19 +++++++++++++++++-- .../apigateway/ApiGatewayExecutorTest.kt | 15 +++++++++++++++ 5 files changed, 57 insertions(+), 12 deletions(-) diff --git a/NEBULA-AGENT.md b/NEBULA-AGENT.md index 6aa6318..9e3ae46 100644 --- a/NEBULA-AGENT.md +++ b/NEBULA-AGENT.md @@ -379,13 +379,14 @@ fun apiGateway(imageName: String = "localstack/localstack:3.0.2", componentName: String = "apiGateway", dsl: ApiGatewayBuilder.(KLogger) -> Unit) -fun restApi(name: String, openApi: String, stage: String = "prod") -fun restApiFromFile(name: String, path: String, stage: String = "prod") +fun restApi(name: String, openApi: String, stage: String = "prod", apiId: String = name.lowercase()) +fun restApiFromFile(name: String, path: String, stage: String = "prod", apiId: String = name.lowercase()) ``` Rules: - `name` must start with a letter and contain only letters, digits, dashes and underscores. It keys the emitted config: `petsApiId`, `petsStage`, `petsInvokeUrl` (dashes are camel-cased: `pet-store` -> `petStoreApiId`). -- Every operation in the spec MUST declare a response schema — Orbital's OpenAPI import needs one to build a return type. +- The API's id is stable across restarts: the name in lower case, or `apiId` if given (lower-case letters, digits, dashes, underscores). Do not generate random ids — an Orbital registry selection records the id. +- Every operation in the spec MUST declare a response schema, as a **named** schema under `components/schemas` referenced with `$ref` — API Gateway only exports named models, so an inline response schema (eg a bare `type: array`) is dropped on import and Orbital then rejects the export. - To make the deployed API answer requests, put `x-amazon-apigateway-integration` blocks in the spec, as on AWS. Without them the API exists for schema export only. - REST APIs only. The community LocalStack image has no HTTP API (API Gateway v2) support. diff --git a/docs/pages/providers/apigateway.mdx b/docs/pages/providers/apigateway.mdx index cfc21a4..20196cd 100644 --- a/docs/pages/providers/apigateway.mdx +++ b/docs/pages/providers/apigateway.mdx @@ -40,7 +40,13 @@ stack { ``` Every operation should declare a response schema, as Orbital's OpenAPI import needs one to -generate a return type. To make the deployed API answer requests, declare integrations in the +generate a return type. Declare it as a named schema under `components/schemas` and reference it: +API Gateway only exports named models, so an inline response schema (a bare array, say) is dropped +on import. + +Each API gets a stable id, the declared name in lower case, so the id is the same on every start of +the stack and anything that recorded it (an Orbital registry selection, a config file) keeps working +after a restart. Pass `apiId = "..."` to choose it. To make the deployed API answer requests, declare integrations in the spec with `x-amazon-apigateway-integration`, the same way you would on AWS. Only REST APIs are supported: the community LocalStack image does not include HTTP APIs (API Gateway v2). @@ -53,7 +59,7 @@ Only REST APIs are supported: the community LocalStack image does not include HT | `secretKey` | The secret key of the running AWS stack | | `region` | The region of the running AWS stack | | `endpointOverride` | The endpoint override of the running AWS stack | -| `ApiId` | The id of the REST API declared as ``, eg `petsApiId` | +| `ApiId` | The id of the REST API declared as ``, eg `petsApiId`. Stable across restarts: the name in lower case unless `apiId` was given | | `Stage` | The stage the API was deployed to | | `InvokeUrl` | The base URL of the deployed stage, eg `http://localhost:4566/restapis/abc123/v1/_user_request_` | diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayDsl.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayDsl.kt index c965e3f..82e8928 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayDsl.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayDsl.kt @@ -40,9 +40,14 @@ class ApiGatewayBuilder( * * [name] is how the API is referred to in the emitted config (eg `petsApiId`), so keep it * to letters and digits. + * + * [apiId] is the id the API gets on the gateway. It defaults to [name] in lower case, so the id + * is the same on every start of the stack: anything that recorded it (an Orbital registry + * selection, an env variable in a config file) keeps working after a restart. LocalStack + * honours the requested id; a real gateway would assign its own. */ - fun restApi(name: String, openApi: String, stage: String = "prod") { - apis.add(RestApiConfig(name, stage, openApi)) + fun restApi(name: String, openApi: String, stage: String = "prod", apiId: String = name.lowercase()) { + apis.add(RestApiConfig(name, stage, openApi, apiId)) } /** @@ -51,8 +56,8 @@ class ApiGatewayBuilder( * A relative path resolves through the stack's bundle (the files shipped alongside the * script). An absolute path is read from the machine running Nebula. */ - fun restApiFromFile(name: String, path: String, stage: String = "prod") { - restApi(name, Files.readString(Path.of(stackResources.resolveFilePath(path))), stage) + fun restApiFromFile(name: String, path: String, stage: String = "prod", apiId: String = name.lowercase()) { + restApi(name, Files.readString(Path.of(stackResources.resolveFilePath(path))), stage, apiId) } fun build(): ApiGatewayConfig = ApiGatewayConfig(imageName, apis, componentName) @@ -60,10 +65,13 @@ class ApiGatewayBuilder( data class ApiGatewayConfig(val imageName: String, val apis: List, val componentName: String) -data class RestApiConfig(val name: String, val stage: String, val openApi: String) { +data class RestApiConfig(val name: String, val stage: String, val openApi: String, val apiId: String) { init { require(name.matches(Regex("[A-Za-z][A-Za-z0-9_-]*"))) { "REST API name '$name' must start with a letter and contain only letters, digits, dashes and underscores" } + require(apiId.matches(Regex("[a-z0-9][a-z0-9_-]*"))) { + "REST API id '$apiId' must be lower-case letters, digits, dashes and underscores" + } } } diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt index 262a38d..6ad6074 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt @@ -11,6 +11,7 @@ import com.orbitalhq.nebula.endpointFor import com.orbitalhq.nebula.events.ComponentLifecycleEventSource import com.orbitalhq.nebula.logging.LogStream import com.orbitalhq.nebula.logging.LoggerName +import mu.KotlinLogging import org.testcontainers.localstack.LocalStackContainer import org.testcontainers.utility.DockerImageName import reactor.core.publisher.Flux @@ -18,6 +19,9 @@ import software.amazon.awssdk.auth.credentials.AwsBasicCredentials import software.amazon.awssdk.core.SdkBytes import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.apigateway.ApiGatewayClient +import software.amazon.awssdk.services.apigateway.model.PutMode + +private val logger = KotlinLogging.logger {} val StackRunner.apiGateway: List get() { @@ -125,9 +129,17 @@ class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List Date: Fri, 25 Sep 2026 10:17:52 +0100 Subject: [PATCH 2/2] feat(tools): let components launch admin tooling, starting with StackPort for LocalStack Components can now offer tools - typically a browser / admin console for the thing they run - that are launched on demand from the management UI. - ProvidesTools: a component describes each tool as a container spec (image, env, port, readiness path). It fills in the coordinates and credentials, since it's the one that knows them. - ToolRunner (owned by StackRunner) launches tools on the Nebula network, maps the port, and stops them when their component stops or the stack shuts down. Tool logs appear in the stack's logs. - S3 and API Gateway offer StackPort (davireis/stackport:0.4.4), pointed at their LocalStack container. - Admin API: POST /api/stacks/{id}/components/{componentId}/tools/{toolId}/launch and /stop. The admin snapshot includes each running component's tools. - UI: a Tools menu on each component, plus a Tools section in the expanded panel. The URL is built from the host the UI was loaded from. Tools reach their component by its container IP (uniqueNetworkHost), not the network alias: stacks share a network and can reuse component names, so an alias like `s3` can resolve to another stack's container. Co-Authored-By: Claude Opus 5.5 --- .../nebula/InfrastructureComponent.kt | 19 ++ .../com/orbitalhq/nebula/StackRunner.kt | 22 ++ .../nebula/apigateway/ApiGatewayExecutor.kt | 9 +- .../com/orbitalhq/nebula/s3/S3Executor.kt | 9 +- .../orbitalhq/nebula/tools/ComponentTools.kt | 45 ++++ .../com/orbitalhq/nebula/tools/StackPort.kt | 33 +++ .../com/orbitalhq/nebula/tools/ToolRunner.kt | 182 ++++++++++++++++ .../nebula/tools/ComponentToolsTest.kt | 102 +++++++++ .../nebula/runtime/server/AdminModels.kt | 5 +- .../nebula/runtime/server/NebulaServer.kt | 40 +++- nebula-ui/src/lib/api.ts | 54 ++++- nebula-ui/src/lib/types/stack.ts | 22 ++ nebula-ui/src/pages/StackDetail.tsx | 1 + .../src/pages/stack-detail/ComponentsView.tsx | 196 +++++++++++++++++- nebula-ui/src/pages/stack-detail/LogsView.tsx | 13 +- 15 files changed, 740 insertions(+), 12 deletions(-) create mode 100644 nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ComponentTools.kt create mode 100644 nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/StackPort.kt create mode 100644 nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ToolRunner.kt create mode 100644 nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/tools/ComponentToolsTest.kt diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfrastructureComponent.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfrastructureComponent.kt index d19e9bd..fb8a61c 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfrastructureComponent.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfrastructureComponent.kt @@ -71,3 +71,22 @@ fun NebulaConfig.endpointFor( ConsumerConnectivity.HOST -> Endpoint(container.host, container.getMappedPort(internalPort)) ConsumerConnectivity.NETWORK -> Endpoint(alias, internalPort) } + +/** + * An address that reaches exactly this container from another container on the Nebula network. + * + * Prefer this over the network alias (conventionally the componentName) when a container + * must reach one specific component: stacks share a network and can reuse component names, + * so an alias like `s3` may resolve to another stack's container. + * + * This is the container's IP rather than its name, as docker's generated names contain + * underscores, which aren't valid in a hostname (and AWS SDKs reject them in endpoint URLs). + */ +val GenericContainer<*>.uniqueNetworkHost: String + get() { + val network = this.network ?: error("Container $containerName is not attached to a Nebula network") + return containerInfo.networkSettings.networks.values + .firstOrNull { it.networkID == network.id } + ?.ipAddress + ?: error("Container $containerName has no address on network ${network.id}") + } diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt index 7c4bfff..289e787 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt @@ -5,6 +5,8 @@ import com.orbitalhq.nebula.core.ComponentState import com.orbitalhq.nebula.core.StackStateEvent import com.orbitalhq.nebula.events.computeStackStateEventFor import com.orbitalhq.nebula.logging.LogMessage +import com.orbitalhq.nebula.tools.ToolRunner +import com.orbitalhq.nebula.tools.ToolView import io.github.oshai.kotlinlogging.KotlinLogging import org.testcontainers.containers.Network import reactor.core.publisher.Flux @@ -57,6 +59,7 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) { private val logger = KotlinLogging.logger {} val stacks = ConcurrentHashMap() private val _stackState = ConcurrentHashMap>>() + private val toolRunner = ToolRunner(config.network) fun submit(submittedStack: NebulaStackWithSource, name: StackName = submittedStack.name, startAsync: Boolean = false): Flux { val storedStack = this.stacks.compute(name) { key, existingSpec -> @@ -152,6 +155,23 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) { ?: error("Component $componentId not found in stack $stackName") } + /** The tools offered by each running component in a stack, keyed by component id. */ + fun tools(stackName: StackName): Map> { + val stack = this.stacks[stackName] ?: return emptyMap() + return toolRunner.toolsFor(stackName, stack.stack.components) + } + + /** Launches a component's tool in the background. See [ToolRunner.launch]. */ + fun launchTool(stackName: StackName, componentId: String, toolId: String): ToolView { + return toolRunner.launch(stackName, findComponent(stackName, componentId), toolId) + } + + /** Stops a component's tool. Returns null if the tool wasn't launched. */ + fun stopTool(stackName: StackName, componentId: String, toolId: String): ToolView? { + findComponent(stackName, componentId) + return toolRunner.stop(stackName, componentId, toolId) + } + /** * Stops a single component within a stack, leaving the rest of the stack running. */ @@ -231,6 +251,8 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) { fun shutDown(name: String) { val spec = this.stacks[name] ?: error("Spec $name not found") logger.info { "Shutting down ${spec.name}" } + // Components stopping would take their tools down anyway - this just doesn't rely on it. + toolRunner.stopAll(name) spec.stack.components.forEach { try { logger.info { "Stopping ${it.name}" } diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt index 6ad6074..1006402 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt @@ -11,6 +11,9 @@ import com.orbitalhq.nebula.endpointFor import com.orbitalhq.nebula.events.ComponentLifecycleEventSource import com.orbitalhq.nebula.logging.LogStream import com.orbitalhq.nebula.logging.LoggerName +import com.orbitalhq.nebula.tools.ProvidesTools +import com.orbitalhq.nebula.tools.ToolDefinition +import com.orbitalhq.nebula.tools.stackPortTool import mu.KotlinLogging import org.testcontainers.localstack.LocalStackContainer import org.testcontainers.utility.DockerImageName @@ -40,7 +43,7 @@ data class DeployedRestApi(val name: String, val apiId: String, val stage: Strin * The emitted config is flat so Orbital can expose every value as an env variable, eg * `NEBULA_API_GATEWAY_ENDPOINT_OVERRIDE` and `NEBULA_API_GATEWAY_PETS_API_ID`. */ -class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List) : InfrastructureComponent> { +class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List) : InfrastructureComponent>, ProvidesTools { private lateinit var localstack: LocalStackContainer lateinit var client: ApiGatewayClient private set @@ -120,6 +123,10 @@ class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List = listOf( + stackPortTool(localstack, LOCALSTACK_INTERNAL_PORT, "Browse the deployed REST APIs") + ) + /** The stage's OpenAPI 3 export, as API Gateway serves it to Orbital. */ fun exportOpenApi(apiName: String): String { val api = deployedApis[apiName] ?: error("No REST API named '$apiName' is declared. Known: ${deployedApis.keys}") diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Executor.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Executor.kt index a79c0a0..baa07ef 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Executor.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Executor.kt @@ -11,6 +11,9 @@ import com.orbitalhq.nebula.endpointFor import com.orbitalhq.nebula.events.ComponentLifecycleEventSource import com.orbitalhq.nebula.logging.LogStream import com.orbitalhq.nebula.logging.LoggerName +import com.orbitalhq.nebula.tools.ProvidesTools +import com.orbitalhq.nebula.tools.ToolDefinition +import com.orbitalhq.nebula.tools.stackPortTool import org.testcontainers.localstack.LocalStackContainer import org.testcontainers.utility.DockerImageName import reactor.core.publisher.Flux @@ -27,7 +30,7 @@ val StackRunner.s3: List return this.component() } -class S3Executor(private val config: S3Config, loggers: List) : InfrastructureComponent { +class S3Executor(private val config: S3Config, loggers: List) : InfrastructureComponent, ProvidesTools { private lateinit var localstack: LocalStackContainer lateinit var s3Client: S3Client private set @@ -98,6 +101,10 @@ class S3Executor(private val config: S3Config, loggers: List) : Infr eventSource.stopContainerAndEmitEvents(localstack) } + override fun tools(): List = listOf( + stackPortTool(localstack, LOCALSTACK_INTERNAL_PORT, "Browse and edit buckets and objects") + ) + private fun createBucket(bucketConfig: BucketConfig) { s3Client.createBucket { it.bucket(bucketConfig.name) } } diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ComponentTools.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ComponentTools.kt new file mode 100644 index 0000000..d032a35 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ComponentTools.kt @@ -0,0 +1,45 @@ +package com.orbitalhq.nebula.tools + +/** + * Implemented by components that can launch extra tooling alongside themselves - + * typically a browser / admin console for poking around in the thing that's running + * (eg: StackPort for LocalStack, kafka-ui for Kafka). + * + * The component only *describes* its tools, since it's the one that knows the + * coordinates and credentials a tool needs. Launching, networking and cleanup are + * handled generically by [ToolRunner], so no tool gets special treatment. + * + * Tools are launched on demand (from the admin UI), never as part of starting a stack. + */ +interface ProvidesTools { + /** + * The tools this component offers. Only called while the component is running, + * so implementations may read state that's only available once started + * (eg: generated credentials). + */ + fun tools(): List +} + +data class ToolDefinition( + /** Unique within the component. Used in URLs, eg `stackport`. */ + val id: String, + val displayName: String, + val description: String, + val launch: ContainerToolSpec +) + +/** + * A tool that runs as a container on the Nebula network, alongside the component it serves. + * Reach the component via [com.orbitalhq.nebula.uniqueNetworkHost] and its *internal* port - the + * tool is always inside the docker network, regardless of [com.orbitalhq.nebula.ConsumerConnectivity]. + */ +data class ContainerToolSpec( + val image: String, + val env: Map = emptyMap(), + /** The port the tool serves its UI on, inside the container. Mapped to a random host port. */ + val httpPort: Int, + /** Where to open the tool's UI. */ + val path: String = "/", + /** An HTTP path that returns 2xx once the tool is ready. Defaults to [path]. */ + val readinessPath: String = path +) diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/StackPort.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/StackPort.kt new file mode 100644 index 0000000..8bd3930 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/StackPort.kt @@ -0,0 +1,33 @@ +package com.orbitalhq.nebula.tools + +import com.orbitalhq.nebula.uniqueNetworkHost +import org.testcontainers.localstack.LocalStackContainer + +const val STACKPORT_IMAGE = "davireis/stackport:0.4.4" + +/** + * StackPort (https://github.com/DaviReisVieira/stackport) - a browser for AWS resources, + * pointed at a LocalStack container over the Nebula network. + * + * @param description what the user would use it for on this component + */ +fun stackPortTool( + localstack: LocalStackContainer, + internalPort: Int, + description: String +) = ToolDefinition( + id = "stackport", + displayName = "StackPort", + description = description, + launch = ContainerToolSpec( + image = STACKPORT_IMAGE, + env = mapOf( + "AWS_ENDPOINT_URL" to "http://${localstack.uniqueNetworkHost}:$internalPort", + "AWS_ACCESS_KEY_ID" to localstack.accessKey, + "AWS_SECRET_ACCESS_KEY" to localstack.secretKey, + "AWS_REGION" to localstack.region, + ), + httpPort = 8080, + readinessPath = "/api/health" + ) +) diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ToolRunner.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ToolRunner.kt new file mode 100644 index 0000000..51b0735 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ToolRunner.kt @@ -0,0 +1,182 @@ +package com.orbitalhq.nebula.tools + +import com.orbitalhq.nebula.InfrastructureComponent +import com.orbitalhq.nebula.StackName +import com.orbitalhq.nebula.core.ComponentState +import io.github.oshai.kotlinlogging.KotlinLogging +import org.testcontainers.containers.GenericContainer +import org.testcontainers.containers.Network +import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.utility.DockerImageName +import reactor.core.Disposable +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap +import kotlin.concurrent.thread + +enum class ToolState { Available, Starting, Running, Failed } + +/** + * A tool offered by a component, and (if it has been launched) where to reach it. + * + * [hostPort] is the host-mapped port only - clients build the URL using whatever + * host they reached Nebula on, so this works when Nebula is on another machine. + */ +data class ToolView( + val id: String, + val displayName: String, + val description: String, + val state: ToolState, + val message: String? = null, + val hostPort: Int? = null, + val path: String +) + +/** Thrown when a tool can't be launched because its component isn't running. */ +class ToolNotAvailableException(message: String) : RuntimeException(message) + +/** + * Launches, tracks and stops the tools offered by [ProvidesTools] components. + * + * Tool instances live outside the stack's component list: they don't count toward + * the stack's state, and aren't restarted with it. A tool is stopped when its + * component stops (for whatever reason), or when its stack is shut down. + */ +class ToolRunner(private val network: Network) { + private val logger = KotlinLogging.logger {} + private val instances = ConcurrentHashMap() + + /** The tools offered by each running component in a stack, keyed by component id. */ + fun toolsFor(stackName: StackName, components: List>): Map> { + return components + .filter { it is ProvidesTools && it.currentState.state == ComponentState.Running } + .associate { component -> + component.id to (component as ProvidesTools).tools().map { definition -> + instances[ToolKey(stackName, component.id, definition.id)]?.view() + ?: definition.availableView() + } + } + .filterValues { it.isNotEmpty() } + } + + /** + * Launches a tool in the background, returning immediately with it Starting. + * Launching a tool that's already starting or running returns the existing instance. + */ + fun launch(stackName: StackName, component: InfrastructureComponent<*>, toolId: String): ToolView { + if (component !is ProvidesTools || component.currentState.state != ComponentState.Running) { + throw ToolNotAvailableException("Component ${component.id} must be running to launch its tools") + } + val definition = component.tools().find { it.id == toolId } + ?: error("Component ${component.id} has no tool $toolId") + val key = ToolKey(stackName, component.id, toolId) + val instance = instances.compute(key) { _, existing -> + if (existing != null && existing.state != ToolState.Failed) existing else ToolInstance(key, definition, component) + }!! + instance.startAsync() + return instance.view() + } + + fun stop(stackName: StackName, componentId: String, toolId: String): ToolView? { + val instance = instances.remove(ToolKey(stackName, componentId, toolId)) ?: return null + instance.stop() + return instance.definition.availableView() + } + + fun stopAll(stackName: StackName) { + instances.keys.filter { it.stackName == stackName }.forEach { stop(it.stackName, it.componentId, it.toolId) } + } + + private fun ToolDefinition.availableView() = ToolView(id, displayName, description, ToolState.Available, path = launch.path) + + private data class ToolKey(val stackName: StackName, val componentId: String, val toolId: String) + + private inner class ToolInstance( + val key: ToolKey, + val definition: ToolDefinition, + val component: InfrastructureComponent<*> + ) { + private val spec = definition.launch + private val container: GenericContainer<*> = GenericContainer(DockerImageName.parse(spec.image)) + .withNetwork(network) + .withExposedPorts(spec.httpPort) + .withEnv(spec.env) + .waitingFor( + Wait.forHttp(spec.readinessPath) + .forPort(spec.httpPort) + // Allows for pulling the image on first launch + .withStartupTimeout(Duration.ofMinutes(3)) + ) + + @Volatile + var state = ToolState.Starting + private set + + @Volatile + private var message: String? = null + + @Volatile + private var hostPort: Int? = null + private var started = false + + @Volatile + private var stopped = false + private var componentSubscription: Disposable? = null + + @Synchronized + fun startAsync() { + if (started) return + started = true + // The tool is only useful while its component is up - stop it when the component goes away. + componentSubscription = component.lifecycleEvents + .filter { it.state != ComponentState.Running } + .subscribe { + logger.info { "Component ${key.componentId} is ${it.state}; stopping tool ${key.toolId}" } + stop(key.stackName, key.componentId, key.toolId) + } + // Tool logs show up in the stack's logs, alongside the component's. + component.logStream.attachToUnstartedContainer(container, "${component.name}/${definition.id}") + thread(name = "tool-${key.componentId}-${key.toolId}") { + try { + logger.info { "Launching tool ${key.toolId} (${spec.image}) for ${key.componentId} in stack ${key.stackName}" } + container.start() + if (stopped) { + // Stopped while we were starting - don't leave the container behind. + stopContainer() + return@thread + } + hostPort = container.getMappedPort(spec.httpPort) + state = ToolState.Running + } catch (e: Exception) { + logger.error(e) { "Tool ${key.toolId} for ${key.componentId} failed to start" } + message = e.message ?: e::class.simpleName + state = ToolState.Failed + stop() + } + } + } + + fun stop() { + stopped = true + componentSubscription?.dispose() + stopContainer() + } + + private fun stopContainer() { + try { + container.stop() + } catch (e: Exception) { + logger.warn(e) { "Error stopping tool ${key.toolId} for ${key.componentId}" } + } + } + + fun view() = ToolView( + id = definition.id, + displayName = definition.displayName, + description = definition.description, + state = state, + message = message, + hostPort = hostPort, + path = spec.path + ) + } +} diff --git a/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/tools/ComponentToolsTest.kt b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/tools/ComponentToolsTest.kt new file mode 100644 index 0000000..d98e861 --- /dev/null +++ b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/tools/ComponentToolsTest.kt @@ -0,0 +1,102 @@ +package com.orbitalhq.nebula.tools + +import com.orbitalhq.nebula.HostConfig +import com.orbitalhq.nebula.NebulaStackWithSource +import com.orbitalhq.nebula.StackRunner +import com.orbitalhq.nebula.s3.s3 +import com.orbitalhq.nebula.stack +import com.orbitalhq.nebula.start +import io.kotest.assertions.nondeterministic.eventually +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.string.shouldNotContain +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import kotlin.time.Duration.Companion.minutes + +class ComponentToolsTest : DescribeSpec({ + lateinit var infra: StackRunner + val http = HttpClient.newHttpClient() + + fun get(url: String): String = http.send( + HttpRequest.newBuilder(URI.create(url)).GET().build(), + HttpResponse.BodyHandlers.ofString() + ).body() + + describe("component tools") { + afterTest { + infra.shutDownAll() + } + + it("launches StackPort against an s3 component, and stops it with the component") { + infra = stack { + s3 { + bucket("tools-test-bucket") { + file("hello.txt", "Hello, world") + } + } + }.start() + val stackName = infra.stacks.keys.single() + val s3 = infra.s3.single() + + val offered = infra.tools(stackName)[s3.id].shouldNotBeNull().single() + offered.id shouldBe "stackport" + offered.state shouldBe ToolState.Available + + infra.launchTool(stackName, s3.id, "stackport").state shouldBe ToolState.Starting + + val running = eventually(3.minutes) { + infra.tools(stackName)[s3.id]!!.single().also { it.state shouldBe ToolState.Running } + } + // StackPort reaches LocalStack over the Nebula network, using the component's credentials + get("http://localhost:${running.hostPort}/api/s3/buckets") shouldContain "tools-test-bucket" + + // Launching again is a no-op + infra.launchTool(stackName, s3.id, "stackport").hostPort shouldBe running.hostPort + + infra.stopComponent(stackName, s3.id) + // Tools are only offered while the component runs, and the tool went down with it + infra.tools(stackName)[s3.id] shouldBe null + infra.startComponent(stackName, s3.id) + infra.tools(stackName)[s3.id]!!.single().state shouldBe ToolState.Available + } + + it("targets its own component when another stack has a component with the same name") { + infra = StackRunner() + // Both stacks get an `s3` component - so the network alias `s3` is ambiguous + listOf("first" to "first-stack-bucket", "second" to "second-stack-bucket").forEach { (stackName, bucket) -> + val stack = stack { s3 { bucket(bucket) {} } } + infra.submit(NebulaStackWithSource(stack, "$stackName-source", HostConfig.UNKNOWN), stackName) + } + val s3Id = infra.s3.first().id + infra.launchTool("second", s3Id, "stackport") + val running = eventually(3.minutes) { + infra.tools("second")[s3Id]!!.single().also { it.state shouldBe ToolState.Running } + } + + val buckets = get("http://localhost:${running.hostPort}/api/s3/buckets") + buckets shouldContain "second-stack-bucket" + buckets shouldNotContain "first-stack-bucket" + } + + it("refuses to launch a tool for a component that isn't running") { + infra = stack { + s3 { bucket("idle-bucket") {} } + }.start() + val stackName = infra.stacks.keys.single() + val s3 = infra.s3.single() + infra.stopComponent(stackName, s3.id) + + infra.tools(stackName).keys.shouldBeEmpty() + shouldThrow { + infra.launchTool(stackName, s3.id, "stackport") + } + } + } +}) diff --git a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/AdminModels.kt b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/AdminModels.kt index c2a40ac..eef082a 100644 --- a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/AdminModels.kt +++ b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/AdminModels.kt @@ -3,6 +3,7 @@ package com.orbitalhq.nebula.runtime.server import com.orbitalhq.nebula.StackName import com.orbitalhq.nebula.core.CompilationError import com.orbitalhq.nebula.core.StackStateEvent +import com.orbitalhq.nebula.tools.ToolView import kotlin.script.experimental.api.ScriptDiagnostic /** @@ -25,7 +26,9 @@ data class AdminStackView( val name: StackName, val stackState: StackStateEvent?, val source: String, - val compilationErrors: List + val compilationErrors: List, + /** Tools offered by each running component, keyed by component id. */ + val tools: Map> = emptyMap() ) { val failedCompilation: Boolean get() = compilationErrors.isNotEmpty() } diff --git a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt index ac3f2be..f4801fc 100644 --- a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt +++ b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt @@ -13,6 +13,7 @@ import com.orbitalhq.nebula.StackRunner import com.orbitalhq.nebula.core.StackBundle import com.orbitalhq.nebula.core.StackStateEvent import com.orbitalhq.nebula.runtime.NebulaScriptExecutor +import com.orbitalhq.nebula.tools.ToolNotAvailableException import io.github.oshai.kotlinlogging.KotlinLogging import io.ktor.http.* import io.ktor.serialization.jackson.* @@ -62,7 +63,8 @@ class NebulaServer( name = event.stackName, stackState = event, source = stackExecutor.sourceFor(event.stackName) ?: "", - compilationErrors = emptyList() + compilationErrors = emptyList(), + tools = stackExecutor.tools(event.stackName) ) } val compiledNames = compiled.map { it.name }.toSet() @@ -214,6 +216,18 @@ class NebulaServer( stackExecutor.startComponent(id, componentId) } } + // Launch one of a component's tools (eg: an admin console). Returns + // straight away with the tool Starting - poll the snapshot for when it's up. + post("/{id}/components/{componentId}/tools/{toolId}/launch") { + handleToolAction(call) { id, componentId, toolId -> + stackExecutor.launchTool(id, componentId, toolId) + } + } + post("/{id}/components/{componentId}/tools/{toolId}/stop") { + handleToolAction(call) { id, componentId, toolId -> + stackExecutor.stopTool(id, componentId, toolId) + } + } } // Live log stream for a stack. webSocket("/api/stacks/{id}/logs") { @@ -371,6 +385,30 @@ class NebulaServer( call.respond(buildAdminSnapshot()) } + private suspend fun handleToolAction( + call: ApplicationCall, + action: (id: String, componentId: String, toolId: String) -> Any? + ) { + val id = call.parameters["id"] + ?: return call.respond(HttpStatusCode.BadRequest, "Missing or malformed id") + val componentId = call.parameters["componentId"] + ?: return call.respond(HttpStatusCode.BadRequest, "Missing or malformed componentId") + val toolId = call.parameters["toolId"] + ?: return call.respond(HttpStatusCode.BadRequest, "Missing or malformed toolId") + val result = try { + action(id, componentId, toolId) + } catch (e: ToolNotAvailableException) { + return call.respond(HttpStatusCode.Conflict, e.message ?: "Tool not available") + } catch (e: IllegalStateException) { + return call.respond(HttpStatusCode.NotFound, e.message ?: "Not found") + } + if (result == null) { + call.respond(HttpStatusCode.NoContent) + } else { + call.respond(result) + } + } + private suspend fun handleComponentAction( call: ApplicationCall, action: (id: String, componentId: String) -> Unit diff --git a/nebula-ui/src/lib/api.ts b/nebula-ui/src/lib/api.ts index 0144b39..97c67b2 100644 --- a/nebula-ui/src/lib/api.ts +++ b/nebula-ui/src/lib/api.ts @@ -1,5 +1,5 @@ import { handleApiError } from './api-error'; -import type { AdminStackView, CompilationError, StackStateEvent } from './types/stack'; +import type { AdminStackView, CompilationError, StackStateEvent, ToolView } from './types/stack'; /** * Client for the Nebula server's admin API. All requests are same-origin: @@ -110,6 +110,58 @@ async function componentAction( return response.json(); } +/** + * Launch one of a component's tools. Returns straight away with the tool + * `Starting` — it shows as `Running` in the snapshot once it's up. + */ +export async function launchTool( + stackName: string, + componentId: string, + toolId: string, +): Promise { + const response = await fetch(toolPath(stackName, componentId, toolId, 'launch'), { + method: 'POST', + }); + if (!response.ok) { + await handleApiError(response, 'Failed to launch tool'); + } + return response.json(); +} + +/** Stop a component's tool. */ +export async function stopTool( + stackName: string, + componentId: string, + toolId: string, +): Promise { + const response = await fetch(toolPath(stackName, componentId, toolId, 'stop'), { + method: 'POST', + }); + if (!response.ok) { + await handleApiError(response, 'Failed to stop tool'); + } +} + +function toolPath( + stackName: string, + componentId: string, + toolId: string, + action: 'launch' | 'stop', +): string { + return `/api/stacks/${encodeURIComponent(stackName)}/components/${encodeURIComponent( + componentId, + )}/tools/${encodeURIComponent(toolId)}/${action}`; +} + +/** + * Where to open a running tool. Uses the host this UI was loaded from, so it + * works when Nebula is running on another machine. + */ +export function toolUrl(tool: ToolView): string | null { + if (tool.hostPort == null) return null; + return `${window.location.protocol}//${window.location.hostname}:${tool.hostPort}${tool.path}`; +} + /** Path for the per-stack log websocket. */ export function logsWebSocketPath(stackName: string): string { return `/api/stacks/${encodeURIComponent(stackName)}/logs`; diff --git a/nebula-ui/src/lib/types/stack.ts b/nebula-ui/src/lib/types/stack.ts index 7a47794..97aaa77 100644 --- a/nebula-ui/src/lib/types/stack.ts +++ b/nebula-ui/src/lib/types/stack.ts @@ -63,6 +63,23 @@ export interface CompilationError { severity: string; } +export type ToolState = 'Available' | 'Starting' | 'Running' | 'Failed'; + +/** + * A tool a component can launch (eg: an admin console) — see + * com.orbitalhq.nebula.tools.ToolView. `hostPort` is set once running; build the + * URL from the host the UI was loaded from. + */ +export interface ToolView { + id: string; + displayName: string; + description: string; + state: ToolState; + message: string | null; + hostPort: number | null; + path: string; +} + /** * Unified admin view of a submitted stack — returned by GET /api/stacks. * Either a compiled stack (with `stackState`) or a failed submission @@ -73,6 +90,8 @@ export interface AdminStackView { stackState: StackStateEvent | null; source: string; compilationErrors: CompilationError[]; + /** Tools offered by each running component, keyed by component id. */ + tools: Record; } /** A single log line — see com.orbitalhq.nebula.logging.LogMessage. */ @@ -100,6 +119,8 @@ export interface StackSummary { compilationErrors: CompilationError[]; /** The script the stack was submitted with. */ source: string; + /** Tools offered by each running component, keyed by component id. */ + tools: Record; } /** @@ -130,5 +151,6 @@ export function toStackSummary(view: AdminStackView): StackSummary { failed, compilationErrors: view.compilationErrors, source: view.source, + tools: view.tools ?? {}, }; } diff --git a/nebula-ui/src/pages/StackDetail.tsx b/nebula-ui/src/pages/StackDetail.tsx index 61a2ca1..3d651df 100644 --- a/nebula-ui/src/pages/StackDetail.tsx +++ b/nebula-ui/src/pages/StackDetail.tsx @@ -158,6 +158,7 @@ export default function StackDetail() { diff --git a/nebula-ui/src/pages/stack-detail/ComponentsView.tsx b/nebula-ui/src/pages/stack-detail/ComponentsView.tsx index 4882486..0315b5e 100644 --- a/nebula-ui/src/pages/stack-detail/ComponentsView.tsx +++ b/nebula-ui/src/pages/stack-detail/ComponentsView.tsx @@ -1,17 +1,194 @@ import { useState, type MouseEvent } from 'react'; -import { ChevronDown, ChevronRight, Copy, Check, Play, Square, Loader2 } from 'lucide-react'; +import { + ChevronDown, + ChevronRight, + Copy, + Check, + Play, + Square, + Loader2, + Wrench, + ExternalLink, + AlertCircle, +} from 'lucide-react'; import { Button } from '@/components/ui/button'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; import { showSuccessToast } from '@/lib/toast'; import { getComponentIcon, StatusDot } from '@/lib/display'; -import { startComponent, stopComponent } from '@/lib/api'; -import { TRANSITION_STATES, type ComponentInfoWithState } from '@/lib/types/stack'; +import { launchTool, startComponent, stopComponent, stopTool, toolUrl } from '@/lib/api'; +import { TRANSITION_STATES, type ComponentInfoWithState, type ToolView } from '@/lib/types/stack'; interface ComponentsViewProps { stackName: string; components: ComponentInfoWithState[]; + /** Tools offered by each running component, keyed by component id. */ + tools: Record; onChanged: () => void; } +/** Launch / stop actions for a component's tools, shared by the menu and the panel. */ +function useToolActions(stackName: string, componentId: string, onChanged: () => void) { + const [pending, setPending] = useState(null); + + const launch = async (tool: ToolView) => { + setPending(tool.id); + try { + await launchTool(stackName, componentId, tool.id); + showSuccessToast(`Launching ${tool.displayName} — it'll be ready to open shortly`); + onChanged(); + } catch { + /* toast already shown */ + } finally { + setPending(null); + } + }; + + const stop = async (tool: ToolView) => { + setPending(tool.id); + try { + await stopTool(stackName, componentId, tool.id); + showSuccessToast(`Stopped ${tool.displayName}`); + onChanged(); + } catch { + /* toast already shown */ + } finally { + setPending(null); + } + }; + + return { pending, launch, stop }; +} + +function ToolsMenu({ + tools, + actions, +}: { + tools: ToolView[]; + actions: ReturnType; +}) { + const anyStarting = tools.some((t) => t.state === 'Starting'); + return ( + + + + + + {tools.map((tool, index) => { + const url = toolUrl(tool); + const busy = actions.pending === tool.id; + return ( +
+ {index > 0 && } + + {tool.displayName} + {tool.description} + + {tool.state === 'Running' && url ? ( + <> + window.open(url, '_blank', 'noopener')}> + Open + + actions.stop(tool)}> + Stop + + + ) : tool.state === 'Starting' ? ( + + Starting… + + ) : ( + actions.launch(tool)}> + {tool.state === 'Failed' ? 'Retry launch' : 'Launch'} + + )} +
+ ); + })} + + + ); +} + +function ToolsPanel({ + tools, + actions, +}: { + tools: ToolView[]; + actions: ReturnType; +}) { + return ( +
+
Tools
+
+ {tools.map((tool) => { + const url = toolUrl(tool); + const busy = actions.pending === tool.id; + return ( +
+
+ {tool.displayName} + {tool.description} + {tool.state === 'Running' && url && ( + + {url} + + )} + {tool.state === 'Failed' && ( + + {tool.message ?? 'Failed to start'} + + )} +
+
+ {tool.state === 'Running' && url ? ( + <> + + + + ) : tool.state === 'Starting' ? ( + + ) : ( + + )} +
+
+ ); + })} +
+
+ ); +} + function CopyableValue({ label, value }: { label: string; value: string | number | boolean }) { const [copied, setCopied] = useState(false); @@ -47,14 +224,17 @@ function CopyableValue({ label, value }: { label: string; value: string | number function ComponentRow({ stackName, component, + tools, onChanged, }: { stackName: string; component: ComponentInfoWithState; + tools: ToolView[]; onChanged: () => void; }) { const [expanded, setExpanded] = useState(false); const [busy, setBusy] = useState(false); + const toolActions = useToolActions(stackName, component.id, onChanged); const state = component.state.state; const isRunning = state === 'Running'; @@ -113,6 +293,7 @@ function ComponentRow({ {state} + {tools.length > 0 && }