feat(tools): let components launch admin tooling, starting with StackPort for LocalStack

NEBULA-AGENT.md

@@ -379,13 +379,14 @@ fun apiGateway(imageName: String = "localstack/localstack:3.0.2",
                componentName: String = "apiGateway",
                dsl: ApiGatewayBuilder.(KLogger) -> Unit)
 
-fun restApi(name: String, openApi: String, stage: String = "prod")
-fun restApiFromFile(name: String, path: String, stage: String = "prod")
+fun restApi(name: String, openApi: String, stage: String = "prod", apiId: String = name.lowercase())
+fun restApiFromFile(name: String, path: String, stage: String = "prod", apiId: String = name.lowercase())
 ```
 
 Rules:
 - `name` must start with a letter and contain only letters, digits, dashes and underscores. It keys the emitted config: `petsApiId`, `petsStage`, `petsInvokeUrl` (dashes are camel-cased: `pet-store` -> `petStoreApiId`).
-- Every operation in the spec MUST declare a response schema — Orbital's OpenAPI import needs one to build a return type.
+- The API's id is stable across restarts: the name in lower case, or `apiId` if given (lower-case letters, digits, dashes, underscores). Do not generate random ids — an Orbital registry selection records the id.
+- Every operation in the spec MUST declare a response schema, as a **named** schema under `components/schemas` referenced with `$ref` — API Gateway only exports named models, so an inline response schema (eg a bare `type: array`) is dropped on import and Orbital then rejects the export.
 - To make the deployed API answer requests, put `x-amazon-apigateway-integration` blocks in the spec, as on AWS. Without them the API exists for schema export only.
 - REST APIs only. The community LocalStack image has no HTTP API (API Gateway v2) support.
 

docs/pages/providers/apigateway.mdx

@@ -40,7 +40,13 @@ stack {
 ```
 
 Every operation should declare a response schema, as Orbital's OpenAPI import needs one to
-generate a return type. To make the deployed API answer requests, declare integrations in the
+generate a return type. Declare it as a named schema under `components/schemas` and reference it:
+API Gateway only exports named models, so an inline response schema (a bare array, say) is dropped
+on import.
+
+Each API gets a stable id, the declared name in lower case, so the id is the same on every start of
+the stack and anything that recorded it (an Orbital registry selection, a config file) keeps working
+after a restart. Pass `apiId = "..."` to choose it. To make the deployed API answer requests, declare integrations in the
 spec with `x-amazon-apigateway-integration`, the same way you would on AWS.
 
 Only REST APIs are supported: the community LocalStack image does not include HTTP APIs (API Gateway v2).
@@ -53,7 +59,7 @@ Only REST APIs are supported: the community LocalStack image does not include HT
 | `secretKey`          | The secret key of the running AWS stack                                              |
 | `region`             | The region of the running AWS stack                                                  |
 | `endpointOverride`   | The endpoint override of the running AWS stack                                       |
-| `<name>ApiId`        | The id of the REST API declared as `<name>`, eg `petsApiId`                          |
+| `<name>ApiId`        | The id of the REST API declared as `<name>`, eg `petsApiId`. Stable across restarts: the name in lower case unless `apiId` was given |
 | `<name>Stage`        | The stage the API was deployed to                                                    |
 | `<name>InvokeUrl`    | The base URL of the deployed stage, eg `http://localhost:4566/restapis/abc123/v1/_user_request_` |
 

nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfrastructureComponent.kt

@@ -71,3 +71,22 @@ fun NebulaConfig.endpointFor(
     ConsumerConnectivity.HOST -> Endpoint(container.host, container.getMappedPort(internalPort))
     ConsumerConnectivity.NETWORK -> Endpoint(alias, internalPort)
 }
+
+/**
+ * An address that reaches exactly this container from another container on the Nebula network.
+ *
+ * Prefer this over the network alias (conventionally the componentName) when a container
+ * must reach one specific component: stacks share a network and can reuse component names,
+ * so an alias like `s3` may resolve to another stack's container.
+ *
+ * This is the container's IP rather than its name, as docker's generated names contain
+ * underscores, which aren't valid in a hostname (and AWS SDKs reject them in endpoint URLs).
+ */
+val GenericContainer<*>.uniqueNetworkHost: String
+    get() {
+        val network = this.network ?: error("Container $containerName is not attached to a Nebula network")
+        return containerInfo.networkSettings.networks.values
+            .firstOrNull { it.networkID == network.id }
+            ?.ipAddress
+            ?: error("Container $containerName has no address on network ${network.id}")
+    }

nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt

@@ -5,6 +5,8 @@ import com.orbitalhq.nebula.core.ComponentState
 import com.orbitalhq.nebula.core.StackStateEvent
 import com.orbitalhq.nebula.events.computeStackStateEventFor
 import com.orbitalhq.nebula.logging.LogMessage
+import com.orbitalhq.nebula.tools.ToolRunner
+import com.orbitalhq.nebula.tools.ToolView
 import io.github.oshai.kotlinlogging.KotlinLogging
 import org.testcontainers.containers.Network
 import reactor.core.publisher.Flux
@@ -57,6 +59,7 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) {
     private val logger = KotlinLogging.logger {}
     val stacks = ConcurrentHashMap<StackName, NebulaStackWithSource>()
     private val _stackState = ConcurrentHashMap<StackName, Map<String, ComponentInfo<*>>>()
+    private val toolRunner = ToolRunner(config.network)
 
     fun submit(submittedStack: NebulaStackWithSource, name: StackName = submittedStack.name, startAsync: Boolean = false): Flux<StackStateEvent> {
         val storedStack = this.stacks.compute(name) { key, existingSpec ->
@@ -152,6 +155,23 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) {
             ?: error("Component $componentId not found in stack $stackName")
     }
 
+    /** The tools offered by each running component in a stack, keyed by component id. */
+    fun tools(stackName: StackName): Map<String, List<ToolView>> {
+        val stack = this.stacks[stackName] ?: return emptyMap()
+        return toolRunner.toolsFor(stackName, stack.stack.components)
+    }
+
+    /** Launches a component's tool in the background. See [ToolRunner.launch]. */
+    fun launchTool(stackName: StackName, componentId: String, toolId: String): ToolView {
+        return toolRunner.launch(stackName, findComponent(stackName, componentId), toolId)
+    }
+
+    /** Stops a component's tool. Returns null if the tool wasn't launched. */
+    fun stopTool(stackName: StackName, componentId: String, toolId: String): ToolView? {
+        findComponent(stackName, componentId)
+        return toolRunner.stop(stackName, componentId, toolId)
+    }
+
     /**
      * Stops a single component within a stack, leaving the rest of the stack running.
      */
@@ -231,6 +251,8 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) {
     fun shutDown(name: String) {
         val spec = this.stacks[name] ?: error("Spec $name not found")
         logger.info { "Shutting down ${spec.name}" }
+        // Components stopping would take their tools down anyway - this just doesn't rely on it.
+        toolRunner.stopAll(name)
         spec.stack.components.forEach {
             try {
                 logger.info { "Stopping ${it.name}" }

nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayDsl.kt

@@ -40,9 +40,14 @@ class ApiGatewayBuilder(
      *
      * [name] is how the API is referred to in the emitted config (eg `petsApiId`), so keep it
      * to letters and digits.
+     *
+     * [apiId] is the id the API gets on the gateway. It defaults to [name] in lower case, so the id
+     * is the same on every start of the stack: anything that recorded it (an Orbital registry
+     * selection, an env variable in a config file) keeps working after a restart. LocalStack
+     * honours the requested id; a real gateway would assign its own.
      */
-    fun restApi(name: String, openApi: String, stage: String = "prod") {
-        apis.add(RestApiConfig(name, stage, openApi))
+    fun restApi(name: String, openApi: String, stage: String = "prod", apiId: String = name.lowercase()) {
+        apis.add(RestApiConfig(name, stage, openApi, apiId))
     }
 
     /**
@@ -51,19 +56,22 @@ class ApiGatewayBuilder(
      * A relative path resolves through the stack's bundle (the files shipped alongside the
      * script). An absolute path is read from the machine running Nebula.
      */
-    fun restApiFromFile(name: String, path: String, stage: String = "prod") {
-        restApi(name, Files.readString(Path.of(stackResources.resolveFilePath(path))), stage)
+    fun restApiFromFile(name: String, path: String, stage: String = "prod", apiId: String = name.lowercase()) {
+        restApi(name, Files.readString(Path.of(stackResources.resolveFilePath(path))), stage, apiId)
     }
 
     fun build(): ApiGatewayConfig = ApiGatewayConfig(imageName, apis, componentName)
 }
 
 data class ApiGatewayConfig(val imageName: String, val apis: List<RestApiConfig>, val componentName: String)
 
-data class RestApiConfig(val name: String, val stage: String, val openApi: String) {
+data class RestApiConfig(val name: String, val stage: String, val openApi: String, val apiId: String) {
     init {
         require(name.matches(Regex("[A-Za-z][A-Za-z0-9_-]*"))) {
             "REST API name '$name' must start with a letter and contain only letters, digits, dashes and underscores"
         }
+        require(apiId.matches(Regex("[a-z0-9][a-z0-9_-]*"))) {
+            "REST API id '$apiId' must be lower-case letters, digits, dashes and underscores"
+        }
     }
 }

nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/apigateway/ApiGatewayExecutor.kt

@@ -11,13 +11,20 @@ import com.orbitalhq.nebula.endpointFor
 import com.orbitalhq.nebula.events.ComponentLifecycleEventSource
 import com.orbitalhq.nebula.logging.LogStream
 import com.orbitalhq.nebula.logging.LoggerName
+import com.orbitalhq.nebula.tools.ProvidesTools
+import com.orbitalhq.nebula.tools.ToolDefinition
+import com.orbitalhq.nebula.tools.stackPortTool
+import mu.KotlinLogging
 import org.testcontainers.localstack.LocalStackContainer
 import org.testcontainers.utility.DockerImageName
 import reactor.core.publisher.Flux
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
 import software.amazon.awssdk.core.SdkBytes
 import software.amazon.awssdk.regions.Region
 import software.amazon.awssdk.services.apigateway.ApiGatewayClient
+import software.amazon.awssdk.services.apigateway.model.PutMode
+
+private val logger = KotlinLogging.logger {}
 
 val StackRunner.apiGateway: List<ApiGatewayExecutor>
     get() {
@@ -36,7 +43,7 @@ data class DeployedRestApi(val name: String, val apiId: String, val stage: Strin
  * The emitted config is flat so Orbital can expose every value as an env variable, eg
  * `NEBULA_API_GATEWAY_ENDPOINT_OVERRIDE` and `NEBULA_API_GATEWAY_PETS_API_ID`.
  */
-class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List<LoggerName>) : InfrastructureComponent<Map<String, String>> {
+class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List<LoggerName>) : InfrastructureComponent<Map<String, String>>, ProvidesTools {
     private lateinit var localstack: LocalStackContainer
     lateinit var client: ApiGatewayClient
         private set
@@ -116,6 +123,10 @@ class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List<Log
         eventSource.stopContainerAndEmitEvents(localstack)
     }
 
+    override fun tools(): List<ToolDefinition> = listOf(
+        stackPortTool(localstack, LOCALSTACK_INTERNAL_PORT, "Browse the deployed REST APIs")
+    )
+
     /** The stage's OpenAPI 3 export, as API Gateway serves it to Orbital. */
     fun exportOpenApi(apiName: String): String {
         val api = deployedApis[apiName] ?: error("No REST API named '$apiName' is declared. Known: ${deployedApis.keys}")
@@ -125,9 +136,17 @@ class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List<Log
     }
 
     private fun importAndDeploy(api: RestApiConfig, endpointOverride: String): DeployedRestApi {
-        val apiId = client.importRestApi {
-            it.body(SdkBytes.fromUtf8String(api.openApi)).failOnWarnings(false)
+        // Create the API with the id we want (LocalStack takes it from the _custom_id_ tag), then
+        // import the spec into it. importRestApi alone would mint a fresh id on every start.
+        val apiId = client.createRestApi {
+            it.name(api.name).tags(mapOf(CUSTOM_ID_TAG to api.apiId))
         }.id()
+        if (apiId != api.apiId) {
+            logger.warn { "Requested id '${api.apiId}' for REST API ${api.name} but the gateway assigned '$apiId'; it will change on the next start" }
+        }
+        client.putRestApi {
+            it.restApiId(apiId).mode(PutMode.OVERWRITE).failOnWarnings(false).body(SdkBytes.fromUtf8String(api.openApi))
+        }
         val deploymentId = client.createDeployment { it.restApiId(apiId).stageName(api.stage) }.id()
         // LocalStack's stage URL. A real gateway uses https://{id}.execute-api.{region}.amazonaws.com/{stage}.
         val invokeUrl = "${endpointOverride.trimEnd('/')}/restapis/$apiId/${api.stage}/_user_request_"
@@ -143,3 +162,6 @@ class ApiGatewayExecutor(private val config: ApiGatewayConfig, loggers: List<Log
 
 // The (edge) port LocalStack listens on inside the container.
 private const val LOCALSTACK_INTERNAL_PORT = 4566
+
+// LocalStack assigns a resource the id given in this tag instead of generating one.
+private const val CUSTOM_ID_TAG = "_custom_id_"

nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Executor.kt

@@ -11,6 +11,9 @@ import com.orbitalhq.nebula.endpointFor
 import com.orbitalhq.nebula.events.ComponentLifecycleEventSource
 import com.orbitalhq.nebula.logging.LogStream
 import com.orbitalhq.nebula.logging.LoggerName
+import com.orbitalhq.nebula.tools.ProvidesTools
+import com.orbitalhq.nebula.tools.ToolDefinition
+import com.orbitalhq.nebula.tools.stackPortTool
 import org.testcontainers.localstack.LocalStackContainer
 import org.testcontainers.utility.DockerImageName
 import reactor.core.publisher.Flux
@@ -27,7 +30,7 @@ val StackRunner.s3: List<S3Executor>
         return this.component<S3Executor>()
     }
 
-class S3Executor(private val config: S3Config, loggers: List<LoggerName>) : InfrastructureComponent<LocalstackContainerConfig> {
+class S3Executor(private val config: S3Config, loggers: List<LoggerName>) : InfrastructureComponent<LocalstackContainerConfig>, ProvidesTools {
     private lateinit var localstack: LocalStackContainer
     lateinit var s3Client: S3Client
         private set
@@ -98,6 +101,10 @@ class S3Executor(private val config: S3Config, loggers: List<LoggerName>) : Infr
         eventSource.stopContainerAndEmitEvents(localstack)
     }
 
+    override fun tools(): List<ToolDefinition> = listOf(
+        stackPortTool(localstack, LOCALSTACK_INTERNAL_PORT, "Browse and edit buckets and objects")
+    )
+
     private fun createBucket(bucketConfig: BucketConfig) {
         s3Client.createBucket { it.bucket(bucketConfig.name) }
     }

nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/ComponentTools.kt

@@ -0,0 +1,45 @@
+package com.orbitalhq.nebula.tools
+
+/**
+ * Implemented by components that can launch extra tooling alongside themselves -
+ * typically a browser / admin console for poking around in the thing that's running
+ * (eg: StackPort for LocalStack, kafka-ui for Kafka).
+ *
+ * The component only *describes* its tools, since it's the one that knows the
+ * coordinates and credentials a tool needs. Launching, networking and cleanup are
+ * handled generically by [ToolRunner], so no tool gets special treatment.
+ *
+ * Tools are launched on demand (from the admin UI), never as part of starting a stack.
+ */
+interface ProvidesTools {
+    /**
+     * The tools this component offers. Only called while the component is running,
+     * so implementations may read state that's only available once started
+     * (eg: generated credentials).
+     */
+    fun tools(): List<ToolDefinition>
+}
+
+data class ToolDefinition(
+    /** Unique within the component. Used in URLs, eg `stackport`. */
+    val id: String,
+    val displayName: String,
+    val description: String,
+    val launch: ContainerToolSpec
+)
+
+/**
+ * A tool that runs as a container on the Nebula network, alongside the component it serves.
+ * Reach the component via [com.orbitalhq.nebula.uniqueNetworkHost] and its *internal* port - the
+ * tool is always inside the docker network, regardless of [com.orbitalhq.nebula.ConsumerConnectivity].
+ */
+data class ContainerToolSpec(
+    val image: String,
+    val env: Map<String, String> = emptyMap(),
+    /** The port the tool serves its UI on, inside the container. Mapped to a random host port. */
+    val httpPort: Int,
+    /** Where to open the tool's UI. */
+    val path: String = "/",
+    /** An HTTP path that returns 2xx once the tool is ready. Defaults to [path]. */
+    val readinessPath: String = path
+)

nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/tools/StackPort.kt

@@ -0,0 +1,33 @@
+package com.orbitalhq.nebula.tools
+
+import com.orbitalhq.nebula.uniqueNetworkHost
+import org.testcontainers.localstack.LocalStackContainer
+
+const val STACKPORT_IMAGE = "davireis/stackport:0.4.4"
+
+/**
+ * StackPort (https://github.com/DaviReisVieira/stackport) - a browser for AWS resources,
+ * pointed at a LocalStack container over the Nebula network.
+ *
+ * @param description what the user would use it for on this component
+ */
+fun stackPortTool(
+    localstack: LocalStackContainer,
+    internalPort: Int,
+    description: String
+) = ToolDefinition(
+    id = "stackport",
+    displayName = "StackPort",
+    description = description,
+    launch = ContainerToolSpec(
+        image = STACKPORT_IMAGE,
+        env = mapOf(
+            "AWS_ENDPOINT_URL" to "http://${localstack.uniqueNetworkHost}:$internalPort",
+            "AWS_ACCESS_KEY_ID" to localstack.accessKey,
+            "AWS_SECRET_ACCESS_KEY" to localstack.secretKey,
+            "AWS_REGION" to localstack.region,
+        ),
+        httpPort = 8080,
+        readinessPath = "/api/health"
+    )
+)