From 0ab698bb59936de6b9ffecf6eb568dba67f5ebe0 Mon Sep 17 00:00:00 2001 From: Marty Pitt Date: Tue, 22 Sep 2026 07:52:40 +0100 Subject: [PATCH] feat(bundles): submit a stack together with the files it reads A stack script that needs an OpenAPI spec or a CSV could only read them from the Nebula server's own filesystem, which is a different machine from the project that defines the stack. Stacks that read files were therefore only usable when someone had put the file on the server by hand. A stack can now be submitted as a bundle: the script plus a map of relative path -> content. Nebula unpacks the resources into a directory belonging to that submission, and the stack reads them through `resources` inside `stack { }`. Providers that read a file (`s3 { file(path) }`, and API Gateway's `restApiFromFile` when it lands) resolve a relative path through the bundle; an absolute path keeps its existing meaning, so stacks written before bundles are unaffected. Additive throughout: - `POST /stacks/bundle` and `PUT /stacks/bundle/{id}` sit alongside the script-only routes, which are unchanged. - The `/stream/stacks` message gained a `bundles` field next to `stacks`. Both default to empty and normalise to the same thing, so an Orbital that predates bundles keeps working, and a bundle with no resources behaves exactly as a script-only submission does. Every resource lookup is confined to the bundle directory: absolute paths and `..` segments are rejected, and the resolved path is turned into a real path (following symlinks) and re-checked to still be inside the bundle. The same check runs at unpack time, so a `../../etc/cron.d` entry is refused before anything is written. Size limits (10MB per file, 50MB per bundle, 500 files) are checked against the decoded bytes before any file is created, and a rejected submission leaves nothing on disk. Resources are text by default, with a BASE64 encoding flag for binary content. Orbital only sends text, because the files reach it through the schema pipeline, which carries additional sources as text. StackRunner now dedupes on the script *and* the resource content, so editing a CSV next to an unchanged script restarts the stack rather than being skipped as a duplicate. A stack's unpacked bundle is deleted when it is replaced or removed; stopping a stack keeps it, since a stopped stack can be started again. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011c7t8dqC37eQzTt9o44x9v --- CLAUDE.md | 10 + NEBULA-AGENT.md | 99 ++++++- docs/pages/_meta.ts | 1 + docs/pages/bundles.mdx | 152 ++++++++++ docs/pages/providers/s3.mdx | 20 ++ docs/pages/stacks.mdx | 7 + .../com/orbitalhq/nebula/core/StackBundle.kt | 103 +++++++ .../kotlin/com/orbitalhq/nebula/InfraDsl.kt | 17 +- .../com/orbitalhq/nebula/NebulaStack.kt | 50 +++- .../com/orbitalhq/nebula/StackRunner.kt | 16 +- .../nebula/resources/StackBundleUnpacker.kt | 149 ++++++++++ .../nebula/resources/StackResources.kt | 243 ++++++++++++++++ .../nebula/resources/StackResourcesContext.kt | 36 +++ .../kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt | 32 ++- .../resources/StackBundleUnpackerTest.kt | 177 ++++++++++++ .../nebula/resources/StackResourcesTest.kt | 181 ++++++++++++ .../nebula/runtime/NebulaScriptExecutor.kt | 54 +++- .../nebula/runtime/server/NebulaServer.kt | 91 +++++- .../runtime/StackBundleExecutionTest.kt | 265 ++++++++++++++++++ .../runtime/server/StreamStacksBundleTest.kt | 152 ++++++++++ .../UpdateStackRequestCompatibilityTest.kt | 101 +++++++ 21 files changed, 1932 insertions(+), 24 deletions(-) create mode 100644 docs/pages/bundles.mdx create mode 100644 nebula-api/src/main/kotlin/com/orbitalhq/nebula/core/StackBundle.kt create mode 100644 nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpacker.kt create mode 100644 nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResources.kt create mode 100644 nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResourcesContext.kt create mode 100644 nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpackerTest.kt create mode 100644 nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackResourcesTest.kt create mode 100644 nebula-runtime/src/test/kotlin/com/orbitalhq/nebula/runtime/StackBundleExecutionTest.kt create mode 100644 nebula-runtime/src/test/kotlin/com/orbitalhq/nebula/runtime/server/StreamStacksBundleTest.kt create mode 100644 nebula-runtime/src/test/kotlin/com/orbitalhq/nebula/runtime/server/UpdateStackRequestCompatibilityTest.kt diff --git a/CLAUDE.md b/CLAUDE.md index bb79204..1d9d57b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,6 +65,16 @@ The DSL supports: Kafka, HTTP servers, SQL databases (PostgreSQL/MySQL/Oracle wi - Default imports provide common utilities - Components implement `InfrastructureComponent<*>` interface +### Stack bundles +- A stack can be submitted with the files it reads (`StackBundle` in nebula-api) + via `POST /stacks/bundle`, `PUT /stacks/bundle/{id}`, or the `bundles` field of + the `/stream/stacks` websocket message. The script-only routes are unchanged. +- Resources unpack into a per-submission temp directory. `StackResources` + (nebula-dsl) confines every lookup to that directory; scripts reach it through + `resources` inside `stack { }`, and providers that read files resolve relative + paths through `StackResources.resolveFilePath`. +- See `docs/pages/bundles.mdx` and §8 of `NEBULA-AGENT.md`. + ### Testing - Uses Kotest framework with descriptive specifications - TestContainers for integration testing diff --git a/NEBULA-AGENT.md b/NEBULA-AGENT.md index 15d815d..f1e3dbb 100644 --- a/NEBULA-AGENT.md +++ b/NEBULA-AGENT.md @@ -30,7 +30,7 @@ version: 0.1.0 sourceRoot: src/ additionalSources: { "@orbital/config" : "orbital/config/*.conf", - "@orbital/nebula" : "orbital/nebula/*.nebula.kts" + "@orbital/nebula" : "orbital/nebula/**" } dependencies: { "com.orbitalhq/core" : "github:orbitalapi/orbital-core-taxi#0.34.0" @@ -40,8 +40,8 @@ dependencies: { **Rules:** - The `@orbital/nebula` key under `additionalSources` is **mandatory**. Without it Orbital will not discover the stack. - By convention the file lives at `orbital/nebula/stack.nebula.kts` (preferred) or `nebula/stack.nebula.kts`. -- Declare the glob with a wildcard (`*.nebula.kts`), not a single explicit file — future stacks in the same folder will be picked up automatically. -- If `taxi.conf` already exists and lacks the `@orbital/nebula` entry, add it. Do not remove unrelated `additionalSources` keys. +- **Declare the glob as `orbital/nebula/**`**, not `orbital/nebula/*.nebula.kts`. The whole directory is the stack's bundle: `*.nebula.kts` files are stacks, and every other file is a resource shipped alongside them (see §8). The narrower `*.nebula.kts` glob still works, but then the stack ships no files. +- If `taxi.conf` already exists with the older `orbital/nebula/*.nebula.kts` entry, widen it to `orbital/nebula/**` when the stack needs to read a file. Do not remove unrelated `additionalSources` keys. **Minimal project layout:** @@ -52,7 +52,9 @@ my-project/ │ └── ... (taxi files) └── orbital/ └── nebula/ - └── stack.nebula.kts + ├── stack.nebula.kts + └── data/ + └── sales.csv <- shipped with the stack, see §8 ``` --- @@ -335,7 +337,8 @@ Empire Strikes Back,LXILO,189 """) } bucket("reports") { - file("/absolute/path/to/local.csv") // read from disk + file("data/sales.csv") // shipped with the stack (see §8) + file("/absolute/path/to/local.csv") // read from the Nebula server's disk file("large.csv", generateSequence()) // Sequence for large/streamed files } } @@ -352,9 +355,11 @@ Inside a `bucket(name)` block: ```kotlin fun file(name: String, content: String) // inline content fun file(name: String, content: Sequence) // streaming — for large files -fun file(path: String) // read file from local disk; uses the filename as the S3 key +fun file(path: String) // read file from disk; uses the filename as the S3 key ``` +`file(path)` resolves a **relative** path against the files shipped with the stack (§8), and an **absolute** path against the Nebula server's own filesystem. Prefer the relative form — the Nebula server is a different machine from the project, so an absolute path almost never points where you think it does. + Use the `Sequence` form for files larger than a few MB (the seeder uses S3 multipart upload — minimum part size is 5 MB, which is already handled by default). ### 5.6 `hazelcast { ... }` — Hazelcast @@ -507,10 +512,88 @@ stack { --- -## 8. Quick checklist before returning a stack +## 8. Shipping files with a stack + +A stack often needs a file: an OpenAPI spec to import, a CSV to seed a bucket, a +fixture to serve. **Those files go in the project's nebula directory, next to the +script**, and are submitted to Nebula along with it. + +This matters because the Nebula server is a different machine from the project. +A path like `/home/jane/project/sales.csv` in a script means *Nebula's* +filesystem, where the file does not exist. + +### Layout + +Everything under the `@orbital/nebula` directory that is **not** a `.nebula.kts` +file is a resource. Resources are shared by every stack in that directory. + +``` +orbital/nebula/ +├── stack.nebula.kts +├── data/ +│ └── sales.csv +└── specs/ + └── orders.yaml +``` + +This requires the wider glob in `taxi.conf` (§2): + +```hocon +"@orbital/nebula" : "orbital/nebula/**" +``` + +With the older `orbital/nebula/*.nebula.kts` glob the script is still found, but +no resources are shipped and `resources` will be empty. + +### Reading them + +Inside `stack { }`, `resources` gives you the files: + +```kotlin +stack { + val sales = resources.readText("data/sales.csv") + + http { + get("/sales") { call -> call.respondText(sales) } + } +} +``` + +| Function | Returns | +|---|---| +| `resources.readText(path)` | the file's text, as UTF-8 | +| `resources.readBytes(path)` | the file's bytes | +| `resources.path(path)` | a `java.nio.file.Path` to the file | +| `resources.exists(path)` | whether the stack shipped that file | +| `resources.names` | every file the stack shipped | +| `resources.hasBundle` | whether this stack was submitted with files at all | + +Paths are relative to the nebula directory — `data/sales.csv` above, not +`orbital/nebula/data/sales.csv`. + +Providers that read a file (`s3 { file(path) }`, and API Gateway's +`restApiFromFile(...)`) resolve a **relative** path the same way, so you can pass +`"data/sales.csv"` straight to them. + +**Rules:** +- Use relative paths. Absolute paths mean the Nebula server's own filesystem, and + almost never point where you want. +- A path may not contain `..`, and may not escape the nebula directory. Both are + rejected, so don't try to reach a file elsewhere in the project — copy it into + the nebula directory instead. +- Limits: 10 MB per file, 50 MB per stack, 500 files. For anything larger, mount + a volume into the Nebula container. +- Resources are text by default. Binary files work, but Orbital ships a project's + files as text, so prefer text formats (CSV, JSON, YAML) for stacks Orbital + submits. + +--- + +## 9. Quick checklist before returning a stack - [ ] File is named `stack.nebula.kts` (unless user specified otherwise) and placed at `orbital/nebula/stack.nebula.kts`. -- [ ] `taxi.conf` contains `"@orbital/nebula" : "orbital/nebula/*.nebula.kts"` under `additionalSources`. +- [ ] `taxi.conf` contains `"@orbital/nebula" : "orbital/nebula/**"` under `additionalSources`. +- [ ] Any file the stack reads sits under `orbital/nebula/`, and is read with a relative path (`resources.readText("data/sales.csv")`, `file("data/sales.csv")`) — never an absolute one. - [ ] Exactly one top-level `stack { }` block. - [ ] No re-imports of `stack`, `Random`, `io.ktor.server.request.*`, `io.ktor.server.response.*`. - [ ] Duplicate component types (two Postgres, two S3, etc.) each have distinct `componentName`s. diff --git a/docs/pages/_meta.ts b/docs/pages/_meta.ts index a015bea..7ce7b6c 100644 --- a/docs/pages/_meta.ts +++ b/docs/pages/_meta.ts @@ -1,6 +1,7 @@ export default { "index": "Getting started", "stacks": "Working with stacks", + "bundles": "Shipping files with a stack", "cli": "CLI & networking", "logging" : "Logging", "examples": "Examples", diff --git a/docs/pages/bundles.mdx b/docs/pages/bundles.mdx new file mode 100644 index 0000000..2eb5694 --- /dev/null +++ b/docs/pages/bundles.mdx @@ -0,0 +1,152 @@ +--- +title: Shipping files with a stack +--- + +## Shipping files with a stack + +A stack script often needs a file: an OpenAPI spec to import into API Gateway, a +CSV to seed an S3 bucket, a fixture to serve from an HTTP endpoint. + +Those files live in the project that defines the stack — on a developer's laptop, +or in a git repo that Orbital has checked out. The Nebula server is somewhere +else entirely, usually a different container. A path in a script like +`file("/home/jane/project/sales.csv")` means *the Nebula server's* filesystem, +which is not where the file is. + +A **stack bundle** fixes that. It is the script plus the files it reads, +submitted together. Nebula unpacks the files into a directory belonging to that +submission, and the stack reads them from there. + +### Submitting a bundle + +`POST /stacks/bundle` creates a stack; `PUT /stacks/bundle/{id}` replaces one. +Both take JSON: + +```json +{ + "script": "stack {\n s3 { bucket(\"tickets\") { file(\"data/sales.csv\") } }\n}", + "resources": { + "data/sales.csv": { "content": "Title,Tickets\nA New Hope,249\n" }, + "specs/orders.yaml": { "content": "openapi: 3.0.0\n..." } + } +} +``` + +```bash +curl --location 'http://localhost:8099/stacks/bundle' \ + --header 'Content-Type: application/json' \ + --data @bundle.json +``` + +The script-only routes (`POST /stacks`, `PUT /stacks/{id}`) are unchanged, and a +bundle with no `resources` behaves exactly as they do. + +Orbital submits bundles over the `/stream/stacks` websocket rather than over +HTTP. That message accepts both shapes: + +```json +{ "stacks": { "films": "stack {}" } } +{ "bundles": { "films": { "script": "stack {}", "resources": { ... } } } } +``` + +### Resource encoding + +Each resource is `{ "content": "...", "encoding": "TEXT" | "BASE64" }`. +`encoding` defaults to `TEXT`, where the content is the file's text and is +written out as UTF-8. Binary files (a `.parquet`, an image) are sent as `BASE64`. + +Orbital only ever sends `TEXT`: the files reach it through the schema pipeline, +which carries a project's additional sources as text. Submit `BASE64` directly if +you need binary content. + +### Limits + +| Limit | Value | +|---|---| +| Largest single file | 10 MB | +| Largest bundle | 50 MB | +| Files per bundle | 500 | + +Exceeding any of these fails the submission with a message naming the file and +the limit. Bundles are for fixtures and specs — if you need gigabytes of seed +data, mount a volume into the Nebula container instead. + +### Reading resources from a script + +Inside `stack { }`, `resources` gives you the files that were shipped: + +```kotlin +stack { + val sales = resources.readText("data/sales.csv") + val spec = resources.readText("specs/orders.yaml") + + http { + get("/sales") { call -> call.respondText(sales) } + } +} +``` + +| Function | Returns | +|---|---| +| `resources.readText(path)` | the file's text, as UTF-8 | +| `resources.readBytes(path)` | the file's bytes | +| `resources.path(path)` | a `java.nio.file.Path` to the unpacked file | +| `resources.exists(path)` | whether the stack shipped that file | +| `resources.names` | every file the stack shipped, as relative paths | +| `resources.hasBundle` | whether this stack was submitted with a bundle | + +Paths are relative to the bundle root, which is the project's nebula directory — +so `data/sales.csv` in the script means `orbital/nebula/data/sales.csv` in the +project. + +### Providers that read files + +Any provider that reads a file resolves a **relative** path through the bundle: + +```kotlin +stack { + s3 { + bucket("tickets") { + file("data/sales.csv") // read from the bundle + file("/mnt/shared/big.csv") // read from the Nebula server, as before + } + } +} +``` + +An **absolute** path keeps its existing meaning — a file on the machine running +Nebula. Stacks written before bundles existed, and stacks run from the CLI +against local files, are unaffected. + +Resolution happens while the script is compiled, so a typo in a resource name +fails the submission with a message listing what the stack actually shipped, +rather than failing later, mid-startup. + +### What a stack may read + +Every resource lookup is confined to the bundle directory: + +- absolute paths are rejected, +- `..` segments are rejected, +- the resolved path is turned into a real path — following symlinks — and + re-checked to still be inside the bundle. + +So a bundle entry named `../../etc/cron.d/evil` is rejected before anything is +written, and a symlink planted inside a bundle that points at +`/etc/ssh/id_rsa` is refused when the stack tries to read it. + +This confines *resource* access. A Nebula script is arbitrary Kotlin and can +always open a file directly; the guarantee is that a bundle-relative path +resolves to a file inside the bundle, nothing more and nothing less. Only submit +stacks you trust to a Nebula server. + +### Lifetime + +Each submission gets its own directory under the system temp directory. It is +deleted when the stack is replaced by a different submission, when the stack is +removed, and when a submission is rejected. Stopping a stack keeps its bundle, +because a stopped stack can be started again. + +A stack is *replaced* when its script **or** any of its resources change — so +editing a CSV next to an unchanged script restarts the stack, rather than being +skipped as a duplicate submission. diff --git a/docs/pages/providers/s3.mdx b/docs/pages/providers/s3.mdx index afd6156..4b221ce 100644 --- a/docs/pages/providers/s3.mdx +++ b/docs/pages/providers/s3.mdx @@ -16,6 +16,26 @@ stack { } ``` +### Seeding a bucket from a file + +`file(path)` uploads a file from disk, using its filename as the S3 key. + +```kotlin +stack { + s3 { + bucket("tickets") { + file("data/sales.csv") // shipped with the stack + file("/mnt/shared/big.csv") // on the machine running Nebula + } + } +} +``` + +A **relative** path is read from the bundle submitted with the stack, so the CSV +travels with the project rather than having to exist on the Nebula server — see +[Shipping files with a stack](/bundles). An **absolute** path is read from the +machine running Nebula, which is what it has always meant. + ### Returned values When an `s3` component is declared, the following data is returned: diff --git a/docs/pages/stacks.mdx b/docs/pages/stacks.mdx index 23f6e54..971d408 100644 --- a/docs/pages/stacks.mdx +++ b/docs/pages/stacks.mdx @@ -32,6 +32,13 @@ curl --location 'http://localhost:8099/stacks' \ }' ``` +### Creating a stack that ships files + +A script that reads a file — a CSV to seed a bucket, an OpenAPI spec — needs that +file to travel with it, because the Nebula server's filesystem is not the +project's. POST a bundle to `/stacks/bundle` instead, and read the files through +`resources` inside the script. See [Shipping files with a stack](/bundles). + ### Listing the state of current stacks Fetch the current stacks (including their configured variables) by GET `/stacks` diff --git a/nebula-api/src/main/kotlin/com/orbitalhq/nebula/core/StackBundle.kt b/nebula-api/src/main/kotlin/com/orbitalhq/nebula/core/StackBundle.kt new file mode 100644 index 0000000..f9ed14b --- /dev/null +++ b/nebula-api/src/main/kotlin/com/orbitalhq/nebula/core/StackBundle.kt @@ -0,0 +1,103 @@ +package com.orbitalhq.nebula.core + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import java.util.Base64 + +/** + * A path to a file inside a [StackBundle], relative to the bundle root. + * + * Always uses `/` as the separator, and must never be absolute or contain + * `..` segments - see `StackResources` in nebula-dsl, which enforces this + * when the bundle is unpacked and again whenever a stack reads from it. + */ +typealias ResourcePath = String + +/** + * How the [ResourceContent.content] string encodes the underlying bytes. + */ +enum class ResourceEncoding { + /** The content is the file's text, and is written out as UTF-8. */ + TEXT, + + /** The content is the file's bytes, Base64 encoded. */ + BASE64 +} + +/** + * The content of a single file shipped alongside a stack script. + * + * Text is carried as text (so bundles stay readable and diffable on the wire); + * binary files are Base64 encoded and flagged via [encoding]. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class ResourceContent( + val content: String, + val encoding: ResourceEncoding = ResourceEncoding.TEXT +) { + companion object { + @JvmStatic + fun text(content: String): ResourceContent = ResourceContent(content, ResourceEncoding.TEXT) + + @JvmStatic + fun binary(bytes: ByteArray): ResourceContent = + ResourceContent(Base64.getEncoder().encodeToString(bytes), ResourceEncoding.BASE64) + } + + /** + * The decoded bytes of this resource, as they will be written to disk when + * the bundle is unpacked. + */ + fun bytes(): ByteArray = when (encoding) { + ResourceEncoding.TEXT -> content.toByteArray(Charsets.UTF_8) + ResourceEncoding.BASE64 -> Base64.getDecoder().decode(content) + } +} + +/** + * A stack script plus the resource files (OpenAPI specs, CSV seed data, ...) + * that it reads at startup. + * + * Submitting a bundle rather than a bare script is what lets a stack read files + * that live in the *submitter's* project, rather than on the Nebula server's own + * filesystem. The resources are unpacked into a per-submission temporary + * directory, and every read from a stack is confined to that directory. + * + * Bundles are a superset of the existing script-only submission: a bundle with + * no resources behaves exactly as `POST /stacks` with the script text does. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class StackBundle( + val script: String, + val resources: Map = emptyMap() +) { + companion object { + /** A bundle carrying just a script, equivalent to the legacy script-only submission. */ + @JvmStatic + fun scriptOnly(script: String): StackBundle = StackBundle(script, emptyMap()) + } + + /** Derived, so it must not be written to (or read from) the wire. */ + @get:JsonIgnore + val hasResources: Boolean + get() = resources.isNotEmpty() +} + +/** + * The limits applied when a [StackBundle] is unpacked. + * + * Bundles travel as JSON over the same websocket that carries stack scripts, and + * are written to the Nebula server's disk, so both the request size and the + * unpacked footprint are capped. Exceeding any limit fails the submission with a + * message naming the offending file and the limit. + */ +object StackBundleLimits { + /** Largest decoded size of any single resource. */ + const val MAX_RESOURCE_BYTES: Long = 10L * 1024 * 1024 + + /** Largest total decoded size of all resources in one bundle. */ + const val MAX_BUNDLE_BYTES: Long = 50L * 1024 * 1024 + + /** Largest number of files in one bundle. */ + const val MAX_RESOURCE_COUNT: Int = 500 +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfraDsl.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfraDsl.kt index 714d0b5..a716aee 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfraDsl.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfraDsl.kt @@ -1,5 +1,7 @@ package com.orbitalhq.nebula +import com.orbitalhq.nebula.resources.StackResources + /** * Base interface for adding DSL support within the script. * @@ -11,5 +13,18 @@ package com.orbitalhq.nebula */ interface InfraDsl { val components: List> + + /** + * The files shipped alongside this stack's script. + * + * Exposed on the DSL (rather than handed to each component) so that any + * provider which reads a file - `s3 { file(...) }`, and API Gateway's + * `restApiFromFile(...)` - can resolve a path through the submitting + * project's bundle rather than the Nebula server's own filesystem. + * + * [StackResources.NONE] when the stack was submitted without a bundle. + */ + val resources: StackResources + fun > add(component: T): T -} \ No newline at end of file +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/NebulaStack.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/NebulaStack.kt index b974ed0..42635e5 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/NebulaStack.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/NebulaStack.kt @@ -9,6 +9,8 @@ import com.orbitalhq.nebula.kafka.KafkaDsl import com.orbitalhq.nebula.logging.LogMessage import com.orbitalhq.nebula.logging.StackLogStream import com.orbitalhq.nebula.mongo.MongoDsl +import com.orbitalhq.nebula.resources.StackResources +import com.orbitalhq.nebula.resources.StackResourcesContext import com.orbitalhq.nebula.s3.S3Dsl import com.orbitalhq.nebula.sql.SqlDsl import com.orbitalhq.nebula.taxi.TaxiPublisherDsl @@ -29,11 +31,34 @@ data class NebulaStackWithSource( } val name = stack.name + + /** + * The files shipped with this submission, unpacked and confined to their own + * directory. [StackResources.NONE] for a script-only submission. + */ + val resources: StackResources + get() = stack.resources + + /** + * Identity of a submission: the script text *plus* the content of the + * resources shipped with it. + * + * [StackRunner] dedupes on this rather than on [source] alone, so editing a + * CSV or an OpenAPI spec next to an unchanged script still replaces the + * running stack instead of being skipped as a duplicate submission. + */ + val submissionKey: String + get() = source + "\u0000" + stack.resources.fingerprint } class NebulaStack( val name: StackName = NameGenerator.generateName(), - initialComponents: List> = emptyList() + initialComponents: List> = emptyList(), + /** + * The files shipped alongside this stack's script. Bound by the executor when + * the stack was submitted as a bundle; [StackResources.NONE] otherwise. + */ + override val resources: StackResources = StackResources.NONE ) : InfraDsl, KafkaDsl, S3Dsl, HttpDsl, SqlDsl, HazelcastDsl, MongoDsl, TaxiPublisherDsl { companion object { private val logger = KotlinLogging.logger {} @@ -53,7 +78,7 @@ class NebulaStack( } fun withName(name: StackName): NebulaStack { - return NebulaStack(name, this._components) + return NebulaStack(name, this._components, this.resources) } private val logStream = StackLogStream() @@ -104,6 +129,25 @@ class NebulaStack( } } +/** + * Declares a stack. + * + * Picks up the resource bundle bound for the current thread by the executor, so + * a script submitted as a bundle can read the files that shipped with it (via + * `resources`, or by giving a relative path to any provider that reads a file) + * without having to declare anything. + */ fun stack(init: NebulaStack.() -> Unit): NebulaStack { - return NebulaStack().apply(init) + return NebulaStack(resources = StackResourcesContext.current()).apply(init) +} + +/** + * Declares a stack with an explicit set of resources. + * + * Scripts don't use this - they get their bundle from the submission. It exists + * for hosts (the CLI, tests) that build a stack in-process and want to point it + * at a directory of files. + */ +fun stack(resources: StackResources, init: NebulaStack.() -> Unit): NebulaStack { + return NebulaStack(resources = resources).apply(init) } \ No newline at end of file diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt index 8f8f08d..7c4bfff 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt @@ -61,12 +61,22 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) { fun submit(submittedStack: NebulaStackWithSource, name: StackName = submittedStack.name, startAsync: Boolean = false): Flux { val storedStack = this.stacks.compute(name) { key, existingSpec -> if (existingSpec != null) { - if (existingSpec.source == submittedStack.source) { + // Compared on the submission key, not the script alone: a stack whose + // script is unchanged but whose bundled resources have been edited is + // a genuinely different submission and must be restarted. + if (existingSpec.submissionKey == submittedStack.submissionKey) { logger.info { "Received duplicate submission for spec $key - reusing existing stack" } + // The incoming bundle was unpacked before we got here, and is now + // redundant - drop it rather than leaving it in the temp directory. + if (submittedStack !== existingSpec) { + submittedStack.resources.delete() + } return@compute existingSpec } else { logger.info { "Replacing spec $key" } shutDown(name) + // The replaced stack is gone for good, so its unpacked bundle goes too. + existingSpec.resources.delete() } } submittedStack @@ -208,7 +218,9 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) { if (this.stacks.containsKey(name)) { shutDown(name) } - this.stacks.remove(name) + // Removal (unlike shutDown, which leaves a stopped stack restartable) means + // the stack is never coming back, so its unpacked bundle is deleted with it. + this.stacks.remove(name)?.resources?.delete() this._stackState.remove(name) } diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpacker.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpacker.kt new file mode 100644 index 0000000..83afe39 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpacker.kt @@ -0,0 +1,149 @@ +package com.orbitalhq.nebula.resources + +import com.orbitalhq.nebula.core.ResourceContent +import com.orbitalhq.nebula.core.StackBundle +import com.orbitalhq.nebula.core.StackBundleLimits +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +/** + * Raised when a submitted bundle is not safe or not small enough to unpack. + * Distinct from [StackResourceException], which is raised when a *running* stack + * asks for something it shouldn't have. + */ +class InvalidStackBundleException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) + +/** + * Writes a submitted [StackBundle]'s resources into a directory of their own. + * + * Every entry is validated before anything is written: the same containment + * rules [StackResources] applies on read are applied here on write, so a + * `../../etc/cron.d/evil` entry is rejected rather than landing on disk (the + * zip-slip class of bug). Size limits are checked against the *decoded* bytes, + * before any file is created, so an oversized bundle leaves nothing behind. + */ +class StackBundleUnpacker( + /** + * The directory under which per-submission bundle directories are created. + * Defaults to a `nebula-stack-bundles` directory in the system temp dir. + */ + private val baseDirectory: Path = defaultBaseDirectory() +) { + companion object { + private val logger = KotlinLogging.logger {} + + private fun defaultBaseDirectory(): Path { + val base = Path.of(System.getProperty("java.io.tmpdir")).resolve("nebula-stack-bundles") + Files.createDirectories(base) + return base.toRealPath() + } + + /** + * A content hash over the bundle's resources. Two bundles with the same + * files in the same state share a fingerprint, regardless of map ordering. + */ + fun fingerprintOf(resources: Map): String { + if (resources.isEmpty()) return "" + val digest = MessageDigest.getInstance("SHA-256") + resources.entries.sortedBy { it.key }.forEach { (path, content) -> + digest.update(path.toByteArray(Charsets.UTF_8)) + digest.update(content.bytes()) + } + return digest.digest().joinToString("") { "%02x".format(it) } + } + } + + /** + * Unpacks [bundle]'s resources for the stack named [stackName], returning a + * [StackResources] confined to the directory they were written to. + * + * A bundle with no resources needs no directory, and yields + * [StackResources.NONE] - so a bundle submission of a bare script behaves + * exactly as the script-only submission does. + * + * @throws InvalidStackBundleException if an entry escapes the bundle, or a + * size limit is exceeded. + */ + fun unpack(stackName: String, bundle: StackBundle): StackResources { + if (!bundle.hasResources) return StackResources.NONE + + validate(stackName, bundle) + + val directory = Files.createTempDirectory(baseDirectory, "${safeDirName(stackName)}-").toRealPath() + try { + bundle.resources.forEach { (relativePath, content) -> + val target = resolveWithin(directory, relativePath) + Files.createDirectories(target.parent) + Files.write(target, content.bytes()) + } + } catch (e: Exception) { + directory.toFile().deleteRecursively() + throw when (e) { + is StackResourceException -> InvalidStackBundleException(e.message ?: "Invalid resource path", e) + is InvalidStackBundleException -> e + else -> InvalidStackBundleException( + "Failed to unpack resources for stack '$stackName' - ${e.message}", e + ) + } + } + logger.info { "Unpacked ${bundle.resources.size} resources for stack '$stackName' into $directory" } + return StackResources( + root = directory, + ownsRoot = true, + fingerprint = fingerprintOf(bundle.resources) + ) + } + + /** + * Checks every entry before a single byte is written, so a bundle is either + * fully unpacked or leaves nothing on disk. + */ + private fun validate(stackName: String, bundle: StackBundle) { + if (bundle.resources.size > StackBundleLimits.MAX_RESOURCE_COUNT) { + throw InvalidStackBundleException( + "Stack '$stackName' ships ${bundle.resources.size} resources, which exceeds the limit of " + + "${StackBundleLimits.MAX_RESOURCE_COUNT} files per stack." + ) + } + // A throwaway root: validation must reject a path before we create the real + // directory, and resolveWithin needs something to resolve against. + val probeRoot = Path.of("/nebula-bundle-probe").normalize() + var totalBytes = 0L + bundle.resources.forEach { (relativePath, content) -> + try { + resolveWithin(probeRoot, relativePath) + } catch (e: StackResourceException) { + throw InvalidStackBundleException( + "Stack '$stackName' ships an unusable resource - ${e.message}", e + ) + } + val bytes = try { + content.bytes().size.toLong() + } catch (e: IllegalArgumentException) { + throw InvalidStackBundleException( + "Resource '$relativePath' in stack '$stackName' is flagged as ${content.encoding} " + + "but is not valid Base64 - ${e.message}", e + ) + } + if (bytes > StackBundleLimits.MAX_RESOURCE_BYTES) { + throw InvalidStackBundleException( + "Resource '$relativePath' in stack '$stackName' is $bytes bytes, which exceeds the " + + "per-file limit of ${StackBundleLimits.MAX_RESOURCE_BYTES} bytes." + ) + } + totalBytes += bytes + if (totalBytes > StackBundleLimits.MAX_BUNDLE_BYTES) { + throw InvalidStackBundleException( + "The resources shipped with stack '$stackName' total more than the bundle limit of " + + "${StackBundleLimits.MAX_BUNDLE_BYTES} bytes." + ) + } + } + } + + /** Stack names carry `/` and `[]` from Orbital's workspace + package prefixes. */ + private fun safeDirName(stackName: String): String = + stackName.replace(Regex("[^A-Za-z0-9._-]"), "_").take(60).ifBlank { "stack" } +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResources.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResources.kt new file mode 100644 index 0000000..72d5636 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResources.kt @@ -0,0 +1,243 @@ +package com.orbitalhq.nebula.resources + +import java.io.IOException +import java.nio.file.Files +import java.nio.file.InvalidPathException +import java.nio.file.Path +import java.nio.file.Paths +import java.security.MessageDigest +import java.util.stream.Collectors +import kotlin.io.path.isDirectory + +/** + * Raised when a stack asks for a resource that doesn't exist, or that would + * escape the bundle directory. + */ +class StackResourceException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) + +/** + * The files shipped alongside a stack script, unpacked into a directory that + * belongs to this submission alone. + * + * Every lookup is confined to [root]: + * + * - absolute paths are rejected, + * - `..` segments are rejected, both before and after normalisation, + * - the resolved path is turned into a *real* path (following symlinks) and + * checked to still be a descendant of the real [root]. + * + * That last step is what stops a symlink planted inside the bundle (or a bundle + * entry crafted to point at one) from reading `/etc/ssh/id_rsa` or any other + * file on the Nebula server. + * + * Note that these checks confine *resource* access. A Nebula script is arbitrary + * Kotlin and can always open files directly; the guarantee here is that a + * bundle-relative path resolves to a file inside the bundle, nothing more and + * nothing less. + */ +class StackResources internal constructor( + /** + * The directory this bundle was unpacked into, or null when the stack was + * submitted without a bundle (a plain script submission, or the CLI). + */ + val root: Path?, + /** + * Whether [delete] should remove [root]. True for directories we created by + * unpacking a bundle; false for a directory handed to us by a caller. + */ + private val ownsRoot: Boolean, + /** + * A stable identity for the content of this bundle. [StackRunner] compares it + * alongside the script source, so resubmitting an unchanged script with + * *changed* resources restarts the stack rather than being treated as a + * duplicate. + */ + val fingerprint: String +) { + companion object { + /** + * The resources of a stack submitted without a bundle. Reading from it + * fails with a message pointing at bundle submission, rather than + * silently resolving against the Nebula server's working directory. + */ + @JvmStatic + val NONE = StackResources(root = null, ownsRoot = false, fingerprint = "") + + /** + * A read-only view over an existing directory. Used by the CLI (where the + * script's own directory is the bundle) and by tests. + */ + @JvmStatic + fun at(directory: Path): StackResources { + require(directory.isDirectory()) { "$directory is not a directory" } + val real = directory.toRealPath() + return StackResources( + root = real, + ownsRoot = false, + fingerprint = fingerprintOfDirectory(real) + ) + } + + private fun fingerprintOfDirectory(root: Path): String { + val digest = MessageDigest.getInstance("SHA-256") + filesUnder(root).sortedBy { root.relativize(it).toString() } + .forEach { file -> + digest.update(root.relativize(file).joinToString("/").toByteArray(Charsets.UTF_8)) + digest.update(Files.readAllBytes(file)) + } + return digest.digest().joinToString("") { "%02x".format(it) } + } + + private fun filesUnder(root: Path): List = + Files.walk(root).use { stream -> + stream.filter { Files.isRegularFile(it) }.collect(Collectors.toList()) + } + } + + /** True when this stack was submitted with a resource bundle. */ + val hasBundle: Boolean + get() = root != null + + /** + * The relative paths of every file in the bundle, `/`-separated and sorted. + * Useful in error messages, and for scripts that want to iterate a directory. + */ + val names: List + get() { + val root = this.root ?: return emptyList() + return filesUnder(root).map { root.relativize(it).joinToString("/") }.sorted() + } + + /** + * Resolves [relativePath] to a real path inside the bundle. + * + * @throws StackResourceException if there is no bundle, the path is absolute + * or escapes the bundle, or no such file exists. + */ + fun path(relativePath: String): Path { + val root = this.root + ?: throw StackResourceException( + "Cannot read resource '$relativePath': this stack was submitted without a resource bundle. " + + "Place the file alongside the stack script in the project's nebula directory, " + + "or submit the stack through the bundle API." + ) + val resolved = resolveWithin(root, relativePath) + if (!Files.exists(resolved)) { + throw StackResourceException( + "No resource named '$relativePath' was shipped with this stack. " + + "Available resources: ${names.joinToString(", ").ifEmpty { "(none)" }}" + ) + } + // toRealPath follows symlinks. A link planted inside the bundle that points + // outside it normalises cleanly above, and is only caught here. + val real = try { + resolved.toRealPath() + } catch (e: IOException) { + throw StackResourceException("Failed to resolve resource '$relativePath' - ${e.message}", e) + } + if (!real.startsWith(root)) { + throw StackResourceException( + "Resource '$relativePath' resolves to $real, which is outside this stack's resource bundle. " + + "A stack may only read the files shipped with it." + ) + } + return real + } + + /** True when [relativePath] names a readable file in the bundle. Never throws. */ + fun exists(relativePath: String): Boolean = try { + path(relativePath) + true + } catch (e: StackResourceException) { + false + } + + /** The UTF-8 text of a resource shipped with this stack. */ + fun readText(relativePath: String): String = Files.readString(path(relativePath)) + + /** The raw bytes of a resource shipped with this stack. */ + fun readBytes(relativePath: String): ByteArray = Files.readAllBytes(path(relativePath)) + + /** + * Resolves a path given to a DSL function that reads a file + * (eg. `s3 { bucket("data") { file("seed/sales.csv") } }`). + * + * Relative paths resolve through the bundle when one is present, so a stack + * ships its own data. Absolute paths keep their existing meaning - a file on + * the machine running Nebula - so stacks written before bundles existed, and + * stacks run from the CLI against local files, behave exactly as they did. + */ + fun resolveFilePath(path: String): String { + if (!hasBundle) return path + return if (isAbsolute(path)) path else path(path).toString() + } + + /** + * Deletes the unpacked bundle, if this instance owns it. Called when the stack + * is removed or replaced. Safe to call more than once. + */ + fun delete() { + val root = this.root ?: return + if (!ownsRoot) return + // deleteRecursively() walks bottom-up and does not follow symlinks, so a + // link inside the bundle is unlinked rather than followed and emptied. + root.toFile().deleteRecursively() + } + + override fun toString(): String = + if (hasBundle) "StackResources($root, ${names.size} files)" else "StackResources(no bundle)" +} + +/** + * Rejects anything that isn't a plain relative path, then resolves it against + * [root] and checks the result is still inside [root]. + * + * Shared by [StackResources] (reads) and [StackBundleUnpacker] (writes), so an + * entry that could not be read can never be written in the first place. + */ +internal fun resolveWithin(root: Path, relativePath: String): Path { + if (relativePath.isBlank()) { + throw StackResourceException("A resource path may not be blank") + } + if (isAbsolute(relativePath)) { + throw StackResourceException( + "Resource path '$relativePath' is absolute. Resources are addressed by their path " + + "relative to the stack's nebula directory." + ) + } + val candidate = try { + Paths.get(relativePath) + } catch (e: InvalidPathException) { + throw StackResourceException("Resource path '$relativePath' is not a valid path - ${e.message}", e) + } + if (candidate.any { it.toString() == ".." }) { + throw StackResourceException( + "Resource path '$relativePath' contains a '..' segment. A stack may not reference " + + "files outside its resource bundle." + ) + } + val resolved = root.resolve(candidate).normalize() + if (!resolved.startsWith(root)) { + throw StackResourceException( + "Resource path '$relativePath' resolves outside the stack's resource bundle." + ) + } + return resolved +} + +/** + * True for paths that address the host filesystem directly. + * + * Deliberately stricter than [Path.isAbsolute]: on Linux a Windows-style + * `C:\keys\id_rsa` or a UNC `\\host\share` is a *valid relative filename*, and + * we'd rather reject it than quietly create a strangely-named file. + */ +internal fun isAbsolute(path: String): Boolean { + if (path.startsWith("/") || path.startsWith("\\")) return true + if (path.length >= 2 && path[1] == ':' && path[0].isLetter()) return true + return try { + Paths.get(path).isAbsolute + } catch (e: InvalidPathException) { + false + } +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResourcesContext.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResourcesContext.kt new file mode 100644 index 0000000..cf23408 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResourcesContext.kt @@ -0,0 +1,36 @@ +package com.orbitalhq.nebula.resources + +/** + * Carries the resources of the stack currently being compiled to the + * `stack { }` block inside the script. + * + * A stack script is a Kotlin script whose `stack { }` block takes no arguments - + * that's the whole point of the DSL, and we don't want every script to have to + * declare where its files come from. The script body runs synchronously on the + * thread that calls the scripting host, so the executor unpacks the bundle, + * binds it here for the duration of the evaluation, and `stack { }` picks it up. + * + * Outside a submission (the CLI, a test constructing a stack by hand) there is + * nothing bound and stacks get [StackResources.NONE]. + */ +object StackResourcesContext { + private val current = ThreadLocal() + + /** + * Binds [resources] for the duration of [block], restoring whatever was + * previously bound. Nesting is supported so a script that (unusually) + * compiles another script doesn't clobber the outer binding. + */ + fun withResources(resources: StackResources, block: () -> T): T { + val previous = current.get() + current.set(resources) + try { + return block() + } finally { + if (previous == null) current.remove() else current.set(previous) + } + } + + /** The resources bound for the current thread, or [StackResources.NONE]. */ + fun current(): StackResources = current.get() ?: StackResources.NONE +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt index 61e13e7..1e0aaad 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt @@ -1,6 +1,7 @@ package com.orbitalhq.nebula.s3 import com.orbitalhq.nebula.InfraDsl +import com.orbitalhq.nebula.resources.StackResources import com.orbitalhq.nebula.utils.NameGenerator import lang.taxi.utils.log import mu.KLogger @@ -10,23 +11,32 @@ private val logger: KLogger = KotlinLogging.logger {} interface S3Dsl : InfraDsl { fun s3(imageName: String = "localstack/localstack:3.0.2", componentName: String = "s3", dsl: S3Builder.(KLogger) -> Unit): S3Executor { - val builder = S3Builder(imageName, componentName) + val builder = S3Builder(imageName, componentName, resources) builder.dsl(logger) return this.add(S3Executor(builder.build(), loggers = listOf(logger.name))) } } -class S3Builder(private val imageName: String, private val componentName: String) { +class S3Builder( + private val imageName: String, + private val componentName: String, + /** The stack's bundle, so `file(path)` can read a file that shipped with the stack. */ + private val stackResources: StackResources +) { private val buckets = mutableListOf() fun bucket(name: String, init: BucketBuilder.() -> Unit) { - buckets.add(BucketBuilder(name).apply(init).build()) + buckets.add(BucketBuilder(name, stackResources).apply(init).build()) } fun build(): S3Config = S3Config(imageName, buckets, componentName = componentName) } -class BucketBuilder(private val name: String) { +class BucketBuilder( + private val name: String, + /** The stack's bundle, so `file(path)` can read a file that shipped with the stack. */ + private val stackResources: StackResources +) { private val resources = mutableListOf() fun file(name: String, content: String) { @@ -36,8 +46,20 @@ class BucketBuilder(private val name: String) { resources.add(SequenceResource(name, content)) } + /** + * Uploads a file from disk, using its filename as the S3 key. + * + * A *relative* path is read from the bundle shipped with the stack, so the + * file travels with the project rather than having to exist on the Nebula + * server. An *absolute* path is read from the machine running Nebula, which + * is what it has always meant. + * + * Resolution happens here, while the script is being evaluated, so a typo in + * a resource name fails the submission with a message listing what the stack + * actually shipped - rather than failing later, mid-startup. + */ fun file(path: String) { - resources.add(FileResource(path)) + resources.add(FileResource(stackResources.resolveFilePath(path))) } fun build(): BucketConfig = BucketConfig(name, resources) diff --git a/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpackerTest.kt b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpackerTest.kt new file mode 100644 index 0000000..b74f583 --- /dev/null +++ b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpackerTest.kt @@ -0,0 +1,177 @@ +package com.orbitalhq.nebula.resources + +import com.orbitalhq.nebula.core.ResourceContent +import com.orbitalhq.nebula.core.StackBundle +import com.orbitalhq.nebula.core.StackBundleLimits +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.listDirectoryEntries + +/** + * Unpacking is where a hostile bundle would land on disk, so the containment + * checks run here too - before a single file is written, not only when a running + * stack reads one back. + */ +class StackBundleUnpackerTest : DescribeSpec({ + + fun unpackerIn(): Pair { + val base = Files.createTempDirectory("unpacker-test").toRealPath() + return StackBundleUnpacker(base) to base + } + + describe("unpacking a bundle") { + it("writes text resources, including nested ones") { + val (unpacker, _) = unpackerIn() + val resources = unpacker.unpack( + "films", StackBundle( + script = "stack {}", + resources = mapOf( + "sales.csv" to ResourceContent.text("Title,Tickets"), + "specs/orders.yaml" to ResourceContent.text("openapi: 3.0.0") + ) + ) + ) + + resources.names.shouldContainExactly("sales.csv", "specs/orders.yaml") + resources.readText("specs/orders.yaml") shouldBe "openapi: 3.0.0" + } + + it("writes binary resources from their base64 encoding") { + val (unpacker, _) = unpackerIn() + val bytes = byteArrayOf(0, 1, 2, -1, -2) + val resources = unpacker.unpack( + "films", StackBundle("stack {}", mapOf("blob.bin" to ResourceContent.binary(bytes))) + ) + + resources.readBytes("blob.bin").toList() shouldBe bytes.toList() + } + + it("gives each submission a directory of its own") { + val (unpacker, base) = unpackerIn() + val bundle = StackBundle("stack {}", mapOf("a.txt" to ResourceContent.text("a"))) + + val first = unpacker.unpack("films", bundle) + val second = unpacker.unpack("films", bundle) + + (first.root == second.root) shouldBe false + base.listDirectoryEntries().size shouldBe 2 + } + + it("needs no directory for a bundle with no resources") { + val (unpacker, base) = unpackerIn() + val resources = unpacker.unpack("films", StackBundle.scriptOnly("stack {}")) + + resources.hasBundle shouldBe false + base.listDirectoryEntries().shouldBeEmpty() + } + + it("deletes the unpacked directory when the stack goes away") { + val (unpacker, _) = unpackerIn() + val resources = unpacker.unpack("films", StackBundle("stack {}", mapOf("a.txt" to ResourceContent.text("a")))) + val root = resources.root!! + + resources.delete() + + root.exists() shouldBe false + } + } + + describe("refusing a bundle that would write outside its directory") { + // This is zip-slip: the attack is in the entry name, and the file lands + // wherever that name points unless the server checks first. + listOf( + "../escaped.txt", + "nested/../../escaped.txt", + "/etc/cron.d/evil", + "C:\\Windows\\evil.txt" + ).forEach { hostilePath -> + it("rejects the entry '$hostilePath'") { + val (unpacker, base) = unpackerIn() + shouldThrow { + unpacker.unpack("films", StackBundle("stack {}", mapOf(hostilePath to ResourceContent.text("pwned")))) + } + // Nothing at all was written - not the hostile entry, and not the + // directory that would have held it. + base.listDirectoryEntries().shouldBeEmpty() + base.parent.resolve("escaped.txt").exists() shouldBe false + } + } + + it("rejects a bundle where one entry is hostile, leaving none of it on disk") { + val (unpacker, base) = unpackerIn() + shouldThrow { + unpacker.unpack( + "films", StackBundle( + "stack {}", mapOf( + "fine.csv" to ResourceContent.text("ok"), + "../escaped.txt" to ResourceContent.text("pwned") + ) + ) + ) + } + base.listDirectoryEntries().shouldBeEmpty() + } + } + + describe("size limits") { + it("rejects a resource over the per-file limit, naming the file and the limit") { + val (unpacker, base) = unpackerIn() + val tooBig = "x".repeat((StackBundleLimits.MAX_RESOURCE_BYTES + 1).toInt()) + + val error = shouldThrow { + unpacker.unpack("films", StackBundle("stack {}", mapOf("huge.csv" to ResourceContent.text(tooBig)))) + } + + error.message!!.shouldContain("huge.csv") + error.message!!.shouldContain(StackBundleLimits.MAX_RESOURCE_BYTES.toString()) + base.listDirectoryEntries().shouldBeEmpty() + } + + it("rejects a bundle with too many files") { + val (unpacker, _) = unpackerIn() + val resources = (0..StackBundleLimits.MAX_RESOURCE_COUNT) + .associate { "file-$it.txt" to ResourceContent.text("x") } + + shouldThrow { + unpacker.unpack("films", StackBundle("stack {}", resources)) + }.message!!.shouldContain(StackBundleLimits.MAX_RESOURCE_COUNT.toString()) + } + + it("rejects content flagged as base64 that isn't") { + val (unpacker, _) = unpackerIn() + shouldThrow { + unpacker.unpack( + "films", + StackBundle("stack {}", mapOf("blob.bin" to ResourceContent("not ** base64", com.orbitalhq.nebula.core.ResourceEncoding.BASE64))) + ) + }.message!!.shouldContain("blob.bin") + } + } + + describe("fingerprints") { + it("is the same for the same files whatever order they arrive in") { + val a = mapOf("a.txt" to ResourceContent.text("1"), "b.txt" to ResourceContent.text("2")) + val b = mapOf("b.txt" to ResourceContent.text("2"), "a.txt" to ResourceContent.text("1")) + + StackBundleUnpacker.fingerprintOf(a) shouldBe StackBundleUnpacker.fingerprintOf(b) + } + + it("differs when a file's content changes") { + val before = StackBundleUnpacker.fingerprintOf(mapOf("a.txt" to ResourceContent.text("1"))) + val after = StackBundleUnpacker.fingerprintOf(mapOf("a.txt" to ResourceContent.text("2"))) + + (before == after) shouldBe false + } + + it("is empty when there are no resources, so script-only submissions dedupe as before") { + StackBundleUnpacker.fingerprintOf(emptyMap()) shouldBe "" + } + } +}) diff --git a/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackResourcesTest.kt b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackResourcesTest.kt new file mode 100644 index 0000000..cdce047 --- /dev/null +++ b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackResourcesTest.kt @@ -0,0 +1,181 @@ +package com.orbitalhq.nebula.resources + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText + +/** + * The containment rules that keep a stack reading only the files that were + * shipped with it. + * + * A Nebula server runs stacks submitted by any project connected to Orbital, and + * its own filesystem holds credentials that have nothing to do with those + * projects. Everything below is a way someone might try to walk out of the + * bundle directory and reach one. + */ +class StackResourcesTest : DescribeSpec({ + + /** A bundle directory with a nested file, plus a "secret" living outside it. */ + fun bundleWithSecretOutside(): Pair { + val tempDir = Files.createTempDirectory("stack-resources-test") + val bundle = tempDir.resolve("bundle").createDirectories() + bundle.resolve("sales.csv").writeText("Title,Tickets\nA New Hope,249\n") + bundle.resolve("specs").createDirectories() + bundle.resolve("specs/orders.yaml").writeText("openapi: 3.0.0") + + val secrets = tempDir.resolve("secrets").createDirectories() + secrets.resolve("id_rsa").writeText("PRIVATE KEY") + + return StackResources.at(bundle) to tempDir + } + + describe("reading resources that were shipped with the stack") { + it("reads a file at the root of the bundle") { + val (resources, _) = bundleWithSecretOutside() + resources.readText("sales.csv") shouldContain "A New Hope" + } + + it("reads a file in a subdirectory") { + val (resources, _) = bundleWithSecretOutside() + resources.readText("specs/orders.yaml") shouldBe "openapi: 3.0.0" + } + + it("lists what the stack shipped with") { + val (resources, _) = bundleWithSecretOutside() + resources.names.shouldContainExactly("sales.csv", "specs/orders.yaml") + } + + it("normalises a path that stays inside the bundle") { + val (resources, _) = bundleWithSecretOutside() + // specs/../sales.csv resolves back inside the bundle, but we reject it + // anyway: allowing ".." at all means relying on normalisation being + // right everywhere, and a legitimate stack never needs it. + shouldThrow { + resources.readText("specs/../sales.csv") + }.message!!.shouldContain("'..' segment") + } + + it("reports what is available when a resource is missing") { + val (resources, _) = bundleWithSecretOutside() + val error = shouldThrow { resources.readText("nope.csv") } + error.message!!.shouldContain("No resource named 'nope.csv'") + error.message!!.shouldContain("sales.csv") + } + } + + describe("refusing to read outside the bundle") { + it("rejects an absolute path") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { + resources.readText("/etc/ssh/id_rsa") + }.message!!.shouldContain("is absolute") + } + + it("rejects a relative path that climbs out with ..") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { + resources.readText("../secrets/id_rsa") + }.message!!.shouldContain("'..' segment") + } + + it("rejects a deeply nested climb that ends outside the bundle") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { + resources.readText("specs/../../secrets/id_rsa") + }.message!!.shouldContain("'..' segment") + } + + it("rejects a Windows-style absolute path, even on a unix host") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { + resources.readText("C:\\Users\\admin\\id_rsa") + }.message!!.shouldContain("is absolute") + } + + it("rejects a blank path") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { resources.readText(" ") } + } + + it("refuses to follow a symlink planted inside the bundle") { + val (resources, tempDir) = bundleWithSecretOutside() + val root = resources.root!! + // Nothing in the path string is suspicious - the escape is entirely in + // the filesystem, which is why normalising the string is not enough. + Files.createSymbolicLink(root.resolve("innocent.pem"), tempDir.resolve("secrets/id_rsa")) + + shouldThrow { + resources.readText("innocent.pem") + }.message!!.shouldContain("outside this stack's resource bundle") + } + + it("refuses to follow a symlinked directory that leads out of the bundle") { + val (resources, tempDir) = bundleWithSecretOutside() + val root = resources.root!! + Files.createSymbolicLink(root.resolve("elsewhere"), tempDir.resolve("secrets")) + + shouldThrow { + resources.readText("elsewhere/id_rsa") + }.message!!.shouldContain("outside this stack's resource bundle") + } + + it("still reads a symlink that stays inside the bundle") { + val (resources, _) = bundleWithSecretOutside() + val root = resources.root!! + Files.createSymbolicLink(root.resolve("latest.csv"), root.resolve("sales.csv")) + + resources.readText("latest.csv") shouldContain "A New Hope" + } + } + + describe("a stack submitted without a bundle") { + it("explains why there is nothing to read") { + shouldThrow { + StackResources.NONE.readText("sales.csv") + }.message!!.shouldContain("submitted without a resource bundle") + } + + it("leaves file paths given to providers exactly as they were") { + // Stacks that predate bundles pass paths on the Nebula server's own + // filesystem; without a bundle nothing changes for them. + StackResources.NONE.resolveFilePath("/data/sales.csv") shouldBe "/data/sales.csv" + StackResources.NONE.resolveFilePath("sales.csv") shouldBe "sales.csv" + } + } + + describe("resolving paths handed to providers") { + it("resolves a relative path through the bundle") { + val (resources, _) = bundleWithSecretOutside() + resources.resolveFilePath("specs/orders.yaml") shouldBe + resources.root!!.resolve("specs/orders.yaml").toString() + } + + it("leaves an absolute path pointing at the host, as it always has") { + val (resources, _) = bundleWithSecretOutside() + resources.resolveFilePath("/mnt/data/sales.csv") shouldBe "/mnt/data/sales.csv" + } + + it("rejects a relative path that escapes the bundle") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { resources.resolveFilePath("../secrets/id_rsa") } + } + } + + describe("fingerprinting") { + it("changes when a resource's content changes") { + val dir = Files.createTempDirectory("fingerprint-test") + dir.resolve("data.csv").writeText("a") + val before = StackResources.at(dir).fingerprint + + dir.resolve("data.csv").writeText("b") + StackResources.at(dir).fingerprint shouldBe StackResources.at(dir).fingerprint + (StackResources.at(dir).fingerprint == before) shouldBe false + } + } +}) diff --git a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/NebulaScriptExecutor.kt b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/NebulaScriptExecutor.kt index e3f81bc..0be7615 100644 --- a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/NebulaScriptExecutor.kt +++ b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/NebulaScriptExecutor.kt @@ -8,6 +8,10 @@ import com.orbitalhq.nebula.HostConfig import com.orbitalhq.nebula.NebulaScript import com.orbitalhq.nebula.NebulaStack import com.orbitalhq.nebula.NebulaStackWithSource +import com.orbitalhq.nebula.core.StackBundle +import com.orbitalhq.nebula.resources.InvalidStackBundleException +import com.orbitalhq.nebula.resources.StackBundleUnpacker +import com.orbitalhq.nebula.resources.StackResourcesContext import io.github.oshai.kotlinlogging.KotlinLogging import java.io.File import kotlin.script.experimental.api.EvaluationResult @@ -22,7 +26,14 @@ import kotlin.script.experimental.jvm.util.isError import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost import kotlin.script.experimental.jvmhost.createJvmCompilationConfigurationFromTemplate -class NebulaScriptExecutor(private val logCompilationErrors: Boolean = true) { +class NebulaScriptExecutor( + private val logCompilationErrors: Boolean = true, + /** + * Writes the resources of a submitted bundle into a directory of their own. + * Injectable so tests can point bundles at a temp directory they control. + */ + private val bundleUnpacker: StackBundleUnpacker = StackBundleUnpacker() +) { companion object { private val logger = KotlinLogging.logger {} } @@ -99,6 +110,47 @@ class NebulaScriptExecutor(private val logCompilationErrors: Boolean = true) { .map { NebulaStackWithSource(it, string, hostConfig) } } + /** + * Compiles a stack submitted as a bundle: the script plus the files that + * shipped with it. + * + * The bundle is unpacked first, and bound for the duration of the evaluation, + * because the `stack { }` block runs *during* compilation - that's when + * `s3 { file("data/sales.csv") }` and friends resolve their paths. + * + * Anything wrong with the bundle itself (an entry that escapes the bundle + * directory, a file over the size limit) is reported as a compilation error, + * so it reaches the submitter through the same channel as a syntax error, + * rather than as an opaque 500. + * + * If compilation fails the unpacked directory is removed - a rejected + * submission must not leave files behind. + */ + fun compileBundle( + stackName: String, + bundle: StackBundle, + hostConfig: HostConfig + ): Either { + val resources = try { + bundleUnpacker.unpack(stackName, bundle) + } catch (e: InvalidStackBundleException) { + logger.warn { "Rejecting bundle for stack '$stackName' - ${e.message}" } + return NebulaCompilationException.forSyntheticDiagnostic( + e.message ?: "The resources submitted with this stack could not be unpacked" + ).left() + } + + return StackResourcesContext.withResources(resources) { + compileStack(bundle.script.toScriptSource()) + }.fold( + { compilationError -> + resources.delete() + compilationError.left() + }, + { stack -> NebulaStackWithSource(stack, bundle.script, hostConfig).right() } + ) + } + @Deprecated("call compileToStackWithSource") fun toStackWithSource(string: String, hostConfig: HostConfig): NebulaStackWithSource { val source = string.toScriptSource() diff --git a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt index 062f59d..ac3f2be 100644 --- a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt +++ b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt @@ -1,5 +1,6 @@ package com.orbitalhq.nebula.runtime.server +import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue @@ -9,6 +10,7 @@ import com.orbitalhq.nebula.NebulaStack import com.orbitalhq.nebula.NebulaStackWithSource import com.orbitalhq.nebula.StackName import com.orbitalhq.nebula.StackRunner +import com.orbitalhq.nebula.core.StackBundle import com.orbitalhq.nebula.core.StackStateEvent import com.orbitalhq.nebula.runtime.NebulaScriptExecutor import io.github.oshai.kotlinlogging.KotlinLogging @@ -96,6 +98,30 @@ class NebulaServer( stackExecutor.submit(stack, startAsync = true) call.respond(stack.name) } + // Submit a stack together with the files it reads at startup. + // Additive: the script-only routes above are untouched, and a + // bundle carrying no resources behaves identically to them. + post("/bundle") { + val bundle = call.receive() + // The name is assigned by the runner, but the unpacker wants + // something to name the directory after before we have one. + submitBundle(call, "unnamed-stack", bundle) { stack -> + stackExecutor.submit(stack, startAsync = true) + call.respond(stack.name) + } + } + put("/bundle/{id}") { + val id = call.parameters["id"] ?: return@put call.respond( + HttpStatusCode.BadRequest, + "Missing or malformed id" + ) + val bundle = call.receive() + submitBundle(call, id, bundle) { stack -> + val named = stack.withName(id) + stackExecutor.submit(named, id) + call.respond(named.name) + } + } put("/{id}") { val id = call.parameters["id"] ?: return@put call.respond( HttpStatusCode.BadRequest, @@ -250,12 +276,12 @@ class NebulaServer( // the other stacks in the submission. Failures are reported back to // the client as a StackStateEvent carrying the compilation errors, // and recorded so the admin snapshot shows them too. - val eventStreams = updateStacksRequest.stacks.mapNotNull { (name, script) -> - scriptExecutor.compileToStackWithSource(script, call.hostConfig()).fold( + val eventStreams = updateStacksRequest.allBundles().mapNotNull { (name, bundle) -> + scriptExecutor.compileBundle(name, bundle, call.hostConfig()).fold( { compilationException -> val errors = compilationException.errors.map { it.toCompilationError() } logger.warn { "Stack $name failed to compile: ${errors.joinToString { it.message }}" } - failedSubmissions[name] = FailedSubmission(name, script, errors) + failedSubmissions[name] = FailedSubmission(name, bundle.script, errors) send( Frame.Text( objectMapper.writeValueAsString( @@ -300,6 +326,37 @@ class NebulaServer( }.start(wait = wait) } + /** + * Compiles a submitted bundle and hands the compiled stack to [onCompiled], + * or answers 422 with the compilation errors. + * + * Bundle problems (a resource that escapes the bundle directory, a file over + * the size limit) arrive here as compilation errors, so the caller gets the + * same shaped response whether the script or its resources were the problem. + */ + private suspend fun submitBundle( + call: ApplicationCall, + stackName: StackName, + bundle: StackBundle, + onCompiled: suspend (NebulaStackWithSource) -> Unit + ) { + scriptExecutor.compileBundle(stackName, bundle, call.hostConfig()).fold( + { exception -> + val failure = FailedSubmission( + stackName, + bundle.script, + exception.errors.map { it.toCompilationError() } + ) + failedSubmissions[stackName] = failure + call.respond(HttpStatusCode.UnprocessableEntity, failure) + }, + { stack -> + failedSubmissions.remove(stackName) + onCompiled(stack) + } + ) + } + private suspend fun handleStackAction( call: ApplicationCall, action: (id: String) -> Unit @@ -341,7 +398,33 @@ data class StackEventStreamRequest(val stackId: StackName) typealias StackScript = String -data class UpdateStackRSocketRequest(val stacks: Map) +/** + * A submission of one or more stacks over `/stream/stacks`. + * + * Accepts both shapes: + * + * - `{"stacks": {"name": "