diff --git a/CLAUDE.md b/CLAUDE.md index bb79204..1d9d57b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,6 +65,16 @@ The DSL supports: Kafka, HTTP servers, SQL databases (PostgreSQL/MySQL/Oracle wi - Default imports provide common utilities - Components implement `InfrastructureComponent<*>` interface +### Stack bundles +- A stack can be submitted with the files it reads (`StackBundle` in nebula-api) + via `POST /stacks/bundle`, `PUT /stacks/bundle/{id}`, or the `bundles` field of + the `/stream/stacks` websocket message. The script-only routes are unchanged. +- Resources unpack into a per-submission temp directory. `StackResources` + (nebula-dsl) confines every lookup to that directory; scripts reach it through + `resources` inside `stack { }`, and providers that read files resolve relative + paths through `StackResources.resolveFilePath`. +- See `docs/pages/bundles.mdx` and §8 of `NEBULA-AGENT.md`. + ### Testing - Uses Kotest framework with descriptive specifications - TestContainers for integration testing diff --git a/NEBULA-AGENT.md b/NEBULA-AGENT.md index 15d815d..f1e3dbb 100644 --- a/NEBULA-AGENT.md +++ b/NEBULA-AGENT.md @@ -30,7 +30,7 @@ version: 0.1.0 sourceRoot: src/ additionalSources: { "@orbital/config" : "orbital/config/*.conf", - "@orbital/nebula" : "orbital/nebula/*.nebula.kts" + "@orbital/nebula" : "orbital/nebula/**" } dependencies: { "com.orbitalhq/core" : "github:orbitalapi/orbital-core-taxi#0.34.0" @@ -40,8 +40,8 @@ dependencies: { **Rules:** - The `@orbital/nebula` key under `additionalSources` is **mandatory**. Without it Orbital will not discover the stack. - By convention the file lives at `orbital/nebula/stack.nebula.kts` (preferred) or `nebula/stack.nebula.kts`. -- Declare the glob with a wildcard (`*.nebula.kts`), not a single explicit file — future stacks in the same folder will be picked up automatically. -- If `taxi.conf` already exists and lacks the `@orbital/nebula` entry, add it. Do not remove unrelated `additionalSources` keys. +- **Declare the glob as `orbital/nebula/**`**, not `orbital/nebula/*.nebula.kts`. The whole directory is the stack's bundle: `*.nebula.kts` files are stacks, and every other file is a resource shipped alongside them (see §8). The narrower `*.nebula.kts` glob still works, but then the stack ships no files. +- If `taxi.conf` already exists with the older `orbital/nebula/*.nebula.kts` entry, widen it to `orbital/nebula/**` when the stack needs to read a file. Do not remove unrelated `additionalSources` keys. **Minimal project layout:** @@ -52,7 +52,9 @@ my-project/ │ └── ... (taxi files) └── orbital/ └── nebula/ - └── stack.nebula.kts + ├── stack.nebula.kts + └── data/ + └── sales.csv <- shipped with the stack, see §8 ``` --- @@ -335,7 +337,8 @@ Empire Strikes Back,LXILO,189 """) } bucket("reports") { - file("/absolute/path/to/local.csv") // read from disk + file("data/sales.csv") // shipped with the stack (see §8) + file("/absolute/path/to/local.csv") // read from the Nebula server's disk file("large.csv", generateSequence()) // Sequence for large/streamed files } } @@ -352,9 +355,11 @@ Inside a `bucket(name)` block: ```kotlin fun file(name: String, content: String) // inline content fun file(name: String, content: Sequence) // streaming — for large files -fun file(path: String) // read file from local disk; uses the filename as the S3 key +fun file(path: String) // read file from disk; uses the filename as the S3 key ``` +`file(path)` resolves a **relative** path against the files shipped with the stack (§8), and an **absolute** path against the Nebula server's own filesystem. Prefer the relative form — the Nebula server is a different machine from the project, so an absolute path almost never points where you think it does. + Use the `Sequence` form for files larger than a few MB (the seeder uses S3 multipart upload — minimum part size is 5 MB, which is already handled by default). ### 5.6 `hazelcast { ... }` — Hazelcast @@ -507,10 +512,88 @@ stack { --- -## 8. Quick checklist before returning a stack +## 8. Shipping files with a stack + +A stack often needs a file: an OpenAPI spec to import, a CSV to seed a bucket, a +fixture to serve. **Those files go in the project's nebula directory, next to the +script**, and are submitted to Nebula along with it. + +This matters because the Nebula server is a different machine from the project. +A path like `/home/jane/project/sales.csv` in a script means *Nebula's* +filesystem, where the file does not exist. + +### Layout + +Everything under the `@orbital/nebula` directory that is **not** a `.nebula.kts` +file is a resource. Resources are shared by every stack in that directory. + +``` +orbital/nebula/ +├── stack.nebula.kts +├── data/ +│ └── sales.csv +└── specs/ + └── orders.yaml +``` + +This requires the wider glob in `taxi.conf` (§2): + +```hocon +"@orbital/nebula" : "orbital/nebula/**" +``` + +With the older `orbital/nebula/*.nebula.kts` glob the script is still found, but +no resources are shipped and `resources` will be empty. + +### Reading them + +Inside `stack { }`, `resources` gives you the files: + +```kotlin +stack { + val sales = resources.readText("data/sales.csv") + + http { + get("/sales") { call -> call.respondText(sales) } + } +} +``` + +| Function | Returns | +|---|---| +| `resources.readText(path)` | the file's text, as UTF-8 | +| `resources.readBytes(path)` | the file's bytes | +| `resources.path(path)` | a `java.nio.file.Path` to the file | +| `resources.exists(path)` | whether the stack shipped that file | +| `resources.names` | every file the stack shipped | +| `resources.hasBundle` | whether this stack was submitted with files at all | + +Paths are relative to the nebula directory — `data/sales.csv` above, not +`orbital/nebula/data/sales.csv`. + +Providers that read a file (`s3 { file(path) }`, and API Gateway's +`restApiFromFile(...)`) resolve a **relative** path the same way, so you can pass +`"data/sales.csv"` straight to them. + +**Rules:** +- Use relative paths. Absolute paths mean the Nebula server's own filesystem, and + almost never point where you want. +- A path may not contain `..`, and may not escape the nebula directory. Both are + rejected, so don't try to reach a file elsewhere in the project — copy it into + the nebula directory instead. +- Limits: 10 MB per file, 50 MB per stack, 500 files. For anything larger, mount + a volume into the Nebula container. +- Resources are text by default. Binary files work, but Orbital ships a project's + files as text, so prefer text formats (CSV, JSON, YAML) for stacks Orbital + submits. + +--- + +## 9. Quick checklist before returning a stack - [ ] File is named `stack.nebula.kts` (unless user specified otherwise) and placed at `orbital/nebula/stack.nebula.kts`. -- [ ] `taxi.conf` contains `"@orbital/nebula" : "orbital/nebula/*.nebula.kts"` under `additionalSources`. +- [ ] `taxi.conf` contains `"@orbital/nebula" : "orbital/nebula/**"` under `additionalSources`. +- [ ] Any file the stack reads sits under `orbital/nebula/`, and is read with a relative path (`resources.readText("data/sales.csv")`, `file("data/sales.csv")`) — never an absolute one. - [ ] Exactly one top-level `stack { }` block. - [ ] No re-imports of `stack`, `Random`, `io.ktor.server.request.*`, `io.ktor.server.response.*`. - [ ] Duplicate component types (two Postgres, two S3, etc.) each have distinct `componentName`s. diff --git a/docs/pages/_meta.ts b/docs/pages/_meta.ts index a015bea..7ce7b6c 100644 --- a/docs/pages/_meta.ts +++ b/docs/pages/_meta.ts @@ -1,6 +1,7 @@ export default { "index": "Getting started", "stacks": "Working with stacks", + "bundles": "Shipping files with a stack", "cli": "CLI & networking", "logging" : "Logging", "examples": "Examples", diff --git a/docs/pages/bundles.mdx b/docs/pages/bundles.mdx new file mode 100644 index 0000000..2eb5694 --- /dev/null +++ b/docs/pages/bundles.mdx @@ -0,0 +1,152 @@ +--- +title: Shipping files with a stack +--- + +## Shipping files with a stack + +A stack script often needs a file: an OpenAPI spec to import into API Gateway, a +CSV to seed an S3 bucket, a fixture to serve from an HTTP endpoint. + +Those files live in the project that defines the stack — on a developer's laptop, +or in a git repo that Orbital has checked out. The Nebula server is somewhere +else entirely, usually a different container. A path in a script like +`file("/home/jane/project/sales.csv")` means *the Nebula server's* filesystem, +which is not where the file is. + +A **stack bundle** fixes that. It is the script plus the files it reads, +submitted together. Nebula unpacks the files into a directory belonging to that +submission, and the stack reads them from there. + +### Submitting a bundle + +`POST /stacks/bundle` creates a stack; `PUT /stacks/bundle/{id}` replaces one. +Both take JSON: + +```json +{ + "script": "stack {\n s3 { bucket(\"tickets\") { file(\"data/sales.csv\") } }\n}", + "resources": { + "data/sales.csv": { "content": "Title,Tickets\nA New Hope,249\n" }, + "specs/orders.yaml": { "content": "openapi: 3.0.0\n..." } + } +} +``` + +```bash +curl --location 'http://localhost:8099/stacks/bundle' \ + --header 'Content-Type: application/json' \ + --data @bundle.json +``` + +The script-only routes (`POST /stacks`, `PUT /stacks/{id}`) are unchanged, and a +bundle with no `resources` behaves exactly as they do. + +Orbital submits bundles over the `/stream/stacks` websocket rather than over +HTTP. That message accepts both shapes: + +```json +{ "stacks": { "films": "stack {}" } } +{ "bundles": { "films": { "script": "stack {}", "resources": { ... } } } } +``` + +### Resource encoding + +Each resource is `{ "content": "...", "encoding": "TEXT" | "BASE64" }`. +`encoding` defaults to `TEXT`, where the content is the file's text and is +written out as UTF-8. Binary files (a `.parquet`, an image) are sent as `BASE64`. + +Orbital only ever sends `TEXT`: the files reach it through the schema pipeline, +which carries a project's additional sources as text. Submit `BASE64` directly if +you need binary content. + +### Limits + +| Limit | Value | +|---|---| +| Largest single file | 10 MB | +| Largest bundle | 50 MB | +| Files per bundle | 500 | + +Exceeding any of these fails the submission with a message naming the file and +the limit. Bundles are for fixtures and specs — if you need gigabytes of seed +data, mount a volume into the Nebula container instead. + +### Reading resources from a script + +Inside `stack { }`, `resources` gives you the files that were shipped: + +```kotlin +stack { + val sales = resources.readText("data/sales.csv") + val spec = resources.readText("specs/orders.yaml") + + http { + get("/sales") { call -> call.respondText(sales) } + } +} +``` + +| Function | Returns | +|---|---| +| `resources.readText(path)` | the file's text, as UTF-8 | +| `resources.readBytes(path)` | the file's bytes | +| `resources.path(path)` | a `java.nio.file.Path` to the unpacked file | +| `resources.exists(path)` | whether the stack shipped that file | +| `resources.names` | every file the stack shipped, as relative paths | +| `resources.hasBundle` | whether this stack was submitted with a bundle | + +Paths are relative to the bundle root, which is the project's nebula directory — +so `data/sales.csv` in the script means `orbital/nebula/data/sales.csv` in the +project. + +### Providers that read files + +Any provider that reads a file resolves a **relative** path through the bundle: + +```kotlin +stack { + s3 { + bucket("tickets") { + file("data/sales.csv") // read from the bundle + file("/mnt/shared/big.csv") // read from the Nebula server, as before + } + } +} +``` + +An **absolute** path keeps its existing meaning — a file on the machine running +Nebula. Stacks written before bundles existed, and stacks run from the CLI +against local files, are unaffected. + +Resolution happens while the script is compiled, so a typo in a resource name +fails the submission with a message listing what the stack actually shipped, +rather than failing later, mid-startup. + +### What a stack may read + +Every resource lookup is confined to the bundle directory: + +- absolute paths are rejected, +- `..` segments are rejected, +- the resolved path is turned into a real path — following symlinks — and + re-checked to still be inside the bundle. + +So a bundle entry named `../../etc/cron.d/evil` is rejected before anything is +written, and a symlink planted inside a bundle that points at +`/etc/ssh/id_rsa` is refused when the stack tries to read it. + +This confines *resource* access. A Nebula script is arbitrary Kotlin and can +always open a file directly; the guarantee is that a bundle-relative path +resolves to a file inside the bundle, nothing more and nothing less. Only submit +stacks you trust to a Nebula server. + +### Lifetime + +Each submission gets its own directory under the system temp directory. It is +deleted when the stack is replaced by a different submission, when the stack is +removed, and when a submission is rejected. Stopping a stack keeps its bundle, +because a stopped stack can be started again. + +A stack is *replaced* when its script **or** any of its resources change — so +editing a CSV next to an unchanged script restarts the stack, rather than being +skipped as a duplicate submission. diff --git a/docs/pages/providers/s3.mdx b/docs/pages/providers/s3.mdx index afd6156..4b221ce 100644 --- a/docs/pages/providers/s3.mdx +++ b/docs/pages/providers/s3.mdx @@ -16,6 +16,26 @@ stack { } ``` +### Seeding a bucket from a file + +`file(path)` uploads a file from disk, using its filename as the S3 key. + +```kotlin +stack { + s3 { + bucket("tickets") { + file("data/sales.csv") // shipped with the stack + file("/mnt/shared/big.csv") // on the machine running Nebula + } + } +} +``` + +A **relative** path is read from the bundle submitted with the stack, so the CSV +travels with the project rather than having to exist on the Nebula server — see +[Shipping files with a stack](/bundles). An **absolute** path is read from the +machine running Nebula, which is what it has always meant. + ### Returned values When an `s3` component is declared, the following data is returned: diff --git a/docs/pages/stacks.mdx b/docs/pages/stacks.mdx index 23f6e54..971d408 100644 --- a/docs/pages/stacks.mdx +++ b/docs/pages/stacks.mdx @@ -32,6 +32,13 @@ curl --location 'http://localhost:8099/stacks' \ }' ``` +### Creating a stack that ships files + +A script that reads a file — a CSV to seed a bucket, an OpenAPI spec — needs that +file to travel with it, because the Nebula server's filesystem is not the +project's. POST a bundle to `/stacks/bundle` instead, and read the files through +`resources` inside the script. See [Shipping files with a stack](/bundles). + ### Listing the state of current stacks Fetch the current stacks (including their configured variables) by GET `/stacks` diff --git a/nebula-api/src/main/kotlin/com/orbitalhq/nebula/core/StackBundle.kt b/nebula-api/src/main/kotlin/com/orbitalhq/nebula/core/StackBundle.kt new file mode 100644 index 0000000..f9ed14b --- /dev/null +++ b/nebula-api/src/main/kotlin/com/orbitalhq/nebula/core/StackBundle.kt @@ -0,0 +1,103 @@ +package com.orbitalhq.nebula.core + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import java.util.Base64 + +/** + * A path to a file inside a [StackBundle], relative to the bundle root. + * + * Always uses `/` as the separator, and must never be absolute or contain + * `..` segments - see `StackResources` in nebula-dsl, which enforces this + * when the bundle is unpacked and again whenever a stack reads from it. + */ +typealias ResourcePath = String + +/** + * How the [ResourceContent.content] string encodes the underlying bytes. + */ +enum class ResourceEncoding { + /** The content is the file's text, and is written out as UTF-8. */ + TEXT, + + /** The content is the file's bytes, Base64 encoded. */ + BASE64 +} + +/** + * The content of a single file shipped alongside a stack script. + * + * Text is carried as text (so bundles stay readable and diffable on the wire); + * binary files are Base64 encoded and flagged via [encoding]. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class ResourceContent( + val content: String, + val encoding: ResourceEncoding = ResourceEncoding.TEXT +) { + companion object { + @JvmStatic + fun text(content: String): ResourceContent = ResourceContent(content, ResourceEncoding.TEXT) + + @JvmStatic + fun binary(bytes: ByteArray): ResourceContent = + ResourceContent(Base64.getEncoder().encodeToString(bytes), ResourceEncoding.BASE64) + } + + /** + * The decoded bytes of this resource, as they will be written to disk when + * the bundle is unpacked. + */ + fun bytes(): ByteArray = when (encoding) { + ResourceEncoding.TEXT -> content.toByteArray(Charsets.UTF_8) + ResourceEncoding.BASE64 -> Base64.getDecoder().decode(content) + } +} + +/** + * A stack script plus the resource files (OpenAPI specs, CSV seed data, ...) + * that it reads at startup. + * + * Submitting a bundle rather than a bare script is what lets a stack read files + * that live in the *submitter's* project, rather than on the Nebula server's own + * filesystem. The resources are unpacked into a per-submission temporary + * directory, and every read from a stack is confined to that directory. + * + * Bundles are a superset of the existing script-only submission: a bundle with + * no resources behaves exactly as `POST /stacks` with the script text does. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class StackBundle( + val script: String, + val resources: Map = emptyMap() +) { + companion object { + /** A bundle carrying just a script, equivalent to the legacy script-only submission. */ + @JvmStatic + fun scriptOnly(script: String): StackBundle = StackBundle(script, emptyMap()) + } + + /** Derived, so it must not be written to (or read from) the wire. */ + @get:JsonIgnore + val hasResources: Boolean + get() = resources.isNotEmpty() +} + +/** + * The limits applied when a [StackBundle] is unpacked. + * + * Bundles travel as JSON over the same websocket that carries stack scripts, and + * are written to the Nebula server's disk, so both the request size and the + * unpacked footprint are capped. Exceeding any limit fails the submission with a + * message naming the offending file and the limit. + */ +object StackBundleLimits { + /** Largest decoded size of any single resource. */ + const val MAX_RESOURCE_BYTES: Long = 10L * 1024 * 1024 + + /** Largest total decoded size of all resources in one bundle. */ + const val MAX_BUNDLE_BYTES: Long = 50L * 1024 * 1024 + + /** Largest number of files in one bundle. */ + const val MAX_RESOURCE_COUNT: Int = 500 +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfraDsl.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfraDsl.kt index 714d0b5..a716aee 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfraDsl.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/InfraDsl.kt @@ -1,5 +1,7 @@ package com.orbitalhq.nebula +import com.orbitalhq.nebula.resources.StackResources + /** * Base interface for adding DSL support within the script. * @@ -11,5 +13,18 @@ package com.orbitalhq.nebula */ interface InfraDsl { val components: List> + + /** + * The files shipped alongside this stack's script. + * + * Exposed on the DSL (rather than handed to each component) so that any + * provider which reads a file - `s3 { file(...) }`, and API Gateway's + * `restApiFromFile(...)` - can resolve a path through the submitting + * project's bundle rather than the Nebula server's own filesystem. + * + * [StackResources.NONE] when the stack was submitted without a bundle. + */ + val resources: StackResources + fun > add(component: T): T -} \ No newline at end of file +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/NebulaStack.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/NebulaStack.kt index b974ed0..42635e5 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/NebulaStack.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/NebulaStack.kt @@ -9,6 +9,8 @@ import com.orbitalhq.nebula.kafka.KafkaDsl import com.orbitalhq.nebula.logging.LogMessage import com.orbitalhq.nebula.logging.StackLogStream import com.orbitalhq.nebula.mongo.MongoDsl +import com.orbitalhq.nebula.resources.StackResources +import com.orbitalhq.nebula.resources.StackResourcesContext import com.orbitalhq.nebula.s3.S3Dsl import com.orbitalhq.nebula.sql.SqlDsl import com.orbitalhq.nebula.taxi.TaxiPublisherDsl @@ -29,11 +31,34 @@ data class NebulaStackWithSource( } val name = stack.name + + /** + * The files shipped with this submission, unpacked and confined to their own + * directory. [StackResources.NONE] for a script-only submission. + */ + val resources: StackResources + get() = stack.resources + + /** + * Identity of a submission: the script text *plus* the content of the + * resources shipped with it. + * + * [StackRunner] dedupes on this rather than on [source] alone, so editing a + * CSV or an OpenAPI spec next to an unchanged script still replaces the + * running stack instead of being skipped as a duplicate submission. + */ + val submissionKey: String + get() = source + "\u0000" + stack.resources.fingerprint } class NebulaStack( val name: StackName = NameGenerator.generateName(), - initialComponents: List> = emptyList() + initialComponents: List> = emptyList(), + /** + * The files shipped alongside this stack's script. Bound by the executor when + * the stack was submitted as a bundle; [StackResources.NONE] otherwise. + */ + override val resources: StackResources = StackResources.NONE ) : InfraDsl, KafkaDsl, S3Dsl, HttpDsl, SqlDsl, HazelcastDsl, MongoDsl, TaxiPublisherDsl { companion object { private val logger = KotlinLogging.logger {} @@ -53,7 +78,7 @@ class NebulaStack( } fun withName(name: StackName): NebulaStack { - return NebulaStack(name, this._components) + return NebulaStack(name, this._components, this.resources) } private val logStream = StackLogStream() @@ -104,6 +129,25 @@ class NebulaStack( } } +/** + * Declares a stack. + * + * Picks up the resource bundle bound for the current thread by the executor, so + * a script submitted as a bundle can read the files that shipped with it (via + * `resources`, or by giving a relative path to any provider that reads a file) + * without having to declare anything. + */ fun stack(init: NebulaStack.() -> Unit): NebulaStack { - return NebulaStack().apply(init) + return NebulaStack(resources = StackResourcesContext.current()).apply(init) +} + +/** + * Declares a stack with an explicit set of resources. + * + * Scripts don't use this - they get their bundle from the submission. It exists + * for hosts (the CLI, tests) that build a stack in-process and want to point it + * at a directory of files. + */ +fun stack(resources: StackResources, init: NebulaStack.() -> Unit): NebulaStack { + return NebulaStack(resources = resources).apply(init) } \ No newline at end of file diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt index 8f8f08d..7c4bfff 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/StackRunner.kt @@ -61,12 +61,22 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) { fun submit(submittedStack: NebulaStackWithSource, name: StackName = submittedStack.name, startAsync: Boolean = false): Flux { val storedStack = this.stacks.compute(name) { key, existingSpec -> if (existingSpec != null) { - if (existingSpec.source == submittedStack.source) { + // Compared on the submission key, not the script alone: a stack whose + // script is unchanged but whose bundled resources have been edited is + // a genuinely different submission and must be restarted. + if (existingSpec.submissionKey == submittedStack.submissionKey) { logger.info { "Received duplicate submission for spec $key - reusing existing stack" } + // The incoming bundle was unpacked before we got here, and is now + // redundant - drop it rather than leaving it in the temp directory. + if (submittedStack !== existingSpec) { + submittedStack.resources.delete() + } return@compute existingSpec } else { logger.info { "Replacing spec $key" } shutDown(name) + // The replaced stack is gone for good, so its unpacked bundle goes too. + existingSpec.resources.delete() } } submittedStack @@ -208,7 +218,9 @@ class StackRunner(private val config: NebulaConfig = NebulaConfig()) { if (this.stacks.containsKey(name)) { shutDown(name) } - this.stacks.remove(name) + // Removal (unlike shutDown, which leaves a stopped stack restartable) means + // the stack is never coming back, so its unpacked bundle is deleted with it. + this.stacks.remove(name)?.resources?.delete() this._stackState.remove(name) } diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpacker.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpacker.kt new file mode 100644 index 0000000..83afe39 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpacker.kt @@ -0,0 +1,149 @@ +package com.orbitalhq.nebula.resources + +import com.orbitalhq.nebula.core.ResourceContent +import com.orbitalhq.nebula.core.StackBundle +import com.orbitalhq.nebula.core.StackBundleLimits +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +/** + * Raised when a submitted bundle is not safe or not small enough to unpack. + * Distinct from [StackResourceException], which is raised when a *running* stack + * asks for something it shouldn't have. + */ +class InvalidStackBundleException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) + +/** + * Writes a submitted [StackBundle]'s resources into a directory of their own. + * + * Every entry is validated before anything is written: the same containment + * rules [StackResources] applies on read are applied here on write, so a + * `../../etc/cron.d/evil` entry is rejected rather than landing on disk (the + * zip-slip class of bug). Size limits are checked against the *decoded* bytes, + * before any file is created, so an oversized bundle leaves nothing behind. + */ +class StackBundleUnpacker( + /** + * The directory under which per-submission bundle directories are created. + * Defaults to a `nebula-stack-bundles` directory in the system temp dir. + */ + private val baseDirectory: Path = defaultBaseDirectory() +) { + companion object { + private val logger = KotlinLogging.logger {} + + private fun defaultBaseDirectory(): Path { + val base = Path.of(System.getProperty("java.io.tmpdir")).resolve("nebula-stack-bundles") + Files.createDirectories(base) + return base.toRealPath() + } + + /** + * A content hash over the bundle's resources. Two bundles with the same + * files in the same state share a fingerprint, regardless of map ordering. + */ + fun fingerprintOf(resources: Map): String { + if (resources.isEmpty()) return "" + val digest = MessageDigest.getInstance("SHA-256") + resources.entries.sortedBy { it.key }.forEach { (path, content) -> + digest.update(path.toByteArray(Charsets.UTF_8)) + digest.update(content.bytes()) + } + return digest.digest().joinToString("") { "%02x".format(it) } + } + } + + /** + * Unpacks [bundle]'s resources for the stack named [stackName], returning a + * [StackResources] confined to the directory they were written to. + * + * A bundle with no resources needs no directory, and yields + * [StackResources.NONE] - so a bundle submission of a bare script behaves + * exactly as the script-only submission does. + * + * @throws InvalidStackBundleException if an entry escapes the bundle, or a + * size limit is exceeded. + */ + fun unpack(stackName: String, bundle: StackBundle): StackResources { + if (!bundle.hasResources) return StackResources.NONE + + validate(stackName, bundle) + + val directory = Files.createTempDirectory(baseDirectory, "${safeDirName(stackName)}-").toRealPath() + try { + bundle.resources.forEach { (relativePath, content) -> + val target = resolveWithin(directory, relativePath) + Files.createDirectories(target.parent) + Files.write(target, content.bytes()) + } + } catch (e: Exception) { + directory.toFile().deleteRecursively() + throw when (e) { + is StackResourceException -> InvalidStackBundleException(e.message ?: "Invalid resource path", e) + is InvalidStackBundleException -> e + else -> InvalidStackBundleException( + "Failed to unpack resources for stack '$stackName' - ${e.message}", e + ) + } + } + logger.info { "Unpacked ${bundle.resources.size} resources for stack '$stackName' into $directory" } + return StackResources( + root = directory, + ownsRoot = true, + fingerprint = fingerprintOf(bundle.resources) + ) + } + + /** + * Checks every entry before a single byte is written, so a bundle is either + * fully unpacked or leaves nothing on disk. + */ + private fun validate(stackName: String, bundle: StackBundle) { + if (bundle.resources.size > StackBundleLimits.MAX_RESOURCE_COUNT) { + throw InvalidStackBundleException( + "Stack '$stackName' ships ${bundle.resources.size} resources, which exceeds the limit of " + + "${StackBundleLimits.MAX_RESOURCE_COUNT} files per stack." + ) + } + // A throwaway root: validation must reject a path before we create the real + // directory, and resolveWithin needs something to resolve against. + val probeRoot = Path.of("/nebula-bundle-probe").normalize() + var totalBytes = 0L + bundle.resources.forEach { (relativePath, content) -> + try { + resolveWithin(probeRoot, relativePath) + } catch (e: StackResourceException) { + throw InvalidStackBundleException( + "Stack '$stackName' ships an unusable resource - ${e.message}", e + ) + } + val bytes = try { + content.bytes().size.toLong() + } catch (e: IllegalArgumentException) { + throw InvalidStackBundleException( + "Resource '$relativePath' in stack '$stackName' is flagged as ${content.encoding} " + + "but is not valid Base64 - ${e.message}", e + ) + } + if (bytes > StackBundleLimits.MAX_RESOURCE_BYTES) { + throw InvalidStackBundleException( + "Resource '$relativePath' in stack '$stackName' is $bytes bytes, which exceeds the " + + "per-file limit of ${StackBundleLimits.MAX_RESOURCE_BYTES} bytes." + ) + } + totalBytes += bytes + if (totalBytes > StackBundleLimits.MAX_BUNDLE_BYTES) { + throw InvalidStackBundleException( + "The resources shipped with stack '$stackName' total more than the bundle limit of " + + "${StackBundleLimits.MAX_BUNDLE_BYTES} bytes." + ) + } + } + } + + /** Stack names carry `/` and `[]` from Orbital's workspace + package prefixes. */ + private fun safeDirName(stackName: String): String = + stackName.replace(Regex("[^A-Za-z0-9._-]"), "_").take(60).ifBlank { "stack" } +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResources.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResources.kt new file mode 100644 index 0000000..72d5636 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResources.kt @@ -0,0 +1,243 @@ +package com.orbitalhq.nebula.resources + +import java.io.IOException +import java.nio.file.Files +import java.nio.file.InvalidPathException +import java.nio.file.Path +import java.nio.file.Paths +import java.security.MessageDigest +import java.util.stream.Collectors +import kotlin.io.path.isDirectory + +/** + * Raised when a stack asks for a resource that doesn't exist, or that would + * escape the bundle directory. + */ +class StackResourceException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) + +/** + * The files shipped alongside a stack script, unpacked into a directory that + * belongs to this submission alone. + * + * Every lookup is confined to [root]: + * + * - absolute paths are rejected, + * - `..` segments are rejected, both before and after normalisation, + * - the resolved path is turned into a *real* path (following symlinks) and + * checked to still be a descendant of the real [root]. + * + * That last step is what stops a symlink planted inside the bundle (or a bundle + * entry crafted to point at one) from reading `/etc/ssh/id_rsa` or any other + * file on the Nebula server. + * + * Note that these checks confine *resource* access. A Nebula script is arbitrary + * Kotlin and can always open files directly; the guarantee here is that a + * bundle-relative path resolves to a file inside the bundle, nothing more and + * nothing less. + */ +class StackResources internal constructor( + /** + * The directory this bundle was unpacked into, or null when the stack was + * submitted without a bundle (a plain script submission, or the CLI). + */ + val root: Path?, + /** + * Whether [delete] should remove [root]. True for directories we created by + * unpacking a bundle; false for a directory handed to us by a caller. + */ + private val ownsRoot: Boolean, + /** + * A stable identity for the content of this bundle. [StackRunner] compares it + * alongside the script source, so resubmitting an unchanged script with + * *changed* resources restarts the stack rather than being treated as a + * duplicate. + */ + val fingerprint: String +) { + companion object { + /** + * The resources of a stack submitted without a bundle. Reading from it + * fails with a message pointing at bundle submission, rather than + * silently resolving against the Nebula server's working directory. + */ + @JvmStatic + val NONE = StackResources(root = null, ownsRoot = false, fingerprint = "") + + /** + * A read-only view over an existing directory. Used by the CLI (where the + * script's own directory is the bundle) and by tests. + */ + @JvmStatic + fun at(directory: Path): StackResources { + require(directory.isDirectory()) { "$directory is not a directory" } + val real = directory.toRealPath() + return StackResources( + root = real, + ownsRoot = false, + fingerprint = fingerprintOfDirectory(real) + ) + } + + private fun fingerprintOfDirectory(root: Path): String { + val digest = MessageDigest.getInstance("SHA-256") + filesUnder(root).sortedBy { root.relativize(it).toString() } + .forEach { file -> + digest.update(root.relativize(file).joinToString("/").toByteArray(Charsets.UTF_8)) + digest.update(Files.readAllBytes(file)) + } + return digest.digest().joinToString("") { "%02x".format(it) } + } + + private fun filesUnder(root: Path): List = + Files.walk(root).use { stream -> + stream.filter { Files.isRegularFile(it) }.collect(Collectors.toList()) + } + } + + /** True when this stack was submitted with a resource bundle. */ + val hasBundle: Boolean + get() = root != null + + /** + * The relative paths of every file in the bundle, `/`-separated and sorted. + * Useful in error messages, and for scripts that want to iterate a directory. + */ + val names: List + get() { + val root = this.root ?: return emptyList() + return filesUnder(root).map { root.relativize(it).joinToString("/") }.sorted() + } + + /** + * Resolves [relativePath] to a real path inside the bundle. + * + * @throws StackResourceException if there is no bundle, the path is absolute + * or escapes the bundle, or no such file exists. + */ + fun path(relativePath: String): Path { + val root = this.root + ?: throw StackResourceException( + "Cannot read resource '$relativePath': this stack was submitted without a resource bundle. " + + "Place the file alongside the stack script in the project's nebula directory, " + + "or submit the stack through the bundle API." + ) + val resolved = resolveWithin(root, relativePath) + if (!Files.exists(resolved)) { + throw StackResourceException( + "No resource named '$relativePath' was shipped with this stack. " + + "Available resources: ${names.joinToString(", ").ifEmpty { "(none)" }}" + ) + } + // toRealPath follows symlinks. A link planted inside the bundle that points + // outside it normalises cleanly above, and is only caught here. + val real = try { + resolved.toRealPath() + } catch (e: IOException) { + throw StackResourceException("Failed to resolve resource '$relativePath' - ${e.message}", e) + } + if (!real.startsWith(root)) { + throw StackResourceException( + "Resource '$relativePath' resolves to $real, which is outside this stack's resource bundle. " + + "A stack may only read the files shipped with it." + ) + } + return real + } + + /** True when [relativePath] names a readable file in the bundle. Never throws. */ + fun exists(relativePath: String): Boolean = try { + path(relativePath) + true + } catch (e: StackResourceException) { + false + } + + /** The UTF-8 text of a resource shipped with this stack. */ + fun readText(relativePath: String): String = Files.readString(path(relativePath)) + + /** The raw bytes of a resource shipped with this stack. */ + fun readBytes(relativePath: String): ByteArray = Files.readAllBytes(path(relativePath)) + + /** + * Resolves a path given to a DSL function that reads a file + * (eg. `s3 { bucket("data") { file("seed/sales.csv") } }`). + * + * Relative paths resolve through the bundle when one is present, so a stack + * ships its own data. Absolute paths keep their existing meaning - a file on + * the machine running Nebula - so stacks written before bundles existed, and + * stacks run from the CLI against local files, behave exactly as they did. + */ + fun resolveFilePath(path: String): String { + if (!hasBundle) return path + return if (isAbsolute(path)) path else path(path).toString() + } + + /** + * Deletes the unpacked bundle, if this instance owns it. Called when the stack + * is removed or replaced. Safe to call more than once. + */ + fun delete() { + val root = this.root ?: return + if (!ownsRoot) return + // deleteRecursively() walks bottom-up and does not follow symlinks, so a + // link inside the bundle is unlinked rather than followed and emptied. + root.toFile().deleteRecursively() + } + + override fun toString(): String = + if (hasBundle) "StackResources($root, ${names.size} files)" else "StackResources(no bundle)" +} + +/** + * Rejects anything that isn't a plain relative path, then resolves it against + * [root] and checks the result is still inside [root]. + * + * Shared by [StackResources] (reads) and [StackBundleUnpacker] (writes), so an + * entry that could not be read can never be written in the first place. + */ +internal fun resolveWithin(root: Path, relativePath: String): Path { + if (relativePath.isBlank()) { + throw StackResourceException("A resource path may not be blank") + } + if (isAbsolute(relativePath)) { + throw StackResourceException( + "Resource path '$relativePath' is absolute. Resources are addressed by their path " + + "relative to the stack's nebula directory." + ) + } + val candidate = try { + Paths.get(relativePath) + } catch (e: InvalidPathException) { + throw StackResourceException("Resource path '$relativePath' is not a valid path - ${e.message}", e) + } + if (candidate.any { it.toString() == ".." }) { + throw StackResourceException( + "Resource path '$relativePath' contains a '..' segment. A stack may not reference " + + "files outside its resource bundle." + ) + } + val resolved = root.resolve(candidate).normalize() + if (!resolved.startsWith(root)) { + throw StackResourceException( + "Resource path '$relativePath' resolves outside the stack's resource bundle." + ) + } + return resolved +} + +/** + * True for paths that address the host filesystem directly. + * + * Deliberately stricter than [Path.isAbsolute]: on Linux a Windows-style + * `C:\keys\id_rsa` or a UNC `\\host\share` is a *valid relative filename*, and + * we'd rather reject it than quietly create a strangely-named file. + */ +internal fun isAbsolute(path: String): Boolean { + if (path.startsWith("/") || path.startsWith("\\")) return true + if (path.length >= 2 && path[1] == ':' && path[0].isLetter()) return true + return try { + Paths.get(path).isAbsolute + } catch (e: InvalidPathException) { + false + } +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResourcesContext.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResourcesContext.kt new file mode 100644 index 0000000..cf23408 --- /dev/null +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/resources/StackResourcesContext.kt @@ -0,0 +1,36 @@ +package com.orbitalhq.nebula.resources + +/** + * Carries the resources of the stack currently being compiled to the + * `stack { }` block inside the script. + * + * A stack script is a Kotlin script whose `stack { }` block takes no arguments - + * that's the whole point of the DSL, and we don't want every script to have to + * declare where its files come from. The script body runs synchronously on the + * thread that calls the scripting host, so the executor unpacks the bundle, + * binds it here for the duration of the evaluation, and `stack { }` picks it up. + * + * Outside a submission (the CLI, a test constructing a stack by hand) there is + * nothing bound and stacks get [StackResources.NONE]. + */ +object StackResourcesContext { + private val current = ThreadLocal() + + /** + * Binds [resources] for the duration of [block], restoring whatever was + * previously bound. Nesting is supported so a script that (unusually) + * compiles another script doesn't clobber the outer binding. + */ + fun withResources(resources: StackResources, block: () -> T): T { + val previous = current.get() + current.set(resources) + try { + return block() + } finally { + if (previous == null) current.remove() else current.set(previous) + } + } + + /** The resources bound for the current thread, or [StackResources.NONE]. */ + fun current(): StackResources = current.get() ?: StackResources.NONE +} diff --git a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt index 61e13e7..1e0aaad 100644 --- a/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt +++ b/nebula-dsl/src/main/kotlin/com/orbitalhq/nebula/s3/S3Dsl.kt @@ -1,6 +1,7 @@ package com.orbitalhq.nebula.s3 import com.orbitalhq.nebula.InfraDsl +import com.orbitalhq.nebula.resources.StackResources import com.orbitalhq.nebula.utils.NameGenerator import lang.taxi.utils.log import mu.KLogger @@ -10,23 +11,32 @@ private val logger: KLogger = KotlinLogging.logger {} interface S3Dsl : InfraDsl { fun s3(imageName: String = "localstack/localstack:3.0.2", componentName: String = "s3", dsl: S3Builder.(KLogger) -> Unit): S3Executor { - val builder = S3Builder(imageName, componentName) + val builder = S3Builder(imageName, componentName, resources) builder.dsl(logger) return this.add(S3Executor(builder.build(), loggers = listOf(logger.name))) } } -class S3Builder(private val imageName: String, private val componentName: String) { +class S3Builder( + private val imageName: String, + private val componentName: String, + /** The stack's bundle, so `file(path)` can read a file that shipped with the stack. */ + private val stackResources: StackResources +) { private val buckets = mutableListOf() fun bucket(name: String, init: BucketBuilder.() -> Unit) { - buckets.add(BucketBuilder(name).apply(init).build()) + buckets.add(BucketBuilder(name, stackResources).apply(init).build()) } fun build(): S3Config = S3Config(imageName, buckets, componentName = componentName) } -class BucketBuilder(private val name: String) { +class BucketBuilder( + private val name: String, + /** The stack's bundle, so `file(path)` can read a file that shipped with the stack. */ + private val stackResources: StackResources +) { private val resources = mutableListOf() fun file(name: String, content: String) { @@ -36,8 +46,20 @@ class BucketBuilder(private val name: String) { resources.add(SequenceResource(name, content)) } + /** + * Uploads a file from disk, using its filename as the S3 key. + * + * A *relative* path is read from the bundle shipped with the stack, so the + * file travels with the project rather than having to exist on the Nebula + * server. An *absolute* path is read from the machine running Nebula, which + * is what it has always meant. + * + * Resolution happens here, while the script is being evaluated, so a typo in + * a resource name fails the submission with a message listing what the stack + * actually shipped - rather than failing later, mid-startup. + */ fun file(path: String) { - resources.add(FileResource(path)) + resources.add(FileResource(stackResources.resolveFilePath(path))) } fun build(): BucketConfig = BucketConfig(name, resources) diff --git a/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpackerTest.kt b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpackerTest.kt new file mode 100644 index 0000000..b74f583 --- /dev/null +++ b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackBundleUnpackerTest.kt @@ -0,0 +1,177 @@ +package com.orbitalhq.nebula.resources + +import com.orbitalhq.nebula.core.ResourceContent +import com.orbitalhq.nebula.core.StackBundle +import com.orbitalhq.nebula.core.StackBundleLimits +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.listDirectoryEntries + +/** + * Unpacking is where a hostile bundle would land on disk, so the containment + * checks run here too - before a single file is written, not only when a running + * stack reads one back. + */ +class StackBundleUnpackerTest : DescribeSpec({ + + fun unpackerIn(): Pair { + val base = Files.createTempDirectory("unpacker-test").toRealPath() + return StackBundleUnpacker(base) to base + } + + describe("unpacking a bundle") { + it("writes text resources, including nested ones") { + val (unpacker, _) = unpackerIn() + val resources = unpacker.unpack( + "films", StackBundle( + script = "stack {}", + resources = mapOf( + "sales.csv" to ResourceContent.text("Title,Tickets"), + "specs/orders.yaml" to ResourceContent.text("openapi: 3.0.0") + ) + ) + ) + + resources.names.shouldContainExactly("sales.csv", "specs/orders.yaml") + resources.readText("specs/orders.yaml") shouldBe "openapi: 3.0.0" + } + + it("writes binary resources from their base64 encoding") { + val (unpacker, _) = unpackerIn() + val bytes = byteArrayOf(0, 1, 2, -1, -2) + val resources = unpacker.unpack( + "films", StackBundle("stack {}", mapOf("blob.bin" to ResourceContent.binary(bytes))) + ) + + resources.readBytes("blob.bin").toList() shouldBe bytes.toList() + } + + it("gives each submission a directory of its own") { + val (unpacker, base) = unpackerIn() + val bundle = StackBundle("stack {}", mapOf("a.txt" to ResourceContent.text("a"))) + + val first = unpacker.unpack("films", bundle) + val second = unpacker.unpack("films", bundle) + + (first.root == second.root) shouldBe false + base.listDirectoryEntries().size shouldBe 2 + } + + it("needs no directory for a bundle with no resources") { + val (unpacker, base) = unpackerIn() + val resources = unpacker.unpack("films", StackBundle.scriptOnly("stack {}")) + + resources.hasBundle shouldBe false + base.listDirectoryEntries().shouldBeEmpty() + } + + it("deletes the unpacked directory when the stack goes away") { + val (unpacker, _) = unpackerIn() + val resources = unpacker.unpack("films", StackBundle("stack {}", mapOf("a.txt" to ResourceContent.text("a")))) + val root = resources.root!! + + resources.delete() + + root.exists() shouldBe false + } + } + + describe("refusing a bundle that would write outside its directory") { + // This is zip-slip: the attack is in the entry name, and the file lands + // wherever that name points unless the server checks first. + listOf( + "../escaped.txt", + "nested/../../escaped.txt", + "/etc/cron.d/evil", + "C:\\Windows\\evil.txt" + ).forEach { hostilePath -> + it("rejects the entry '$hostilePath'") { + val (unpacker, base) = unpackerIn() + shouldThrow { + unpacker.unpack("films", StackBundle("stack {}", mapOf(hostilePath to ResourceContent.text("pwned")))) + } + // Nothing at all was written - not the hostile entry, and not the + // directory that would have held it. + base.listDirectoryEntries().shouldBeEmpty() + base.parent.resolve("escaped.txt").exists() shouldBe false + } + } + + it("rejects a bundle where one entry is hostile, leaving none of it on disk") { + val (unpacker, base) = unpackerIn() + shouldThrow { + unpacker.unpack( + "films", StackBundle( + "stack {}", mapOf( + "fine.csv" to ResourceContent.text("ok"), + "../escaped.txt" to ResourceContent.text("pwned") + ) + ) + ) + } + base.listDirectoryEntries().shouldBeEmpty() + } + } + + describe("size limits") { + it("rejects a resource over the per-file limit, naming the file and the limit") { + val (unpacker, base) = unpackerIn() + val tooBig = "x".repeat((StackBundleLimits.MAX_RESOURCE_BYTES + 1).toInt()) + + val error = shouldThrow { + unpacker.unpack("films", StackBundle("stack {}", mapOf("huge.csv" to ResourceContent.text(tooBig)))) + } + + error.message!!.shouldContain("huge.csv") + error.message!!.shouldContain(StackBundleLimits.MAX_RESOURCE_BYTES.toString()) + base.listDirectoryEntries().shouldBeEmpty() + } + + it("rejects a bundle with too many files") { + val (unpacker, _) = unpackerIn() + val resources = (0..StackBundleLimits.MAX_RESOURCE_COUNT) + .associate { "file-$it.txt" to ResourceContent.text("x") } + + shouldThrow { + unpacker.unpack("films", StackBundle("stack {}", resources)) + }.message!!.shouldContain(StackBundleLimits.MAX_RESOURCE_COUNT.toString()) + } + + it("rejects content flagged as base64 that isn't") { + val (unpacker, _) = unpackerIn() + shouldThrow { + unpacker.unpack( + "films", + StackBundle("stack {}", mapOf("blob.bin" to ResourceContent("not ** base64", com.orbitalhq.nebula.core.ResourceEncoding.BASE64))) + ) + }.message!!.shouldContain("blob.bin") + } + } + + describe("fingerprints") { + it("is the same for the same files whatever order they arrive in") { + val a = mapOf("a.txt" to ResourceContent.text("1"), "b.txt" to ResourceContent.text("2")) + val b = mapOf("b.txt" to ResourceContent.text("2"), "a.txt" to ResourceContent.text("1")) + + StackBundleUnpacker.fingerprintOf(a) shouldBe StackBundleUnpacker.fingerprintOf(b) + } + + it("differs when a file's content changes") { + val before = StackBundleUnpacker.fingerprintOf(mapOf("a.txt" to ResourceContent.text("1"))) + val after = StackBundleUnpacker.fingerprintOf(mapOf("a.txt" to ResourceContent.text("2"))) + + (before == after) shouldBe false + } + + it("is empty when there are no resources, so script-only submissions dedupe as before") { + StackBundleUnpacker.fingerprintOf(emptyMap()) shouldBe "" + } + } +}) diff --git a/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackResourcesTest.kt b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackResourcesTest.kt new file mode 100644 index 0000000..cdce047 --- /dev/null +++ b/nebula-dsl/src/test/kotlin/com/orbitalhq/nebula/resources/StackResourcesTest.kt @@ -0,0 +1,181 @@ +package com.orbitalhq.nebula.resources + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText + +/** + * The containment rules that keep a stack reading only the files that were + * shipped with it. + * + * A Nebula server runs stacks submitted by any project connected to Orbital, and + * its own filesystem holds credentials that have nothing to do with those + * projects. Everything below is a way someone might try to walk out of the + * bundle directory and reach one. + */ +class StackResourcesTest : DescribeSpec({ + + /** A bundle directory with a nested file, plus a "secret" living outside it. */ + fun bundleWithSecretOutside(): Pair { + val tempDir = Files.createTempDirectory("stack-resources-test") + val bundle = tempDir.resolve("bundle").createDirectories() + bundle.resolve("sales.csv").writeText("Title,Tickets\nA New Hope,249\n") + bundle.resolve("specs").createDirectories() + bundle.resolve("specs/orders.yaml").writeText("openapi: 3.0.0") + + val secrets = tempDir.resolve("secrets").createDirectories() + secrets.resolve("id_rsa").writeText("PRIVATE KEY") + + return StackResources.at(bundle) to tempDir + } + + describe("reading resources that were shipped with the stack") { + it("reads a file at the root of the bundle") { + val (resources, _) = bundleWithSecretOutside() + resources.readText("sales.csv") shouldContain "A New Hope" + } + + it("reads a file in a subdirectory") { + val (resources, _) = bundleWithSecretOutside() + resources.readText("specs/orders.yaml") shouldBe "openapi: 3.0.0" + } + + it("lists what the stack shipped with") { + val (resources, _) = bundleWithSecretOutside() + resources.names.shouldContainExactly("sales.csv", "specs/orders.yaml") + } + + it("normalises a path that stays inside the bundle") { + val (resources, _) = bundleWithSecretOutside() + // specs/../sales.csv resolves back inside the bundle, but we reject it + // anyway: allowing ".." at all means relying on normalisation being + // right everywhere, and a legitimate stack never needs it. + shouldThrow { + resources.readText("specs/../sales.csv") + }.message!!.shouldContain("'..' segment") + } + + it("reports what is available when a resource is missing") { + val (resources, _) = bundleWithSecretOutside() + val error = shouldThrow { resources.readText("nope.csv") } + error.message!!.shouldContain("No resource named 'nope.csv'") + error.message!!.shouldContain("sales.csv") + } + } + + describe("refusing to read outside the bundle") { + it("rejects an absolute path") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { + resources.readText("/etc/ssh/id_rsa") + }.message!!.shouldContain("is absolute") + } + + it("rejects a relative path that climbs out with ..") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { + resources.readText("../secrets/id_rsa") + }.message!!.shouldContain("'..' segment") + } + + it("rejects a deeply nested climb that ends outside the bundle") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { + resources.readText("specs/../../secrets/id_rsa") + }.message!!.shouldContain("'..' segment") + } + + it("rejects a Windows-style absolute path, even on a unix host") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { + resources.readText("C:\\Users\\admin\\id_rsa") + }.message!!.shouldContain("is absolute") + } + + it("rejects a blank path") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { resources.readText(" ") } + } + + it("refuses to follow a symlink planted inside the bundle") { + val (resources, tempDir) = bundleWithSecretOutside() + val root = resources.root!! + // Nothing in the path string is suspicious - the escape is entirely in + // the filesystem, which is why normalising the string is not enough. + Files.createSymbolicLink(root.resolve("innocent.pem"), tempDir.resolve("secrets/id_rsa")) + + shouldThrow { + resources.readText("innocent.pem") + }.message!!.shouldContain("outside this stack's resource bundle") + } + + it("refuses to follow a symlinked directory that leads out of the bundle") { + val (resources, tempDir) = bundleWithSecretOutside() + val root = resources.root!! + Files.createSymbolicLink(root.resolve("elsewhere"), tempDir.resolve("secrets")) + + shouldThrow { + resources.readText("elsewhere/id_rsa") + }.message!!.shouldContain("outside this stack's resource bundle") + } + + it("still reads a symlink that stays inside the bundle") { + val (resources, _) = bundleWithSecretOutside() + val root = resources.root!! + Files.createSymbolicLink(root.resolve("latest.csv"), root.resolve("sales.csv")) + + resources.readText("latest.csv") shouldContain "A New Hope" + } + } + + describe("a stack submitted without a bundle") { + it("explains why there is nothing to read") { + shouldThrow { + StackResources.NONE.readText("sales.csv") + }.message!!.shouldContain("submitted without a resource bundle") + } + + it("leaves file paths given to providers exactly as they were") { + // Stacks that predate bundles pass paths on the Nebula server's own + // filesystem; without a bundle nothing changes for them. + StackResources.NONE.resolveFilePath("/data/sales.csv") shouldBe "/data/sales.csv" + StackResources.NONE.resolveFilePath("sales.csv") shouldBe "sales.csv" + } + } + + describe("resolving paths handed to providers") { + it("resolves a relative path through the bundle") { + val (resources, _) = bundleWithSecretOutside() + resources.resolveFilePath("specs/orders.yaml") shouldBe + resources.root!!.resolve("specs/orders.yaml").toString() + } + + it("leaves an absolute path pointing at the host, as it always has") { + val (resources, _) = bundleWithSecretOutside() + resources.resolveFilePath("/mnt/data/sales.csv") shouldBe "/mnt/data/sales.csv" + } + + it("rejects a relative path that escapes the bundle") { + val (resources, _) = bundleWithSecretOutside() + shouldThrow { resources.resolveFilePath("../secrets/id_rsa") } + } + } + + describe("fingerprinting") { + it("changes when a resource's content changes") { + val dir = Files.createTempDirectory("fingerprint-test") + dir.resolve("data.csv").writeText("a") + val before = StackResources.at(dir).fingerprint + + dir.resolve("data.csv").writeText("b") + StackResources.at(dir).fingerprint shouldBe StackResources.at(dir).fingerprint + (StackResources.at(dir).fingerprint == before) shouldBe false + } + } +}) diff --git a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/NebulaScriptExecutor.kt b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/NebulaScriptExecutor.kt index e3f81bc..0be7615 100644 --- a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/NebulaScriptExecutor.kt +++ b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/NebulaScriptExecutor.kt @@ -8,6 +8,10 @@ import com.orbitalhq.nebula.HostConfig import com.orbitalhq.nebula.NebulaScript import com.orbitalhq.nebula.NebulaStack import com.orbitalhq.nebula.NebulaStackWithSource +import com.orbitalhq.nebula.core.StackBundle +import com.orbitalhq.nebula.resources.InvalidStackBundleException +import com.orbitalhq.nebula.resources.StackBundleUnpacker +import com.orbitalhq.nebula.resources.StackResourcesContext import io.github.oshai.kotlinlogging.KotlinLogging import java.io.File import kotlin.script.experimental.api.EvaluationResult @@ -22,7 +26,14 @@ import kotlin.script.experimental.jvm.util.isError import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost import kotlin.script.experimental.jvmhost.createJvmCompilationConfigurationFromTemplate -class NebulaScriptExecutor(private val logCompilationErrors: Boolean = true) { +class NebulaScriptExecutor( + private val logCompilationErrors: Boolean = true, + /** + * Writes the resources of a submitted bundle into a directory of their own. + * Injectable so tests can point bundles at a temp directory they control. + */ + private val bundleUnpacker: StackBundleUnpacker = StackBundleUnpacker() +) { companion object { private val logger = KotlinLogging.logger {} } @@ -99,6 +110,47 @@ class NebulaScriptExecutor(private val logCompilationErrors: Boolean = true) { .map { NebulaStackWithSource(it, string, hostConfig) } } + /** + * Compiles a stack submitted as a bundle: the script plus the files that + * shipped with it. + * + * The bundle is unpacked first, and bound for the duration of the evaluation, + * because the `stack { }` block runs *during* compilation - that's when + * `s3 { file("data/sales.csv") }` and friends resolve their paths. + * + * Anything wrong with the bundle itself (an entry that escapes the bundle + * directory, a file over the size limit) is reported as a compilation error, + * so it reaches the submitter through the same channel as a syntax error, + * rather than as an opaque 500. + * + * If compilation fails the unpacked directory is removed - a rejected + * submission must not leave files behind. + */ + fun compileBundle( + stackName: String, + bundle: StackBundle, + hostConfig: HostConfig + ): Either { + val resources = try { + bundleUnpacker.unpack(stackName, bundle) + } catch (e: InvalidStackBundleException) { + logger.warn { "Rejecting bundle for stack '$stackName' - ${e.message}" } + return NebulaCompilationException.forSyntheticDiagnostic( + e.message ?: "The resources submitted with this stack could not be unpacked" + ).left() + } + + return StackResourcesContext.withResources(resources) { + compileStack(bundle.script.toScriptSource()) + }.fold( + { compilationError -> + resources.delete() + compilationError.left() + }, + { stack -> NebulaStackWithSource(stack, bundle.script, hostConfig).right() } + ) + } + @Deprecated("call compileToStackWithSource") fun toStackWithSource(string: String, hostConfig: HostConfig): NebulaStackWithSource { val source = string.toScriptSource() diff --git a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt index 062f59d..ac3f2be 100644 --- a/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt +++ b/nebula-runtime/src/main/kotlin/com/orbitalhq/nebula/runtime/server/NebulaServer.kt @@ -1,5 +1,6 @@ package com.orbitalhq.nebula.runtime.server +import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue @@ -9,6 +10,7 @@ import com.orbitalhq.nebula.NebulaStack import com.orbitalhq.nebula.NebulaStackWithSource import com.orbitalhq.nebula.StackName import com.orbitalhq.nebula.StackRunner +import com.orbitalhq.nebula.core.StackBundle import com.orbitalhq.nebula.core.StackStateEvent import com.orbitalhq.nebula.runtime.NebulaScriptExecutor import io.github.oshai.kotlinlogging.KotlinLogging @@ -96,6 +98,30 @@ class NebulaServer( stackExecutor.submit(stack, startAsync = true) call.respond(stack.name) } + // Submit a stack together with the files it reads at startup. + // Additive: the script-only routes above are untouched, and a + // bundle carrying no resources behaves identically to them. + post("/bundle") { + val bundle = call.receive() + // The name is assigned by the runner, but the unpacker wants + // something to name the directory after before we have one. + submitBundle(call, "unnamed-stack", bundle) { stack -> + stackExecutor.submit(stack, startAsync = true) + call.respond(stack.name) + } + } + put("/bundle/{id}") { + val id = call.parameters["id"] ?: return@put call.respond( + HttpStatusCode.BadRequest, + "Missing or malformed id" + ) + val bundle = call.receive() + submitBundle(call, id, bundle) { stack -> + val named = stack.withName(id) + stackExecutor.submit(named, id) + call.respond(named.name) + } + } put("/{id}") { val id = call.parameters["id"] ?: return@put call.respond( HttpStatusCode.BadRequest, @@ -250,12 +276,12 @@ class NebulaServer( // the other stacks in the submission. Failures are reported back to // the client as a StackStateEvent carrying the compilation errors, // and recorded so the admin snapshot shows them too. - val eventStreams = updateStacksRequest.stacks.mapNotNull { (name, script) -> - scriptExecutor.compileToStackWithSource(script, call.hostConfig()).fold( + val eventStreams = updateStacksRequest.allBundles().mapNotNull { (name, bundle) -> + scriptExecutor.compileBundle(name, bundle, call.hostConfig()).fold( { compilationException -> val errors = compilationException.errors.map { it.toCompilationError() } logger.warn { "Stack $name failed to compile: ${errors.joinToString { it.message }}" } - failedSubmissions[name] = FailedSubmission(name, script, errors) + failedSubmissions[name] = FailedSubmission(name, bundle.script, errors) send( Frame.Text( objectMapper.writeValueAsString( @@ -300,6 +326,37 @@ class NebulaServer( }.start(wait = wait) } + /** + * Compiles a submitted bundle and hands the compiled stack to [onCompiled], + * or answers 422 with the compilation errors. + * + * Bundle problems (a resource that escapes the bundle directory, a file over + * the size limit) arrive here as compilation errors, so the caller gets the + * same shaped response whether the script or its resources were the problem. + */ + private suspend fun submitBundle( + call: ApplicationCall, + stackName: StackName, + bundle: StackBundle, + onCompiled: suspend (NebulaStackWithSource) -> Unit + ) { + scriptExecutor.compileBundle(stackName, bundle, call.hostConfig()).fold( + { exception -> + val failure = FailedSubmission( + stackName, + bundle.script, + exception.errors.map { it.toCompilationError() } + ) + failedSubmissions[stackName] = failure + call.respond(HttpStatusCode.UnprocessableEntity, failure) + }, + { stack -> + failedSubmissions.remove(stackName) + onCompiled(stack) + } + ) + } + private suspend fun handleStackAction( call: ApplicationCall, action: (id: String) -> Unit @@ -341,7 +398,33 @@ data class StackEventStreamRequest(val stackId: StackName) typealias StackScript = String -data class UpdateStackRSocketRequest(val stacks: Map) +/** + * A submission of one or more stacks over `/stream/stacks`. + * + * Accepts both shapes: + * + * - `{"stacks": {"name": "