From af24666762940dc45734300692b8c4ed4732eec0 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Wed, 29 Jul 2026 12:04:55 +1000 Subject: [PATCH] publish the interoperability testbed --- .../schemas/connect-protocol.v1.schema.json | 2 +- .../schemas/encrypted-relay.v1.schema.json | 2 +- public/testbed/schemas/v0.1/README.md | 15 + .../v0.1/adapter-description.schema.json | 73 +++ .../testbed/schemas/v0.1/evidence.schema.json | 49 ++ .../schemas/v0.1/fixture-catalog.schema.json | 30 + .../schemas/v0.1/run-request.schema.json | 29 + .../testbed/schemas/v0.1/scenario.schema.json | 139 +++++ .../schemas/v0.1/transcript.schema.json | 86 +++ public/testbed/v0.1/README.md | 87 +++ public/testbed/v0.1/fixtures/catalog.json | 175 ++++++ .../core.shared-contract-consumers.json | 88 +++ .../scenarios/interop.action-ambiguous.json | 52 ++ .../interop.action-explicit-provider.json | 65 ++ .../interop.action-single-provider.json | 57 ++ .../interop.application-event-multicast.json | 75 +++ ...nterop.application-provider-lifecycle.json | 84 +++ .../scenarios/interop.authority-boundary.json | 51 ++ .../interop.contract-digest-drift.json | 53 ++ .../scenarios/interop.event-multicast.json | 78 +++ .../scenarios/interop.invalid-payload.json | 63 ++ .../interop.provider-disappearance.json | 52 ++ .../scenarios/interop.request-replay.json | 63 ++ .../runtime.application-execution.json | 75 +++ .../scenarios/runtime.competing-workers.json | 82 +++ .../scenarios/runtime.crash-recovery.json | 73 +++ scripts/sync-sources.mjs | 13 + site-sources.json | 8 +- src/components/SiteFooter.astro | 1 + src/components/SiteHeader.astro | 3 +- src/data/conformance.json | 6 +- src/layouts/BaseLayout.astro | 2 +- src/pages/testbed/index.astro | 573 ++++++++++++++++++ 33 files changed, 2293 insertions(+), 11 deletions(-) create mode 100644 public/testbed/schemas/v0.1/README.md create mode 100644 public/testbed/schemas/v0.1/adapter-description.schema.json create mode 100644 public/testbed/schemas/v0.1/evidence.schema.json create mode 100644 public/testbed/schemas/v0.1/fixture-catalog.schema.json create mode 100644 public/testbed/schemas/v0.1/run-request.schema.json create mode 100644 public/testbed/schemas/v0.1/scenario.schema.json create mode 100644 public/testbed/schemas/v0.1/transcript.schema.json create mode 100644 public/testbed/v0.1/README.md create mode 100644 public/testbed/v0.1/fixtures/catalog.json create mode 100644 public/testbed/v0.1/scenarios/core.shared-contract-consumers.json create mode 100644 public/testbed/v0.1/scenarios/interop.action-ambiguous.json create mode 100644 public/testbed/v0.1/scenarios/interop.action-explicit-provider.json create mode 100644 public/testbed/v0.1/scenarios/interop.action-single-provider.json create mode 100644 public/testbed/v0.1/scenarios/interop.application-event-multicast.json create mode 100644 public/testbed/v0.1/scenarios/interop.application-provider-lifecycle.json create mode 100644 public/testbed/v0.1/scenarios/interop.authority-boundary.json create mode 100644 public/testbed/v0.1/scenarios/interop.contract-digest-drift.json create mode 100644 public/testbed/v0.1/scenarios/interop.event-multicast.json create mode 100644 public/testbed/v0.1/scenarios/interop.invalid-payload.json create mode 100644 public/testbed/v0.1/scenarios/interop.provider-disappearance.json create mode 100644 public/testbed/v0.1/scenarios/interop.request-replay.json create mode 100644 public/testbed/v0.1/scenarios/runtime.application-execution.json create mode 100644 public/testbed/v0.1/scenarios/runtime.competing-workers.json create mode 100644 public/testbed/v0.1/scenarios/runtime.crash-recovery.json create mode 100644 src/pages/testbed/index.astro diff --git a/public/connect/schemas/connect-protocol.v1.schema.json b/public/connect/schemas/connect-protocol.v1.schema.json index 9cc90cf..96e9863 100644 --- a/public/connect/schemas/connect-protocol.v1.schema.json +++ b/public/connect/schemas/connect-protocol.v1.schema.json @@ -18,7 +18,7 @@ "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$" }, "operation": { - "enum": ["describe", "changes", "read", "query", "list_views", "execute_view", "read_view_source", "create_view_source", "update_view_source", "delete_view_source", "validate", "create", "update", "delete", "rename", "read_type", "create_type", "update_type", "list_timers", "put_timer", "cancel_timer", "reconcile_timers", "sync"] + "enum": ["describe", "changes", "read", "query", "list_views", "execute_view", "read_view_source", "create_view_source", "update_view_source", "delete_view_source", "validate", "create", "update", "delete", "rename", "read_type", "create_type", "update_type", "install_type_pack", "list_timers", "put_timer", "cancel_timer", "reconcile_timers", "sync"] }, "contractRequirement": { "type": "object", diff --git a/public/connect/schemas/encrypted-relay.v1.schema.json b/public/connect/schemas/encrypted-relay.v1.schema.json index 1657525..4d639c5 100644 --- a/public/connect/schemas/encrypted-relay.v1.schema.json +++ b/public/connect/schemas/encrypted-relay.v1.schema.json @@ -14,7 +14,7 @@ "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$" }, "operation": { - "enum": ["describe", "changes", "read", "query", "list_views", "execute_view", "read_view_source", "create_view_source", "update_view_source", "delete_view_source", "validate", "create", "update", "delete", "rename", "read_type", "create_type", "update_type", "list_timers", "put_timer", "cancel_timer", "reconcile_timers"] + "enum": ["describe", "changes", "read", "query", "list_views", "execute_view", "read_view_source", "create_view_source", "update_view_source", "delete_view_source", "validate", "create", "update", "delete", "rename", "read_type", "create_type", "update_type", "install_type_pack", "list_timers", "put_timer", "cancel_timer", "reconcile_timers"] }, "envelope": { "type": "object", diff --git a/public/testbed/schemas/v0.1/README.md b/public/testbed/schemas/v0.1/README.md new file mode 100644 index 0000000..f10be5b --- /dev/null +++ b/public/testbed/schemas/v0.1/README.md @@ -0,0 +1,15 @@ +# mdbase interoperability testbed schemas + +These Draft 2020-12 schemas define portable testbed protocol `0.1`: + +- `scenario.schema.json` validates neutral, spec-owned scenario definitions. +- `fixture-catalog.schema.json` validates the shared input catalog. +- `adapter-description.schema.json` validates the capabilities reported by a + black-box adapter. +- `run-request.schema.json` validates the complete input sent to an adapter. +- `transcript.schema.json` validates deterministic observable results. +- `evidence.schema.json` validates the digest-bound run summary suitable for a + conformance claim. + +The process protocol and conformance rules are defined in +[`testbed/v0.1/README.md`](../../../testbed/v0.1/README.md). diff --git a/public/testbed/schemas/v0.1/adapter-description.schema.json b/public/testbed/schemas/v0.1/adapter-description.schema.json new file mode 100644 index 0000000..d2fc432 --- /dev/null +++ b/public/testbed/schemas/v0.1/adapter-description.schema.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mdbase.dev/schemas/testbed/v0.1/adapter-description.schema.json", + "title": "mdbase interoperability testbed adapter description", + "type": "object", + "required": [ + "kind", + "protocol_version", + "implementation", + "profiles", + "roles", + "scenarios" + ], + "properties": { + "kind": { "const": "mdbase.testbed.adapter" }, + "protocol_version": { "const": "0.1" }, + "implementation": { + "type": "object", + "required": ["id", "name", "version"], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9._:/-]*$" + }, + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "language": { "type": "string", "minLength": 1 }, + "target": { "type": "string", "minLength": 1 } + }, + "patternProperties": { + "^x-[A-Za-z0-9._:-]+$": true + }, + "additionalProperties": false + }, + "profiles": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "core_read", + "event_action_interop/0.1", + "runtime/0.2" + ] + } + }, + "roles": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "contract_store", + "record_consumer", + "event_source", + "event_consumer", + "action_caller", + "action_provider", + "bridge", + "runtime", + "runtime_store" + ] + } + }, + "scenarios": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + } + } + }, + "additionalProperties": false +} diff --git a/public/testbed/schemas/v0.1/evidence.schema.json b/public/testbed/schemas/v0.1/evidence.schema.json new file mode 100644 index 0000000..8597081 --- /dev/null +++ b/public/testbed/schemas/v0.1/evidence.schema.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mdbase.dev/schemas/testbed/v0.1/evidence.schema.json", + "title": "mdbase interoperability testbed evidence", + "type": "object", + "required": [ + "kind", + "protocol_version", + "implementation", + "result", + "scenarios" + ], + "properties": { + "kind": { "const": "mdbase.testbed.evidence" }, + "protocol_version": { "const": "0.1" }, + "implementation": { + "$ref": "adapter-description.schema.json#/properties/implementation" + }, + "result": { "enum": ["pass", "fail"] }, + "scenarios": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "profile", "result", "transcript_digest"], + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + }, + "profile": { + "enum": [ + "core_read", + "event_action_interop/0.1", + "runtime/0.2" + ] + }, + "result": { "enum": ["pass", "fail"] }, + "transcript_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/public/testbed/schemas/v0.1/fixture-catalog.schema.json b/public/testbed/schemas/v0.1/fixture-catalog.schema.json new file mode 100644 index 0000000..3ad1ea7 --- /dev/null +++ b/public/testbed/schemas/v0.1/fixture-catalog.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mdbase.dev/schemas/testbed/v0.1/fixture-catalog.schema.json", + "title": "mdbase interoperability testbed fixture catalog", + "type": "object", + "required": ["kind", "protocol_version", "fixtures"], + "properties": { + "kind": { "const": "mdbase.testbed.fixtures" }, + "protocol_version": { "const": "0.1" }, + "fixtures": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + }, + "additionalProperties": { + "type": "object", + "required": ["kind", "value"], + "properties": { + "kind": { + "enum": ["contract", "type", "record", "workflow", "policy"] + }, + "value": {} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/public/testbed/schemas/v0.1/run-request.schema.json b/public/testbed/schemas/v0.1/run-request.schema.json new file mode 100644 index 0000000..7306f68 --- /dev/null +++ b/public/testbed/schemas/v0.1/run-request.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mdbase.dev/schemas/testbed/v0.1/run-request.schema.json", + "title": "mdbase interoperability testbed run request", + "type": "object", + "required": ["kind", "protocol_version", "scenario", "fixtures"], + "properties": { + "kind": { "const": "mdbase.testbed.run" }, + "protocol_version": { "const": "0.1" }, + "scenario": { + "$ref": "scenario.schema.json" + }, + "fixtures": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["kind", "value"], + "properties": { + "kind": { + "enum": ["contract", "type", "record", "workflow", "policy"] + }, + "value": {} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/public/testbed/schemas/v0.1/scenario.schema.json b/public/testbed/schemas/v0.1/scenario.schema.json new file mode 100644 index 0000000..ceec8dc --- /dev/null +++ b/public/testbed/schemas/v0.1/scenario.schema.json @@ -0,0 +1,139 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mdbase.dev/schemas/testbed/v0.1/scenario.schema.json", + "title": "mdbase interoperability testbed scenario", + "type": "object", + "required": [ + "kind", + "protocol_version", + "id", + "name", + "ring", + "profile", + "roles", + "operation", + "fixtures", + "parameters", + "covers", + "expect" + ], + "properties": { + "kind": { "const": "mdbase.testbed.scenario" }, + "protocol_version": { "const": "0.1" }, + "id": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + }, + "name": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "ring": { + "enum": ["contract", "interop", "runtime"] + }, + "profile": { + "enum": [ + "core_read", + "event_action_interop/0.1", + "runtime/0.2" + ] + }, + "roles": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": [ + "contract_store", + "record_consumer", + "event_source", + "event_consumer", + "action_caller", + "action_provider", + "bridge", + "runtime", + "runtime_store" + ] + } + }, + "operation": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + }, + "fixtures": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + } + }, + "parameters": { + "type": "object", + "additionalProperties": true + }, + "covers": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9._/-]+$" + } + }, + "expect": { + "type": "object", + "required": ["entries"], + "properties": { + "entries": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/transcriptEntry" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "$defs": { + "transcriptEntry": { + "type": "object", + "required": [ + "sequence", + "phase", + "actor", + "operation", + "outcome", + "facts" + ], + "properties": { + "sequence": { "type": "integer", "minimum": 1 }, + "phase": { + "enum": ["arrange", "act", "observe", "recover"] + }, + "actor": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$" + }, + "operation": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + }, + "outcome": { + "enum": [ + "succeeded", + "rejected", + "failed", + "indeterminate", + "skipped" + ] + }, + "facts": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + } + } +} diff --git a/public/testbed/schemas/v0.1/transcript.schema.json b/public/testbed/schemas/v0.1/transcript.schema.json new file mode 100644 index 0000000..51a35e6 --- /dev/null +++ b/public/testbed/schemas/v0.1/transcript.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mdbase.dev/schemas/testbed/v0.1/transcript.schema.json", + "title": "mdbase interoperability testbed transcript", + "type": "object", + "required": [ + "kind", + "protocol_version", + "scenario_id", + "implementation", + "entries" + ], + "properties": { + "kind": { "const": "mdbase.testbed.transcript" }, + "protocol_version": { "const": "0.1" }, + "scenario_id": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + }, + "implementation": { + "type": "object", + "required": ["id", "name", "version"], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9._:/-]*$" + }, + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "language": { "type": "string", "minLength": 1 }, + "target": { "type": "string", "minLength": 1 } + }, + "patternProperties": { + "^x-[A-Za-z0-9._:-]+$": true + }, + "additionalProperties": false + }, + "entries": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/entry" } + } + }, + "additionalProperties": false, + "$defs": { + "entry": { + "type": "object", + "required": [ + "sequence", + "phase", + "actor", + "operation", + "outcome", + "facts" + ], + "properties": { + "sequence": { "type": "integer", "minimum": 1 }, + "phase": { + "enum": ["arrange", "act", "observe", "recover"] + }, + "actor": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$" + }, + "operation": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$" + }, + "outcome": { + "enum": [ + "succeeded", + "rejected", + "failed", + "indeterminate", + "skipped" + ] + }, + "facts": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + } + } +} diff --git a/public/testbed/v0.1/README.md b/public/testbed/v0.1/README.md new file mode 100644 index 0000000..864d201 --- /dev/null +++ b/public/testbed/v0.1/README.md @@ -0,0 +1,87 @@ +# mdbase interoperability testbed 0.1 + +The testbed verifies observable behavior through a small process boundary. It +does not require implementations to share a language, runtime, database, or +plugin host, and it does not make fixture code authoritative. + +## The three rings + +1. **Contract** — load one record contract and type implementation, project one + record, and prove that multiple independent consumers receive the same + contract view. +2. **Interop** — register live sources, consumers, callers, and providers; then + verify CloudEvents multicast, exact action admission, ambiguity, unload, + validation, authorization, and replay behavior. +3. **Runtime** — exercise a durable store through crash recovery and competing + workers, including invocation reuse and lease fencing. + +Scenario JSON, not a particular implementation, defines the oracle. Every +scenario names neutral fixtures from `fixtures/catalog.json` and contains its +complete expected transcript. Volatile implementation details such as database +keys, wall-clock time, and generated IDs are reduced to stable facts before +comparison. + +## Adapter process protocol + +An adapter is an executable supporting two commands: + +```text +adapter describe +adapter run +``` + +`describe` writes one JSON document conforming to +`adapter-description.schema.json`. For `run`, the testbed writes this JSON +request to standard input: + +```json +{ + "kind": "mdbase.testbed.run", + "protocol_version": "0.1", + "scenario": {}, + "fixtures": {} +} +``` + +The adapter writes one transcript conforming to `transcript.schema.json`. +Diagnostic logging belongs on standard error; standard output contains only the +protocol document. Each `run` command is a fresh process, so scenarios cannot +pass because of hidden state left by another scenario. + +Adapters MUST exercise public or documented implementation boundaries. They +MUST NOT copy the expected transcript without observing the behavior it +describes. A transcript is evidence of a named scenario, not authority to read, +write, publish, subscribe, invoke, or provide. + +## Running + +From this repository: + +```bash +npm test --prefix packages/testbed +npm exec --prefix packages/testbed mdbase-testbed -- run --adapter reference +``` + +Against another implementation: + +```bash +mdbase-testbed run --adapter ./path/to/adapter +``` + +If the adapter is launched through a language tool, use `command:NAME` and pass +its fixed arguments separately. The runner never invokes a shell: + +```bash +mdbase-testbed run --adapter command:cargo \ + --adapter-arg run --adapter-arg --quiet \ + --adapter-arg -p --adapter-arg my-testbed-adapter --adapter-arg -- +``` + +Use `mdbase-testbed list` to inspect the portable scenario inventory and +`mdbase-testbed validate` to validate every schema, fixture, scenario, and +canonical expected transcript. + +Passing the testbed supplements the full v0.3 conformance suites. It does not by +itself establish a complete profile claim: implementations still run every +normative test for the profile they claim and attach the testbed transcript as +machine-readable evidence. diff --git a/public/testbed/v0.1/fixtures/catalog.json b/public/testbed/v0.1/fixtures/catalog.json new file mode 100644 index 0000000..7d5507f --- /dev/null +++ b/public/testbed/v0.1/fixtures/catalog.json @@ -0,0 +1,175 @@ +{ + "kind": "mdbase.testbed.fixtures", + "protocol_version": "0.1", + "fixtures": { + "contract.example-note": { + "kind": "contract", + "value": { + "kind": "mdbase.contract", + "contract_type": "record", + "id": "example.note", + "version": "1.0.0", + "name": "Portable example note", + "record_schema": { + "dialect": "json-schema-2020-12", + "value": { + "type": "object", + "required": ["title"], + "additionalProperties": false, + "properties": { + "title": { "type": "string", "minLength": 1 } + } + } + } + } + }, + "type.shared-note": { + "kind": "type", + "value": { + "kind": "mdbase.type", + "name": "shared_note", + "version": 1, + "schema": { + "dialect": "json-schema-2020-12", + "value": { + "type": "object", + "required": ["type", "headline"], + "properties": { + "type": { "const": "shared_note" }, + "headline": { "type": "string", "minLength": 1 } + } + } + }, + "implements": [ + { + "contract": "example.note", + "version": "1.0.0", + "fields": { + "title": "headline" + } + } + ] + } + }, + "record.shared-note": { + "kind": "record", + "value": { + "type": "shared_note", + "headline": "One record, two consumers" + } + }, + "contract.record-changed": { + "kind": "contract", + "value": { + "kind": "mdbase.contract", + "contract_type": "event", + "id": "example.record.changed", + "version": "1.0.0", + "name": "Example record changed", + "data_schema": { + "dialect": "json-schema-2020-12", + "value": { + "type": "object", + "required": ["record_id", "revision"], + "additionalProperties": false, + "properties": { + "record_id": { "type": "string", "minLength": 1 }, + "revision": { "type": "integer", "minimum": 1 } + } + } + } + } + }, + "contract.record-annotate": { + "kind": "contract", + "value": { + "kind": "mdbase.contract", + "contract_type": "action", + "id": "example.record.annotate", + "version": "1.0.0", + "name": "Annotate example record", + "input_schema": { + "dialect": "json-schema-2020-12", + "value": { + "type": "object", + "required": ["record_id", "label"], + "additionalProperties": false, + "properties": { + "record_id": { "type": "string", "minLength": 1 }, + "label": { "type": "string", "minLength": 1 } + } + } + }, + "output_schema": { + "dialect": "json-schema-2020-12", + "value": { + "type": "object", + "required": ["record_id", "applied"], + "additionalProperties": false, + "properties": { + "record_id": { "type": "string", "minLength": 1 }, + "applied": { "const": true } + } + } + }, + "behavior": { + "idempotency": "required", + "cancellation": "cooperative" + } + } + }, + "workflow.annotate-changed": { + "kind": "workflow", + "value": { + "type": "runtime_workflow", + "id": "example.annotate-changed", + "version": "1.0.0", + "name": "Annotate changed records", + "enabled": true, + "triggers": [ + { + "id": "changed", + "event": { + "id": "example.record.changed", + "version": "^1.0.0" + } + } + ], + "steps": [ + { + "id": "annotate", + "action": { + "id": "example.record.annotate", + "version": "^1.0.0" + }, + "requires": { + "capabilities": ["record.write"] + }, + "input": { + "record_id": { "$expr": "event.data.record_id" }, + "label": "observed" + } + } + ] + } + }, + "policy.allow-record-write": { + "kind": "policy", + "value": { + "type": "runtime_policy", + "id": "example.testbed-policy", + "version": "1.0.0", + "enabled": true, + "executors": { + "default": "testbed-runtime" + }, + "grants": [ + { + "capability": "record.write", + "mode": "allow" + } + ] + } + } + } +} diff --git a/public/testbed/v0.1/scenarios/core.shared-contract-consumers.json b/public/testbed/v0.1/scenarios/core.shared-contract-consumers.json new file mode 100644 index 0000000..d6f713b --- /dev/null +++ b/public/testbed/v0.1/scenarios/core.shared-contract-consumers.json @@ -0,0 +1,88 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "core.shared-contract-consumers", + "name": "Two applications consume one type implementation", + "description": "One type implements one record contract and two independent consumers observe the same projected contract view.", + "ring": "contract", + "profile": "core_read", + "roles": ["contract_store", "record_consumer"], + "operation": "contract.shared-consumers", + "fixtures": [ + "contract.example-note", + "type.shared-note", + "record.shared-note" + ], + "parameters": { + "consumers": ["consumer-alpha", "consumer-beta"] + }, + "covers": [ + "core_read.data_contract_discovery", + "core_read.data_contract_implementation_validation", + "core_read.data_contract_projection" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "contract-store", + "operation": "contract.load", + "outcome": "succeeded", + "facts": { + "contract": "example.note", + "version": "1.0.0" + } + }, + { + "sequence": 2, + "phase": "arrange", + "actor": "contract-store", + "operation": "type.load", + "outcome": "succeeded", + "facts": { + "type": "shared_note", + "implements": "example.note" + } + }, + { + "sequence": 3, + "phase": "act", + "actor": "consumer-alpha", + "operation": "contract-view.read", + "outcome": "succeeded", + "facts": { + "contract": "example.note", + "view": { + "title": "One record, two consumers" + } + } + }, + { + "sequence": 4, + "phase": "act", + "actor": "consumer-beta", + "operation": "contract-view.read", + "outcome": "succeeded", + "facts": { + "contract": "example.note", + "view": { + "title": "One record, two consumers" + } + } + }, + { + "sequence": 5, + "phase": "observe", + "actor": "testbed", + "operation": "contract-view.compare", + "outcome": "succeeded", + "facts": { + "consumers": 2, + "same_contract": true, + "same_view": true + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.action-ambiguous.json b/public/testbed/v0.1/scenarios/interop.action-ambiguous.json new file mode 100644 index 0000000..69c1387 --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.action-ambiguous.json @@ -0,0 +1,52 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.action-ambiguous", + "name": "Multiple eligible providers are explicitly ambiguous", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["action_caller", "action_provider", "bridge"], + "operation": "interop.action-ambiguous", + "fixtures": ["contract.record-annotate"], + "parameters": { + "providers": ["provider-alpha", "provider-beta"] + }, + "covers": [ + "event_action_interop/0.1.explicit_provider_ambiguity" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "provider-alpha", + "operation": "action-provider.register", + "outcome": "succeeded", + "facts": { + "contract": "example.record.annotate" + } + }, + { + "sequence": 2, + "phase": "arrange", + "actor": "provider-beta", + "operation": "action-provider.register", + "outcome": "succeeded", + "facts": { + "contract": "example.record.annotate" + } + }, + { + "sequence": 3, + "phase": "act", + "actor": "caller", + "operation": "action.invoke", + "outcome": "rejected", + "facts": { + "code": "ambiguous_provider", + "provider_calls": 0 + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.action-explicit-provider.json b/public/testbed/v0.1/scenarios/interop.action-explicit-provider.json new file mode 100644 index 0000000..aa399bb --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.action-explicit-provider.json @@ -0,0 +1,65 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.action-explicit-provider", + "name": "An explicit selector resolves provider ambiguity", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["action_caller", "action_provider", "bridge"], + "operation": "interop.action-explicit-provider", + "fixtures": ["contract.record-annotate"], + "parameters": { + "providers": ["provider-alpha", "provider-beta"], + "selected": "provider-beta" + }, + "covers": [ + "event_action_interop/0.1.explicit_provider_ambiguity", + "event_action_interop/0.1.exact_contract_and_provider_evidence" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "provider-alpha", + "operation": "action-provider.register", + "outcome": "succeeded", + "facts": { + "contract": "example.record.annotate" + } + }, + { + "sequence": 2, + "phase": "arrange", + "actor": "provider-beta", + "operation": "action-provider.register", + "outcome": "succeeded", + "facts": { + "contract": "example.record.annotate" + } + }, + { + "sequence": 3, + "phase": "act", + "actor": "caller", + "operation": "action.invoke", + "outcome": "succeeded", + "facts": { + "selected": "provider-beta" + } + }, + { + "sequence": 4, + "phase": "observe", + "actor": "bridge", + "operation": "action.outcome", + "outcome": "succeeded", + "facts": { + "provider": "provider-beta", + "provider_alpha_calls": 0, + "provider_beta_calls": 1 + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.action-single-provider.json b/public/testbed/v0.1/scenarios/interop.action-single-provider.json new file mode 100644 index 0000000..33a1999 --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.action-single-provider.json @@ -0,0 +1,57 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.action-single-provider", + "name": "A single eligible provider is invoked", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["action_caller", "action_provider", "bridge"], + "operation": "interop.action-single-provider", + "fixtures": ["contract.record-annotate"], + "parameters": { + "request_id": "req-single-1", + "provider": "provider-alpha" + }, + "covers": [ + "event_action_interop/0.1.action_request_admission_and_outcome", + "event_action_interop/0.1.exact_contract_and_provider_evidence" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "provider-alpha", + "operation": "action-provider.register", + "outcome": "succeeded", + "facts": { + "contract": "example.record.annotate", + "idempotency": "request" + } + }, + { + "sequence": 2, + "phase": "act", + "actor": "caller", + "operation": "action.invoke", + "outcome": "succeeded", + "facts": { + "request_id": "req-single-1" + } + }, + { + "sequence": 3, + "phase": "observe", + "actor": "bridge", + "operation": "action.outcome", + "outcome": "succeeded", + "facts": { + "exact_contract": true, + "provider": "provider-alpha", + "provider_calls": 1, + "request_id": "req-single-1" + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.application-event-multicast.json b/public/testbed/v0.1/scenarios/interop.application-event-multicast.json new file mode 100644 index 0000000..dafd078 --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.application-event-multicast.json @@ -0,0 +1,75 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.application-event-multicast", + "name": "An application event source reaches two independent consumers", + "description": "A real application-owned event contract is multicast without making its product-specific payload part of the neutral oracle.", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["event_source", "event_consumer", "bridge"], + "operation": "interop.application-event-multicast", + "fixtures": [], + "parameters": {}, + "covers": [ + "event_action_interop/0.1.cloudevents_event_envelope", + "event_action_interop/0.1.event_multicast_and_version_resolution", + "event_action_interop/0.1.boundary_validation" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "application", + "operation": "contract.describe", + "outcome": "succeeded", + "facts": { + "contract_type": "event", + "schema": "json-schema-2020-12" + } + }, + { + "sequence": 2, + "phase": "arrange", + "actor": "application", + "operation": "event-source.register", + "outcome": "succeeded", + "facts": { + "sources": 1 + } + }, + { + "sequence": 3, + "phase": "arrange", + "actor": "consumers", + "operation": "event.subscribe", + "outcome": "succeeded", + "facts": { + "consumers": 2 + } + }, + { + "sequence": 4, + "phase": "act", + "actor": "application", + "operation": "event.publish", + "outcome": "succeeded", + "facts": { + "exact_contract": true, + "specversion": "1.0" + } + }, + { + "sequence": 5, + "phase": "observe", + "actor": "bridge", + "operation": "event.deliver", + "outcome": "succeeded", + "facts": { + "deliveries": 2, + "distinct_consumers": 2 + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.application-provider-lifecycle.json b/public/testbed/v0.1/scenarios/interop.application-provider-lifecycle.json new file mode 100644 index 0000000..90b250e --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.application-provider-lifecycle.json @@ -0,0 +1,84 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.application-provider-lifecycle", + "name": "An application provider validates, invokes, and unloads cleanly", + "description": "A real application-owned action contract is exercised without making its product-specific contract part of the neutral oracle.", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["action_caller", "action_provider", "bridge"], + "operation": "interop.application-provider-lifecycle", + "fixtures": [], + "parameters": {}, + "covers": [ + "event_action_interop/0.1.action_request_admission_and_outcome", + "event_action_interop/0.1.boundary_validation", + "event_action_interop/0.1.cancellation_and_unload" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "application", + "operation": "contract.describe", + "outcome": "succeeded", + "facts": { + "contract_type": "action", + "schema": "json-schema-2020-12" + } + }, + { + "sequence": 2, + "phase": "arrange", + "actor": "application", + "operation": "action-provider.register", + "outcome": "succeeded", + "facts": { + "providers": 1 + } + }, + { + "sequence": 3, + "phase": "act", + "actor": "caller", + "operation": "action.invoke", + "outcome": "succeeded", + "facts": { + "exact_contract": true, + "provider_calls": 1 + } + }, + { + "sequence": 4, + "phase": "act", + "actor": "caller", + "operation": "action.invoke", + "outcome": "rejected", + "facts": { + "code": "invalid_action_input" + } + }, + { + "sequence": 5, + "phase": "act", + "actor": "application", + "operation": "client.dispose", + "outcome": "succeeded", + "facts": { + "providers": 0 + } + }, + { + "sequence": 6, + "phase": "observe", + "actor": "caller", + "operation": "action.invoke", + "outcome": "rejected", + "facts": { + "code": "no_provider" + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.authority-boundary.json b/public/testbed/v0.1/scenarios/interop.authority-boundary.json new file mode 100644 index 0000000..16815c3 --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.authority-boundary.json @@ -0,0 +1,51 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.authority-boundary", + "name": "Contract compatibility never grants provider authority", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["action_provider", "bridge"], + "operation": "interop.authority-boundary", + "fixtures": ["contract.record-annotate"], + "parameters": { + "denied_application": "untrusted-provider" + }, + "covers": [ + "event_action_interop/0.1.independent_authorization" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "untrusted-provider", + "operation": "contract.validate", + "outcome": "succeeded", + "facts": { + "contract": "example.record.annotate" + } + }, + { + "sequence": 2, + "phase": "act", + "actor": "untrusted-provider", + "operation": "action-provider.register", + "outcome": "rejected", + "facts": { + "code": "unauthorized" + } + }, + { + "sequence": 3, + "phase": "observe", + "actor": "bridge", + "operation": "registry.inspect", + "outcome": "succeeded", + "facts": { + "providers": 0 + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.contract-digest-drift.json b/public/testbed/v0.1/scenarios/interop.contract-digest-drift.json new file mode 100644 index 0000000..8afd818 --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.contract-digest-drift.json @@ -0,0 +1,53 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.contract-digest-drift", + "name": "Digest drift for one exact contract identity fails atomically", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["event_source", "bridge"], + "operation": "interop.contract-digest-drift", + "fixtures": ["contract.record-changed"], + "parameters": { + "mutate": "data_schema.value.properties.record_id.minLength" + }, + "covers": [ + "event_action_interop/0.1.first_class_event_and_action_contracts", + "event_action_interop/0.1.boundary_validation" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "source-alpha", + "operation": "event-source.register", + "outcome": "succeeded", + "facts": { + "sources": 1 + } + }, + { + "sequence": 2, + "phase": "act", + "actor": "source-beta", + "operation": "event-source.register", + "outcome": "rejected", + "facts": { + "code": "contract_digest_conflict" + } + }, + { + "sequence": 3, + "phase": "observe", + "actor": "bridge", + "operation": "registry.inspect", + "outcome": "succeeded", + "facts": { + "atomic": true, + "sources": 1 + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.event-multicast.json b/public/testbed/v0.1/scenarios/interop.event-multicast.json new file mode 100644 index 0000000..9badddf --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.event-multicast.json @@ -0,0 +1,78 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.event-multicast", + "name": "One event is multicast to every compatible consumer", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["event_source", "event_consumer", "bridge"], + "operation": "interop.event-multicast", + "fixtures": ["contract.record-changed"], + "parameters": { + "event_id": "evt-shared-1", + "consumers": ["consumer-alpha", "consumer-beta"] + }, + "covers": [ + "event_action_interop/0.1.cloudevents_event_envelope", + "event_action_interop/0.1.event_multicast_and_version_resolution" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "source", + "operation": "event-source.register", + "outcome": "succeeded", + "facts": { + "contract": "example.record.changed", + "version": "1.0.0" + } + }, + { + "sequence": 2, + "phase": "arrange", + "actor": "consumer-alpha", + "operation": "event.subscribe", + "outcome": "succeeded", + "facts": { + "version": "^1.0.0" + } + }, + { + "sequence": 3, + "phase": "arrange", + "actor": "consumer-beta", + "operation": "event.subscribe", + "outcome": "succeeded", + "facts": { + "version": ">=1.0.0 <2.0.0" + } + }, + { + "sequence": 4, + "phase": "act", + "actor": "source", + "operation": "event.publish", + "outcome": "succeeded", + "facts": { + "event_id": "evt-shared-1" + } + }, + { + "sequence": 5, + "phase": "observe", + "actor": "bridge", + "operation": "event.deliver", + "outcome": "succeeded", + "facts": { + "consumers": ["consumer-alpha", "consumer-beta"], + "deliveries": 2, + "duplicate": false, + "exact_contract": true, + "specversion": "1.0" + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.invalid-payload.json b/public/testbed/v0.1/scenarios/interop.invalid-payload.json new file mode 100644 index 0000000..781e26d --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.invalid-payload.json @@ -0,0 +1,63 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.invalid-payload", + "name": "Invalid event data is rejected before delivery", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["event_source", "event_consumer", "bridge"], + "operation": "interop.invalid-payload", + "fixtures": ["contract.record-changed"], + "parameters": { + "data": { + "record_id": "note-1" + } + }, + "covers": [ + "event_action_interop/0.1.boundary_validation" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "source", + "operation": "event-source.register", + "outcome": "succeeded", + "facts": { + "contract": "example.record.changed" + } + }, + { + "sequence": 2, + "phase": "arrange", + "actor": "consumer", + "operation": "event.subscribe", + "outcome": "succeeded", + "facts": { + "version": "^1.0.0" + } + }, + { + "sequence": 3, + "phase": "act", + "actor": "source", + "operation": "event.publish", + "outcome": "rejected", + "facts": { + "code": "invalid_event_data" + } + }, + { + "sequence": 4, + "phase": "observe", + "actor": "consumer", + "operation": "event.deliver", + "outcome": "skipped", + "facts": { + "deliveries": 0 + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.provider-disappearance.json b/public/testbed/v0.1/scenarios/interop.provider-disappearance.json new file mode 100644 index 0000000..e42ff65 --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.provider-disappearance.json @@ -0,0 +1,52 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.provider-disappearance", + "name": "Unloading a provider removes it from admission", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["action_caller", "action_provider", "bridge"], + "operation": "interop.provider-disappearance", + "fixtures": ["contract.record-annotate"], + "parameters": { + "provider": "provider-alpha" + }, + "covers": [ + "event_action_interop/0.1.cancellation_and_unload" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "provider-alpha", + "operation": "action-provider.register", + "outcome": "succeeded", + "facts": { + "providers": 1 + } + }, + { + "sequence": 2, + "phase": "act", + "actor": "provider-alpha", + "operation": "client.dispose", + "outcome": "succeeded", + "facts": { + "providers": 0 + } + }, + { + "sequence": 3, + "phase": "observe", + "actor": "caller", + "operation": "action.invoke", + "outcome": "rejected", + "facts": { + "code": "no_provider", + "provider_calls": 0 + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/interop.request-replay.json b/public/testbed/v0.1/scenarios/interop.request-replay.json new file mode 100644 index 0000000..ac3338d --- /dev/null +++ b/public/testbed/v0.1/scenarios/interop.request-replay.json @@ -0,0 +1,63 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "interop.request-replay", + "name": "A declared idempotent request reuses its recorded outcome", + "ring": "interop", + "profile": "event_action_interop/0.1", + "roles": ["action_caller", "action_provider", "bridge"], + "operation": "interop.request-replay", + "fixtures": ["contract.record-annotate"], + "parameters": { + "request_id": "req-replay-1", + "idempotency_key": "run-1:annotate" + }, + "covers": [ + "event_action_interop/0.1.duplicate_and_indeterminate_semantics" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "provider-alpha", + "operation": "action-provider.register", + "outcome": "succeeded", + "facts": { + "idempotency": "request" + } + }, + { + "sequence": 2, + "phase": "act", + "actor": "caller", + "operation": "action.invoke", + "outcome": "succeeded", + "facts": { + "request_id": "req-replay-1" + } + }, + { + "sequence": 3, + "phase": "act", + "actor": "caller", + "operation": "action.replay", + "outcome": "succeeded", + "facts": { + "request_id": "req-replay-1" + } + }, + { + "sequence": 4, + "phase": "observe", + "actor": "bridge", + "operation": "action.outcome", + "outcome": "succeeded", + "facts": { + "handler_calls": 1, + "same_outcome": true + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/runtime.application-execution.json b/public/testbed/v0.1/scenarios/runtime.application-execution.json new file mode 100644 index 0000000..c913726 --- /dev/null +++ b/public/testbed/v0.1/scenarios/runtime.application-execution.json @@ -0,0 +1,75 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "runtime.application-execution", + "name": "An application-owned runtime executes one exact admitted effect", + "description": "A product runtime composes its real contracts, admits an exact source/provider plan, authorizes dispatch, and persists one successful run.", + "ring": "runtime", + "profile": "runtime/0.2", + "roles": ["event_source", "action_provider", "runtime", "runtime_store"], + "operation": "runtime.application-execution", + "fixtures": [], + "parameters": {}, + "covers": [ + "runtime/0.2.exact_contract_source_and_provider_admission", + "runtime/0.2.dispatch_authorization", + "runtime/0.2.durable_event_admission" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "application-runtime", + "operation": "catalog.compose", + "outcome": "succeeded", + "facts": { + "contracts_verified": true, + "executor_selected": true + } + }, + { + "sequence": 2, + "phase": "act", + "actor": "event-source", + "operation": "event.publish", + "outcome": "succeeded", + "facts": { + "exact_contract": true + } + }, + { + "sequence": 3, + "phase": "act", + "actor": "application-runtime", + "operation": "event.admit", + "outcome": "succeeded", + "facts": { + "runs": 1 + } + }, + { + "sequence": 4, + "phase": "act", + "actor": "action-provider", + "operation": "action.dispatch", + "outcome": "succeeded", + "facts": { + "authorization_checked": true, + "provider_calls": 1 + } + }, + { + "sequence": 5, + "phase": "observe", + "actor": "runtime-store", + "operation": "run.inspect", + "outcome": "succeeded", + "facts": { + "exact_bindings": true, + "status": "succeeded" + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/runtime.competing-workers.json b/public/testbed/v0.1/scenarios/runtime.competing-workers.json new file mode 100644 index 0000000..4277796 --- /dev/null +++ b/public/testbed/v0.1/scenarios/runtime.competing-workers.json @@ -0,0 +1,82 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "runtime.competing-workers", + "name": "Competing workers are fenced by one durable lease", + "ring": "runtime", + "profile": "runtime/0.2", + "roles": ["runtime", "runtime_store"], + "operation": "runtime.competing-workers", + "fixtures": [ + "contract.record-changed", + "contract.record-annotate", + "workflow.annotate-changed", + "policy.allow-record-write" + ], + "parameters": { + "event_id": "evt-fence-1", + "workers": ["worker-alpha", "worker-beta"] + }, + "covers": [ + "runtime/0.2.idempotency_and_concurrency", + "runtime/0.2.lease_and_crash_recovery", + "runtime/0.2.run_state_machine" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "runtime", + "operation": "event.admit", + "outcome": "succeeded", + "facts": { + "event_id": "evt-fence-1", + "runs": 1 + } + }, + { + "sequence": 2, + "phase": "act", + "actor": "worker-alpha", + "operation": "run.claim", + "outcome": "succeeded", + "facts": { + "claims": 1 + } + }, + { + "sequence": 3, + "phase": "act", + "actor": "worker-beta", + "operation": "run.claim", + "outcome": "skipped", + "facts": { + "claims": 0 + } + }, + { + "sequence": 4, + "phase": "recover", + "actor": "worker-alpha", + "operation": "run.commit", + "outcome": "rejected", + "facts": { + "code": "stale_lease" + } + }, + { + "sequence": 5, + "phase": "observe", + "actor": "runtime-store", + "operation": "run.inspect", + "outcome": "succeeded", + "facts": { + "concurrent_claims": 1, + "runs": 1, + "winner": "worker-beta" + } + } + ] + } +} diff --git a/public/testbed/v0.1/scenarios/runtime.crash-recovery.json b/public/testbed/v0.1/scenarios/runtime.crash-recovery.json new file mode 100644 index 0000000..8bcf9b8 --- /dev/null +++ b/public/testbed/v0.1/scenarios/runtime.crash-recovery.json @@ -0,0 +1,73 @@ +{ + "kind": "mdbase.testbed.scenario", + "protocol_version": "0.1", + "id": "runtime.crash-recovery", + "name": "An unknown idempotent dispatch recovers without a duplicate effect", + "ring": "runtime", + "profile": "runtime/0.2", + "roles": ["runtime", "runtime_store", "action_provider"], + "operation": "runtime.crash-recovery", + "fixtures": [ + "contract.record-changed", + "contract.record-annotate", + "workflow.annotate-changed", + "policy.allow-record-write" + ], + "parameters": { + "event_id": "evt-crash-1", + "failpoint": "after_provider_effect_before_result_commit", + "expire_lease": true + }, + "covers": [ + "runtime/0.2.durable_action_attempts", + "runtime/0.2.lease_and_crash_recovery" + ], + "expect": { + "entries": [ + { + "sequence": 1, + "phase": "arrange", + "actor": "runtime", + "operation": "event.admit", + "outcome": "succeeded", + "facts": { + "event_id": "evt-crash-1", + "runs": 1 + } + }, + { + "sequence": 2, + "phase": "act", + "actor": "worker-alpha", + "operation": "action.dispatch", + "outcome": "indeterminate", + "facts": { + "provider_effects": 1 + } + }, + { + "sequence": 3, + "phase": "recover", + "actor": "worker-beta", + "operation": "lease.recover", + "outcome": "succeeded", + "facts": { + "same_attempt_id": true, + "same_invocation_id": true + } + }, + { + "sequence": 4, + "phase": "observe", + "actor": "runtime-store", + "operation": "run.inspect", + "outcome": "succeeded", + "facts": { + "dispatches": 2, + "logical_effects": 1, + "status": "succeeded" + } + } + ] + } +} diff --git a/scripts/sync-sources.mjs b/scripts/sync-sources.mjs index 60be47a..d73b3ef 100644 --- a/scripts/sync-sources.mjs +++ b/scripts/sync-sources.mjs @@ -28,6 +28,18 @@ required(interopSchemaSource, "mdbase interoperability schemas"); mkdirSync(interopSchemaDestination, { recursive: true }); cpSync(interopSchemaSource, interopSchemaDestination, { recursive: true, force: true }); +const testbedSchemaSource = join(specDir, "schemas", "testbed", "v0.1"); +const testbedSchemaDestination = join(root, "public", "testbed", "schemas", "v0.1"); +required(testbedSchemaSource, "mdbase testbed schemas"); +mkdirSync(testbedSchemaDestination, { recursive: true }); +cpSync(testbedSchemaSource, testbedSchemaDestination, { recursive: true, force: true }); + +const testbedSuiteSource = join(specDir, "testbed", "v0.1"); +const testbedSuiteDestination = join(root, "public", "testbed", "v0.1"); +required(testbedSuiteSource, "mdbase testbed suite"); +mkdirSync(testbedSuiteDestination, { recursive: true }); +cpSync(testbedSuiteSource, testbedSuiteDestination, { recursive: true, force: true }); + const runtimeSchemaSource = join( specDir, "standard-packs", @@ -150,6 +162,7 @@ writeFileSync( console.log(`Copied Connect schemas from ${schemaSource}`); console.log(`Copied interoperability schemas from ${interopSchemaSource}`); +console.log(`Copied testbed schemas and scenarios from ${testbedSuiteSource}`); console.log(`Copied Runtime 0.2 schemas from ${runtimeSchemaSource}`); console.log(`Copied the homepage design and murmuration from ${introductionSource}`); console.log(`Copied shared theme values from ${connectThemeSource}`); diff --git a/site-sources.json b/site-sources.json index a98abd3..38b1d80 100644 --- a/site-sources.json +++ b/site-sources.json @@ -1,11 +1,11 @@ { "specification": { "repository": "mdbase-dev/mdbase-spec", - "ref": "92f0032a086cc265d3415189cbf6245e369b13c4" + "ref": "fe1f0c854a58565a2dc6499505f95d3294ce314a" }, "connect": { "repository": "mdbase-dev/mdbase-connect", - "ref": "111ca0a68636414167d9ffcbc32c37f14a6cd804" + "ref": "1f332e95784942163652100d4edf78216efa5419" }, "contracts": { "repository": "mdbase-dev/mdbase-contracts", @@ -15,12 +15,12 @@ { "id": "rust", "repository": "callumalpass/mdbase-rs", - "ref": "fd3808c0897882e37e21ab369b541a5aaa0b676c" + "ref": "281629ef0a54ed5f7868366c5a8a7f69f41eb479" }, { "id": "typescript", "repository": "callumalpass/mdbase", - "ref": "83746e98825a02635ec10a2cad2ea28d5e4b0223" + "ref": "82298b258e5fbc6613c6fc76072d57d4b3028bb4" } ] } diff --git a/src/components/SiteFooter.astro b/src/components/SiteFooter.astro index 8181050..7a9e78c 100644 --- a/src/components/SiteFooter.astro +++ b/src/components/SiteFooter.astro @@ -9,6 +9,7 @@ Connect SDK docs Interoperability + Testbed Runtime Contracts Implementations diff --git a/src/components/SiteHeader.astro b/src/components/SiteHeader.astro index 8c28c98..d8fdc2a 100644 --- a/src/components/SiteHeader.astro +++ b/src/components/SiteHeader.astro @@ -2,7 +2,7 @@ import BrandMark from "./BrandMark.astro"; interface Props { - current?: "home" | "connect" | "sdk" | "interop" | "runtime" | "implementations" | "spec"; + current?: "home" | "connect" | "sdk" | "interop" | "testbed" | "runtime" | "implementations" | "spec"; showSceneCount?: boolean; } @@ -12,6 +12,7 @@ const links = [ { href: "/connect/", label: "Connect", key: "connect" }, { href: "/sdk/quickstart/", label: "Build", key: "sdk" }, { href: "/interop/", label: "Interop", key: "interop" }, + { href: "/testbed/", label: "Test", key: "testbed" }, { href: "/implementations/", label: "Implement", key: "implementations" }, { href: "/spec/", label: "Spec", key: "spec" } ] as const; diff --git a/src/data/conformance.json b/src/data/conformance.json index 52f5759..d818d2d 100644 --- a/src/data/conformance.json +++ b/src/data/conformance.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-07-28", + "generated_at": "2026-07-29", "spec_version": "0.3.0", "implementations": [ { @@ -7,7 +7,7 @@ "language": "Rust", "version": "0.4.0-rc.3", "repository": "https://github.com/callumalpass/mdbase-rs", - "evidence_date": "2026-07-28", + "evidence_date": "2026-07-29", "profiles": [ "core_read", "collection_semantics", @@ -28,7 +28,7 @@ "language": "TypeScript", "version": "0.3.0-rc.4", "repository": "https://github.com/callumalpass/mdbase", - "evidence_date": "2026-07-28", + "evidence_date": "2026-07-29", "profiles": [ "core_read", "collection_semantics", diff --git a/src/layouts/BaseLayout.astro b/src/layouts/BaseLayout.astro index 2c632c8..99535a4 100644 --- a/src/layouts/BaseLayout.astro +++ b/src/layouts/BaseLayout.astro @@ -12,7 +12,7 @@ import ThemeControlScript from "../components/ThemeControlScript.astro"; interface Props { title: string; description: string; - current?: "home" | "connect" | "sdk" | "interop" | "runtime" | "implementations" | "spec"; + current?: "home" | "connect" | "sdk" | "interop" | "testbed" | "runtime" | "implementations" | "spec"; noIndex?: boolean; } diff --git a/src/pages/testbed/index.astro b/src/pages/testbed/index.astro new file mode 100644 index 0000000..b01002a --- /dev/null +++ b/src/pages/testbed/index.astro @@ -0,0 +1,573 @@ +--- +import BaseLayout from "../../layouts/BaseLayout.astro"; +import CodeBlock from "../../components/CodeBlock.astro"; + +const runCommand = `npm install --save-dev @callumalpass/mdbase-testbed + +npx mdbase-testbed validate +npx mdbase-testbed run \\ + --adapter ./scripts/testbed-adapter.mjs \\ + --evidence ./conformance/testbed-evidence.json`; + +const protocol = `your-adapter describe +# writes one adapter-description JSON document + +your-adapter run +# reads one scenario request from stdin +# writes one observable transcript to stdout`; + +const rings = [ + { + id: "contract", + number: "01", + title: "Contract", + count: 1, + summary: "One contract, one implementing type, one record, two consumers.", + scenarios: [ + ["core.shared-contract-consumers", "Both consumers receive the same validated contract view."] + ] + }, + { + id: "interop", + number: "02", + title: "Interop", + count: 11, + summary: "Live sources, consumers, callers, and providers cross an authorization boundary.", + scenarios: [ + ["interop.event-multicast", "One event reaches every compatible consumer."], + ["interop.action-single-provider", "One eligible provider is selected."], + ["interop.action-ambiguous", "Two eligible providers stop implicit dispatch."], + ["interop.action-explicit-provider", "An explicit selector resolves ambiguity."], + ["interop.provider-disappearance", "An unloaded provider is unavailable."], + ["interop.contract-digest-drift", "Same version with a different digest is rejected."], + ["interop.invalid-payload", "Schema-invalid data never reaches a handler."], + ["interop.authority-boundary", "Compatibility cannot grant authority."], + ["interop.request-replay", "A logical request is not applied twice."], + ["interop.application-provider-lifecycle", "A real app provider registers, validates, runs, and unloads."], + ["interop.application-event-multicast", "A real app source reaches two independent consumers."] + ] + }, + { + id: "runtime", + number: "03", + title: "Runtime", + count: 3, + summary: "Durable admission, recovery, and worker fencing are observed through the store.", + scenarios: [ + ["runtime.crash-recovery", "A lost outcome reuses the same attempt and invocation."], + ["runtime.competing-workers", "A stale worker cannot commit after its lease expires."], + ["runtime.application-execution", "A real app catalog produces one exact successful run."] + ] + } +]; +--- + + +
+
+
+

Portable testbed 0.1

+

Make the same promise. Prove the same behavior.

+
+
+

+ One neutral suite tests TypeScript, Rust, plugins, desktop services, + and future implementations through a process boundary. Products keep + their own architecture; users get comparable evidence. +

+
+ Run the testbed + + Open a scenario + +
+
+
+
+ +
+
+
+

The black-box boundary

+

The spec asks. Your implementation answers.

+
+

+ Each scenario starts a fresh adapter process. Only stable facts cross + the boundary; generated IDs, database keys, and wall-clock timing stay + inside. +

+
+ +
+ +
+

What the testbed separates

+
+
+ Owned by the specification +

The question and the oracle

+

+ Draft 2020-12 schemas define neutral fixtures, run requests, adapter + descriptions, transcripts, and evidence. Every expected transcript + lives beside its scenario. +

+
+
+ Owned by the implementation +

The behavior under test

+

+ An adapter drives public collection, bridge, provider, runtime, and + store boundaries. It reduces observations to stable facts; it does + not reimplement the feature. +

+
+
+ Owned by neither +

Authority

+

+ Passing proves behavior, not permission. A transcript cannot grant + access, register a provider, publish an event, or authorize a side + effect. +

+
+
+
+ +
+
+
+

15 scenarios · 3 rings

+

The portable inventory

+
+

+ Small scenarios isolate failures. Application scenarios then repeat + the critical paths with real product contracts and providers. +

+
+
+ { + rings.map((ring) => ( +
+
+ {ring.number} +
+

{ring.title}

+

{ring.summary}

+
+ {ring.count} +
+
    + { + ring.scenarios.map(([id, description]) => ( +
  • + {id} + {description} +
    • + )) + } +
+
+ )) + } +
+
+ +
+
+
+

Fresh process per scenario

+

Run it from any language

+
+

+ The runner launches an executable directly—never through a shell—and + validates both sides of every exchange before comparing observations. +

+
+
+
+

Install and verify

+ +
+
+

Implement two commands

+ +
+
+
+ + Read the adapter protocol + + + Scenario JSON Schema + + + Reference implementation + +
+
+ +
+

Evidence is precise—and deliberately limited

+

+ A passing run can emit a machine-readable evidence document containing + the implementation identity, scenario IDs, profile names, and canonical + transcript digests. Attach it to a conformance claim to make the tested + behavior reproducible. +

+
+ Supplemental evidence, not a blanket badge +

+ A complete profile claim still runs every normative conformance test + for that profile. The testbed adds cross-language behavioral evidence; + it does not silently widen the claim. +

+
+
+ + +