From f641ea80d8f961a2927ed3c572281083b75f2662 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Wed, 16 Sep 2026 15:06:58 -0700 Subject: [PATCH 01/34] Added the changelog entry for the Workflow Streams provider. The changelog checkpoint requires an entry for any user-facing change. --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f9e500d63..551a3002ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,10 @@ to include examples, links to docs, or any other relevant information. `temporalio.contrib.workflow_streams` without naming it. Records are the `StreamRecord` proto inside the shipped item payload, and a handle without a run id follows continue-as-new run by run. +- **Experimental**: `temporalio.streams.providers.workflow_streams` serves the + stream interface over the shipped Workflow Streams transport, so a workflow + reads and publishes through `temporalio.contrib.workflow_streams` without + naming it. ### Changed From f35717e157284c444055cb873628fcef38db8db5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 15 Sep 2026 15:55:24 -0700 Subject: [PATCH 02/34] Added the Nexus front that hides the store behind one endpoint. --- temporalio/streams/providers/nexus.py | 398 ++++++++++++++++++++++++++ tests/streams/test_nexus_provider.py | 118 ++++++++ 2 files changed, 516 insertions(+) create mode 100644 temporalio/streams/providers/nexus.py create mode 100644 tests/streams/test_nexus_provider.py diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py new file mode 100644 index 0000000000..b718e9d14e --- /dev/null +++ b/temporalio/streams/providers/nexus.py @@ -0,0 +1,398 @@ +"""The Nexus front: the outside surface behind one Temporal-authenticated endpoint. + +Two halves in one module. The provider half implements only :func:`producer` +and :func:`consumer`; the workflow side raises, naming a storage provider, +because a workflow's publishes and reads ride the Workflow Task and cannot +cross an RPC. The handler half runs in a worker configured with any storage +provider and serves two sync operations, ``append`` and ``read``, so the +store behind the endpoint is invisible to callers and an operator switches +it without touching them. + +Reads hand out batches and an append carries one batch per call, because a +Nexus operation per record costs too much for token streams. Supersession +records are not transported: the caller's reader re-synthesizes them from +the attempts it observes, which is the policy module's job on every +provider. Cursors pass through opaque, so the caller cannot tell which store +produced them. + +One stated prototype limit: append deduplication holds at the handler by +batch index per producer, so a handler restart during a producer's life can +double-append. The store-level fix, a sequence-explicit append in the IDL +contract, is listed in the design doc's work table. +""" + +from __future__ import annotations + +import asyncio +import base64 +import dataclasses +import json +import urllib.error +import urllib.request +from collections.abc import AsyncIterator +from datetime import timedelta +from typing import Any + +import nexusrpc +import nexusrpc.handler + +import temporalio.converter +from temporalio.api.common.v1 import Payload +from temporalio.streams import _frame, _provider +from temporalio.streams._handles import ReadSource, WriteSink +from temporalio.streams._policy import AttemptTracker +from temporalio.streams._record import BEGINNING, Cursor, RecordKind, StreamRecord + +_WORKFLOW_SIDE_ERROR = ( + "the nexus provider is an outside transport; a worker publishes and reads " + "through a storage provider, so configure one of those on the worker" +) + + +@dataclasses.dataclass +class AppendInput: + workflow_id: str + stream: str + producer_id: str + attempt: int + batch_index: int + payloads: list[str] = dataclasses.field(default_factory=list) + finish: bool = False + + +@dataclasses.dataclass +class AppendOutput: + cursor: str = "" + + +@dataclasses.dataclass +class ReadInput: + workflow_id: str + stream: str = "" + from_token: str = "" + max_records: int = 100 + wait_ms: int = 5000 + + +@dataclasses.dataclass +class RecordWire: + token: str + frame: str + + +@dataclasses.dataclass +class ReadOutput: + records: list[RecordWire] = dataclasses.field(default_factory=list) + next_token: str = "" + + +@nexusrpc.service +class TemporalStreams: + append: nexusrpc.Operation[AppendInput, AppendOutput] + read: nexusrpc.Operation[ReadInput, ReadOutput] + + +@nexusrpc.handler.service_handler(service=TemporalStreams) +class TemporalStreamsHandler: + """Serves the stream endpoint, delegating to one storage provider. + + The delegate is an explicit instance rather than the process default, so + a caller and a handler can coexist in one process, and so an operator can + run handlers for a new store next to handlers for the old one. + """ + + def __init__(self, client: Any, provider: str, **provider_options: Any) -> None: + self._client = client + self._delegate = _provider.instance(provider, **provider_options) + self._producers: dict[tuple[str, str, str, int], tuple[Any, int]] = {} + + @nexusrpc.handler.sync_operation + async def append( + self, _ctx: nexusrpc.handler.StartOperationContext, input: AppendInput + ) -> AppendOutput: + key = (input.workflow_id, input.stream, input.producer_id, input.attempt) + cached = self._producers.get(key) + if cached is not None and input.batch_index <= cached[1]: + return AppendOutput() + if cached is None: + delegate = await self._delegate.producer( + self._client, + workflow_id=input.workflow_id, + stream=input.stream, + producer_id=input.producer_id, + attempt=input.attempt, + ) + else: + delegate = cached[0] + self._producers[key] = (delegate, input.batch_index) + cursor = "" + if input.payloads: + payloads = [] + for encoded in input.payloads: + payload = Payload() + payload.ParseFromString(base64.b64decode(encoded)) + payloads.append(payload) + cursor = (await delegate.append(*payloads)).token + if input.finish: + await delegate.finish() + return AppendOutput(cursor=cursor) + + @nexusrpc.handler.sync_operation + async def read( + self, _ctx: nexusrpc.handler.StartOperationContext, input: ReadInput + ) -> ReadOutput: + delegate = await self._delegate.consumer( + self._client, workflow_id=input.workflow_id, stream=input.stream + ) + converter = temporalio.converter.DataConverter.default.payload_converter + records: list[RecordWire] = [] + next_token = input.from_token + subscription = delegate.read( + start=Cursor(input.from_token) if input.from_token else BEGINNING + ) + try: + async with asyncio.timeout(input.wait_ms / 1000): + async for record in subscription: + if record.kind is RecordKind.SUPERSEDED: + continue + body = b"" + if record.kind is RecordKind.DATA: + body = converter.to_payloads([record.value])[0].SerializeToString() + frame = _frame.encode( + topic=record.topic, + kind=record.kind, + producer=record.producer, + attempt=record.attempt, + sequence=record.sequence, + body=body, + ) + records.append( + RecordWire( + token=record.cursor.token, + frame=base64.b64encode(frame).decode("ascii"), + ) + ) + next_token = record.cursor.token + if len(records) >= input.max_records: + break + except TimeoutError: + pass + finally: + await subscription.aclose() + return ReadOutput(records=records, next_token=next_token) + + +def _post(url: str, body: dict) -> dict: + request = urllib.request.Request( + url, + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + raw = response.read() + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace") + raise RuntimeError(f"stream endpoint call failed ({error.code}): {detail}") + return json.loads(raw) if raw else {} + + +class NexusProducer: + """Appends through the stream endpoint; framing happens behind it.""" + + def __init__( + self, base_url: str, converter: Any, workflow_id: str, stream: str, + producer_id: str, attempt: int, + ) -> None: + self._base_url = base_url + self._converter = converter + self._workflow_id = workflow_id + self._stream = stream + self._producer_id = producer_id + self._attempt = attempt + self._batch_index = 0 + + @property + def attempt(self) -> int: + return self._attempt + + async def append(self, *values: Any) -> Cursor: + payloads = [] + for value in values: + payload = ( + value + if isinstance(value, Payload) + else self._converter.to_payloads([value])[0] + ) + payloads.append(base64.b64encode(payload.SerializeToString()).decode()) + return Cursor((await self._call(payloads=payloads)).get("cursor", "")) + + async def finish(self) -> None: + await self._call(finish=True) + + async def _call(self, payloads: list[str] | None = None, finish: bool = False) -> dict: + self._batch_index += 1 + return await asyncio.to_thread( + _post, + f"{self._base_url}/append", + dataclasses.asdict( + AppendInput( + workflow_id=self._workflow_id, + stream=self._stream, + producer_id=self._producer_id, + attempt=self._attempt, + batch_index=self._batch_index, + payloads=payloads or [], + finish=finish, + ) + ), + ) + + +class NexusConsumer: + """Reads batches from the stream endpoint, re-synthesizing supersession.""" + + def __init__(self, base_url: str, converter: Any, workflow_id: str, stream: str) -> None: + self._base_url = base_url + self._converter = converter + self._workflow_id = workflow_id + self._stream = stream + + async def read( + self, + *, + start: Cursor = BEGINNING, + topic: str | None = None, + type: type | None = None, + ) -> AsyncIterator[StreamRecord[Any]]: + attempts = AttemptTracker() + token = start.token + last_yielded: str | None = None + while True: + raw = await asyncio.to_thread( + _post, + f"{self._base_url}/read", + dataclasses.asdict( + ReadInput( + workflow_id=self._workflow_id, + stream=self._stream, + from_token=token, + ) + ), + ) + for wire in raw.get("records", []): + if wire["token"] == last_yielded: + # An opaque token cannot be advanced past itself, so a + # resumed batch starts with the record already yielded. + continue + try: + kind, frame_topic, source, attempt, sequence, body = _frame.decode( + base64.b64decode(wire["frame"]) + ) + except ValueError: + continue + if topic is not None and frame_topic != topic: + last_yielded = wire["token"] + continue + cursor = Cursor(wire["token"]) + superseded = attempts.note(source, attempt, cursor) + if superseded is not None: + yield superseded + yield StreamRecord( + value=self._decode(body, type) if kind is RecordKind.DATA else None, + cursor=cursor, + kind=kind, + topic=frame_topic, + producer=source, + attempt=attempt, + sequence=sequence, + ) + last_yielded = wire["token"] + token = raw.get("next_token", token) or token + + def _decode(self, body: bytes, as_type: type | None) -> Any: + payload = Payload() + payload.ParseFromString(body) + if as_type is None: + return self._converter.from_payloads([payload])[0] + return self._converter.from_payloads([payload], [as_type])[0] + + +class _NexusProvider: + name = "nexus" + + def __init__(self) -> None: + self._base_url: str | None = None + + def configure(self, **options: Any) -> None: + endpoint = options.pop("endpoint", None) + http_address = options.pop("http_address", "http://127.0.0.1:7243") + service = options.pop("service", "TemporalStreams") + if options: + raise TypeError( + "the nexus provider takes endpoint, http_address and service, " + f"got {sorted(options)}" + ) + if not endpoint: + raise TypeError("the nexus provider needs endpoint=") + self._base_url = ( + f"{http_address}/nexus/endpoints/{endpoint}/services/{service}" + ) + + def worker_options(self) -> dict[str, Any]: + raise RuntimeError(_WORKFLOW_SIDE_ERROR) + + def open_read( + self, + stream: str, + *, + start: Cursor = BEGINNING, + idle_timeout: timedelta | None = None, + ) -> ReadSource: + raise RuntimeError(_WORKFLOW_SIDE_ERROR) + + def open_write(self, topic: str) -> WriteSink: + raise RuntimeError(_WORKFLOW_SIDE_ERROR) + + def _url(self) -> str: + if self._base_url is None: + raise RuntimeError("configure the nexus provider before opening handles") + return self._base_url + + async def producer( + self, + client: Any, + *, + workflow_id: str, + stream: str, + producer_id: str = "", + attempt: int = 0, + ) -> NexusProducer: + if not producer_id: + from temporalio import activity + + producer_id = activity.info().activity_id + attempt = attempt or activity.info().attempt + return NexusProducer( + self._url(), + _converter(client), + workflow_id, + stream, + producer_id, + attempt, + ) + + async def consumer( + self, client: Any, *, workflow_id: str, stream: str = "" + ) -> NexusConsumer: + return NexusConsumer(self._url(), _converter(client), workflow_id, stream) + + +def _converter(client: Any) -> Any: + if client is None: + return temporalio.converter.DataConverter.default.payload_converter + return client.data_converter.payload_converter + + +_provider.register("nexus", _NexusProvider) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py new file mode 100644 index 0000000000..7e2c1074e9 --- /dev/null +++ b/tests/streams/test_nexus_provider.py @@ -0,0 +1,118 @@ +"""Live conformance for the Nexus front. + +The caller talks only to the stream endpoint over the server's Nexus HTTP +ingress; the handler worker delegates to the workflow_streams provider, so +this test is the provider-hiding demonstration: nothing on the caller side +names or could name the store. Gated behind ``STREAMS_LIVE=nexus`` because it +needs a dev server with an HTTP port and a registered Nexus endpoint. + +Environment: ``TEMPORAL_ADDRESS`` (default ``localhost:7233``), +``TEMPORAL_HTTP`` (default ``http://127.0.0.1:7243``), and an endpoint named +``streams-e2e`` targeting task queue ``streams-handlers-e2e``. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import uuid + +import pytest + +from temporalio import streams +from temporalio.client import Client +from temporalio.streams import RecordKind +from temporalio.streams._provider import instance +from temporalio.streams.providers.nexus import TemporalStreamsHandler +from temporalio.worker import Worker + +from tests.streams.test_workflow_streams_provider import EchoLoop, take + +pytestmark = pytest.mark.skipif( + os.environ.get("STREAMS_LIVE") != "nexus", + reason="needs a live server and nexus endpoint; run with STREAMS_LIVE=nexus", +) + +ENDPOINT = "streams-e2e" +HANDLER_TQ = "streams-handlers-e2e" + + +def _endpoint_id() -> str: + # The HTTP ingress dispatches by endpoint id, not name. + out = subprocess.run( + [ + "temporal", "operator", "nexus", "endpoint", "get", + "--name", ENDPOINT, "-o", "json", + "--address", os.environ.get("TEMPORAL_ADDRESS", "localhost:7233"), + ], + capture_output=True, + text=True, + check=True, + ) + return json.loads(out.stdout)["id"] + + +async def test_interface_loop_through_the_nexus_front(): + # The workflow worker and the handler worker both use the storage + # provider; only the caller goes through the front. + streams.configure(provider="workflow_streams") + client = await Client.connect( + os.environ.get("TEMPORAL_ADDRESS", "localhost:7233") + ) + front = instance( + "nexus", + endpoint=_endpoint_id(), + http_address=os.environ.get("TEMPORAL_HTTP", "http://127.0.0.1:7243"), + ) + workflow_id = f"streams-nexus-live-{uuid.uuid4().hex}" + + async with Worker( + client, + task_queue=HANDLER_TQ, + nexus_service_handlers=[ + TemporalStreamsHandler(client, provider="workflow_streams") + ], + ): + async with Worker( + client, + task_queue=f"tq-{workflow_id}", + workflows=[EchoLoop], + **streams.worker_options(), + ): + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + + producer = await front.producer( + None, + workflow_id=workflow_id, + stream="inputs", + producer_id="model", + attempt=1, + ) + await producer.append({"n": 1}, {"n": 2}) + await producer.append({"n": 3}) + await producer.finish() + + consumer = await front.consumer(None, workflow_id=workflow_id) + records = await take(consumer.read(type=dict), 4, timeout=60) + assert [r.kind for r in records] == [ + RecordKind.DATA, + RecordKind.DATA, + RecordKind.DATA, + RecordKind.FINISH, + ] + assert [r.value["echo"] for r in records[:3]] == [1, 2, 3] + + # An opaque cursor from behind the front resumes a fresh reader. + checkpoint = records[1].cursor + resumed = await front.consumer(None, workflow_id=workflow_id) + again = await take( + resumed.read(type=dict, start=checkpoint), 2, timeout=60 + ) + assert [r.value["echo"] for r in again[:2]] == [2, 3] + + await handle.signal(EchoLoop.release) + assert await handle.result() == 3 From 832b8c3918d897dc842c53a1092865908684f98d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 15 Sep 2026 18:01:04 -0700 Subject: [PATCH 03/34] Made cursors exclusive, added latest(), and allowed topic appends. Porting the agent harness onto the interface surfaced three things the small example never needed. A reader resumes after the record a cursor names, so it can store the last cursor it handled without advancing an opaque token. A consumer reports its latest position, so a client can follow a turn it is about to start without the workflow reporting a position. And a producer can append onto a topic of the stream the workflow publishes, so an activity's live output lands next to the workflow's own records for one outside reader. --- temporalio/streams/providers/nexus.py | 63 +++++++++++++++++++-------- tests/streams/test_nexus_provider.py | 7 +-- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index b718e9d14e..37a2f16753 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -56,6 +56,7 @@ class AppendInput: producer_id: str attempt: int batch_index: int + topic: str = "" payloads: list[str] = dataclasses.field(default_factory=list) finish: bool = False @@ -69,9 +70,13 @@ class AppendOutput: class ReadInput: workflow_id: str stream: str = "" - from_token: str = "" + topic: str = "" + after_token: str = "" max_records: int = 100 wait_ms: int = 5000 + # Answer with the newest position and no records, for a reader that + # wants to follow from now. + latest_only: bool = False @dataclasses.dataclass @@ -110,7 +115,12 @@ def __init__(self, client: Any, provider: str, **provider_options: Any) -> None: async def append( self, _ctx: nexusrpc.handler.StartOperationContext, input: AppendInput ) -> AppendOutput: - key = (input.workflow_id, input.stream, input.producer_id, input.attempt) + key = ( + input.workflow_id, + input.stream or f"@{input.topic}", + input.producer_id, + input.attempt, + ) cached = self._producers.get(key) if cached is not None and input.batch_index <= cached[1]: return AppendOutput() @@ -119,6 +129,7 @@ async def append( self._client, workflow_id=input.workflow_id, stream=input.stream, + topic=input.topic, producer_id=input.producer_id, attempt=input.attempt, ) @@ -144,11 +155,15 @@ async def read( delegate = await self._delegate.consumer( self._client, workflow_id=input.workflow_id, stream=input.stream ) + topic = input.topic or None + if input.latest_only: + return ReadOutput(next_token=(await delegate.latest(topic=topic)).token) converter = temporalio.converter.DataConverter.default.payload_converter records: list[RecordWire] = [] - next_token = input.from_token + next_token = input.after_token subscription = delegate.read( - start=Cursor(input.from_token) if input.from_token else BEGINNING + after=Cursor(input.after_token) if input.after_token else BEGINNING, + topic=topic, ) try: async with asyncio.timeout(input.wait_ms / 1000): @@ -203,12 +218,13 @@ class NexusProducer: def __init__( self, base_url: str, converter: Any, workflow_id: str, stream: str, - producer_id: str, attempt: int, + topic: str, producer_id: str, attempt: int, ) -> None: self._base_url = base_url self._converter = converter self._workflow_id = workflow_id self._stream = stream + self._topic = topic self._producer_id = producer_id self._attempt = attempt self._batch_index = 0 @@ -243,6 +259,7 @@ async def _call(self, payloads: list[str] | None = None, finish: bool = False) - producer_id=self._producer_id, attempt=self._attempt, batch_index=self._batch_index, + topic=self._topic, payloads=payloads or [], finish=finish, ) @@ -262,13 +279,12 @@ def __init__(self, base_url: str, converter: Any, workflow_id: str, stream: str) async def read( self, *, - start: Cursor = BEGINNING, + after: Cursor = BEGINNING, topic: str | None = None, type: type | None = None, ) -> AsyncIterator[StreamRecord[Any]]: attempts = AttemptTracker() - token = start.token - last_yielded: str | None = None + token = after.token while True: raw = await asyncio.to_thread( _post, @@ -277,15 +293,12 @@ async def read( ReadInput( workflow_id=self._workflow_id, stream=self._stream, - from_token=token, + topic=topic or "", + after_token=token, ) ), ) for wire in raw.get("records", []): - if wire["token"] == last_yielded: - # An opaque token cannot be advanced past itself, so a - # resumed batch starts with the record already yielded. - continue try: kind, frame_topic, source, attempt, sequence, body = _frame.decode( base64.b64decode(wire["frame"]) @@ -293,7 +306,6 @@ async def read( except ValueError: continue if topic is not None and frame_topic != topic: - last_yielded = wire["token"] continue cursor = Cursor(wire["token"]) superseded = attempts.note(source, attempt, cursor) @@ -308,9 +320,24 @@ async def read( attempt=attempt, sequence=sequence, ) - last_yielded = wire["token"] token = raw.get("next_token", token) or token + async def latest(self, *, topic: str | None = None) -> Cursor: + raw = await asyncio.to_thread( + _post, + f"{self._base_url}/read", + dataclasses.asdict( + ReadInput( + workflow_id=self._workflow_id, + stream=self._stream, + topic=topic or "", + latest_only=True, + ) + ), + ) + token = raw.get("next_token", "") + return Cursor(token) if token else BEGINNING + def _decode(self, body: bytes, as_type: type | None) -> Any: payload = Payload() payload.ParseFromString(body) @@ -347,7 +374,7 @@ def open_read( self, stream: str, *, - start: Cursor = BEGINNING, + after: Cursor = BEGINNING, idle_timeout: timedelta | None = None, ) -> ReadSource: raise RuntimeError(_WORKFLOW_SIDE_ERROR) @@ -365,7 +392,8 @@ async def producer( client: Any, *, workflow_id: str, - stream: str, + stream: str = "", + topic: str = "", producer_id: str = "", attempt: int = 0, ) -> NexusProducer: @@ -379,6 +407,7 @@ async def producer( _converter(client), workflow_id, stream, + topic, producer_id, attempt, ) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 7e2c1074e9..29bd8ca4da 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -106,11 +106,12 @@ async def test_interface_loop_through_the_nexus_front(): ] assert [r.value["echo"] for r in records[:3]] == [1, 2, 3] - # An opaque cursor from behind the front resumes a fresh reader. - checkpoint = records[1].cursor + # An opaque cursor from behind the front resumes a fresh reader + # just past the record it names. + checkpoint = records[0].cursor resumed = await front.consumer(None, workflow_id=workflow_id) again = await take( - resumed.read(type=dict, start=checkpoint), 2, timeout=60 + resumed.read(type=dict, after=checkpoint), 2, timeout=60 ) assert [r.value["echo"] for r in again[:2]] == [2, 3] From 50b1dab01b8a340acac6a043d9467f3a2b939f14 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Wed, 16 Sep 2026 15:13:57 -0700 Subject: [PATCH 04/34] Made the Nexus stream front pass the linters. The handler closes the delegate's subscription as soon as it answers, and the interface only promises an iterator, so the close is guarded by what the delegate actually returned. --- temporalio/streams/providers/nexus.py | 58 ++++++++++++++++++++++----- tests/streams/test_nexus_provider.py | 24 ++++++----- 2 files changed, 61 insertions(+), 21 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 37a2f16753..aad67538e5 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -29,7 +29,7 @@ import json import urllib.error import urllib.request -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator, AsyncIterator from datetime import timedelta from typing import Any @@ -51,6 +51,8 @@ @dataclasses.dataclass class AppendInput: + """One append call: who is writing, where, and what.""" + workflow_id: str stream: str producer_id: str @@ -63,11 +65,15 @@ class AppendInput: @dataclasses.dataclass class AppendOutput: + """Where the append landed, empty when it wrote nothing.""" + cursor: str = "" @dataclasses.dataclass class ReadInput: + """One read call: where to resume from and how long to wait.""" + workflow_id: str stream: str = "" topic: str = "" @@ -81,18 +87,24 @@ class ReadInput: @dataclasses.dataclass class RecordWire: + """One record on the wire: its cursor and its base64 frame.""" + token: str frame: str @dataclasses.dataclass class ReadOutput: + """What one read call answered, and where to resume.""" + records: list[RecordWire] = dataclasses.field(default_factory=list) next_token: str = "" @nexusrpc.service class TemporalStreams: + """The stream endpoint's two operations.""" + append: nexusrpc.Operation[AppendInput, AppendOutput] read: nexusrpc.Operation[ReadInput, ReadOutput] @@ -107,6 +119,7 @@ class TemporalStreamsHandler: """ def __init__(self, client: Any, provider: str, **provider_options: Any) -> None: + """Serve the endpoint out of the provider named by ``provider``.""" self._client = client self._delegate = _provider.instance(provider, **provider_options) self._producers: dict[tuple[str, str, str, int], tuple[Any, int]] = {} @@ -115,6 +128,7 @@ def __init__(self, client: Any, provider: str, **provider_options: Any) -> None: async def append( self, _ctx: nexusrpc.handler.StartOperationContext, input: AppendInput ) -> AppendOutput: + """Append on the caller's account, dropping a repeated batch.""" key = ( input.workflow_id, input.stream or f"@{input.topic}", @@ -152,6 +166,7 @@ async def append( async def read( self, _ctx: nexusrpc.handler.StartOperationContext, input: ReadInput ) -> ReadOutput: + """Answer with the records after the caller's token, or time out.""" delegate = await self._delegate.consumer( self._client, workflow_id=input.workflow_id, stream=input.stream ) @@ -172,7 +187,9 @@ async def read( continue body = b"" if record.kind is RecordKind.DATA: - body = converter.to_payloads([record.value])[0].SerializeToString() + body = converter.to_payloads([record.value])[ + 0 + ].SerializeToString() frame = _frame.encode( topic=record.topic, kind=record.kind, @@ -193,7 +210,11 @@ async def read( except TimeoutError: pass finally: - await subscription.aclose() + # The delegate's read parks against its store, and this call + # answers before the caller asks again, so let it go now rather + # than when the collector is next swept. + if isinstance(subscription, AsyncGenerator): + await subscription.aclose() return ReadOutput(records=records, next_token=next_token) @@ -217,9 +238,16 @@ class NexusProducer: """Appends through the stream endpoint; framing happens behind it.""" def __init__( - self, base_url: str, converter: Any, workflow_id: str, stream: str, - topic: str, producer_id: str, attempt: int, + self, + base_url: str, + converter: Any, + workflow_id: str, + stream: str, + topic: str, + producer_id: str, + attempt: int, ) -> None: + """Bind this producer to one stream or topic on the endpoint.""" self._base_url = base_url self._converter = converter self._workflow_id = workflow_id @@ -231,9 +259,11 @@ def __init__( @property def attempt(self) -> int: + """The generation this producer is writing.""" return self._attempt async def append(self, *values: Any) -> Cursor: + """Append ``values`` through the endpoint.""" payloads = [] for value in values: payload = ( @@ -245,9 +275,12 @@ async def append(self, *values: Any) -> Cursor: return Cursor((await self._call(payloads=payloads)).get("cursor", "")) async def finish(self) -> None: + """Mark this producer done, so a reader stops waiting on it.""" await self._call(finish=True) - async def _call(self, payloads: list[str] | None = None, finish: bool = False) -> dict: + async def _call( + self, payloads: list[str] | None = None, finish: bool = False + ) -> dict: self._batch_index += 1 return await asyncio.to_thread( _post, @@ -270,7 +303,10 @@ async def _call(self, payloads: list[str] | None = None, finish: bool = False) - class NexusConsumer: """Reads batches from the stream endpoint, re-synthesizing supersession.""" - def __init__(self, base_url: str, converter: Any, workflow_id: str, stream: str) -> None: + def __init__( + self, base_url: str, converter: Any, workflow_id: str, stream: str + ) -> None: + """Read what ``workflow_id`` publishes, through the endpoint.""" self._base_url = base_url self._converter = converter self._workflow_id = workflow_id @@ -283,6 +319,7 @@ async def read( topic: str | None = None, type: type | None = None, ) -> AsyncIterator[StreamRecord[Any]]: + """Yield records after ``after``, one endpoint batch at a time.""" attempts = AttemptTracker() token = after.token while True: @@ -323,6 +360,7 @@ async def read( token = raw.get("next_token", token) or token async def latest(self, *, topic: str | None = None) -> Cursor: + """The cursor of the last record written, for following from now.""" raw = await asyncio.to_thread( _post, f"{self._base_url}/read", @@ -363,9 +401,7 @@ def configure(self, **options: Any) -> None: ) if not endpoint: raise TypeError("the nexus provider needs endpoint=") - self._base_url = ( - f"{http_address}/nexus/endpoints/{endpoint}/services/{service}" - ) + self._base_url = f"{http_address}/nexus/endpoints/{endpoint}/services/{service}" def worker_options(self) -> dict[str, Any]: raise RuntimeError(_WORKFLOW_SIDE_ERROR) @@ -377,9 +413,11 @@ def open_read( after: Cursor = BEGINNING, idle_timeout: timedelta | None = None, ) -> ReadSource: + del stream, after, idle_timeout raise RuntimeError(_WORKFLOW_SIDE_ERROR) def open_write(self, topic: str) -> WriteSink: + del topic raise RuntimeError(_WORKFLOW_SIDE_ERROR) def _url(self) -> str: diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 29bd8ca4da..f07e4956c6 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -13,7 +13,6 @@ from __future__ import annotations -import asyncio import json import os import subprocess @@ -27,7 +26,6 @@ from temporalio.streams._provider import instance from temporalio.streams.providers.nexus import TemporalStreamsHandler from temporalio.worker import Worker - from tests.streams.test_workflow_streams_provider import EchoLoop, take pytestmark = pytest.mark.skipif( @@ -43,9 +41,17 @@ def _endpoint_id() -> str: # The HTTP ingress dispatches by endpoint id, not name. out = subprocess.run( [ - "temporal", "operator", "nexus", "endpoint", "get", - "--name", ENDPOINT, "-o", "json", - "--address", os.environ.get("TEMPORAL_ADDRESS", "localhost:7233"), + "temporal", + "operator", + "nexus", + "endpoint", + "get", + "--name", + ENDPOINT, + "-o", + "json", + "--address", + os.environ.get("TEMPORAL_ADDRESS", "localhost:7233"), ], capture_output=True, text=True, @@ -58,9 +64,7 @@ async def test_interface_loop_through_the_nexus_front(): # The workflow worker and the handler worker both use the storage # provider; only the caller goes through the front. streams.configure(provider="workflow_streams") - client = await Client.connect( - os.environ.get("TEMPORAL_ADDRESS", "localhost:7233") - ) + client = await Client.connect(os.environ.get("TEMPORAL_ADDRESS", "localhost:7233")) front = instance( "nexus", endpoint=_endpoint_id(), @@ -110,9 +114,7 @@ async def test_interface_loop_through_the_nexus_front(): # just past the record it names. checkpoint = records[0].cursor resumed = await front.consumer(None, workflow_id=workflow_id) - again = await take( - resumed.read(type=dict, after=checkpoint), 2, timeout=60 - ) + again = await take(resumed.read(type=dict, after=checkpoint), 2, timeout=60) assert [r.value["echo"] for r in again[:2]] == [2, 3] await handle.signal(EchoLoop.release) From 1ee4c7a20774f5a456a6192415506ed30839b39e Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Wed, 16 Sep 2026 15:13:57 -0700 Subject: [PATCH 05/34] Added the changelog entry for the Nexus stream front. The changelog checkpoint requires an entry for any user-facing change. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 551a3002ae..923da8ccc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,9 @@ to include examples, links to docs, or any other relevant information. stream interface over the shipped Workflow Streams transport, so a workflow reads and publishes through `temporalio.contrib.workflow_streams` without naming it. +- **Experimental**: `temporalio.streams.providers.nexus` puts one Nexus + endpoint in front of a storage provider, so a caller reaches a stream + through the endpoint and never names the store. ### Changed From 93f87cfec11d7ae820063d393645f0c91af82780 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Wed, 16 Sep 2026 16:09:41 -0700 Subject: [PATCH 06/34] Bounded the Nexus read with a timeout 3.10 has. `asyncio.timeout` arrived in 3.11 and this package supports 3.10, so the collecting loop moves into a coroutine and `wait_for` bounds it. --- temporalio/streams/providers/nexus.py | 58 ++++++++++++++------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index aad67538e5..3701a70cb1 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -180,34 +180,38 @@ async def read( after=Cursor(input.after_token) if input.after_token else BEGINNING, topic=topic, ) - try: - async with asyncio.timeout(input.wait_ms / 1000): - async for record in subscription: - if record.kind is RecordKind.SUPERSEDED: - continue - body = b"" - if record.kind is RecordKind.DATA: - body = converter.to_payloads([record.value])[ - 0 - ].SerializeToString() - frame = _frame.encode( - topic=record.topic, - kind=record.kind, - producer=record.producer, - attempt=record.attempt, - sequence=record.sequence, - body=body, - ) - records.append( - RecordWire( - token=record.cursor.token, - frame=base64.b64encode(frame).decode("ascii"), - ) + + async def collect() -> None: + nonlocal next_token + async for record in subscription: + if record.kind is RecordKind.SUPERSEDED: + continue + body = b"" + if record.kind is RecordKind.DATA: + body = converter.to_payloads([record.value])[0].SerializeToString() + frame = _frame.encode( + topic=record.topic, + kind=record.kind, + producer=record.producer, + attempt=record.attempt, + sequence=record.sequence, + body=body, + ) + records.append( + RecordWire( + token=record.cursor.token, + frame=base64.b64encode(frame).decode("ascii"), ) - next_token = record.cursor.token - if len(records) >= input.max_records: - break - except TimeoutError: + ) + next_token = record.cursor.token + if len(records) >= input.max_records: + return + + try: + # `wait_for` rather than `asyncio.timeout`, which 3.10 does not + # have. Whatever was collected before the deadline is the answer. + await asyncio.wait_for(collect(), input.wait_ms / 1000) + except asyncio.TimeoutError: pass finally: # The delegate's read parks against its store, and this call From b6637c9ae22f48e3dd4d248d1c3fb0f6c644f875 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 18 Sep 2026 09:26:33 -0700 Subject: [PATCH 07/34] Added the IDL contract and generation for the stream endpoint. The stream endpoint's shape lived only in hand-written Python, so no other language could be given the contract. This defines it once in a nexusrpc yaml contract and generates the Python bindings from it. The generated _definitions helper is hidden from the API docs and skipped by pydocstyle, the same way the other generated trees are. --- pyproject.toml | 6 +- scripts/gen_streams_nexus_api.py | 80 +++ .../providers/_nexus_generated/__init__.py | 23 + .../_nexus_generated/_definitions.py | 542 ++++++++++++++ .../providers/_nexus_generated/models.py | 670 ++++++++++++++++++ .../providers/_nexus_generated/services.py | 39 + .../providers/temporal_streams.nexusrpc.yaml | 123 ++++ 7 files changed, 1482 insertions(+), 1 deletion(-) create mode 100644 scripts/gen_streams_nexus_api.py create mode 100644 temporalio/streams/providers/_nexus_generated/__init__.py create mode 100644 temporalio/streams/providers/_nexus_generated/_definitions.py create mode 100644 temporalio/streams/providers/_nexus_generated/models.py create mode 100644 temporalio/streams/providers/_nexus_generated/services.py create mode 100644 temporalio/streams/providers/temporal_streams.nexusrpc.yaml diff --git a/pyproject.toml b/pyproject.toml index b7077785f5..e55c392193 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,9 +125,11 @@ format = [ ] gen-docs = "uv run scripts/gen_docs.py" gen-nexus-system-api = "uv run scripts/gen_nexus_system_api.py" +gen-streams-nexus-api = "uv run scripts/gen_streams_nexus_api.py" gen-protos = [ { cmd = "uv run scripts/gen_protos.py" }, { ref = "gen-nexus-system-api" }, + { ref = "gen-streams-nexus-api" }, { cmd = "uv run scripts/gen_payload_visitor.py" }, { cmd = "uv run scripts/gen_bridge_client.py" }, { ref = "format" }, @@ -135,6 +137,7 @@ gen-protos = [ gen-protos-docker = [ { cmd = "uv run scripts/gen_protos_docker.py" }, { ref = "gen-nexus-system-api" }, + { ref = "gen-streams-nexus-api" }, { cmd = "uv run scripts/gen_payload_visitor.py" }, { cmd = "uv run scripts/gen_bridge_client.py" }, { ref = "format" }, @@ -207,7 +210,7 @@ ignore_errors = true [tool.pydocstyle] convention = "google" # https://github.com/PyCQA/pydocstyle/issues/363#issuecomment-625563088 -match_dir = "^(?!(docs|scripts|tests|api|proto|system|\\.)).*" +match_dir = "^(?!(docs|scripts|tests|api|proto|system|_nexus_generated|\\.)).*" add_ignore = [ # We like to wrap at a certain number of chars, even long summary sentences. # https://github.com/PyCQA/pydocstyle/issues/184 @@ -241,6 +244,7 @@ privacy = [ "HIDDEN:temporalio.worker.workflow_sandbox.importer", "HIDDEN:temporalio.worker.workflow_sandbox.in_sandbox", "HIDDEN:**.*_pb2*", + "HIDDEN:temporalio.streams.providers._nexus_generated._definitions", ] project-name = "Temporal Python" sidebar-expand-depth = 2 diff --git a/scripts/gen_streams_nexus_api.py b/scripts/gen_streams_nexus_api.py new file mode 100644 index 0000000000..800aba9448 --- /dev/null +++ b/scripts/gen_streams_nexus_api.py @@ -0,0 +1,80 @@ +import os +import shutil +import subprocess +import sys +from pathlib import Path + +base_dir = Path(__file__).parent.parent +providers_dir = base_dir / "temporalio" / "streams" / "providers" +contract_path = providers_dir / "temporal_streams.nexusrpc.yaml" +output_dir = providers_dir / "_nexus_generated" +# Pinned to the version CI installs, because nexgen's output changes between +# releases and check-protos compares against what is committed here. +NEX_GEN_VERSION = "0.2.4" + + +def nex_gen_command() -> list[str]: + if bin_path := os.environ.get("NEX_GEN_BIN"): + return [bin_path] + + if shutil.which("nexgen") is None: + subprocess.check_call( + [ + "cargo", + "install", + "--locked", + "nexgen", + "--version", + NEX_GEN_VERSION, + # Same build as the system API script installs, so one binary + # serves both and neither can overwrite the other's output. + "--features", + "advanced", + "--force", + ] + ) + return ["nexgen"] + + +def generate_streams_nexus_api() -> None: + if not contract_path.exists(): + raise RuntimeError(f"missing stream contract: {contract_path}") + + command = nex_gen_command() + shutil.rmtree(output_dir, ignore_errors=True) + subprocess.check_call( + [ + *command, + "python", + str(contract_path), + "--output", + str(output_dir), + ] + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "ruff", + "check", + "--select", + "I", + "--fix", + str(output_dir), + ] + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "ruff", + "format", + str(output_dir), + ] + ) + + +if __name__ == "__main__": + print("Generating stream endpoint Nexus API...", file=sys.stderr) + generate_streams_nexus_api() + print("Done", file=sys.stderr) diff --git a/temporalio/streams/providers/_nexus_generated/__init__.py b/temporalio/streams/providers/_nexus_generated/__init__.py new file mode 100644 index 0000000000..c62eaa6dd3 --- /dev/null +++ b/temporalio/streams/providers/_nexus_generated/__init__.py @@ -0,0 +1,23 @@ +# Generated by nexgen v0.2.4. DO NOT EDIT! + +from __future__ import annotations + +from ._definitions import Violation +from .models import ( + AppendInput, + AppendOutput, + ReadInput, + ReadOutput, + RecordWire, +) +from .services import TemporalStreams + +__all__ = [ + "Violation", + "AppendInput", + "AppendOutput", + "ReadInput", + "ReadOutput", + "RecordWire", + "TemporalStreams", +] diff --git a/temporalio/streams/providers/_nexus_generated/_definitions.py b/temporalio/streams/providers/_nexus_generated/_definitions.py new file mode 100644 index 0000000000..dc8de5bb57 --- /dev/null +++ b/temporalio/streams/providers/_nexus_generated/_definitions.py @@ -0,0 +1,542 @@ +# Generated by nexgen v0.2.4. DO NOT EDIT! + +from __future__ import annotations + +import base64 +import collections.abc +import dataclasses +import datetime +import json +import re +import typing + +import temporalio.converter +import temporalio.exceptions + +__all__ = [ + "Violation", + "_binary64", + "_check_contains", + "_check_date_time", + "_check_duration", + "_check_time", + "_check_unique_items", + "_collect", + "_json_values_equal", + "_format_base64", + "_format_base64url", + "_format_date", + "_format_date_time", + "_format_duration", + "_format_time", + "_parse_base64", + "_parse_base64url", + "_parse_date", + "_parse_date_time", + "_parse_duration", + "_parse_spec_integer", + "_parse_time", + "_quote", + "_transfer_type_convertible", +] + + +@dataclasses.dataclass(frozen=True, slots=True) +class Violation: + """A single constraint failure, located by JSON path.""" + + path: str + reason: str + + +def _quote(value: object) -> str: + """Renders a value in the JSON form every target quotes offending values in.""" + + try: + return json.dumps(value, ensure_ascii=False) + except (TypeError, ValueError): + return repr(value) + + +def _collect( + violations: list[Violation], + path: str, + error: temporalio.exceptions.ApplicationError, +) -> None: + """Re-paths a nested model's violations under `path` and appends them.""" + + if error.type != "PayloadValidationError" or not error.details: + raise error + # Generated failures retain the original list as their first detail. The + # cast is type-checker-only and performs no serialization. + nested_violations = typing.cast(list[Violation], error.details[0]) + for inner in nested_violations: + # A nested violation about the value *itself* carries no path of its own + # (a union branch's own constraint, an element-level check), so the + # prefix is the whole path -- never a dangling separator (P11). + nested = f"{path}.{inner.path}" if inner.path else path + violations.append(Violation(path=nested, reason=inner.reason)) + + +_ModelT = typing.TypeVar("_ModelT") + + +def _transfer_type_convertible( + converter: type[temporalio.converter.TransferTypeConverter[typing.Any, typing.Any]], +) -> collections.abc.Callable[[type[_ModelT]], type[_ModelT]]: + """Registers a transfer type converter on a model class. + + Wraps `temporalio.converter.transfer_type_convertible` to erase the + converter's value-type parameter. Binding it directly on the decorated class + is circular for a static type checker -- the class's type depends on the + decorator, whose value type depends on the class -- which degrades the model + to `Unknown`. Erasing it here keeps the decorator idiomatic at each model and + resolves the cycle. + """ + + return temporalio.converter.transfer_type_convertible(converter) + + +_INTEGER_CAP = (1 << 53) - 1 + + +def _parse_spec_integer( + value: object, path: str, violations: list[Violation] +) -> int | None: + """Parses a JSON number as a spec integer (`1.0` accepted, `1.5` rejected).""" + + # `bool` is a subclass of `int`, so it must be excluded before the int check. + if isinstance(value, bool) or not isinstance(value, (int, float)): + violations.append(Violation(path=path, reason="expected integer")) + return None + if isinstance(value, float): + if not value.is_integer(): + violations.append(Violation(path=path, reason="expected integer")) + return None + out = int(value) + else: + out = value + if abs(out) > _INTEGER_CAP: + violations.append(Violation(path=path, reason="expected integer")) + return None + return out + + +def _binary64(value: float) -> float: + """Narrows a `number` value to the binary64 domain shared by every target.""" + + try: + return float(value) + except OverflowError: + # Past the binary64 range there is nothing to narrow to. The caller's + # finiteness check has already recorded that violation and will raise, so + # hand the value back rather than throwing out of turn and losing the + # other violations collected alongside it. + return value + + +def _json_values_equal(left: typing.Any, right: typing.Any) -> bool: + """Compares JSON values without Python's bool-as-int equality leak.""" + + left_is_number = isinstance(left, (int, float)) and not isinstance(left, bool) + right_is_number = isinstance(right, (int, float)) and not isinstance(right, bool) + if left_is_number or right_is_number: + return left_is_number and right_is_number and left == right + if type(left) is not type(right): + return False + if isinstance(left, list): + left_list = typing.cast("list[typing.Any]", left) + right_list = typing.cast("list[typing.Any]", right) + return len(left_list) == len(right_list) and all( + _json_values_equal(a, b) for a, b in zip(left_list, right_list) + ) + if isinstance(left, dict): + left_dict = typing.cast("dict[typing.Any, typing.Any]", left) + right_dict = typing.cast("dict[typing.Any, typing.Any]", right) + return left_dict.keys() == right_dict.keys() and all( + _json_values_equal(left_dict[key], right_dict[key]) for key in left_dict + ) + return left == right + + +def _check_unique_items( + value: list[typing.Any], path: str, violations: list[Violation] +) -> None: + """Asserts an array's elements are pairwise distinct.""" + + seen: list[typing.Any] = [] + for index, element in enumerate(value): + for earlier, previous in enumerate(seen): + if _json_values_equal(previous, element): + violations.append( + Violation( + path=path, + reason=( + f"duplicate items: element at index {index} " + f"equals index {earlier}" + ), + ) + ) + break + seen.append(element) + + +def _check_contains( + value: list[typing.Any], + matches: typing.Callable[[typing.Any], bool], + min_contains: int, + max_contains: int | None, + bounded_min: bool, + path: str, + violations: list[Violation], +) -> None: + """Asserts how many of an array's elements match the `contains` schema.""" + + match_count = sum(1 for element in value if matches(element)) + if match_count < min_contains: + if bounded_min: + violations.append( + Violation( + path=path, + reason=( + f"too few matching items: at least {min_contains}, " + f"got {match_count}" + ), + ) + ) + else: + violations.append( + Violation(path=path, reason="no element matches the required schema") + ) + if max_contains is not None and match_count > max_contains: + violations.append( + Violation( + path=path, + reason=( + f"too many matching items: at most {max_contains}, " + f"got {match_count}" + ), + ) + ) + + +_TEMPORAL_DATE_TIME_RE = re.compile( + r"^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?([Zz]|[+-]([01][0-9]|2[0-3]):[0-5][0-9])\Z" +) +_TEMPORAL_DATE_RE = re.compile(r"^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])\Z") +_TEMPORAL_TIME_RE = re.compile( + r"^([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?([Zz]|[+-]([01][0-9]|2[0-3]):[0-5][0-9])?\Z" +) +_TEMPORAL_DURATION_RE = re.compile( + r"^PT(?:[0-9]+H(?:[0-9]+M(?:[0-9]+S)?)?|[0-9]+M(?:[0-9]+S)?|[0-9]+S)\Z" +) +_TEMPORAL_MAX_DURATION_SECONDS = ((1 << 63) - 1) // 1_000_000_000 +# A duration component with more digits than the cap itself is over the cap +# whatever those digits are, which is how the magnitude is bounded before `int()` +# sees it: CPython refuses to convert a string of more than 4300 digits. +_TEMPORAL_MAX_DURATION_DIGITS = len(str(_TEMPORAL_MAX_DURATION_SECONDS)) +# `datetime` resolves to microseconds, and `fromisoformat` before Python 3.11 +# parses only the fraction widths `isoformat` writes. +_TEMPORAL_FRACTION_DIGITS = 6 + + +def _days_in_month(year: int, month: int) -> int: + if month in (1, 3, 5, 7, 8, 10, 12): + return 31 + if month in (4, 6, 9, 11): + return 30 + if month == 2: + return 29 if (year % 4 == 0 and year % 100 != 0) or year % 400 == 0 else 28 + return 0 + + +def _valid_temporal_calendar(value: str) -> bool: + if len(value) < 10: + return False + try: + year, month, day = int(value[0:4]), int(value[5:7]), int(value[8:10]) + except ValueError: + return False + # `datetime.MINYEAR` is 1, which is also the shared cross-language floor. + # Year 0000 is rejected rather than shifted into range, and + # `_temporal_reason` says so. + if year < datetime.MINYEAR: + return False + maximum = _days_in_month(year, month) + return maximum > 0 and 1 <= day <= maximum + + +def _temporal_reason(name: str, value: str) -> str: + """The reason a rejected temporal string is reported under. + + Year 0000 earns its own clause so the caller sees the shared calendar floor + rather than only a generic malformed-timestamp reason. + """ + + if value[0:4] == "0000": + return ( + f"must be a valid {name}, got {_quote(value)}: year 0000 is not" + f" representable (datetime.MINYEAR is {datetime.MINYEAR})" + ) + return f"must be a valid {name}, got {_quote(value)}" + + +def _temporal_isoformat(value: str) -> str: + """Rewrites a wire temporal into the spelling `fromisoformat` accepts. + + `Z` becomes `+00:00`, and the fractional second is padded or truncated to + exactly `_TEMPORAL_FRACTION_DIGITS`: before Python 3.11 `fromisoformat` + parses only what `isoformat` writes, so an RFC 3339 `.1` or `.1234567` -- + which every other target accepts -- would otherwise raise. Digits past the + sixth are dropped, the loss at `datetime`'s own resolution that P1 allows; + the canonical output re-trims the padding, so `.1` still writes as `.1`. + """ + + normalized = value.upper() + if normalized.endswith("Z"): + normalized = normalized[:-1] + "+00:00" + dot = normalized.find(".") + if dot < 0: + return normalized + end = dot + 1 + while end < len(normalized) and normalized[end].isdigit(): + end += 1 + fraction = normalized[dot + 1 : end].ljust(_TEMPORAL_FRACTION_DIGITS, "0") + return ( + normalized[: dot + 1] + fraction[:_TEMPORAL_FRACTION_DIGITS] + normalized[end:] + ) + + +def _parse_date_time( + value: str, path: str, violations: list[Violation] +) -> datetime.datetime | None: + if _TEMPORAL_DATE_TIME_RE.match(value) is None or not _valid_temporal_calendar( + value + ): + violations.append( + Violation(path=path, reason=_temporal_reason("date-time", value)) + ) + return None + return datetime.datetime.fromisoformat(_temporal_isoformat(value)) + + +def _parse_date( + value: str, path: str, violations: list[Violation] +) -> datetime.date | None: + if _TEMPORAL_DATE_RE.match(value) is None or not _valid_temporal_calendar(value): + violations.append(Violation(path=path, reason=_temporal_reason("date", value))) + return None + return datetime.date.fromisoformat(value) + + +def _parse_time( + value: str, path: str, violations: list[Violation] +) -> datetime.time | None: + if _TEMPORAL_TIME_RE.match(value) is None: + violations.append(Violation(path=path, reason=_temporal_reason("time", value))) + return None + return datetime.time.fromisoformat(_temporal_isoformat(value)) + + +def _parse_duration( + value: str, path: str, violations: list[Violation] +) -> datetime.timedelta | None: + if _TEMPORAL_DURATION_RE.match(value) is None: + violations.append( + Violation(path=path, reason=_temporal_reason("duration", value)) + ) + return None + total = 0 + number = "" + for char in value[2:]: + if char.isdigit(): + number += char + continue + digits = number.lstrip("0") + number = "" + if len(digits) > _TEMPORAL_MAX_DURATION_DIGITS: + # Over the cap by digit count alone (see the constant), so the + # conversion `int()` would refuse is never attempted. + total = _TEMPORAL_MAX_DURATION_SECONDS + 1 + break + total += int(digits or "0") * {"H": 3600, "M": 60, "S": 1}[char] + if total > _TEMPORAL_MAX_DURATION_SECONDS: + break + if total > _TEMPORAL_MAX_DURATION_SECONDS: + violations.append( + Violation(path=path, reason=_temporal_reason("duration", value)) + ) + return None + return datetime.timedelta(seconds=total) + + +def _check_temporal_offset( + name: str, + value: datetime.datetime | datetime.time, + offset: datetime.timedelta, + path: str, + violations: list[Violation], +) -> None: + """Asserts a UTC offset is a whole number of minutes, the finest the wire + form spells (`tzinfo` allows seconds, which the offset would silently lose). + """ + + if offset % datetime.timedelta(minutes=1): + violations.append( + Violation( + path=path, + reason=( + f"must be a valid {name}, got {_quote(str(value))}: " + f"the UTC offset {offset} is not a whole number of minutes" + ), + ) + ) + + +def _check_date_time( + value: datetime.datetime, path: str, violations: list[Violation] +) -> None: + """Asserts a datetime is writable as a wire date-time (P12). + + A dataclass is constructed unchecked, so a naive datetime -- with no offset + the required wire form could carry -- reaches serialize; without this it + would emit a value this module's own parser rejects. + """ + + offset = value.utcoffset() + if offset is None: + violations.append( + Violation( + path=path, + reason=( + f"must be a valid date-time, got {_quote(str(value))}: " + "a naive datetime carries no UTC offset" + ), + ) + ) + return + _check_temporal_offset("date-time", value, offset, path, violations) + + +def _check_time(value: datetime.time, path: str, violations: list[Violation]) -> None: + """Asserts a time is writable as a wire time (P12). The offset is optional in + the grammar, so only its precision is held to anything.""" + + offset = value.utcoffset() + if offset is not None: + _check_temporal_offset("time", value, offset, path, violations) + + +def _check_duration( + value: datetime.timedelta, path: str, violations: list[Violation] +) -> None: + """Asserts a timedelta is writable as a wire duration (P12): the grammar is + unsigned, whole-second and capped, and a `timedelta` is none of those.""" + + if value < datetime.timedelta(0): + reason = "a duration cannot be negative" + elif value % datetime.timedelta(seconds=1): + reason = "a duration cannot carry a fraction of a second" + elif value.total_seconds() > _TEMPORAL_MAX_DURATION_SECONDS: + reason = f"a duration cannot exceed {_TEMPORAL_MAX_DURATION_SECONDS} seconds" + else: + return + violations.append( + Violation( + path=path, + reason=f"must be a valid duration, got {_quote(str(value))}: {reason}", + ) + ) + + +def _temporal_frac(microsecond: int) -> str: + if microsecond == 0: + return "" + return "." + f"{microsecond:06d}".rstrip("0") + + +def _temporal_offset(value: datetime.datetime | datetime.time) -> str: + offset = value.utcoffset() + if offset is None: + return "" + total = int(offset.total_seconds()) + if total == 0: + return "Z" + sign = "+" if total > 0 else "-" + total = abs(total) + return f"{sign}{total // 3600:02d}:{(total % 3600) // 60:02d}" + + +def _format_date_time(value: datetime.datetime) -> str: + return ( + f"{value.year:04d}-{value.month:02d}-{value.day:02d}" + f"T{value.hour:02d}:{value.minute:02d}:{value.second:02d}" + f"{_temporal_frac(value.microsecond)}{_temporal_offset(value)}" + ) + + +def _format_date(value: datetime.date) -> str: + return f"{value.year:04d}-{value.month:02d}-{value.day:02d}" + + +def _format_time(value: datetime.time) -> str: + return ( + f"{value.hour:02d}:{value.minute:02d}:{value.second:02d}" + f"{_temporal_frac(value.microsecond)}{_temporal_offset(value)}" + ) + + +def _format_duration(value: datetime.timedelta) -> str: + total = int(value.total_seconds()) + if total == 0: + return "PT0S" + hours, remainder = divmod(total, 3600) + minutes, seconds = divmod(remainder, 60) + out = "PT" + if hours: + out += f"{hours}H" + if minutes: + out += f"{minutes}M" + if seconds: + out += f"{seconds}S" + return out + + +_BASE64_RE = re.compile( + "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?\\Z", + re.ASCII, +) +_BASE64URL_RE = re.compile( + "^(?:[A-Za-z0-9_-]{4})*(?:[A-Za-z0-9_-][AQgw]|[A-Za-z0-9_-]{2}[AEIMQUYcgkosw048])?\\Z", + re.ASCII, +) + + +def _parse_base64(value: str, path: str, violations: list[Violation]) -> bytes | None: + if _BASE64_RE.match(value) is None: + violations.append( + Violation(path=path, reason=f"must be base64-encoded, got {_quote(value)}") + ) + return None + return base64.b64decode(value, validate=True) + + +def _format_base64(value: bytes) -> str: + return base64.b64encode(value).decode("ascii") + + +def _parse_base64url( + value: str, path: str, violations: list[Violation] +) -> bytes | None: + if _BASE64URL_RE.match(value) is None: + violations.append( + Violation( + path=path, reason=f"must be base64url-encoded, got {_quote(value)}" + ) + ) + return None + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def _format_base64url(value: bytes) -> str: + return base64.urlsafe_b64encode(value).rstrip(b"=").decode("ascii") diff --git a/temporalio/streams/providers/_nexus_generated/models.py b/temporalio/streams/providers/_nexus_generated/models.py new file mode 100644 index 0000000000..f22711a4f4 --- /dev/null +++ b/temporalio/streams/providers/_nexus_generated/models.py @@ -0,0 +1,670 @@ +# Generated by nexgen v0.2.4. DO NOT EDIT! + +from __future__ import annotations + +import dataclasses +import typing + +import typing_extensions + +import temporalio.converter +import temporalio.exceptions + +from ._definitions import ( + Violation, + _collect, + _parse_spec_integer, + _transfer_type_convertible, +) + + +class _AppendInputTransferTypeConverter( + temporalio.converter.TransferTypeConverter["AppendInput", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["AppendInput"] + ) -> "AppendInput": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + workflow_id_value: str = typing.cast("typing.Any", None) + if "workflow_id" not in raw or raw["workflow_id"] is None: + violations.append(Violation(path="workflow_id", reason="required")) + else: + workflow_id_value_raw = raw["workflow_id"] + if not isinstance(workflow_id_value_raw, str): + violations.append( + Violation(path="workflow_id", reason="expected string") + ) + else: + workflow_id_value = workflow_id_value_raw + + stream_value: str = typing.cast("typing.Any", None) + if "stream" not in raw or raw["stream"] is None: + violations.append(Violation(path="stream", reason="required")) + else: + stream_value_raw = raw["stream"] + if not isinstance(stream_value_raw, str): + violations.append(Violation(path="stream", reason="expected string")) + else: + stream_value = stream_value_raw + + producer_id_value: str = typing.cast("typing.Any", None) + if "producer_id" not in raw or raw["producer_id"] is None: + violations.append(Violation(path="producer_id", reason="required")) + else: + producer_id_value_raw = raw["producer_id"] + if not isinstance(producer_id_value_raw, str): + violations.append( + Violation(path="producer_id", reason="expected string") + ) + else: + producer_id_value = producer_id_value_raw + + attempt_value: int = typing.cast("typing.Any", None) + if "attempt" not in raw or raw["attempt"] is None: + violations.append(Violation(path="attempt", reason="required")) + else: + attempt_value_raw = raw["attempt"] + attempt_value_parsed = _parse_spec_integer( + attempt_value_raw, "attempt", violations + ) + if attempt_value_parsed is not None: + attempt_value = attempt_value_parsed + + batch_index_value: int = typing.cast("typing.Any", None) + if "batch_index" not in raw or raw["batch_index"] is None: + violations.append(Violation(path="batch_index", reason="required")) + else: + batch_index_value_raw = raw["batch_index"] + batch_index_value_parsed = _parse_spec_integer( + batch_index_value_raw, "batch_index", violations + ) + if batch_index_value_parsed is not None: + batch_index_value = batch_index_value_parsed + + topic_value: str | None = None + if "topic" in raw: + topic_value_raw = raw["topic"] + if topic_value_raw is None: + violations.append( + Violation(path="topic", reason="explicit null not allowed") + ) + else: + if not isinstance(topic_value_raw, str): + violations.append(Violation(path="topic", reason="expected string")) + else: + topic_value = topic_value_raw + + payloads_value: list[str] | None = None + if "payloads" in raw: + payloads_value_raw = raw["payloads"] + if payloads_value_raw is None: + violations.append( + Violation(path="payloads", reason="explicit null not allowed") + ) + else: + if not isinstance(payloads_value_raw, list): + violations.append( + Violation(path="payloads", reason="expected array") + ) + else: + payloads_value_list: list[str] = [] + for payloads_value_index, payloads_value_element in enumerate( + typing.cast("list[typing.Any]", payloads_value_raw) + ): + payloads_value_item_path = f"payloads[{payloads_value_index}]" + payloads_value_item_violation_count = len(violations) + payloads_value_item: str = typing.cast("typing.Any", None) + if not isinstance(payloads_value_element, str): + violations.append( + Violation( + path=payloads_value_item_path, + reason="expected string", + ) + ) + else: + payloads_value_item = payloads_value_element + if len(violations) == payloads_value_item_violation_count: + payloads_value_list.append(payloads_value_item) + payloads_value = payloads_value_list + + finish_value: bool | None = None + if "finish" in raw: + finish_value_raw = raw["finish"] + if finish_value_raw is None: + violations.append( + Violation(path="finish", reason="explicit null not allowed") + ) + else: + if not isinstance(finish_value_raw, bool): + violations.append( + Violation(path="finish", reason="expected boolean") + ) + else: + finish_value = finish_value_raw + + for key in raw: + if ( + key != "workflow_id" + and key != "stream" + and key != "producer_id" + and key != "attempt" + and key != "batch_index" + and key != "topic" + and key != "payloads" + and key != "finish" + ): + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return AppendInput( + workflow_id=workflow_id_value, + stream=stream_value, + producer_id=producer_id_value, + attempt=attempt_value, + batch_index=batch_index_value, + topic=topic_value, + payloads=payloads_value, + finish=finish_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "AppendInput") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + out["workflow_id"] = value.workflow_id + out["stream"] = value.stream + out["producer_id"] = value.producer_id + if abs(value.attempt) > 9007199254740991: + violations.append( + Violation(path="attempt", reason="exceeds ±(2^53-1) integer cap") + ) + out["attempt"] = value.attempt + if abs(value.batch_index) > 9007199254740991: + violations.append( + Violation(path="batch_index", reason="exceeds ±(2^53-1) integer cap") + ) + out["batch_index"] = value.batch_index + if value.topic is not None: + out["topic"] = value.topic + if value.payloads is not None: + out["payloads"] = value.payloads + if value.finish is not None: + out["finish"] = value.finish + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_AppendInputTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class AppendInput: + """One append call: who is writing, where, and what.""" + + workflow_id: str + """The workflow whose stream is being written.""" + + stream: str + """The named stream to append to. Empty when the producer writes to a topic instead.""" + + producer_id: str + """Identifies the writer across its retries, so its attempts can be ordered.""" + + attempt: int + """The generation this producer is writing. A later attempt supersedes an earlier one.""" + + batch_index: int + """Counts this producer's batches from 1. The handler drops an index it has already + written for this producer attempt. + """ + + topic: str | None = None + """The topic to append to, when stream is empty.""" + + payloads: list[str] | None = None + """The records to write, each a base64-encoded serialized Temporal Payload. Empty on a + call that only finishes. + """ + + finish: bool | None = None + """Mark this producer done after writing the batch, so a reader stops waiting on it.""" + + +class _AppendOutputTransferTypeConverter( + temporalio.converter.TransferTypeConverter["AppendOutput", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["AppendOutput"] + ) -> "AppendOutput": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + cursor_value: str | None = None + if "cursor" in raw: + cursor_value_raw = raw["cursor"] + if cursor_value_raw is None: + violations.append( + Violation(path="cursor", reason="explicit null not allowed") + ) + else: + if not isinstance(cursor_value_raw, str): + violations.append( + Violation(path="cursor", reason="expected string") + ) + else: + cursor_value = cursor_value_raw + + for key in raw: + if key != "cursor": + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return AppendOutput( + cursor=cursor_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "AppendOutput") -> typing.Any: + out: dict[str, typing.Any] = {} + if value.cursor is not None: + out["cursor"] = value.cursor + return out + + +@_transfer_type_convertible(_AppendOutputTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class AppendOutput: + """Where the append landed, empty when it wrote nothing.""" + + cursor: str | None = None + """Opaque token naming the last record written by this call. Empty when the call + carried no payloads. + """ + + +class _ReadInputTransferTypeConverter( + temporalio.converter.TransferTypeConverter["ReadInput", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["ReadInput"] + ) -> "ReadInput": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + workflow_id_value: str = typing.cast("typing.Any", None) + if "workflow_id" not in raw or raw["workflow_id"] is None: + violations.append(Violation(path="workflow_id", reason="required")) + else: + workflow_id_value_raw = raw["workflow_id"] + if not isinstance(workflow_id_value_raw, str): + violations.append( + Violation(path="workflow_id", reason="expected string") + ) + else: + workflow_id_value = workflow_id_value_raw + + stream_value: str | None = None + if "stream" in raw: + stream_value_raw = raw["stream"] + if stream_value_raw is None: + violations.append( + Violation(path="stream", reason="explicit null not allowed") + ) + else: + if not isinstance(stream_value_raw, str): + violations.append( + Violation(path="stream", reason="expected string") + ) + else: + stream_value = stream_value_raw + + topic_value: str | None = None + if "topic" in raw: + topic_value_raw = raw["topic"] + if topic_value_raw is None: + violations.append( + Violation(path="topic", reason="explicit null not allowed") + ) + else: + if not isinstance(topic_value_raw, str): + violations.append(Violation(path="topic", reason="expected string")) + else: + topic_value = topic_value_raw + + after_token_value: str | None = None + if "after_token" in raw: + after_token_value_raw = raw["after_token"] + if after_token_value_raw is None: + violations.append( + Violation(path="after_token", reason="explicit null not allowed") + ) + else: + if not isinstance(after_token_value_raw, str): + violations.append( + Violation(path="after_token", reason="expected string") + ) + else: + after_token_value = after_token_value_raw + + max_records_value: int | None = None + if "max_records" in raw: + max_records_value_raw = raw["max_records"] + if max_records_value_raw is None: + violations.append( + Violation(path="max_records", reason="explicit null not allowed") + ) + else: + max_records_value_parsed = _parse_spec_integer( + max_records_value_raw, "max_records", violations + ) + if max_records_value_parsed is not None: + max_records_value = max_records_value_parsed + + wait_ms_value: int | None = None + if "wait_ms" in raw: + wait_ms_value_raw = raw["wait_ms"] + if wait_ms_value_raw is None: + violations.append( + Violation(path="wait_ms", reason="explicit null not allowed") + ) + else: + wait_ms_value_parsed = _parse_spec_integer( + wait_ms_value_raw, "wait_ms", violations + ) + if wait_ms_value_parsed is not None: + wait_ms_value = wait_ms_value_parsed + + latest_only_value: bool | None = None + if "latest_only" in raw: + latest_only_value_raw = raw["latest_only"] + if latest_only_value_raw is None: + violations.append( + Violation(path="latest_only", reason="explicit null not allowed") + ) + else: + if not isinstance(latest_only_value_raw, bool): + violations.append( + Violation(path="latest_only", reason="expected boolean") + ) + else: + latest_only_value = latest_only_value_raw + + for key in raw: + if ( + key != "workflow_id" + and key != "stream" + and key != "topic" + and key != "after_token" + and key != "max_records" + and key != "wait_ms" + and key != "latest_only" + ): + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return ReadInput( + workflow_id=workflow_id_value, + stream=stream_value, + topic=topic_value, + after_token=after_token_value, + max_records=max_records_value, + wait_ms=wait_ms_value, + latest_only=latest_only_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "ReadInput") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + out["workflow_id"] = value.workflow_id + if value.stream is not None: + out["stream"] = value.stream + if value.topic is not None: + out["topic"] = value.topic + if value.after_token is not None: + out["after_token"] = value.after_token + if value.max_records is not None: + if abs(value.max_records) > 9007199254740991: + violations.append( + Violation( + path="max_records", reason="exceeds ±(2^53-1) integer cap" + ) + ) + out["max_records"] = value.max_records + if value.wait_ms is not None: + if abs(value.wait_ms) > 9007199254740991: + violations.append( + Violation(path="wait_ms", reason="exceeds ±(2^53-1) integer cap") + ) + out["wait_ms"] = value.wait_ms + if value.latest_only is not None: + out["latest_only"] = value.latest_only + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_ReadInputTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class ReadInput: + """One read call: where to resume from and how long to wait.""" + + workflow_id: str + """The workflow whose stream is being read.""" + + stream: str | None = None + """The named stream to read. Empty reads every stream the workflow publishes.""" + + topic: str | None = None + """Yield only records on this topic. Empty yields every topic.""" + + after_token: str | None = None + """Opaque cursor from an earlier record or append. The read resumes strictly after the + record it names, so a caller never sees that record twice. Empty starts at the + beginning. The token is produced by whichever store sits behind the endpoint, so a + caller cannot tell which one that is. + """ + + max_records: int | None = None + """Return at most this many records. Defaults to 100 when omitted.""" + + wait_ms: int | None = None + """Wait at most this long for records to arrive before answering. Defaults to 5000 when + omitted. A call that collects nothing answers with no records and the caller's own + token. + """ + + latest_only: bool | None = None + """Answer with the newest position and no records, for a reader that wants to follow + from now. + """ + + +class _ReadOutputTransferTypeConverter( + temporalio.converter.TransferTypeConverter["ReadOutput", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["ReadOutput"] + ) -> "ReadOutput": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + records_value: list[RecordWire] | None = None + if "records" in raw: + records_value_raw = raw["records"] + if records_value_raw is None: + violations.append( + Violation(path="records", reason="explicit null not allowed") + ) + else: + if not isinstance(records_value_raw, list): + violations.append( + Violation(path="records", reason="expected array") + ) + else: + records_value_list: list[RecordWire] = [] + for records_value_index, records_value_element in enumerate( + typing.cast("list[typing.Any]", records_value_raw) + ): + records_value_item_path = f"records[{records_value_index}]" + records_value_item_violation_count = len(violations) + records_value_item: RecordWire = typing.cast("typing.Any", None) + try: + records_value_item = ( + _RecordWireTransferTypeConverter().from_transfer_type( + records_value_element, RecordWire + ) + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, records_value_item_path, error) + if len(violations) == records_value_item_violation_count: + records_value_list.append(records_value_item) + records_value = records_value_list + + next_token_value: str | None = None + if "next_token" in raw: + next_token_value_raw = raw["next_token"] + if next_token_value_raw is None: + violations.append( + Violation(path="next_token", reason="explicit null not allowed") + ) + else: + if not isinstance(next_token_value_raw, str): + violations.append( + Violation(path="next_token", reason="expected string") + ) + else: + next_token_value = next_token_value_raw + + for key in raw: + if key != "records" and key != "next_token": + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return ReadOutput( + records=records_value, + next_token=next_token_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "ReadOutput") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + if value.records is not None: + records_out: list[typing.Any] = [] + for records_index, records_element in enumerate(value.records): + try: + records_out.append( + _RecordWireTransferTypeConverter().to_transfer_type( + records_element + ) + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, f"records[{records_index}]", error) + out["records"] = records_out + if value.next_token is not None: + out["next_token"] = value.next_token + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_ReadOutputTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class ReadOutput: + """What one read call answered, and where to resume.""" + + records: list[RecordWire] | None = None + """The records after the caller's token, in stream order.""" + + next_token: str | None = None + """Opaque cursor to pass as after_token on the following call. Echoes the caller's own + token when the call collected nothing. + """ + + +class _RecordWireTransferTypeConverter( + temporalio.converter.TransferTypeConverter["RecordWire", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["RecordWire"] + ) -> "RecordWire": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + token_value: str = typing.cast("typing.Any", None) + if "token" not in raw or raw["token"] is None: + violations.append(Violation(path="token", reason="required")) + else: + token_value_raw = raw["token"] + if not isinstance(token_value_raw, str): + violations.append(Violation(path="token", reason="expected string")) + else: + token_value = token_value_raw + + frame_value: str = typing.cast("typing.Any", None) + if "frame" not in raw or raw["frame"] is None: + violations.append(Violation(path="frame", reason="required")) + else: + frame_value_raw = raw["frame"] + if not isinstance(frame_value_raw, str): + violations.append(Violation(path="frame", reason="expected string")) + else: + frame_value = frame_value_raw + + for key in raw: + if key != "token" and key != "frame": + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return RecordWire( + token=token_value, + frame=frame_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "RecordWire") -> typing.Any: + out: dict[str, typing.Any] = {} + out["token"] = value.token + out["frame"] = value.frame + return out + + +@_transfer_type_convertible(_RecordWireTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class RecordWire: + """One record on the wire: its cursor and its base64 frame.""" + + token: str + """Opaque cursor naming this record. Pass it as after_token to resume just past it.""" + + frame: str + """The base64-encoded record frame, carrying the topic, kind, producer, attempt, + sequence and body. + """ diff --git a/temporalio/streams/providers/_nexus_generated/services.py b/temporalio/streams/providers/_nexus_generated/services.py new file mode 100644 index 0000000000..ce5604d815 --- /dev/null +++ b/temporalio/streams/providers/_nexus_generated/services.py @@ -0,0 +1,39 @@ +# Generated by nexgen v0.2.4. DO NOT EDIT! + +from __future__ import annotations + +from nexusrpc import Operation, service + +from .models import ( + AppendInput, + AppendOutput, + ReadInput, + ReadOutput, +) + + +@service +class TemporalStreams: + """The stream endpoint's two operations. One Temporal-authenticated endpoint hides the + store behind it, so an operator switches storage without touching callers. Reads + hand out batches and an append carries one batch per call, because a Nexus operation + per record costs too much for token streams. + """ + + append: Operation[ + AppendInput, + AppendOutput, + ] = Operation(name="append") + """Append one batch on the caller's account. A repeated batch_index for the same + producer attempt is dropped, so a retried call writes once. Supersession records are + not transported: a reader re-synthesizes them from the attempts it observes. + """ + + read: Operation[ + ReadInput, + ReadOutput, + ] = Operation(name="read") + """Answer with the records after the caller's token, or time out. The call parks until + it has max_records records or wait_ms elapses, whichever comes first, and returns + whatever it collected. + """ diff --git a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml new file mode 100644 index 0000000000..ffa8495097 --- /dev/null +++ b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml @@ -0,0 +1,123 @@ +nexusrpc: "1.0.0" + +services: + TemporalStreams: + fqn: TemporalStreams + description: "The stream endpoint's two operations. One Temporal-authenticated endpoint hides the store behind it, so an operator switches storage without touching callers. Reads hand out batches and an append carries one batch per call, because a Nexus operation per record costs too much for token streams." + operations: + append: + fqn: append + description: "Append one batch on the caller's account. A repeated batch_index for the same producer attempt is dropped, so a retried call writes once. Supersession records are not transported: a reader re-synthesizes them from the attempts it observes." + input: { $ref: "#/$defs/AppendInput" } + output: { $ref: "#/$defs/AppendOutput" } + read: + fqn: read + description: "Answer with the records after the caller's token, or time out. The call parks until it has max_records records or wait_ms elapses, whichever comes first, and returns whatever it collected." + input: { $ref: "#/$defs/ReadInput" } + output: { $ref: "#/$defs/ReadOutput" } + +$defs: + AppendInput: + type: object + description: "One append call: who is writing, where, and what." + properties: + workflow_id: + type: string + description: "The workflow whose stream is being written." + stream: + type: string + description: "The named stream to append to. Empty when the producer writes to a topic instead." + producer_id: + type: string + description: "Identifies the writer across its retries, so its attempts can be ordered." + attempt: + type: integer + description: "The generation this producer is writing. A later attempt supersedes an earlier one." + batch_index: + type: integer + description: "Counts this producer's batches from 1. The handler drops an index it has already written for this producer attempt." + topic: + type: string + description: "The topic to append to, when stream is empty." + payloads: + type: array + items: + type: string + description: "The records to write, each a base64-encoded serialized Temporal Payload. Empty on a call that only finishes." + finish: + type: boolean + description: "Mark this producer done after writing the batch, so a reader stops waiting on it." + required: + - workflow_id + - stream + - producer_id + - attempt + - batch_index + additionalProperties: false + + AppendOutput: + type: object + description: "Where the append landed, empty when it wrote nothing." + properties: + cursor: + type: string + description: "Opaque token naming the last record written by this call. Empty when the call carried no payloads." + additionalProperties: false + + ReadInput: + type: object + description: "One read call: where to resume from and how long to wait." + properties: + workflow_id: + type: string + description: "The workflow whose stream is being read." + stream: + type: string + description: "The named stream to read. Empty reads every stream the workflow publishes." + topic: + type: string + description: "Yield only records on this topic. Empty yields every topic." + after_token: + type: string + description: "Opaque cursor from an earlier record or append. The read resumes strictly after the record it names, so a caller never sees that record twice. Empty starts at the beginning. The token is produced by whichever store sits behind the endpoint, so a caller cannot tell which one that is." + max_records: + type: integer + description: "Return at most this many records. Defaults to 100 when omitted." + wait_ms: + type: integer + description: "Wait at most this long for records to arrive before answering. Defaults to 5000 when omitted. A call that collects nothing answers with no records and the caller's own token." + latest_only: + type: boolean + description: "Answer with the newest position and no records, for a reader that wants to follow from now." + required: + - workflow_id + additionalProperties: false + + RecordWire: + type: object + description: "One record on the wire: its cursor and its base64 frame." + properties: + token: + type: string + description: "Opaque cursor naming this record. Pass it as after_token to resume just past it." + frame: + type: string + description: "The base64-encoded record frame, carrying the topic, kind, producer, attempt, sequence and body." + required: + - token + - frame + additionalProperties: false + + ReadOutput: + type: object + description: "What one read call answered, and where to resume." + properties: + records: + type: array + items: + $ref: "#/$defs/RecordWire" + description: "The records after the caller's token, in stream order." + next_token: + type: string + description: "Opaque cursor to pass as after_token on the following call. Echoes the caller's own token when the call collected nothing." + additionalProperties: false From b90346aebb31a1da7449bfc5bd29ce8b620d67f6 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 18 Sep 2026 09:30:01 -0700 Subject: [PATCH 08/34] Moved the nexus provider onto the generated contract. The five wire dataclasses and the service class now come from the generated module, so the contract has one source. Both halves read the operation and service names off the generated definition instead of hard-coding paths. Field names and operation names on the wire are unchanged. The one wire difference is that a read no longer sends max_records, wait_ms and latest_only when the caller leaves them at their defaults, because the contract marks them optional and the handler applies the documented defaults. Nothing outside this module and its tests speaks this wire. --- temporalio/streams/providers/nexus.py | 205 ++++++++++++-------------- 1 file changed, 93 insertions(+), 112 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 3701a70cb1..8e19c2d4d0 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -8,6 +8,12 @@ store behind the endpoint is invisible to callers and an operator switches it without touching them. +The wire types and the service definition come from +``temporal_streams.nexusrpc.yaml``, so any language nexgen targets can be +handed the same contract. What stays hand-written here is what the +generator cannot express yet: the handler's dedupe and long-poll collect +loop, and the caller's batching, cursor handling and codec. + Reads hand out batches and an append carries one batch per call, because a Nexus operation per record costs too much for token streams. Supersession records are not transported: the caller's reader re-synthesizes them from @@ -25,13 +31,11 @@ import asyncio import base64 -import dataclasses -import json import urllib.error import urllib.request from collections.abc import AsyncGenerator, AsyncIterator from datetime import timedelta -from typing import Any +from typing import Any, TypeVar import nexusrpc import nexusrpc.handler @@ -42,71 +46,40 @@ from temporalio.streams._handles import ReadSource, WriteSink from temporalio.streams._policy import AttemptTracker from temporalio.streams._record import BEGINNING, Cursor, RecordKind, StreamRecord +from temporalio.streams.providers._nexus_generated import ( + AppendInput, + AppendOutput, + ReadInput, + ReadOutput, + RecordWire, + TemporalStreams, +) _WORKFLOW_SIDE_ERROR = ( "the nexus provider is an outside transport; a worker publishes and reads " "through a storage provider, so configure one of those on the worker" ) +# The contract leaves these unset, so the handler is the one place that says +# what an omitted read bound means. +_DEFAULT_MAX_RECORDS = 100 +_DEFAULT_WAIT_MS = 5000 -@dataclasses.dataclass -class AppendInput: - """One append call: who is writing, where, and what.""" - - workflow_id: str - stream: str - producer_id: str - attempt: int - batch_index: int - topic: str = "" - payloads: list[str] = dataclasses.field(default_factory=list) - finish: bool = False - - -@dataclasses.dataclass -class AppendOutput: - """Where the append landed, empty when it wrote nothing.""" - - cursor: str = "" - - -@dataclasses.dataclass -class ReadInput: - """One read call: where to resume from and how long to wait.""" - - workflow_id: str - stream: str = "" - topic: str = "" - after_token: str = "" - max_records: int = 100 - wait_ms: int = 5000 - # Answer with the newest position and no records, for a reader that - # wants to follow from now. - latest_only: bool = False - - -@dataclasses.dataclass -class RecordWire: - """One record on the wire: its cursor and its base64 frame.""" - - token: str - frame: str - - -@dataclasses.dataclass -class ReadOutput: - """What one read call answered, and where to resume.""" - - records: list[RecordWire] = dataclasses.field(default_factory=list) - next_token: str = "" - - -@nexusrpc.service -class TemporalStreams: - """The stream endpoint's two operations.""" +_definition = nexusrpc.get_service_definition(TemporalStreams) +if _definition is None: # pragma: no cover + raise RuntimeError("the generated stream service carries no nexus definition") +# Both halves read the wire names off the generated definition, so a rename in +# the contract cannot leave the caller posting to a path the handler no longer +# serves. +_SERVICE_NAME = _definition.name +_OPERATION_NAMES = { + operation.method_name: operation.name + for operation in _definition.operation_definitions.values() +} +_APPEND_OPERATION = _OPERATION_NAMES["append"] +_READ_OPERATION = _OPERATION_NAMES["read"] - append: nexusrpc.Operation[AppendInput, AppendOutput] - read: nexusrpc.Operation[ReadInput, ReadOutput] +_OutputT = TypeVar("_OutputT") @nexusrpc.handler.service_handler(service=TemporalStreams) @@ -129,9 +102,10 @@ async def append( self, _ctx: nexusrpc.handler.StartOperationContext, input: AppendInput ) -> AppendOutput: """Append on the caller's account, dropping a repeated batch.""" + topic = input.topic or "" key = ( input.workflow_id, - input.stream or f"@{input.topic}", + input.stream or f"@{topic}", input.producer_id, input.attempt, ) @@ -143,7 +117,7 @@ async def append( self._client, workflow_id=input.workflow_id, stream=input.stream, - topic=input.topic, + topic=topic, producer_id=input.producer_id, attempt=input.attempt, ) @@ -168,14 +142,18 @@ async def read( ) -> ReadOutput: """Answer with the records after the caller's token, or time out.""" delegate = await self._delegate.consumer( - self._client, workflow_id=input.workflow_id, stream=input.stream + self._client, workflow_id=input.workflow_id, stream=input.stream or "" ) topic = input.topic or None if input.latest_only: return ReadOutput(next_token=(await delegate.latest(topic=topic)).token) + max_records = ( + _DEFAULT_MAX_RECORDS if input.max_records is None else input.max_records + ) + wait_ms = _DEFAULT_WAIT_MS if input.wait_ms is None else input.wait_ms converter = temporalio.converter.DataConverter.default.payload_converter records: list[RecordWire] = [] - next_token = input.after_token + next_token = input.after_token or "" subscription = delegate.read( after=Cursor(input.after_token) if input.after_token else BEGINNING, topic=topic, @@ -204,13 +182,13 @@ async def collect() -> None: ) ) next_token = record.cursor.token - if len(records) >= input.max_records: + if len(records) >= max_records: return try: # `wait_for` rather than `asyncio.timeout`, which 3.10 does not # have. Whatever was collected before the deadline is the answer. - await asyncio.wait_for(collect(), input.wait_ms / 1000) + await asyncio.wait_for(collect(), wait_ms / 1000) except asyncio.TimeoutError: pass finally: @@ -222,20 +200,29 @@ async def collect() -> None: return ReadOutput(records=records, next_token=next_token) -def _post(url: str, body: dict) -> dict: +def _post(url: str, body: bytes) -> bytes: request = urllib.request.Request( url, - data=json.dumps(body).encode(), + data=body, headers={"Content-Type": "application/json"}, method="POST", ) try: with urllib.request.urlopen(request, timeout=30) as response: - raw = response.read() + return response.read() except urllib.error.HTTPError as error: detail = error.read().decode(errors="replace") raise RuntimeError(f"stream endpoint call failed ({error.code}): {detail}") - return json.loads(raw) if raw else {} + + +async def _invoke(url: str, request: Any, output: type[_OutputT]) -> _OutputT: + # The contract types carry their own JSON encoding, so the raw caller and + # the worker serving the operation agree on the body without either of + # them spelling the fields out. + contract = temporalio.converter.DataConverter.default.payload_converter + raw = await asyncio.to_thread(_post, url, contract.to_payloads([request])[0].data) + payload = Payload(metadata={"encoding": b"json/plain"}, data=raw or b"{}") + return contract.from_payloads([payload], [output])[0] class NexusProducer: @@ -276,7 +263,7 @@ async def append(self, *values: Any) -> Cursor: else self._converter.to_payloads([value])[0] ) payloads.append(base64.b64encode(payload.SerializeToString()).decode()) - return Cursor((await self._call(payloads=payloads)).get("cursor", "")) + return Cursor((await self._call(payloads=payloads)).cursor or "") async def finish(self) -> None: """Mark this producer done, so a reader stops waiting on it.""" @@ -284,23 +271,21 @@ async def finish(self) -> None: async def _call( self, payloads: list[str] | None = None, finish: bool = False - ) -> dict: + ) -> AppendOutput: self._batch_index += 1 - return await asyncio.to_thread( - _post, - f"{self._base_url}/append", - dataclasses.asdict( - AppendInput( - workflow_id=self._workflow_id, - stream=self._stream, - producer_id=self._producer_id, - attempt=self._attempt, - batch_index=self._batch_index, - topic=self._topic, - payloads=payloads or [], - finish=finish, - ) + return await _invoke( + f"{self._base_url}/{_APPEND_OPERATION}", + AppendInput( + workflow_id=self._workflow_id, + stream=self._stream, + producer_id=self._producer_id, + attempt=self._attempt, + batch_index=self._batch_index, + topic=self._topic, + payloads=payloads or [], + finish=finish, ), + AppendOutput, ) @@ -327,28 +312,26 @@ async def read( attempts = AttemptTracker() token = after.token while True: - raw = await asyncio.to_thread( - _post, - f"{self._base_url}/read", - dataclasses.asdict( - ReadInput( - workflow_id=self._workflow_id, - stream=self._stream, - topic=topic or "", - after_token=token, - ) + answer = await _invoke( + f"{self._base_url}/{_READ_OPERATION}", + ReadInput( + workflow_id=self._workflow_id, + stream=self._stream, + topic=topic or "", + after_token=token, ), + ReadOutput, ) - for wire in raw.get("records", []): + for wire in answer.records or []: try: kind, frame_topic, source, attempt, sequence, body = _frame.decode( - base64.b64decode(wire["frame"]) + base64.b64decode(wire.frame) ) except ValueError: continue if topic is not None and frame_topic != topic: continue - cursor = Cursor(wire["token"]) + cursor = Cursor(wire.token) superseded = attempts.note(source, attempt, cursor) if superseded is not None: yield superseded @@ -361,23 +344,21 @@ async def read( attempt=attempt, sequence=sequence, ) - token = raw.get("next_token", token) or token + token = answer.next_token or token async def latest(self, *, topic: str | None = None) -> Cursor: """The cursor of the last record written, for following from now.""" - raw = await asyncio.to_thread( - _post, - f"{self._base_url}/read", - dataclasses.asdict( - ReadInput( - workflow_id=self._workflow_id, - stream=self._stream, - topic=topic or "", - latest_only=True, - ) + answer = await _invoke( + f"{self._base_url}/{_READ_OPERATION}", + ReadInput( + workflow_id=self._workflow_id, + stream=self._stream, + topic=topic or "", + latest_only=True, ), + ReadOutput, ) - token = raw.get("next_token", "") + token = answer.next_token or "" return Cursor(token) if token else BEGINNING def _decode(self, body: bytes, as_type: type | None) -> Any: @@ -397,7 +378,7 @@ def __init__(self) -> None: def configure(self, **options: Any) -> None: endpoint = options.pop("endpoint", None) http_address = options.pop("http_address", "http://127.0.0.1:7243") - service = options.pop("service", "TemporalStreams") + service = options.pop("service", _SERVICE_NAME) if options: raise TypeError( "the nexus provider takes endpoint, http_address and service, " From 299a90bb86719715f6b26a9a67fc58d112e2450d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 18 Sep 2026 09:37:16 -0700 Subject: [PATCH 09/34] Applied the payload codec on the nexus provider's caller side. The caller ran the payload converter but never the codec, so records left the process in the clear even when one was configured. The producer now encodes before posting and the consumer decodes after reading, while the handler stays codec-blind, which is what keeps plaintext out of whoever hosts the endpoint. A data_converter option on the provider says which one to use when there is no client to take it from. --- CHANGELOG.md | 5 +- temporalio/streams/providers/nexus.py | 72 ++++++---- tests/streams/test_nexus_provider.py | 182 ++++++++++++++++++++++++-- 3 files changed, 223 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 923da8ccc0..2ad3811ebe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,10 @@ to include examples, links to docs, or any other relevant information. naming it. - **Experimental**: `temporalio.streams.providers.nexus` puts one Nexus endpoint in front of a storage provider, so a caller reaches a stream - through the endpoint and never names the store. + through the endpoint and never names the store. Its contract is defined in + `temporal_streams.nexusrpc.yaml` and the bindings are generated from it. + Configure it with `data_converter=` to run a payload codec on the caller + side, so records are encoded before they leave the process. ### Changed diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 8e19c2d4d0..a73c830ef9 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -231,7 +231,7 @@ class NexusProducer: def __init__( self, base_url: str, - converter: Any, + data_converter: temporalio.converter.DataConverter, workflow_id: str, stream: str, topic: str, @@ -240,7 +240,8 @@ def __init__( ) -> None: """Bind this producer to one stream or topic on the endpoint.""" self._base_url = base_url - self._converter = converter + self._converter = data_converter.payload_converter + self._codec = data_converter.payload_codec self._workflow_id = workflow_id self._stream = stream self._topic = topic @@ -255,15 +256,22 @@ def attempt(self) -> int: async def append(self, *values: Any) -> Cursor: """Append ``values`` through the endpoint.""" - payloads = [] - for value in values: - payload = ( - value - if isinstance(value, Payload) - else self._converter.to_payloads([value])[0] - ) - payloads.append(base64.b64encode(payload.SerializeToString()).decode()) - return Cursor((await self._call(payloads=payloads)).cursor or "") + payloads = [ + value + if isinstance(value, Payload) + else self._converter.to_payloads([value])[0] + for value in values + ] + if self._codec is not None: + # The endpoint is the edge of this process, so a configured codec + # runs here rather than at the handler: whoever hosts the endpoint + # never holds the plaintext. + payloads = await self._codec.encode(payloads) + encoded = [ + base64.b64encode(payload.SerializeToString()).decode() + for payload in payloads + ] + return Cursor((await self._call(payloads=encoded)).cursor or "") async def finish(self) -> None: """Mark this producer done, so a reader stops waiting on it.""" @@ -293,11 +301,16 @@ class NexusConsumer: """Reads batches from the stream endpoint, re-synthesizing supersession.""" def __init__( - self, base_url: str, converter: Any, workflow_id: str, stream: str + self, + base_url: str, + data_converter: temporalio.converter.DataConverter, + workflow_id: str, + stream: str, ) -> None: """Read what ``workflow_id`` publishes, through the endpoint.""" self._base_url = base_url - self._converter = converter + self._converter = data_converter.payload_converter + self._codec = data_converter.payload_codec self._workflow_id = workflow_id self._stream = stream @@ -335,8 +348,11 @@ async def read( superseded = attempts.note(source, attempt, cursor) if superseded is not None: yield superseded + value = ( + await self._decode(body, type) if kind is RecordKind.DATA else None + ) yield StreamRecord( - value=self._decode(body, type) if kind is RecordKind.DATA else None, + value=value, cursor=cursor, kind=kind, topic=frame_topic, @@ -361,9 +377,11 @@ async def latest(self, *, topic: str | None = None) -> Cursor: token = answer.next_token or "" return Cursor(token) if token else BEGINNING - def _decode(self, body: bytes, as_type: type | None) -> Any: + async def _decode(self, body: bytes, as_type: type | None) -> Any: payload = Payload() payload.ParseFromString(body) + if self._codec is not None: + payload = (await self._codec.decode([payload]))[0] if as_type is None: return self._converter.from_payloads([payload])[0] return self._converter.from_payloads([payload], [as_type])[0] @@ -374,19 +392,22 @@ class _NexusProvider: def __init__(self) -> None: self._base_url: str | None = None + self._data_converter: temporalio.converter.DataConverter | None = None def configure(self, **options: Any) -> None: endpoint = options.pop("endpoint", None) http_address = options.pop("http_address", "http://127.0.0.1:7243") service = options.pop("service", _SERVICE_NAME) + data_converter = options.pop("data_converter", None) if options: raise TypeError( - "the nexus provider takes endpoint, http_address and service, " - f"got {sorted(options)}" + "the nexus provider takes endpoint, http_address, service and " + f"data_converter, got {sorted(options)}" ) if not endpoint: raise TypeError("the nexus provider needs endpoint=") self._base_url = f"{http_address}/nexus/endpoints/{endpoint}/services/{service}" + self._data_converter = data_converter def worker_options(self) -> dict[str, Any]: raise RuntimeError(_WORKFLOW_SIDE_ERROR) @@ -427,7 +448,7 @@ async def producer( attempt = attempt or activity.info().attempt return NexusProducer( self._url(), - _converter(client), + self._converter(client), workflow_id, stream, topic, @@ -438,13 +459,14 @@ async def producer( async def consumer( self, client: Any, *, workflow_id: str, stream: str = "" ) -> NexusConsumer: - return NexusConsumer(self._url(), _converter(client), workflow_id, stream) - - -def _converter(client: Any) -> Any: - if client is None: - return temporalio.converter.DataConverter.default.payload_converter - return client.data_converter.payload_converter + return NexusConsumer(self._url(), self._converter(client), workflow_id, stream) + + def _converter(self, client: Any) -> temporalio.converter.DataConverter: + if self._data_converter is not None: + return self._data_converter + if client is None: + return temporalio.converter.DataConverter.default + return client.data_converter _provider.register("nexus", _NexusProvider) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index f07e4956c6..18679ad5f1 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -1,40 +1,55 @@ -"""Live conformance for the Nexus front. +"""Conformance for the Nexus front. -The caller talks only to the stream endpoint over the server's Nexus HTTP -ingress; the handler worker delegates to the workflow_streams provider, so -this test is the provider-hiding demonstration: nothing on the caller side -names or could name the store. Gated behind ``STREAMS_LIVE=nexus`` because it -needs a dev server with an HTTP port and a registered Nexus endpoint. +The caller talks only to the stream endpoint; the handler delegates to a +storage provider, so these tests are the provider-hiding demonstration: +nothing on the caller side names or could name the store. -Environment: ``TEMPORAL_ADDRESS`` (default ``localhost:7233``), -``TEMPORAL_HTTP`` (default ``http://127.0.0.1:7243``), and an endpoint named -``streams-e2e`` targeting task queue ``streams-handlers-e2e``. +The loop test runs over the server's Nexus HTTP ingress and is gated behind +``STREAMS_LIVE=nexus`` because it needs a dev server with an HTTP port and a +registered Nexus endpoint. Environment: ``TEMPORAL_ADDRESS`` (default +``localhost:7233``), ``TEMPORAL_HTTP`` (default ``http://127.0.0.1:7243``), +and an endpoint named ``streams-e2e`` targeting task queue +``streams-handlers-e2e``. + +The codec test stands the endpoint up in this process instead, because what +it checks is the bytes the caller puts on the wire. """ from __future__ import annotations +import asyncio +import base64 +import dataclasses import json import os import subprocess import uuid +from collections.abc import Sequence +from typing import Any, cast import pytest +import temporalio.converter from temporalio import streams +from temporalio.api.common.v1 import Payload from temporalio.client import Client from temporalio.streams import RecordKind +from temporalio.streams import _frame as frame from temporalio.streams._provider import instance +from temporalio.streams.providers import memory, nexus +from temporalio.streams.providers._nexus_generated import AppendInput, ReadInput from temporalio.streams.providers.nexus import TemporalStreamsHandler from temporalio.worker import Worker from tests.streams.test_workflow_streams_provider import EchoLoop, take -pytestmark = pytest.mark.skipif( +live_only = pytest.mark.skipif( os.environ.get("STREAMS_LIVE") != "nexus", reason="needs a live server and nexus endpoint; run with STREAMS_LIVE=nexus", ) ENDPOINT = "streams-e2e" HANDLER_TQ = "streams-handlers-e2e" +APPEND_OPERATION = nexus._APPEND_OPERATION # pyright: ignore[reportPrivateUsage] def _endpoint_id() -> str: @@ -60,6 +75,7 @@ def _endpoint_id() -> str: return json.loads(out.stdout)["id"] +@live_only async def test_interface_loop_through_the_nexus_front(): # The workflow worker and the handler worker both use the storage # provider; only the caller goes through the front. @@ -119,3 +135,149 @@ async def test_interface_loop_through_the_nexus_front(): await handle.signal(EchoLoop.release) assert await handle.result() == 3 + + +class ScrambleCodec(temporalio.converter.PayloadCodec): + """Flips every byte of a payload, so the middle holds nothing readable.""" + + KEY = 0x5A + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + """Wrap each payload as opaque bytes nothing downstream can read.""" + return [ + Payload( + metadata={"encoding": b"binary/plain"}, + data=self._flip(payload.SerializeToString()), + ) + for payload in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + """Recover the payloads :meth:`encode` wrapped.""" + out: list[Payload] = [] + for payload in payloads: + inner = Payload() + inner.ParseFromString(self._flip(payload.data)) + out.append(inner) + return out + + @classmethod + def _flip(cls, data: bytes) -> bytes: + return bytes(byte ^ cls.KEY for byte in data) + + +async def _dispatch( + handler: TemporalStreamsHandler, operation: str, request: Any +) -> Any: + context = cast("Any", None) # neither operation looks at it + if operation == APPEND_OPERATION: + return await handler.append(context, request) + return await handler.read(context, request) + + +def _in_process_endpoint( + handler: TemporalStreamsHandler, + posted: list[bytes], + answered: list[bytes], +) -> Any: + """Serve the caller's posts from ``handler``, recording both directions.""" + contract = temporalio.converter.DataConverter.default.payload_converter + loop = asyncio.get_running_loop() + + def post(url: str, body: bytes) -> bytes: + posted.append(body) + operation = url.rsplit("/", 1)[1] + request_type = AppendInput if operation == APPEND_OPERATION else ReadInput + request: Any = contract.from_payloads( + [Payload(metadata={"encoding": b"json/plain"}, data=body)], [request_type] + )[0] + if request_type is ReadInput: + # This endpoint has no other traffic to wait for, so park briefly + # rather than for the contract's default. + request.wait_ms = 200 + answer = asyncio.run_coroutine_threadsafe( + _dispatch(handler, operation, request), loop + ).result() + raw = contract.to_payloads([answer])[0].data + answered.append(raw) + return raw + + return post + + +async def _loop_through_an_endpoint( + monkeypatch: pytest.MonkeyPatch, + codec: temporalio.converter.PayloadCodec | None, +) -> tuple[list[Any], list[bytes], list[bytes]]: + memory.reset() + streams.configure(provider="memory") + posted: list[bytes] = [] + answered: list[bytes] = [] + handler = TemporalStreamsHandler(None, provider="memory") + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, posted, answered)) + front = instance( + "nexus", + endpoint="in-process", + data_converter=dataclasses.replace( + temporalio.converter.DataConverter.default, payload_codec=codec + ), + ) + workflow_id = "wf-codec" + + producer = await front.producer( + None, + workflow_id=workflow_id, + stream="inputs", + producer_id="model", + attempt=1, + ) + await producer.append({"secret": "tuna"}) + await producer.finish() + + consumer = await front.consumer(None, workflow_id=workflow_id, stream="inputs") + records = await take(consumer.read(type=dict), 2, timeout=30) + memory.reset() + return records, posted, answered + + +def _appended_bytes(posted: list[bytes]) -> bytes: + out = b"" + for body in posted: + for encoded in json.loads(body).get("payloads", []): + out += base64.b64decode(encoded) + return out + + +def _answered_bodies(answered: list[bytes]) -> bytes: + out = b"" + for body in answered: + for record in json.loads(body).get("records", []): + out += frame.decode(base64.b64decode(record["frame"]))[5] + return out + + +async def test_the_caller_encodes_records_with_the_configured_codec( + monkeypatch: pytest.MonkeyPatch, +): + records, posted, answered = await _loop_through_an_endpoint( + monkeypatch, ScrambleCodec() + ) + + # The value survives the hop, so the codec is applied on both sides. + assert [record.kind for record in records] == [RecordKind.DATA, RecordKind.FINISH] + assert records[0].value == {"secret": "tuna"} + # Neither direction carried the plaintext. + assert b"tuna" not in _appended_bytes(posted) + assert b"tuna" not in _answered_bodies(answered) + + +async def test_without_a_codec_the_same_records_go_out_in_the_clear( + monkeypatch: pytest.MonkeyPatch, +): + # The contrast is the point: the bytes above differ because of the codec, + # not because the transport obscures them anyway. + records, posted, answered = await _loop_through_an_endpoint(monkeypatch, None) + + assert records[0].value == {"secret": "tuna"} + assert b"tuna" in _appended_bytes(posted) + assert b"tuna" in _answered_bodies(answered) From 534bd596daf0b5deabacfe5c8c2de71c8a2feba6 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 18 Sep 2026 17:02:21 -0700 Subject: [PATCH 10/34] Made the stream contract generator refuse a nexgen other than the pinned one. The output differs between releases, so a developer with another version on PATH would regenerate different code and only see the drift in CI. --- scripts/gen_streams_nexus_api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/gen_streams_nexus_api.py b/scripts/gen_streams_nexus_api.py index 800aba9448..2d4f16c6d4 100644 --- a/scripts/gen_streams_nexus_api.py +++ b/scripts/gen_streams_nexus_api.py @@ -36,11 +36,26 @@ def nex_gen_command() -> list[str]: return ["nexgen"] +def check_version(command: list[str]) -> None: + # A different release on PATH would regenerate different code locally + # and the drift would only show in CI, so refuse before writing anything. + reported = subprocess.check_output([*command, "--version"], text=True).strip() + found = reported.split()[-1] if reported else "" + if found != NEX_GEN_VERSION: + raise SystemExit( + f"found nexgen {found or '?'} at {command[0]}, but the stream contract is " + f"generated with {NEX_GEN_VERSION}. Install it with `cargo install --locked " + f"nexgen --version {NEX_GEN_VERSION} --features advanced` or point " + "NEX_GEN_BIN at that binary." + ) + + def generate_streams_nexus_api() -> None: if not contract_path.exists(): raise RuntimeError(f"missing stream contract: {contract_path}") command = nex_gen_command() + check_version(command) shutil.rmtree(output_dir, ignore_errors=True) subprocess.check_call( [ From 3cb7d1a93bf64ce4d07a49632819f7d0e80a4b20 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 18 Sep 2026 17:02:21 -0700 Subject: [PATCH 11/34] Regenerated the stream contract with byte payloads, read bounds and tolerant outputs. Payload fields carry contentEncoding base64 so both halves pass bytes and the generated code does the encoding. max_records and wait_ms carry the bounds the validator enforces, outputs accept unknown fields so a handler can add one before every caller is rebuilt, and descriptions are folded at 100 columns. --- .../providers/_nexus_generated/models.py | 201 +++++++++++++++--- .../providers/_nexus_generated/services.py | 8 +- .../providers/temporal_streams.nexusrpc.yaml | 108 ++++++++-- 3 files changed, 261 insertions(+), 56 deletions(-) diff --git a/temporalio/streams/providers/_nexus_generated/models.py b/temporalio/streams/providers/_nexus_generated/models.py index f22711a4f4..d00cbcc637 100644 --- a/temporalio/streams/providers/_nexus_generated/models.py +++ b/temporalio/streams/providers/_nexus_generated/models.py @@ -13,10 +13,20 @@ from ._definitions import ( Violation, _collect, + _format_base64, + _parse_base64, _parse_spec_integer, _transfer_type_convertible, ) +_APPEND_OUTPUT_DECLARED: frozenset[str] = frozenset({"cursor"}) + + +_READ_OUTPUT_DECLARED: frozenset[str] = frozenset({"records", "next_token"}) + + +_RECORD_WIRE_DECLARED: frozenset[str] = frozenset({"token", "frame"}) + class _AppendInputTransferTypeConverter( temporalio.converter.TransferTypeConverter["AppendInput", typing.Any] @@ -87,6 +97,13 @@ def from_transfer_type( ) if batch_index_value_parsed is not None: batch_index_value = batch_index_value_parsed + if batch_index_value < 1: + violations.append( + Violation( + path="batch_index", + reason=f"must be >= 1, got {batch_index_value}", + ) + ) topic_value: str | None = None if "topic" in raw: @@ -101,7 +118,7 @@ def from_transfer_type( else: topic_value = topic_value_raw - payloads_value: list[str] | None = None + payloads_value: list[bytes] | None = None if "payloads" in raw: payloads_value_raw = raw["payloads"] if payloads_value_raw is None: @@ -114,13 +131,13 @@ def from_transfer_type( Violation(path="payloads", reason="expected array") ) else: - payloads_value_list: list[str] = [] + payloads_value_list: list[bytes] = [] for payloads_value_index, payloads_value_element in enumerate( typing.cast("list[typing.Any]", payloads_value_raw) ): payloads_value_item_path = f"payloads[{payloads_value_index}]" payloads_value_item_violation_count = len(violations) - payloads_value_item: str = typing.cast("typing.Any", None) + payloads_value_item: bytes = typing.cast("typing.Any", None) if not isinstance(payloads_value_element, str): violations.append( Violation( @@ -129,7 +146,13 @@ def from_transfer_type( ) ) else: - payloads_value_item = payloads_value_element + payloads_value_item_parsed = _parse_base64( + payloads_value_element, + payloads_value_item_path, + violations, + ) + if payloads_value_item_parsed is not None: + payloads_value_item = payloads_value_item_parsed if len(violations) == payloads_value_item_violation_count: payloads_value_list.append(payloads_value_item) payloads_value = payloads_value_list @@ -190,11 +213,17 @@ def to_transfer_type(self, value: "AppendInput") -> typing.Any: violations.append( Violation(path="batch_index", reason="exceeds ±(2^53-1) integer cap") ) + if value.batch_index < 1: + violations.append( + Violation( + path="batch_index", reason=f"must be >= 1, got {value.batch_index}" + ) + ) out["batch_index"] = value.batch_index if value.topic is not None: out["topic"] = value.topic if value.payloads is not None: - out["payloads"] = value.payloads + out["payloads"] = [_format_base64(element) for element in value.payloads] if value.finish is not None: out["finish"] = value.finish if violations: @@ -221,15 +250,16 @@ class AppendInput: batch_index: int """Counts this producer's batches from 1. The handler drops an index it has already - written for this producer attempt. + written for this producer attempt, and rejects one that skips ahead or that resumes + an attempt it never saw start. """ topic: str | None = None """The topic to append to, when stream is empty.""" - payloads: list[str] | None = None - """The records to write, each a base64-encoded serialized Temporal Payload. Empty on a - call that only finishes. + payloads: list[bytes] | None = None + """The records to write, each a serialized Temporal Payload. Empty on a call that only + finishes. """ finish: bool | None = None @@ -265,33 +295,53 @@ def from_transfer_type( else: cursor_value = cursor_value_raw + additional_properties: dict[str, typing.Any] = {} for key in raw: - if key != "cursor": - violations.append(Violation(path=key, reason="unknown field")) + if key not in _APPEND_OUTPUT_DECLARED: + additional_properties[key] = raw[key] if violations: raise temporalio.converter.create_payload_validation_error(violations) return AppendOutput( cursor=cursor_value, + additional_properties=additional_properties, ) @typing_extensions.override def to_transfer_type(self, value: "AppendOutput") -> typing.Any: + violations: list[Violation] = [] out: dict[str, typing.Any] = {} if value.cursor is not None: out["cursor"] = value.cursor + for key, entry in value.additional_properties.items(): + if key in _APPEND_OUTPUT_DECLARED: + violations.append( + Violation( + path=key, + reason="additional property collides with declared property", + ) + ) + else: + out[key] = entry + if violations: + raise temporalio.converter.create_payload_validation_error(violations) return out @_transfer_type_convertible(_AppendOutputTransferTypeConverter) @dataclasses.dataclass(slots=True, kw_only=True) class AppendOutput: - """Where the append landed, empty when it wrote nothing.""" + """Where the append landed, when the store can say.""" cursor: str | None = None - """Opaque token naming the last record written by this call. Empty when the call - carried no payloads. + """Opaque token naming the last record written by this call. Absent when the call + carried no payloads, when the batch was a repeat the handler dropped, or when the + store learns positions only at read time. """ + additional_properties: dict[str, typing.Any] = dataclasses.field( + default_factory=dict + ) + class _ReadInputTransferTypeConverter( temporalio.converter.TransferTypeConverter["ReadInput", typing.Any] @@ -375,6 +425,20 @@ def from_transfer_type( ) if max_records_value_parsed is not None: max_records_value = max_records_value_parsed + if max_records_value < 1: + violations.append( + Violation( + path="max_records", + reason=f"must be >= 1, got {max_records_value}", + ) + ) + if max_records_value > 1000: + violations.append( + Violation( + path="max_records", + reason=f"must be <= 1000, got {max_records_value}", + ) + ) wait_ms_value: int | None = None if "wait_ms" in raw: @@ -389,6 +453,20 @@ def from_transfer_type( ) if wait_ms_value_parsed is not None: wait_ms_value = wait_ms_value_parsed + if wait_ms_value < 0: + violations.append( + Violation( + path="wait_ms", + reason=f"must be >= 0, got {wait_ms_value}", + ) + ) + if wait_ms_value > 60000: + violations.append( + Violation( + path="wait_ms", + reason=f"must be <= 60000, got {wait_ms_value}", + ) + ) latest_only_value: bool | None = None if "latest_only" in raw: @@ -446,12 +524,38 @@ def to_transfer_type(self, value: "ReadInput") -> typing.Any: path="max_records", reason="exceeds ±(2^53-1) integer cap" ) ) + if value.max_records < 1: + violations.append( + Violation( + path="max_records", + reason=f"must be >= 1, got {value.max_records}", + ) + ) + if value.max_records > 1000: + violations.append( + Violation( + path="max_records", + reason=f"must be <= 1000, got {value.max_records}", + ) + ) out["max_records"] = value.max_records if value.wait_ms is not None: if abs(value.wait_ms) > 9007199254740991: violations.append( Violation(path="wait_ms", reason="exceeds ±(2^53-1) integer cap") ) + if value.wait_ms < 0: + violations.append( + Violation( + path="wait_ms", reason=f"must be >= 0, got {value.wait_ms}" + ) + ) + if value.wait_ms > 60000: + violations.append( + Violation( + path="wait_ms", reason=f"must be <= 60000, got {value.wait_ms}" + ) + ) out["wait_ms"] = value.wait_ms if value.latest_only is not None: out["latest_only"] = value.latest_only @@ -472,7 +576,9 @@ class ReadInput: """The named stream to read. Empty reads every stream the workflow publishes.""" topic: str | None = None - """Yield only records on this topic. Empty yields every topic.""" + """Yield only records on this topic. Empty yields every topic. Only meaningful when + stream is empty, because an inbound stream has no topics. + """ after_token: str | None = None """Opaque cursor from an earlier record or append. The read resumes strictly after the @@ -485,9 +591,9 @@ class ReadInput: """Return at most this many records. Defaults to 100 when omitted.""" wait_ms: int | None = None - """Wait at most this long for records to arrive before answering. Defaults to 5000 when - omitted. A call that collects nothing answers with no records and the caller's own - token. + """Wait at most this long for records to arrive before answering. The handler shortens + it to fit the request deadline. A call that collects nothing answers with no records + and the caller's own token. """ latest_only: bool | None = None @@ -557,14 +663,16 @@ def from_transfer_type( else: next_token_value = next_token_value_raw + additional_properties: dict[str, typing.Any] = {} for key in raw: - if key != "records" and key != "next_token": - violations.append(Violation(path=key, reason="unknown field")) + if key not in _READ_OUTPUT_DECLARED: + additional_properties[key] = raw[key] if violations: raise temporalio.converter.create_payload_validation_error(violations) return ReadOutput( records=records_value, next_token=next_token_value, + additional_properties=additional_properties, ) @typing_extensions.override @@ -585,6 +693,16 @@ def to_transfer_type(self, value: "ReadOutput") -> typing.Any: out["records"] = records_out if value.next_token is not None: out["next_token"] = value.next_token + for key, entry in value.additional_properties.items(): + if key in _READ_OUTPUT_DECLARED: + violations.append( + Violation( + path=key, + reason="additional property collides with declared property", + ) + ) + else: + out[key] = entry if violations: raise temporalio.converter.create_payload_validation_error(violations) return out @@ -603,6 +721,10 @@ class ReadOutput: token when the call collected nothing. """ + additional_properties: dict[str, typing.Any] = dataclasses.field( + default_factory=dict + ) + class _RecordWireTransferTypeConverter( temporalio.converter.TransferTypeConverter["RecordWire", typing.Any] @@ -628,7 +750,7 @@ def from_transfer_type( else: token_value = token_value_raw - frame_value: str = typing.cast("typing.Any", None) + frame_value: bytes = typing.cast("typing.Any", None) if "frame" not in raw or raw["frame"] is None: violations.append(Violation(path="frame", reason="required")) else: @@ -636,35 +758,54 @@ def from_transfer_type( if not isinstance(frame_value_raw, str): violations.append(Violation(path="frame", reason="expected string")) else: - frame_value = frame_value_raw + frame_value_parsed = _parse_base64(frame_value_raw, "frame", violations) + if frame_value_parsed is not None: + frame_value = frame_value_parsed + additional_properties: dict[str, typing.Any] = {} for key in raw: - if key != "token" and key != "frame": - violations.append(Violation(path=key, reason="unknown field")) + if key not in _RECORD_WIRE_DECLARED: + additional_properties[key] = raw[key] if violations: raise temporalio.converter.create_payload_validation_error(violations) return RecordWire( token=token_value, frame=frame_value, + additional_properties=additional_properties, ) @typing_extensions.override def to_transfer_type(self, value: "RecordWire") -> typing.Any: + violations: list[Violation] = [] out: dict[str, typing.Any] = {} out["token"] = value.token - out["frame"] = value.frame + out["frame"] = _format_base64(value.frame) + for key, entry in value.additional_properties.items(): + if key in _RECORD_WIRE_DECLARED: + violations.append( + Violation( + path=key, + reason="additional property collides with declared property", + ) + ) + else: + out[key] = entry + if violations: + raise temporalio.converter.create_payload_validation_error(violations) return out @_transfer_type_convertible(_RecordWireTransferTypeConverter) @dataclasses.dataclass(slots=True, kw_only=True) class RecordWire: - """One record on the wire: its cursor and its base64 frame.""" + """One record on the wire: its cursor and its frame.""" token: str """Opaque cursor naming this record. Pass it as after_token to resume just past it.""" - frame: str - """The base64-encoded record frame, carrying the topic, kind, producer, attempt, - sequence and body. - """ + frame: bytes + """The record frame, carrying the topic, kind, producer, attempt, sequence and body.""" + + additional_properties: dict[str, typing.Any] = dataclasses.field( + default_factory=dict + ) diff --git a/temporalio/streams/providers/_nexus_generated/services.py b/temporalio/streams/providers/_nexus_generated/services.py index ce5604d815..e0673aeab7 100644 --- a/temporalio/streams/providers/_nexus_generated/services.py +++ b/temporalio/streams/providers/_nexus_generated/services.py @@ -24,9 +24,11 @@ class TemporalStreams: AppendInput, AppendOutput, ] = Operation(name="append") - """Append one batch on the caller's account. A repeated batch_index for the same - producer attempt is dropped, so a retried call writes once. Supersession records are - not transported: a reader re-synthesizes them from the attempts it observes. + """Append one batch on the caller's account. The handler drops a batch_index it has + already written for this producer attempt, so a retried call writes once, and + rejects an index that skips ahead or a producer attempt it has no state for. + Supersession records are not transported: a reader re-synthesizes them from the + attempts it observes. """ read: Operation[ diff --git a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml index ffa8495097..a1cc346317 100644 --- a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml +++ b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml @@ -1,18 +1,35 @@ +# The stream endpoint's wire contract. Inputs reject unknown fields so a typo +# fails loudly at the handler; outputs accept them so a handler can add a +# field before every caller has been rebuilt from the newer contract. nexusrpc: "1.0.0" services: TemporalStreams: fqn: TemporalStreams - description: "The stream endpoint's two operations. One Temporal-authenticated endpoint hides the store behind it, so an operator switches storage without touching callers. Reads hand out batches and an append carries one batch per call, because a Nexus operation per record costs too much for token streams." + description: >- + The stream endpoint's two operations. One Temporal-authenticated endpoint + hides the store behind it, so an operator switches storage without + touching callers. Reads hand out batches and an append carries one batch + per call, because a Nexus operation per record costs too much for token + streams. operations: append: fqn: append - description: "Append one batch on the caller's account. A repeated batch_index for the same producer attempt is dropped, so a retried call writes once. Supersession records are not transported: a reader re-synthesizes them from the attempts it observes." + description: >- + Append one batch on the caller's account. The handler drops a + batch_index it has already written for this producer attempt, so a + retried call writes once, and rejects an index that skips ahead or a + producer attempt it has no state for. Supersession records are not + transported: a reader re-synthesizes them from the attempts it + observes. input: { $ref: "#/$defs/AppendInput" } output: { $ref: "#/$defs/AppendOutput" } read: fqn: read - description: "Answer with the records after the caller's token, or time out. The call parks until it has max_records records or wait_ms elapses, whichever comes first, and returns whatever it collected." + description: >- + Answer with the records after the caller's token, or time out. The + call parks until it has max_records records or wait_ms elapses, + whichever comes first, and returns whatever it collected. input: { $ref: "#/$defs/ReadInput" } output: { $ref: "#/$defs/ReadOutput" } @@ -26,16 +43,26 @@ $defs: description: "The workflow whose stream is being written." stream: type: string - description: "The named stream to append to. Empty when the producer writes to a topic instead." + description: >- + The named stream to append to. Empty when the producer writes to a + topic instead. producer_id: type: string - description: "Identifies the writer across its retries, so its attempts can be ordered." + description: >- + Identifies the writer across its retries, so its attempts can be + ordered. attempt: type: integer - description: "The generation this producer is writing. A later attempt supersedes an earlier one." + description: >- + The generation this producer is writing. A later attempt supersedes + an earlier one. batch_index: type: integer - description: "Counts this producer's batches from 1. The handler drops an index it has already written for this producer attempt." + minimum: 1 + description: >- + Counts this producer's batches from 1. The handler drops an index it + has already written for this producer attempt, and rejects one that + skips ahead or that resumes an attempt it never saw start. topic: type: string description: "The topic to append to, when stream is empty." @@ -43,10 +70,15 @@ $defs: type: array items: type: string - description: "The records to write, each a base64-encoded serialized Temporal Payload. Empty on a call that only finishes." + contentEncoding: base64 + description: >- + The records to write, each a serialized Temporal Payload. Empty on a + call that only finishes. finish: type: boolean - description: "Mark this producer done after writing the batch, so a reader stops waiting on it." + description: >- + Mark this producer done after writing the batch, so a reader stops + waiting on it. required: - workflow_id - stream @@ -57,12 +89,16 @@ $defs: AppendOutput: type: object - description: "Where the append landed, empty when it wrote nothing." + description: "Where the append landed, when the store can say." properties: cursor: type: string - description: "Opaque token naming the last record written by this call. Empty when the call carried no payloads." - additionalProperties: false + description: >- + Opaque token naming the last record written by this call. Absent + when the call carried no payloads, when the batch was a repeat the + handler dropped, or when the store learns positions only at read + time. + additionalProperties: true ReadInput: type: object @@ -73,40 +109,64 @@ $defs: description: "The workflow whose stream is being read." stream: type: string - description: "The named stream to read. Empty reads every stream the workflow publishes." + description: >- + The named stream to read. Empty reads every stream the workflow + publishes. topic: type: string - description: "Yield only records on this topic. Empty yields every topic." + description: >- + Yield only records on this topic. Empty yields every topic. Only + meaningful when stream is empty, because an inbound stream has no + topics. after_token: type: string - description: "Opaque cursor from an earlier record or append. The read resumes strictly after the record it names, so a caller never sees that record twice. Empty starts at the beginning. The token is produced by whichever store sits behind the endpoint, so a caller cannot tell which one that is." + description: >- + Opaque cursor from an earlier record or append. The read resumes + strictly after the record it names, so a caller never sees that + record twice. Empty starts at the beginning. The token is produced by + whichever store sits behind the endpoint, so a caller cannot tell + which one that is. max_records: type: integer + minimum: 1 + maximum: 1000 description: "Return at most this many records. Defaults to 100 when omitted." wait_ms: type: integer - description: "Wait at most this long for records to arrive before answering. Defaults to 5000 when omitted. A call that collects nothing answers with no records and the caller's own token." + minimum: 0 + maximum: 60000 + description: >- + Wait at most this long for records to arrive before answering. The + handler shortens it to fit the request deadline. A call that collects + nothing answers with no records and the caller's own token. latest_only: type: boolean - description: "Answer with the newest position and no records, for a reader that wants to follow from now." + description: >- + Answer with the newest position and no records, for a reader that + wants to follow from now. required: - workflow_id additionalProperties: false RecordWire: type: object - description: "One record on the wire: its cursor and its base64 frame." + description: "One record on the wire: its cursor and its frame." properties: token: type: string - description: "Opaque cursor naming this record. Pass it as after_token to resume just past it." + description: >- + Opaque cursor naming this record. Pass it as after_token to resume + just past it. frame: type: string - description: "The base64-encoded record frame, carrying the topic, kind, producer, attempt, sequence and body." + contentEncoding: base64 + description: >- + The record frame, carrying the topic, kind, producer, attempt, + sequence and body. required: - token - frame - additionalProperties: false + additionalProperties: true ReadOutput: type: object @@ -119,5 +179,7 @@ $defs: description: "The records after the caller's token, in stream order." next_token: type: string - description: "Opaque cursor to pass as after_token on the following call. Echoes the caller's own token when the call collected nothing." - additionalProperties: false + description: >- + Opaque cursor to pass as after_token on the following call. Echoes + the caller's own token when the call collected nothing. + additionalProperties: true From c54d7506e58c1a24d624832a7f6b1e43d82c87c2 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 18 Sep 2026 17:02:21 -0700 Subject: [PATCH 12/34] Reworked the Nexus front's dedupe, reads, error mapping and endpoint lookup. The handler refuses to continue a producer attempt it has no state for and rejects an index that skips ahead, so a restarted or load-balanced handler cannot lose records behind a success answer. Reads come from one parked subscription per stream, pass bodies through as raw payloads so a codec survives the hop, and stop at the request deadline. The caller commits its batch index only after the endpoint answered, resends a failed batch under the same index, maps transport failures to one error type, and resolves an endpoint name through the operator service. --- temporalio/streams/providers/nexus.py | 685 +++++++++++++++++++------- 1 file changed, 508 insertions(+), 177 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index a73c830ef9..8c311d1d9a 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -15,33 +15,51 @@ loop, and the caller's batching, cursor handling and codec. Reads hand out batches and an append carries one batch per call, because a -Nexus operation per record costs too much for token streams. Supersession -records are not transported: the caller's reader re-synthesizes them from -the attempts it observes, which is the policy module's job on every -provider. Cursors pass through opaque, so the caller cannot tell which store -produced them. - -One stated prototype limit: append deduplication holds at the handler by -batch index per producer, so a handler restart during a producer's life can -double-append. The store-level fix, a sequence-explicit append in the IDL -contract, is listed in the design doc's work table. +Nexus operation per record costs too much for token streams. Record bodies +cross the handler untouched: it reads the store as raw payloads and frames +them as they are, so a codec that changes the payload encoding survives the +hop and the handler's worker never needs the key. Supersession records are +not transported: the caller's reader re-synthesizes them from the attempts +it observes, which is the policy module's job on every provider. Cursors +pass through opaque, so the caller cannot tell which store produced them. + +The handler keeps one parked read per ``(workflow, stream, topic)`` and +serves consecutive calls from it, so an idle caller does not leave one +abandoned long poll on the store per call. A call whose token does not +match the parked position replaces the subscription, and an idle one is +released after a minute; that is the residual cost. + +Two stated prototype limits. Append deduplication lives in handler memory, +by batch index per producer attempt, so a handler that has no state for a +producer attempt refuses to continue it rather than starting a fresh +delegate whose numbering the store would drop as a repeat; the caller opens +a new attempt, which readers report as a supersession. The store-level fix, +a sequence-explicit append in the contract, is listed in the design doc's +work table. And any caller the endpoint admits may touch any workflow's +streams in the namespace; the endpoint's own authorization is the boundary. """ from __future__ import annotations import asyncio -import base64 +import logging +import time import urllib.error import urllib.request -from collections.abc import AsyncGenerator, AsyncIterator -from datetime import timedelta +from collections import OrderedDict +from collections.abc import AsyncGenerator, Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone from typing import Any, TypeVar import nexusrpc import nexusrpc.handler +from google.protobuf.message import DecodeError import temporalio.converter from temporalio.api.common.v1 import Payload +from temporalio.api.operatorservice.v1 import ListNexusEndpointsRequest +from temporalio.common import RawValue from temporalio.streams import _frame, _provider from temporalio.streams._handles import ReadSource, WriteSink from temporalio.streams._policy import AttemptTracker @@ -61,9 +79,16 @@ ) # The contract leaves these unset, so the handler is the one place that says -# what an omitted read bound means. +# what an omitted read bound means. The wait is long because the handler +# shortens it to the request deadline anyway, and a shorter default only +# means more round trips on an idle stream. _DEFAULT_MAX_RECORDS = 100 -_DEFAULT_WAIT_MS = 5000 +_DEFAULT_WAIT_MS = 30_000 +_DEADLINE_MARGIN = timedelta(milliseconds=500) +_DEFAULT_MAX_PRODUCERS = 10_000 +_DEFAULT_SUBSCRIPTION_IDLE = timedelta(seconds=60) +_QUEUE_DEPTH = 1000 +_APPEND_TIMEOUT_MS = 30_000 _definition = nexusrpc.get_service_definition(TemporalStreams) if _definition is None: # pragma: no cover @@ -81,6 +106,34 @@ _OutputT = TypeVar("_OutputT") +logger = logging.getLogger(__name__) + + +def _bad_request(message: str) -> nexusrpc.HandlerError: + return nexusrpc.HandlerError( + message, + type=nexusrpc.HandlerErrorType.BAD_REQUEST, + retryable_override=False, + ) + + +@dataclass +class _Subscription: + """One parked read on the store, shared by consecutive calls.""" + + records: asyncio.Queue[StreamRecord[Any]] + position: str + last_used: float + pump: asyncio.Task[None] | None = None + failure: BaseException | None = None + done: bool = False + + +@dataclass +class _ProducerState: + delegate: Any + batch_index: int + @nexusrpc.handler.service_handler(service=TemporalStreams) class TemporalStreamsHandler: @@ -91,17 +144,45 @@ class TemporalStreamsHandler: run handlers for a new store next to handlers for the old one. """ - def __init__(self, client: Any, provider: str, **provider_options: Any) -> None: - """Serve the endpoint out of the provider named by ``provider``.""" + def __init__( + self, + client: Any, + provider: str, + *, + max_producers: int = _DEFAULT_MAX_PRODUCERS, + subscription_idle: timedelta = _DEFAULT_SUBSCRIPTION_IDLE, + **provider_options: Any, + ) -> None: + """Serve the endpoint out of the provider named by ``provider``. + + ``max_producers`` bounds the dedupe state kept per producer attempt; + the oldest is evicted, and a producer evicted mid-life gets a clear + refusal on its next batch rather than a silent drop. + ``subscription_idle`` is how long a parked read outlives its last + caller. + """ self._client = client self._delegate = _provider.instance(provider, **provider_options) - self._producers: dict[tuple[str, str, str, int], tuple[Any, int]] = {} + self._producers: OrderedDict[tuple[str, str, str, int], _ProducerState] = ( + OrderedDict() + ) + self._max_producers = max_producers + self._subscriptions: dict[tuple[str, str, str], _Subscription] = {} + self._read_locks: dict[tuple[str, str, str], asyncio.Lock] = {} + self._subscription_idle = subscription_idle.total_seconds() @nexusrpc.handler.sync_operation async def append( self, _ctx: nexusrpc.handler.StartOperationContext, input: AppendInput ) -> AppendOutput: - """Append on the caller's account, dropping a repeated batch.""" + """Append on the caller's account, dropping a repeated batch. + + Raises: + nexusrpc.HandlerError: ``BAD_REQUEST`` when the batch index skips + ahead, when it continues a producer attempt this handler has + no state for, or when a payload is not a serialized + ``Payload``. + """ topic = input.topic or "" key = ( input.workflow_id, @@ -109,10 +190,18 @@ async def append( input.producer_id, input.attempt, ) - cached = self._producers.get(key) - if cached is not None and input.batch_index <= cached[1]: - return AppendOutput() - if cached is None: + state = self._producers.get(key) + if state is None: + if input.batch_index != 1: + # A fresh delegate would restart its numbering at zero, and + # the store would drop that as a repeat of the first batch. + # Refusing here turns a silent loss into a failure the caller + # can act on by opening a new attempt. + raise _bad_request( + f"producer {input.producer_id!r} attempt {input.attempt} resumed " + f"at batch {input.batch_index} on a handler that has no state for " + "it; open a new attempt" + ) delegate = await self._delegate.producer( self._client, workflow_id=input.workflow_id, @@ -121,117 +210,304 @@ async def append( producer_id=input.producer_id, attempt=input.attempt, ) - else: - delegate = cached[0] - self._producers[key] = (delegate, input.batch_index) - cursor = "" - if input.payloads: - payloads = [] - for encoded in input.payloads: - payload = Payload() - payload.ParseFromString(base64.b64decode(encoded)) - payloads.append(payload) - cursor = (await delegate.append(*payloads)).token + state = _ProducerState(delegate, 0) + elif input.batch_index <= state.batch_index: + # A repeat, or a producer that restarted its numbering: either + # way the store already holds the batch. + self._producers.move_to_end(key) + return AppendOutput() + elif input.batch_index != state.batch_index + 1: + raise _bad_request( + f"batch {input.batch_index} skips ahead of {state.batch_index} for " + f"producer {input.producer_id!r} attempt {input.attempt}" + ) + payloads = self._payloads(input.payloads) + cursor: str | None = None + if payloads: + appended = await state.delegate.append(*payloads) + cursor = appended.token if appended is not None else None if input.finish: - await delegate.finish() + await state.delegate.finish() + self._producers.pop(key, None) + return AppendOutput(cursor=cursor) + # Recorded only once the store accepted the batch, so a failed append + # is not mistaken for a repeat when the caller retries it. + state.batch_index = input.batch_index + self._producers[key] = state + self._producers.move_to_end(key) + while len(self._producers) > self._max_producers: + self._producers.popitem(last=False) return AppendOutput(cursor=cursor) + @staticmethod + def _payloads(encoded: list[bytes] | None) -> list[Payload]: + payloads = [] + for index, raw in enumerate(encoded or []): + payload = Payload() + try: + payload.ParseFromString(raw) + except DecodeError as error: + raise _bad_request( + f"payloads[{index}] is not a serialized Temporal Payload: {error}" + ) from error + payloads.append(payload) + return payloads + @nexusrpc.handler.sync_operation async def read( - self, _ctx: nexusrpc.handler.StartOperationContext, input: ReadInput + self, ctx: nexusrpc.handler.StartOperationContext, input: ReadInput ) -> ReadOutput: """Answer with the records after the caller's token, or time out.""" - delegate = await self._delegate.consumer( - self._client, workflow_id=input.workflow_id, stream=input.stream or "" - ) + stream = input.stream or "" topic = input.topic or None if input.latest_only: + delegate = await self._delegate.consumer( + self._client, workflow_id=input.workflow_id, stream=stream + ) return ReadOutput(next_token=(await delegate.latest(topic=topic)).token) max_records = ( _DEFAULT_MAX_RECORDS if input.max_records is None else input.max_records ) - wait_ms = _DEFAULT_WAIT_MS if input.wait_ms is None else input.wait_ms - converter = temporalio.converter.DataConverter.default.payload_converter - records: list[RecordWire] = [] - next_token = input.after_token or "" - subscription = delegate.read( - after=Cursor(input.after_token) if input.after_token else BEGINNING, - topic=topic, - ) - - async def collect() -> None: - nonlocal next_token - async for record in subscription: - if record.kind is RecordKind.SUPERSEDED: - continue - body = b"" - if record.kind is RecordKind.DATA: - body = converter.to_payloads([record.value])[0].SerializeToString() - frame = _frame.encode( - topic=record.topic, - kind=record.kind, - producer=record.producer, - attempt=record.attempt, - sequence=record.sequence, - body=body, - ) - records.append( - RecordWire( - token=record.cursor.token, - frame=base64.b64encode(frame).decode("ascii"), - ) + wait = (_DEFAULT_WAIT_MS if input.wait_ms is None else input.wait_ms) / 1000 + if ctx.request_deadline is not None: + # The server answers the caller with a timeout at the deadline + # whatever this call does, so collecting past it only holds the + # slot for an answer nobody receives. + deadline = ctx.request_deadline + if deadline.tzinfo is None: + deadline = deadline.replace(tzinfo=timezone.utc) + left = deadline - datetime.now(timezone.utc) - _DEADLINE_MARGIN + wait = max(0.0, min(wait, left.total_seconds())) + after = input.after_token or "" + key = (input.workflow_id, stream, topic or "") + await self._expire_subscriptions() + lock = self._read_locks.setdefault(key, asyncio.Lock()) + async with lock: + subscription = self._subscriptions.get(key) + if subscription is None or subscription.position != after: + if subscription is not None: + await self._drop(key) + subscription = await self._subscribe( + key, input.workflow_id, stream, topic, after ) - next_token = record.cursor.token - if len(records) >= max_records: - return + records, next_token = await self._drain(subscription, max_records, wait) + subscription.position = next_token + subscription.last_used = time.monotonic() + if subscription.done and subscription.records.empty(): + # The store ended the read, as the Workflow Streams delegate + # does when the run closes. Nothing more will arrive on it. + await self._drop(key) + return ReadOutput(records=records, next_token=next_token) + + async def _subscribe( + self, + key: tuple[str, str, str], + workflow_id: str, + stream: str, + topic: str | None, + after: str, + ) -> _Subscription: + delegate = await self._delegate.consumer( + self._client, workflow_id=workflow_id, stream=stream + ) + # Raw payloads: the handler frames what the store holds without + # decoding it, so an encoding only the caller's codec understands + # passes through untouched. + source = delegate.read( + after=Cursor(after) if after else BEGINNING, topic=topic, type=RawValue + ) + subscription = _Subscription( + records=asyncio.Queue(maxsize=_QUEUE_DEPTH), + position=after, + last_used=time.monotonic(), + ) + async def pump() -> None: + try: + async for record in source: + if record.kind is RecordKind.SUPERSEDED: + continue + await subscription.records.put(record) + except asyncio.CancelledError: + raise + except Exception as error: + subscription.failure = error + finally: + subscription.done = True + if isinstance(source, AsyncGenerator): + await source.aclose() + + subscription.pump = asyncio.create_task(pump()) + self._subscriptions[key] = subscription + return subscription + + async def _drain( + self, subscription: _Subscription, max_records: int, wait: float + ) -> tuple[list[RecordWire], str]: + records: list[RecordWire] = [] + next_token = subscription.position + queue = subscription.records + deadline = time.monotonic() + wait + while len(records) < max_records: + if queue.empty(): + remaining = deadline - time.monotonic() + if records or remaining <= 0: + break + if subscription.done: + # Honour the wait even though nothing can arrive, so a + # caller polling a closed stream does not spin on the + # endpoint. + await asyncio.sleep(remaining) + break + try: + record = await asyncio.wait_for(queue.get(), remaining) + except asyncio.TimeoutError: + break + else: + record = queue.get_nowait() + records.append(self._wire(record)) + next_token = record.cursor.token + if not records and subscription.failure is not None: + failure, subscription.failure = subscription.failure, None + raise failure + return records, next_token + + @staticmethod + def _wire(record: StreamRecord[Any]) -> RecordWire: + body = ( + record.value.payload.SerializeToString() + if isinstance(record.value, RawValue) + else b"" + ) + frame = _frame.encode( + topic=record.topic, + kind=record.kind, + producer=record.producer, + attempt=record.attempt, + sequence=record.sequence, + body=body, + ) + return RecordWire(token=record.cursor.token, frame=frame) + + async def close(self) -> None: + """Release every parked read. + + Call it when the worker hosting this handler stops, so subscriptions + waiting on the store do not outlive it. + """ + for key in list(self._subscriptions): + await self._drop(key) + self._read_locks.clear() + + async def _drop(self, key: tuple[str, str, str]) -> None: + subscription = self._subscriptions.pop(key, None) + if subscription is None or subscription.pump is None: + return + subscription.pump.cancel() try: - # `wait_for` rather than `asyncio.timeout`, which 3.10 does not - # have. Whatever was collected before the deadline is the answer. - await asyncio.wait_for(collect(), wait_ms / 1000) - except asyncio.TimeoutError: + await subscription.pump + except (asyncio.CancelledError, Exception): pass - finally: - # The delegate's read parks against its store, and this call - # answers before the caller asks again, so let it go now rather - # than when the collector is next swept. - if isinstance(subscription, AsyncGenerator): - await subscription.aclose() - return ReadOutput(records=records, next_token=next_token) + async def _expire_subscriptions(self) -> None: + cutoff = time.monotonic() - self._subscription_idle + for key in [k for k, s in self._subscriptions.items() if s.last_used < cutoff]: + lock = self._read_locks.get(key) + if lock is not None and lock.locked(): + continue + await self._drop(key) + self._read_locks.pop(key, None) + + +class StreamEndpointError(RuntimeError): + """The stream endpoint refused a call or could not be reached. -def _post(url: str, body: bytes) -> bytes: + ``status`` is the HTTP status when the endpoint answered, and ``None`` + when the connection itself failed. + """ + + def __init__(self, message: str, *, status: int | None = None) -> None: + """Carry the endpoint's answer, or the reason it gave none.""" + super().__init__(message) + self.status = status + + +def _post(url: str, body: bytes, headers: Mapping[str, str], timeout_ms: int) -> bytes: request = urllib.request.Request( url, data=body, - headers={"Content-Type": "application/json"}, + headers={ + "Content-Type": "application/json", + # Tells the server how long the handler may park, so it does not + # time the call out ahead of a wait the caller asked for. + "Request-Timeout": f"{timeout_ms}ms", + **headers, + }, method="POST", ) try: - with urllib.request.urlopen(request, timeout=30) as response: + with urllib.request.urlopen(request, timeout=timeout_ms / 1000 + 5) as response: return response.read() except urllib.error.HTTPError as error: detail = error.read().decode(errors="replace") - raise RuntimeError(f"stream endpoint call failed ({error.code}): {detail}") + raise StreamEndpointError( + f"stream endpoint call failed ({error.code}): {detail}", status=error.code + ) from error + except urllib.error.URLError as error: + raise StreamEndpointError( + f"stream endpoint unreachable at {url}: {error.reason}" + ) from error + +class _Front: + """Everything the caller half needs to reach one endpoint.""" -async def _invoke(url: str, request: Any, output: type[_OutputT]) -> _OutputT: - # The contract types carry their own JSON encoding, so the raw caller and - # the worker serving the operation agree on the body without either of - # them spelling the fields out. - contract = temporalio.converter.DataConverter.default.payload_converter - raw = await asyncio.to_thread(_post, url, contract.to_payloads([request])[0].data) - payload = Payload(metadata={"encoding": b"json/plain"}, data=raw or b"{}") - return contract.from_payloads([payload], [output])[0] + def __init__( + self, + base_url: str, + data_converter: temporalio.converter.DataConverter, + headers: Mapping[str, str], + wait_ms: int, + max_records: int, + ) -> None: + self.base_url = base_url + self.converter = data_converter.payload_converter + self.codec = data_converter.payload_codec + self.headers = dict(headers) + self.wait_ms = wait_ms + self.max_records = max_records + + async def invoke( + self, operation: str, request: Any, output: type[_OutputT], *, timeout_ms: int + ) -> _OutputT: + # The contract types carry their own JSON encoding, so the raw caller + # and the worker serving the operation agree on the body without + # either of them spelling the fields out. + contract = temporalio.converter.DataConverter.default.payload_converter + raw = await asyncio.to_thread( + _post, + f"{self.base_url}/{operation}", + contract.to_payloads([request])[0].data, + self.headers, + timeout_ms, + ) + payload = Payload(metadata={"encoding": b"json/plain"}, data=raw or b"{}") + return contract.from_payloads([payload], [output])[0] class NexusProducer: - """Appends through the stream endpoint; framing happens behind it.""" + """Appends through the stream endpoint; framing happens behind it. + + Calls are serialized and the batch index is committed only once the + endpoint answered. A batch whose call raised stays pending and is sent + again under the same index, either when the caller retries the same + values or ahead of whatever the caller sends next, so an ambiguous + failure writes the batch once and loses nothing. + """ def __init__( self, - base_url: str, - data_converter: temporalio.converter.DataConverter, + front: _Front, workflow_id: str, stream: str, topic: str, @@ -239,78 +515,92 @@ def __init__( attempt: int, ) -> None: """Bind this producer to one stream or topic on the endpoint.""" - self._base_url = base_url - self._converter = data_converter.payload_converter - self._codec = data_converter.payload_codec + self._front = front self._workflow_id = workflow_id self._stream = stream self._topic = topic self._producer_id = producer_id self._attempt = attempt self._batch_index = 0 + self._pending: tuple[AppendInput, tuple[bytes, ...], bool] | None = None + self._lock = asyncio.Lock() @property def attempt(self) -> int: """The generation this producer is writing.""" return self._attempt - async def append(self, *values: Any) -> Cursor: + async def append(self, *values: Any) -> Cursor | None: """Append ``values`` through the endpoint.""" + if not values: + return None payloads = [ value if isinstance(value, Payload) - else self._converter.to_payloads([value])[0] + else self._front.converter.to_payloads([value])[0] for value in values ] - if self._codec is not None: - # The endpoint is the edge of this process, so a configured codec - # runs here rather than at the handler: whoever hosts the endpoint - # never holds the plaintext. - payloads = await self._codec.encode(payloads) - encoded = [ - base64.b64encode(payload.SerializeToString()).decode() - for payload in payloads - ] - return Cursor((await self._call(payloads=encoded)).cursor or "") + answer = await self._call(payloads, finish=False) + return Cursor(answer.cursor) if answer.cursor else None async def finish(self) -> None: """Mark this producer done, so a reader stops waiting on it.""" - await self._call(finish=True) + await self._call([], finish=True) + + async def _call(self, payloads: list[Payload], *, finish: bool) -> AppendOutput: + # Identity is taken before the codec runs: a codec may encrypt with a + # fresh nonce each time, and the retry has to be recognised anyway. + identity = tuple(payload.SerializeToString() for payload in payloads) + if self._front.codec is not None: + # The endpoint is the edge of this process, so a configured codec + # runs here rather than at the handler: whoever hosts the endpoint + # never holds the plaintext. + payloads = await self._front.codec.encode(payloads) + encoded = [payload.SerializeToString() for payload in payloads] + async with self._lock: + if self._pending is not None and self._pending[1:] != (identity, finish): + # The caller moved on from a batch the endpoint may or may + # not have applied. It goes first under its own index, so the + # handler drops it if it landed and writes it if not, and the + # new batch takes the index after it. + await self._send(*self._pending) + elif self._pending is not None: + return await self._send(*self._pending) + request = self._request(self._batch_index + 1, encoded, finish) + return await self._send(request, identity, finish) + + def _request( + self, batch_index: int, payloads: list[bytes], finish: bool + ) -> AppendInput: + return AppendInput( + workflow_id=self._workflow_id, + stream=self._stream, + producer_id=self._producer_id, + attempt=self._attempt, + batch_index=batch_index, + topic=self._topic, + payloads=payloads, + finish=finish, + ) - async def _call( - self, payloads: list[str] | None = None, finish: bool = False + async def _send( + self, request: AppendInput, identity: tuple[bytes, ...], finish: bool ) -> AppendOutput: - self._batch_index += 1 - return await _invoke( - f"{self._base_url}/{_APPEND_OPERATION}", - AppendInput( - workflow_id=self._workflow_id, - stream=self._stream, - producer_id=self._producer_id, - attempt=self._attempt, - batch_index=self._batch_index, - topic=self._topic, - payloads=payloads or [], - finish=finish, - ), - AppendOutput, + self._pending = (request, identity, finish) + answer = await self._front.invoke( + _APPEND_OPERATION, request, AppendOutput, timeout_ms=_APPEND_TIMEOUT_MS ) + self._batch_index = request.batch_index + self._pending = None + return answer class NexusConsumer: """Reads batches from the stream endpoint, re-synthesizing supersession.""" - def __init__( - self, - base_url: str, - data_converter: temporalio.converter.DataConverter, - workflow_id: str, - stream: str, - ) -> None: + def __init__(self, front: _Front, workflow_id: str, stream: str) -> None: """Read what ``workflow_id`` publishes, through the endpoint.""" - self._base_url = base_url - self._converter = data_converter.payload_converter - self._codec = data_converter.payload_codec + self._front = front self._workflow_id = workflow_id self._stream = stream @@ -320,31 +610,37 @@ async def read( after: Cursor = BEGINNING, topic: str | None = None, type: type | None = None, - ) -> AsyncIterator[StreamRecord[Any]]: + ) -> AsyncGenerator[StreamRecord[Any], None]: """Yield records after ``after``, one endpoint batch at a time.""" + _provider.check_topic(self._stream, topic) attempts = AttemptTracker() token = after.token while True: - answer = await _invoke( - f"{self._base_url}/{_READ_OPERATION}", + answer = await self._front.invoke( + _READ_OPERATION, ReadInput( workflow_id=self._workflow_id, stream=self._stream, topic=topic or "", after_token=token, + max_records=self._front.max_records, + wait_ms=self._front.wait_ms, ), ReadOutput, + timeout_ms=self._front.wait_ms + 5000, ) for wire in answer.records or []: + cursor = Cursor(wire.token) try: kind, frame_topic, source, attempt, sequence, body = _frame.decode( - base64.b64decode(wire.frame) + wire.frame ) - except ValueError: + except ValueError as error: + # Same answer as every other reader: skip and say so. + logger.warning("skipping stream record at %s: %s", cursor, error) continue if topic is not None and frame_topic != topic: continue - cursor = Cursor(wire.token) superseded = attempts.note(source, attempt, cursor) if superseded is not None: yield superseded @@ -364,8 +660,8 @@ async def read( async def latest(self, *, topic: str | None = None) -> Cursor: """The cursor of the last record written, for following from now.""" - answer = await _invoke( - f"{self._base_url}/{_READ_OPERATION}", + answer = await self._front.invoke( + _READ_OPERATION, ReadInput( workflow_id=self._workflow_id, stream=self._stream, @@ -373,6 +669,7 @@ async def latest(self, *, topic: str | None = None) -> Cursor: latest_only=True, ), ReadOutput, + timeout_ms=_APPEND_TIMEOUT_MS, ) token = answer.next_token or "" return Cursor(token) if token else BEGINNING @@ -380,34 +677,58 @@ async def latest(self, *, topic: str | None = None) -> Cursor: async def _decode(self, body: bytes, as_type: type | None) -> Any: payload = Payload() payload.ParseFromString(body) - if self._codec is not None: - payload = (await self._codec.decode([payload]))[0] + if self._front.codec is not None: + payload = (await self._front.codec.decode([payload]))[0] if as_type is None: - return self._converter.from_payloads([payload])[0] - return self._converter.from_payloads([payload], [as_type])[0] + return self._front.converter.from_payloads([payload])[0] + return self._front.converter.from_payloads([payload], [as_type])[0] class _NexusProvider: name = "nexus" def __init__(self) -> None: - self._base_url: str | None = None + self._endpoint: str | None = None + self._endpoint_id: str | None = None + self._http_address = "http://127.0.0.1:7243" + self._service = _SERVICE_NAME self._data_converter: temporalio.converter.DataConverter | None = None + self._client: Any = None + self._headers: dict[str, str] = {} + self._wait_ms = _DEFAULT_WAIT_MS + self._max_records = _DEFAULT_MAX_RECORDS def configure(self, **options: Any) -> None: + """Point the caller half at one endpoint. + + ``endpoint`` is the endpoint's name when ``client`` is given, and is + resolved to its id through the operator service on first use; without + a client it has to be the id, because the HTTP ingress dispatches by + id. ``headers`` go on every request, which is where an authorization + header belongs. ``wait_ms`` and ``max_records`` are the read bounds + the caller asks the handler for. + """ endpoint = options.pop("endpoint", None) - http_address = options.pop("http_address", "http://127.0.0.1:7243") - service = options.pop("service", _SERVICE_NAME) - data_converter = options.pop("data_converter", None) + self._http_address = options.pop("http_address", self._http_address) + self._service = options.pop("service", _SERVICE_NAME) + self._data_converter = options.pop("data_converter", None) + self._client = options.pop("client", None) + self._headers = dict(options.pop("headers", None) or {}) + self._wait_ms = int(options.pop("wait_ms", _DEFAULT_WAIT_MS)) + self._max_records = int(options.pop("max_records", _DEFAULT_MAX_RECORDS)) if options: raise TypeError( - "the nexus provider takes endpoint, http_address, service and " - f"data_converter, got {sorted(options)}" + "the nexus provider takes endpoint, http_address, service, " + f"data_converter, client, headers, wait_ms and max_records, got " + f"{sorted(options)}" ) if not endpoint: - raise TypeError("the nexus provider needs endpoint=") - self._base_url = f"{http_address}/nexus/endpoints/{endpoint}/services/{service}" - self._data_converter = data_converter + raise TypeError( + "the nexus provider needs endpoint= together with " + "client=, or endpoint= on its own" + ) + self._endpoint = endpoint + self._endpoint_id = None def worker_options(self) -> dict[str, Any]: raise RuntimeError(_WORKFLOW_SIDE_ERROR) @@ -426,10 +747,31 @@ def open_write(self, topic: str) -> WriteSink: del topic raise RuntimeError(_WORKFLOW_SIDE_ERROR) - def _url(self) -> str: - if self._base_url is None: + async def _front(self, client: Any) -> _Front: + if self._endpoint is None: raise RuntimeError("configure the nexus provider before opening handles") - return self._base_url + if self._endpoint_id is None: + resolver = self._client if self._client is not None else client + if resolver is None: + self._endpoint_id = self._endpoint + else: + found = await resolver.operator_service.list_nexus_endpoints( + ListNexusEndpointsRequest(name=self._endpoint) + ) + if not found.endpoints: + raise RuntimeError(f"no nexus endpoint is named {self._endpoint!r}") + self._endpoint_id = found.endpoints[0].id + base_url = ( + f"{self._http_address}/nexus/endpoints/{self._endpoint_id}" + f"/services/{self._service}" + ) + return _Front( + base_url, + self._converter(client), + self._headers, + self._wait_ms, + self._max_records, + ) async def producer( self, @@ -441,25 +783,14 @@ async def producer( producer_id: str = "", attempt: int = 0, ) -> NexusProducer: - if not producer_id: - from temporalio import activity - - producer_id = activity.info().activity_id - attempt = attempt or activity.info().attempt return NexusProducer( - self._url(), - self._converter(client), - workflow_id, - stream, - topic, - producer_id, - attempt, + await self._front(client), workflow_id, stream, topic, producer_id, attempt ) async def consumer( self, client: Any, *, workflow_id: str, stream: str = "" ) -> NexusConsumer: - return NexusConsumer(self._url(), self._converter(client), workflow_id, stream) + return NexusConsumer(await self._front(client), workflow_id, stream) def _converter(self, client: Any) -> temporalio.converter.DataConverter: if self._data_converter is not None: From 769cfd058966f212962c9704ccfbac200ed49d7c Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 18 Sep 2026 17:02:21 -0700 Subject: [PATCH 13/34] Covered the Nexus front's refusals, retries, subscription reuse and deadline clamp. --- tests/streams/test_nexus_provider.py | 334 ++++++++++++++++++++++----- 1 file changed, 274 insertions(+), 60 deletions(-) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 18679ad5f1..77cec6f4eb 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -11,8 +11,8 @@ and an endpoint named ``streams-e2e`` targeting task queue ``streams-handlers-e2e``. -The codec test stands the endpoint up in this process instead, because what -it checks is the bytes the caller puts on the wire. +Everything else stands the endpoint up in this process, because what it +checks is the bytes the caller puts on the wire and the handler's own rules. """ from __future__ import annotations @@ -22,11 +22,13 @@ import dataclasses import json import os -import subprocess import uuid -from collections.abc import Sequence -from typing import Any, cast +from collections.abc import Mapping, Sequence +from datetime import datetime, timedelta, timezone +from typing import Any +import nexusrpc +import nexusrpc.handler import pytest import temporalio.converter @@ -38,7 +40,10 @@ from temporalio.streams._provider import instance from temporalio.streams.providers import memory, nexus from temporalio.streams.providers._nexus_generated import AppendInput, ReadInput -from temporalio.streams.providers.nexus import TemporalStreamsHandler +from temporalio.streams.providers.nexus import ( + StreamEndpointError, + TemporalStreamsHandler, +) from temporalio.worker import Worker from tests.streams.test_workflow_streams_provider import EchoLoop, take @@ -50,51 +55,27 @@ ENDPOINT = "streams-e2e" HANDLER_TQ = "streams-handlers-e2e" APPEND_OPERATION = nexus._APPEND_OPERATION # pyright: ignore[reportPrivateUsage] - - -def _endpoint_id() -> str: - # The HTTP ingress dispatches by endpoint id, not name. - out = subprocess.run( - [ - "temporal", - "operator", - "nexus", - "endpoint", - "get", - "--name", - ENDPOINT, - "-o", - "json", - "--address", - os.environ.get("TEMPORAL_ADDRESS", "localhost:7233"), - ], - capture_output=True, - text=True, - check=True, - ) - return json.loads(out.stdout)["id"] +READ_OPERATION = nexus._READ_OPERATION # pyright: ignore[reportPrivateUsage] @live_only async def test_interface_loop_through_the_nexus_front(): # The workflow worker and the handler worker both use the storage - # provider; only the caller goes through the front. + # provider; only the caller goes through the front, and it names the + # endpoint the way an operator does, by name. streams.configure(provider="workflow_streams") client = await Client.connect(os.environ.get("TEMPORAL_ADDRESS", "localhost:7233")) front = instance( "nexus", - endpoint=_endpoint_id(), + endpoint=ENDPOINT, + client=client, http_address=os.environ.get("TEMPORAL_HTTP", "http://127.0.0.1:7243"), + wait_ms=5000, ) workflow_id = f"streams-nexus-live-{uuid.uuid4().hex}" + handler = TemporalStreamsHandler(client, provider="workflow_streams") - async with Worker( - client, - task_queue=HANDLER_TQ, - nexus_service_handlers=[ - TemporalStreamsHandler(client, provider="workflow_streams") - ], - ): + async with Worker(client, task_queue=HANDLER_TQ, nexus_service_handlers=[handler]): async with Worker( client, task_queue=f"tq-{workflow_id}", @@ -135,10 +116,11 @@ async def test_interface_loop_through_the_nexus_front(): await handle.signal(EchoLoop.release) assert await handle.result() == 3 + await handler.close() class ScrambleCodec(temporalio.converter.PayloadCodec): - """Flips every byte of a payload, so the middle holds nothing readable.""" + """Flips every byte and labels the result with an encoding only it can read.""" KEY = 0x5A @@ -146,7 +128,7 @@ async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: """Wrap each payload as opaque bytes nothing downstream can read.""" return [ Payload( - metadata={"encoding": b"binary/plain"}, + metadata={"encoding": b"binary/scrambled"}, data=self._flip(payload.SerializeToString()), ) for payload in payloads @@ -166,45 +148,98 @@ def _flip(cls, data: bytes) -> bytes: return bytes(byte ^ cls.KEY for byte in data) +class _NeverCancelled(nexusrpc.handler.OperationTaskCancellation): + def is_cancelled(self) -> bool: + return False + + def cancellation_reason(self) -> str | None: + return None + + def wait_until_cancelled_sync(self, timeout: float | None = None) -> bool: + del timeout + return False + + async def wait_until_cancelled(self) -> None: + await asyncio.Event().wait() + + +def _context( + operation: str, deadline: datetime | None = None +) -> nexusrpc.handler.StartOperationContext: + return nexusrpc.handler.StartOperationContext( + service=nexus._SERVICE_NAME, # pyright: ignore[reportPrivateUsage] + operation=operation, + headers={}, + task_cancellation=_NeverCancelled(), + request_deadline=deadline, + request_id=uuid.uuid4().hex, + ) + + async def _dispatch( handler: TemporalStreamsHandler, operation: str, request: Any ) -> Any: - context = cast("Any", None) # neither operation looks at it if operation == APPEND_OPERATION: - return await handler.append(context, request) - return await handler.read(context, request) + return await handler.append(_context(operation), request) + return await handler.read(_context(operation), request) def _in_process_endpoint( handler: TemporalStreamsHandler, posted: list[bytes], answered: list[bytes], + *, + fail_after_applying: list[str] | None = None, ) -> Any: - """Serve the caller's posts from ``handler``, recording both directions.""" + """Serve the caller's posts from ``handler``, recording both directions. + + ``fail_after_applying`` names operations whose first call is applied and + then reported as a transport failure, the ambiguous case a retry has to + survive. + """ contract = temporalio.converter.DataConverter.default.payload_converter loop = asyncio.get_running_loop() + failing = set(fail_after_applying or []) - def post(url: str, body: bytes) -> bytes: + def post( + url: str, body: bytes, headers: Mapping[str, str], timeout_ms: int + ) -> bytes: + del headers, timeout_ms posted.append(body) operation = url.rsplit("/", 1)[1] request_type = AppendInput if operation == APPEND_OPERATION else ReadInput request: Any = contract.from_payloads( [Payload(metadata={"encoding": b"json/plain"}, data=body)], [request_type] )[0] - if request_type is ReadInput: - # This endpoint has no other traffic to wait for, so park briefly - # rather than for the contract's default. - request.wait_ms = 200 - answer = asyncio.run_coroutine_threadsafe( - _dispatch(handler, operation, request), loop - ).result() + try: + answer = asyncio.run_coroutine_threadsafe( + _dispatch(handler, operation, request), loop + ).result() + except nexusrpc.HandlerError as error: + raise StreamEndpointError(str(error), status=400) from error raw = contract.to_payloads([answer])[0].data answered.append(raw) + if operation in failing: + failing.discard(operation) + raise StreamEndpointError("connection reset after the handler answered") return raw return post +def _front(codec: temporalio.converter.PayloadCodec | None) -> Any: + return instance( + "nexus", + endpoint="in-process", + # This endpoint has no other traffic to wait for, so park briefly + # rather than for the contract's default. + wait_ms=200, + data_converter=dataclasses.replace( + temporalio.converter.DataConverter.default, payload_codec=codec + ), + ) + + async def _loop_through_an_endpoint( monkeypatch: pytest.MonkeyPatch, codec: temporalio.converter.PayloadCodec | None, @@ -215,13 +250,7 @@ async def _loop_through_an_endpoint( answered: list[bytes] = [] handler = TemporalStreamsHandler(None, provider="memory") monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, posted, answered)) - front = instance( - "nexus", - endpoint="in-process", - data_converter=dataclasses.replace( - temporalio.converter.DataConverter.default, payload_codec=codec - ), - ) + front = _front(codec) workflow_id = "wf-codec" producer = await front.producer( @@ -263,7 +292,9 @@ async def test_the_caller_encodes_records_with_the_configured_codec( monkeypatch, ScrambleCodec() ) - # The value survives the hop, so the codec is applied on both sides. + # The value survives the hop although the handler cannot read the + # encoding, so the body passed through it untouched and the codec ran on + # the caller alone. assert [record.kind for record in records] == [RecordKind.DATA, RecordKind.FINISH] assert records[0].value == {"secret": "tuna"} # Neither direction carried the plaintext. @@ -281,3 +312,186 @@ async def test_without_a_codec_the_same_records_go_out_in_the_clear( assert records[0].value == {"secret": "tuna"} assert b"tuna" in _appended_bytes(posted) assert b"tuna" in _answered_bodies(answered) + + +def _append( + workflow_id: str, batch_index: int, *values: Any, finish: bool = False +) -> AppendInput: + converter = temporalio.converter.DataConverter.default.payload_converter + return AppendInput( + workflow_id=workflow_id, + stream="inputs", + producer_id="model", + attempt=1, + batch_index=batch_index, + payloads=[ + converter.to_payloads([value])[0].SerializeToString() for value in values + ], + finish=finish, + ) + + +async def _stored(workflow_id: str) -> list[Any]: + consumer = await streams.consumer(None, workflow_id=workflow_id, stream="inputs") + out: list[Any] = [] + async for record in consumer.read(type=dict): + if record.kind is RecordKind.DATA: + out.append(record.value) + if await consumer.latest() == record.cursor: + break + return out + + +@pytest.fixture +def _memory_store(): # pyright: ignore[reportUnusedFunction] + memory.reset() + streams.configure(provider="memory") + yield + memory.reset() + + +@pytest.mark.usefixtures("_memory_store") +async def test_a_handler_without_state_for_a_producer_refuses_to_continue_it(): + # A second handler instance stands in for a restarted or load-balanced + # handler worker: it shares the store but not the dedupe state. + first = TemporalStreamsHandler(None, provider="memory") + second = TemporalStreamsHandler(None, provider="memory") + await _dispatch(first, APPEND_OPERATION, _append("wf", 1, {"n": 1})) + with pytest.raises(nexusrpc.HandlerError) as failed: + await _dispatch(second, APPEND_OPERATION, _append("wf", 2, {"n": 2})) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + assert failed.value.retryable_override is False + # The store holds the first batch and nothing was silently lost or doubled. + assert await _stored("wf") == [{"n": 1}] + + +@pytest.mark.usefixtures("_memory_store") +async def test_the_handler_drops_repeats_and_rejects_gaps(): + handler = TemporalStreamsHandler(None, provider="memory") + await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})) + await _dispatch(handler, APPEND_OPERATION, _append("wf", 2, {"n": 2})) + # An exact repeat is dropped with a success answer and no position. + repeat = await _dispatch(handler, APPEND_OPERATION, _append("wf", 2, {"n": 2})) + assert repeat.cursor is None + with pytest.raises(nexusrpc.HandlerError) as failed: + await _dispatch(handler, APPEND_OPERATION, _append("wf", 4, {"n": 4})) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + assert await _stored("wf") == [{"n": 1}, {"n": 2}] + + +@pytest.mark.usefixtures("_memory_store") +async def test_a_malformed_payload_is_the_callers_fault(): + handler = TemporalStreamsHandler(None, provider="memory") + request = _append("wf", 1) + request.payloads = [b"\xff\xfe not a payload"] + with pytest.raises(nexusrpc.HandlerError) as failed: + await _dispatch(handler, APPEND_OPERATION, request) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + + +@pytest.mark.usefixtures("_memory_store") +async def test_the_caller_retries_an_ambiguous_append_under_the_same_index( + monkeypatch: pytest.MonkeyPatch, +): + posted: list[bytes] = [] + handler = TemporalStreamsHandler(None, provider="memory") + monkeypatch.setattr( + nexus, + "_post", + _in_process_endpoint( + handler, posted, [], fail_after_applying=[APPEND_OPERATION] + ), + ) + front = _front(None) + producer = await front.producer( + None, workflow_id="wf", stream="inputs", producer_id="model", attempt=1 + ) + with pytest.raises(StreamEndpointError): + await producer.append({"n": 1}) + await producer.append({"n": 1}) + await producer.append({"n": 2}) + await producer.finish() + indexes = [json.loads(body)["batch_index"] for body in posted] + # The retry re-sent index 1, which the handler dropped as a repeat, so + # the store holds each record once. + assert indexes == [1, 1, 2, 3] + assert await _stored("wf") == [{"n": 1}, {"n": 2}] + + +@pytest.mark.usefixtures("_memory_store") +async def test_a_failed_append_goes_out_before_the_next_one( + monkeypatch: pytest.MonkeyPatch, +): + posted: list[bytes] = [] + handler = TemporalStreamsHandler(None, provider="memory") + monkeypatch.setattr( + nexus, + "_post", + _in_process_endpoint( + handler, posted, [], fail_after_applying=[APPEND_OPERATION] + ), + ) + front = _front(None) + producer = await front.producer( + None, workflow_id="wf", stream="inputs", producer_id="model", attempt=1 + ) + with pytest.raises(StreamEndpointError): + await producer.append({"n": 1}) + # The caller moves on without retrying; the pending batch is replayed + # first under its own index and the new one takes the next. + await producer.append({"n": 2}) + indexes = [json.loads(body)["batch_index"] for body in posted] + assert indexes == [1, 1, 2] + assert await _stored("wf") == [{"n": 1}, {"n": 2}] + + +@pytest.mark.usefixtures("_memory_store") +async def test_consecutive_reads_share_one_parked_subscription(): + handler = TemporalStreamsHandler(None, provider="memory") + await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1}, {"n": 2})) + first = await _dispatch( + handler, + READ_OPERATION, + ReadInput(workflow_id="wf", stream="inputs", max_records=1, wait_ms=200), + ) + assert len(first.records) == 1 + parked = handler._subscriptions # pyright: ignore[reportPrivateUsage] + pump = next(iter(parked.values())).pump + # A call that resumes where the last one stopped drains the same + # subscription instead of parking a second read on the store. + second = await _dispatch( + handler, + READ_OPERATION, + ReadInput( + workflow_id="wf", + stream="inputs", + after_token=first.next_token, + max_records=1, + wait_ms=200, + ), + ) + assert len(second.records) == 1 + assert second.next_token != first.next_token + assert len(parked) == 1 and next(iter(parked.values())).pump is pump + # A call from somewhere else replaces it. + await _dispatch( + handler, + READ_OPERATION, + ReadInput(workflow_id="wf", stream="inputs", wait_ms=200), + ) + assert len(parked) == 1 and next(iter(parked.values())).pump is not pump + await handler.close() + assert not parked + + +@pytest.mark.usefixtures("_memory_store") +async def test_the_read_wait_is_cut_to_the_request_deadline(): + handler = TemporalStreamsHandler(None, provider="memory") + started = asyncio.get_running_loop().time() + answer = await handler.read( + _context(READ_OPERATION, datetime.now(timezone.utc) + timedelta(seconds=1)), + ReadInput(workflow_id="wf", stream="inputs", wait_ms=60000), + ) + assert answer.records == [] and answer.next_token == "" + assert asyncio.get_running_loop().time() - started < 5 + await handler.close() From ff52038e56b8741a7e4cdd39cab44f9d9ed9400f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 21 Sep 2026 02:15:21 -0700 Subject: [PATCH 14/34] Carried the StreamRecord proto and producer identity on the stream contract. A record crosses the endpoint as the serialized proto every store keeps, an append names its topic, run and sequence, and a read says when the store ended it. --- .../providers/_nexus_generated/models.py | 245 +++++++++++------- .../providers/_nexus_generated/services.py | 17 +- .../providers/temporal_streams.nexusrpc.yaml | 96 ++++--- 3 files changed, 224 insertions(+), 134 deletions(-) diff --git a/temporalio/streams/providers/_nexus_generated/models.py b/temporalio/streams/providers/_nexus_generated/models.py index d00cbcc637..2677843efa 100644 --- a/temporalio/streams/providers/_nexus_generated/models.py +++ b/temporalio/streams/providers/_nexus_generated/models.py @@ -22,10 +22,10 @@ _APPEND_OUTPUT_DECLARED: frozenset[str] = frozenset({"cursor"}) -_READ_OUTPUT_DECLARED: frozenset[str] = frozenset({"records", "next_token"}) +_READ_OUTPUT_DECLARED: frozenset[str] = frozenset({"records", "next_token", "done"}) -_RECORD_WIRE_DECLARED: frozenset[str] = frozenset({"token", "frame"}) +_RECORD_WIRE_DECLARED: frozenset[str] = frozenset({"token", "record"}) class _AppendInputTransferTypeConverter( @@ -54,15 +54,30 @@ def from_transfer_type( else: workflow_id_value = workflow_id_value_raw - stream_value: str = typing.cast("typing.Any", None) - if "stream" not in raw or raw["stream"] is None: - violations.append(Violation(path="stream", reason="required")) + run_id_value: str | None = None + if "run_id" in raw: + run_id_value_raw = raw["run_id"] + if run_id_value_raw is None: + violations.append( + Violation(path="run_id", reason="explicit null not allowed") + ) + else: + if not isinstance(run_id_value_raw, str): + violations.append( + Violation(path="run_id", reason="expected string") + ) + else: + run_id_value = run_id_value_raw + + topic_value: str = typing.cast("typing.Any", None) + if "topic" not in raw or raw["topic"] is None: + violations.append(Violation(path="topic", reason="required")) else: - stream_value_raw = raw["stream"] - if not isinstance(stream_value_raw, str): - violations.append(Violation(path="stream", reason="expected string")) + topic_value_raw = raw["topic"] + if not isinstance(topic_value_raw, str): + violations.append(Violation(path="topic", reason="expected string")) else: - stream_value = stream_value_raw + topic_value = topic_value_raw producer_id_value: str = typing.cast("typing.Any", None) if "producer_id" not in raw or raw["producer_id"] is None: @@ -87,6 +102,24 @@ def from_transfer_type( if attempt_value_parsed is not None: attempt_value = attempt_value_parsed + sequence_value: int = typing.cast("typing.Any", None) + if "sequence" not in raw or raw["sequence"] is None: + violations.append(Violation(path="sequence", reason="required")) + else: + sequence_value_raw = raw["sequence"] + sequence_value_parsed = _parse_spec_integer( + sequence_value_raw, "sequence", violations + ) + if sequence_value_parsed is not None: + sequence_value = sequence_value_parsed + if sequence_value < 0: + violations.append( + Violation( + path="sequence", + reason=f"must be >= 0, got {sequence_value}", + ) + ) + batch_index_value: int = typing.cast("typing.Any", None) if "batch_index" not in raw or raw["batch_index"] is None: violations.append(Violation(path="batch_index", reason="required")) @@ -105,19 +138,6 @@ def from_transfer_type( ) ) - topic_value: str | None = None - if "topic" in raw: - topic_value_raw = raw["topic"] - if topic_value_raw is None: - violations.append( - Violation(path="topic", reason="explicit null not allowed") - ) - else: - if not isinstance(topic_value_raw, str): - violations.append(Violation(path="topic", reason="expected string")) - else: - topic_value = topic_value_raw - payloads_value: list[bytes] | None = None if "payloads" in raw: payloads_value_raw = raw["payloads"] @@ -175,11 +195,12 @@ def from_transfer_type( for key in raw: if ( key != "workflow_id" - and key != "stream" + and key != "run_id" + and key != "topic" and key != "producer_id" and key != "attempt" + and key != "sequence" and key != "batch_index" - and key != "topic" and key != "payloads" and key != "finish" ): @@ -188,11 +209,12 @@ def from_transfer_type( raise temporalio.converter.create_payload_validation_error(violations) return AppendInput( workflow_id=workflow_id_value, - stream=stream_value, + run_id=run_id_value, + topic=topic_value, producer_id=producer_id_value, attempt=attempt_value, + sequence=sequence_value, batch_index=batch_index_value, - topic=topic_value, payloads=payloads_value, finish=finish_value, ) @@ -202,13 +224,24 @@ def to_transfer_type(self, value: "AppendInput") -> typing.Any: violations: list[Violation] = [] out: dict[str, typing.Any] = {} out["workflow_id"] = value.workflow_id - out["stream"] = value.stream + if value.run_id is not None: + out["run_id"] = value.run_id + out["topic"] = value.topic out["producer_id"] = value.producer_id if abs(value.attempt) > 9007199254740991: violations.append( Violation(path="attempt", reason="exceeds ±(2^53-1) integer cap") ) out["attempt"] = value.attempt + if abs(value.sequence) > 9007199254740991: + violations.append( + Violation(path="sequence", reason="exceeds ±(2^53-1) integer cap") + ) + if value.sequence < 0: + violations.append( + Violation(path="sequence", reason=f"must be >= 0, got {value.sequence}") + ) + out["sequence"] = value.sequence if abs(value.batch_index) > 9007199254740991: violations.append( Violation(path="batch_index", reason="exceeds ±(2^53-1) integer cap") @@ -220,8 +253,6 @@ def to_transfer_type(self, value: "AppendInput") -> typing.Any: ) ) out["batch_index"] = value.batch_index - if value.topic is not None: - out["topic"] = value.topic if value.payloads is not None: out["payloads"] = [_format_base64(element) for element in value.payloads] if value.finish is not None: @@ -239,31 +270,45 @@ class AppendInput: workflow_id: str """The workflow whose stream is being written.""" - stream: str - """The named stream to append to. Empty when the producer writes to a topic instead.""" + run_id: str | None = None + """The run to address. Empty addresses the workflow's execution chain, which is what a + producer normally wants. + """ + + topic: str + """The topic of the workflow's stream to append to.""" producer_id: str - """Identifies the writer across its retries, so its attempts can be ordered.""" + """Identifies the writer across its retries, so its attempts can be ordered. Never + empty here: the caller resolved it, from the activity context when it was not given. + """ attempt: int """The generation this producer is writing. A later attempt supersedes an earlier one.""" - batch_index: int - """Counts this producer's batches from 1. The handler drops an index it has already - written for this producer attempt, and rejects one that skips ahead or that resumes - an attempt it never saw start. + sequence: int + """The producer's sequence of the first record in this batch; the batch is numbered + from it and a finish takes the next number. It has to continue where the previous + batch ended, so the handler and the store agree on every record's position within + the attempt. """ - topic: str | None = None - """The topic to append to, when stream is empty.""" + batch_index: int + """Counts this producer's batches from 1. The handler answers a repeat of the last + index with the original's position, and refuses one that skips ahead, one already + behind the last, or one that resumes an attempt it never saw start. + """ payloads: list[bytes] | None = None - """The records to write, each a serialized Temporal Payload. Empty on a call that only - finishes. + """The record bodies to write, in order, each a serialized + temporal.api.common.v1.Payload. A payload codec configured on the caller has already + run on them. Empty on a call that only finishes. """ finish: bool | None = None - """Mark this producer done after writing the batch, so a reader stops waiting on it.""" + """Write FINISH for this producer after the batch: it will write nothing more on the + topic. Says nothing about the producer's outcome. + """ class _AppendOutputTransferTypeConverter( @@ -333,9 +378,9 @@ class AppendOutput: """Where the append landed, when the store can say.""" cursor: str | None = None - """Opaque token naming the last record written by this call. Absent when the call - carried no payloads, when the batch was a repeat the handler dropped, or when the - store learns positions only at read time. + """Opaque token naming the last record written by this call, or by the original when + the call repeated the last batch. Absent when the store learns positions only at + read time; such a caller positions itself with a latest_only read. """ additional_properties: dict[str, typing.Any] = dataclasses.field( @@ -369,33 +414,30 @@ def from_transfer_type( else: workflow_id_value = workflow_id_value_raw - stream_value: str | None = None - if "stream" in raw: - stream_value_raw = raw["stream"] - if stream_value_raw is None: + run_id_value: str | None = None + if "run_id" in raw: + run_id_value_raw = raw["run_id"] + if run_id_value_raw is None: violations.append( - Violation(path="stream", reason="explicit null not allowed") + Violation(path="run_id", reason="explicit null not allowed") ) else: - if not isinstance(stream_value_raw, str): + if not isinstance(run_id_value_raw, str): violations.append( - Violation(path="stream", reason="expected string") + Violation(path="run_id", reason="expected string") ) else: - stream_value = stream_value_raw + run_id_value = run_id_value_raw - topic_value: str | None = None - if "topic" in raw: + topic_value: str = typing.cast("typing.Any", None) + if "topic" not in raw or raw["topic"] is None: + violations.append(Violation(path="topic", reason="required")) + else: topic_value_raw = raw["topic"] - if topic_value_raw is None: - violations.append( - Violation(path="topic", reason="explicit null not allowed") - ) + if not isinstance(topic_value_raw, str): + violations.append(Violation(path="topic", reason="expected string")) else: - if not isinstance(topic_value_raw, str): - violations.append(Violation(path="topic", reason="expected string")) - else: - topic_value = topic_value_raw + topic_value = topic_value_raw after_token_value: str | None = None if "after_token" in raw: @@ -486,7 +528,7 @@ def from_transfer_type( for key in raw: if ( key != "workflow_id" - and key != "stream" + and key != "run_id" and key != "topic" and key != "after_token" and key != "max_records" @@ -498,7 +540,7 @@ def from_transfer_type( raise temporalio.converter.create_payload_validation_error(violations) return ReadInput( workflow_id=workflow_id_value, - stream=stream_value, + run_id=run_id_value, topic=topic_value, after_token=after_token_value, max_records=max_records_value, @@ -511,10 +553,9 @@ def to_transfer_type(self, value: "ReadInput") -> typing.Any: violations: list[Violation] = [] out: dict[str, typing.Any] = {} out["workflow_id"] = value.workflow_id - if value.stream is not None: - out["stream"] = value.stream - if value.topic is not None: - out["topic"] = value.topic + if value.run_id is not None: + out["run_id"] = value.run_id + out["topic"] = value.topic if value.after_token is not None: out["after_token"] = value.after_token if value.max_records is not None: @@ -572,19 +613,19 @@ class ReadInput: workflow_id: str """The workflow whose stream is being read.""" - stream: str | None = None - """The named stream to read. Empty reads every stream the workflow publishes.""" - - topic: str | None = None - """Yield only records on this topic. Empty yields every topic. Only meaningful when - stream is empty, because an inbound stream has no topics. + run_id: str | None = None + """Pin the read to this run. Empty follows the workflow's execution chain across + continue-as-new. """ + topic: str + """The topic of the workflow's stream to read.""" + after_token: str | None = None """Opaque cursor from an earlier record or append. The read resumes strictly after the record it names, so a caller never sees that record twice. Empty starts at the beginning. The token is produced by whichever store sits behind the endpoint, so a - caller cannot tell which one that is. + caller cannot tell which one that is, and a token from another store is refused. """ max_records: int | None = None @@ -663,6 +704,19 @@ def from_transfer_type( else: next_token_value = next_token_value_raw + done_value: bool | None = None + if "done" in raw: + done_value_raw = raw["done"] + if done_value_raw is None: + violations.append( + Violation(path="done", reason="explicit null not allowed") + ) + else: + if not isinstance(done_value_raw, bool): + violations.append(Violation(path="done", reason="expected boolean")) + else: + done_value = done_value_raw + additional_properties: dict[str, typing.Any] = {} for key in raw: if key not in _READ_OUTPUT_DECLARED: @@ -672,6 +726,7 @@ def from_transfer_type( return ReadOutput( records=records_value, next_token=next_token_value, + done=done_value, additional_properties=additional_properties, ) @@ -693,6 +748,8 @@ def to_transfer_type(self, value: "ReadOutput") -> typing.Any: out["records"] = records_out if value.next_token is not None: out["next_token"] = value.next_token + if value.done is not None: + out["done"] = value.done for key, entry in value.additional_properties.items(): if key in _READ_OUTPUT_DECLARED: violations.append( @@ -721,6 +778,12 @@ class ReadOutput: token when the call collected nothing. """ + done: bool | None = None + """The store ended the read: the owning execution, or its chain, is closed and every + retained record after the caller's token has been delivered. Nothing more will + arrive, so the caller stops. + """ + additional_properties: dict[str, typing.Any] = dataclasses.field( default_factory=dict ) @@ -750,17 +813,19 @@ def from_transfer_type( else: token_value = token_value_raw - frame_value: bytes = typing.cast("typing.Any", None) - if "frame" not in raw or raw["frame"] is None: - violations.append(Violation(path="frame", reason="required")) + record_value: bytes = typing.cast("typing.Any", None) + if "record" not in raw or raw["record"] is None: + violations.append(Violation(path="record", reason="required")) else: - frame_value_raw = raw["frame"] - if not isinstance(frame_value_raw, str): - violations.append(Violation(path="frame", reason="expected string")) + record_value_raw = raw["record"] + if not isinstance(record_value_raw, str): + violations.append(Violation(path="record", reason="expected string")) else: - frame_value_parsed = _parse_base64(frame_value_raw, "frame", violations) - if frame_value_parsed is not None: - frame_value = frame_value_parsed + record_value_parsed = _parse_base64( + record_value_raw, "record", violations + ) + if record_value_parsed is not None: + record_value = record_value_parsed additional_properties: dict[str, typing.Any] = {} for key in raw: @@ -770,7 +835,7 @@ def from_transfer_type( raise temporalio.converter.create_payload_validation_error(violations) return RecordWire( token=token_value, - frame=frame_value, + record=record_value, additional_properties=additional_properties, ) @@ -779,7 +844,7 @@ def to_transfer_type(self, value: "RecordWire") -> typing.Any: violations: list[Violation] = [] out: dict[str, typing.Any] = {} out["token"] = value.token - out["frame"] = _format_base64(value.frame) + out["record"] = _format_base64(value.record) for key, entry in value.additional_properties.items(): if key in _RECORD_WIRE_DECLARED: violations.append( @@ -798,13 +863,15 @@ def to_transfer_type(self, value: "RecordWire") -> typing.Any: @_transfer_type_convertible(_RecordWireTransferTypeConverter) @dataclasses.dataclass(slots=True, kw_only=True) class RecordWire: - """One record on the wire: its cursor and its frame.""" + """One record on the wire: its cursor and the record itself.""" token: str """Opaque cursor naming this record. Pass it as after_token to resume just past it.""" - frame: bytes - """The record frame, carrying the topic, kind, producer, attempt, sequence and body.""" + record: bytes + """The serialized temporal.api.stream.v1.StreamRecord: topic, kind, producer_id, + attempt, sequence and body, as the store holds it. + """ additional_properties: dict[str, typing.Any] = dataclasses.field( default_factory=dict diff --git a/temporalio/streams/providers/_nexus_generated/services.py b/temporalio/streams/providers/_nexus_generated/services.py index e0673aeab7..591e310466 100644 --- a/temporalio/streams/providers/_nexus_generated/services.py +++ b/temporalio/streams/providers/_nexus_generated/services.py @@ -17,18 +17,20 @@ class TemporalStreams: """The stream endpoint's two operations. One Temporal-authenticated endpoint hides the store behind it, so an operator switches storage without touching callers. Reads hand out batches and an append carries one batch per call, because a Nexus operation - per record costs too much for token streams. + per record costs too much for token streams. A record crosses as the serialized + temporal.api.stream.v1.StreamRecord, the same bytes every store keeps, so a caller + in any language decodes it with the api protos alone. """ append: Operation[ AppendInput, AppendOutput, ] = Operation(name="append") - """Append one batch on the caller's account. The handler drops a batch_index it has - already written for this producer attempt, so a retried call writes once, and - rejects an index that skips ahead or a producer attempt it has no state for. - Supersession records are not transported: a reader re-synthesizes them from the - attempts it observes. + """Append one batch on the caller's account. The handler writes a batch once: a repeat + of the last batch_index answers with where the original landed, and an index that + skips ahead, one already behind the last, a sequence that does not continue, or a + producer attempt the handler has no state for is refused. Supersession records are + not transported: a reader re-synthesizes them from the attempts it observes. """ read: Operation[ @@ -37,5 +39,6 @@ class TemporalStreams: ] = Operation(name="read") """Answer with the records after the caller's token, or time out. The call parks until it has max_records records or wait_ms elapses, whichever comes first, and returns - whatever it collected. + whatever it collected. It says when the store has ended the read, so the caller can + stop. """ diff --git a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml index a1cc346317..bf97209b35 100644 --- a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml +++ b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml @@ -11,15 +11,18 @@ services: hides the store behind it, so an operator switches storage without touching callers. Reads hand out batches and an append carries one batch per call, because a Nexus operation per record costs too much for token - streams. + streams. A record crosses as the serialized + temporal.api.stream.v1.StreamRecord, the same bytes every store keeps, so + a caller in any language decodes it with the api protos alone. operations: append: fqn: append description: >- - Append one batch on the caller's account. The handler drops a - batch_index it has already written for this producer attempt, so a - retried call writes once, and rejects an index that skips ahead or a - producer attempt it has no state for. Supersession records are not + Append one batch on the caller's account. The handler writes a batch + once: a repeat of the last batch_index answers with where the + original landed, and an index that skips ahead, one already behind + the last, a sequence that does not continue, or a producer attempt + the handler has no state for is refused. Supersession records are not transported: a reader re-synthesizes them from the attempts it observes. input: { $ref: "#/$defs/AppendInput" } @@ -29,7 +32,8 @@ services: description: >- Answer with the records after the caller's token, or time out. The call parks until it has max_records records or wait_ms elapses, - whichever comes first, and returns whatever it collected. + whichever comes first, and returns whatever it collected. It says + when the store has ended the read, so the caller can stop. input: { $ref: "#/$defs/ReadInput" } output: { $ref: "#/$defs/ReadOutput" } @@ -41,49 +45,61 @@ $defs: workflow_id: type: string description: "The workflow whose stream is being written." - stream: + run_id: type: string description: >- - The named stream to append to. Empty when the producer writes to a - topic instead. + The run to address. Empty addresses the workflow's execution chain, + which is what a producer normally wants. + topic: + type: string + description: "The topic of the workflow's stream to append to." producer_id: type: string description: >- Identifies the writer across its retries, so its attempts can be - ordered. + ordered. Never empty here: the caller resolved it, from the activity + context when it was not given. attempt: type: integer description: >- The generation this producer is writing. A later attempt supersedes an earlier one. + sequence: + type: integer + minimum: 0 + description: >- + The producer's sequence of the first record in this batch; the batch + is numbered from it and a finish takes the next number. It has to + continue where the previous batch ended, so the handler and the + store agree on every record's position within the attempt. batch_index: type: integer minimum: 1 description: >- - Counts this producer's batches from 1. The handler drops an index it - has already written for this producer attempt, and rejects one that - skips ahead or that resumes an attempt it never saw start. - topic: - type: string - description: "The topic to append to, when stream is empty." + Counts this producer's batches from 1. The handler answers a repeat + of the last index with the original's position, and refuses one that + skips ahead, one already behind the last, or one that resumes an + attempt it never saw start. payloads: type: array items: type: string contentEncoding: base64 description: >- - The records to write, each a serialized Temporal Payload. Empty on a - call that only finishes. + The record bodies to write, in order, each a serialized + temporal.api.common.v1.Payload. A payload codec configured on the + caller has already run on them. Empty on a call that only finishes. finish: type: boolean description: >- - Mark this producer done after writing the batch, so a reader stops - waiting on it. + Write FINISH for this producer after the batch: it will write nothing + more on the topic. Says nothing about the producer's outcome. required: - workflow_id - - stream + - topic - producer_id - attempt + - sequence - batch_index additionalProperties: false @@ -94,10 +110,10 @@ $defs: cursor: type: string description: >- - Opaque token naming the last record written by this call. Absent - when the call carried no payloads, when the batch was a repeat the - handler dropped, or when the store learns positions only at read - time. + Opaque token naming the last record written by this call, or by the + original when the call repeated the last batch. Absent when the + store learns positions only at read time; such a caller positions + itself with a latest_only read. additionalProperties: true ReadInput: @@ -107,17 +123,14 @@ $defs: workflow_id: type: string description: "The workflow whose stream is being read." - stream: + run_id: type: string description: >- - The named stream to read. Empty reads every stream the workflow - publishes. + Pin the read to this run. Empty follows the workflow's execution + chain across continue-as-new. topic: type: string - description: >- - Yield only records on this topic. Empty yields every topic. Only - meaningful when stream is empty, because an inbound stream has no - topics. + description: "The topic of the workflow's stream to read." after_token: type: string description: >- @@ -125,7 +138,7 @@ $defs: strictly after the record it names, so a caller never sees that record twice. Empty starts at the beginning. The token is produced by whichever store sits behind the endpoint, so a caller cannot tell - which one that is. + which one that is, and a token from another store is refused. max_records: type: integer minimum: 1 @@ -146,26 +159,27 @@ $defs: wants to follow from now. required: - workflow_id + - topic additionalProperties: false RecordWire: type: object - description: "One record on the wire: its cursor and its frame." + description: "One record on the wire: its cursor and the record itself." properties: token: type: string description: >- Opaque cursor naming this record. Pass it as after_token to resume just past it. - frame: + record: type: string contentEncoding: base64 description: >- - The record frame, carrying the topic, kind, producer, attempt, - sequence and body. + The serialized temporal.api.stream.v1.StreamRecord: topic, kind, + producer_id, attempt, sequence and body, as the store holds it. required: - token - - frame + - record additionalProperties: true ReadOutput: @@ -182,4 +196,10 @@ $defs: description: >- Opaque cursor to pass as after_token on the following call. Echoes the caller's own token when the call collected nothing. + done: + type: boolean + description: >- + The store ended the read: the owning execution, or its chain, is + closed and every retained record after the caller's token has been + delivered. Nothing more will arrive, so the caller stops. additionalProperties: true From 46ae99be4596532e159e29b5ced0ebf652154963 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 21 Sep 2026 02:15:21 -0700 Subject: [PATCH 15/34] Rewrote the Nexus front on the plugin surface. NexusStreams is the outside half only and the handler fronts a storage provider's own handles. Endpoint failures reach the caller as the stream condition the store raised or as an RPCError, never as an HTTP error. --- temporalio/streams/providers/nexus.py | 813 +++++++++++++++----------- 1 file changed, 468 insertions(+), 345 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 8c311d1d9a..07da0318b8 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -1,47 +1,58 @@ """The Nexus front: the outside surface behind one Temporal-authenticated endpoint. -Two halves in one module. The provider half implements only :func:`producer` -and :func:`consumer`; the workflow side raises, naming a storage provider, -because a workflow's publishes and reads ride the Workflow Task and cannot -cross an RPC. The handler half runs in a worker configured with any storage -provider and serves two sync operations, ``append`` and ``read``, so the -store behind the endpoint is invisible to callers and an operator switches -it without touching them. +Two halves in one module. :class:`NexusStreams` is a provider that implements +the outside half only: it hands out handles that append and read through the +endpoint, and it has no workflow half, because a workflow's publishes and +reads ride the Workflow Task and cannot cross an RPC. +:class:`TemporalStreamsHandler` runs in a worker next to any storage provider +and serves two sync operations, ``append`` and ``read``, by fronting that +provider's own handles, so the store behind the endpoint is invisible to +callers and an operator switches it without touching them. The wire types and the service definition come from ``temporal_streams.nexusrpc.yaml``, so any language nexgen targets can be -handed the same contract. What stays hand-written here is what the -generator cannot express yet: the handler's dedupe and long-poll collect -loop, and the caller's batching, cursor handling and codec. +handed the same contract. A record crosses as the serialized +``temporal.api.stream.v1.StreamRecord``, the same bytes every store keeps, so +a caller in another language decodes it with the api protos alone. What stays +hand-written here is what the generator cannot express yet: the handler's +dedupe and long-poll collect loop, and the caller's batching, cursor handling +and codec. Reads hand out batches and an append carries one batch per call, because a Nexus operation per record costs too much for token streams. Record bodies -cross the handler untouched: it reads the store as raw payloads and frames +cross the handler untouched: it reads the store as raw payloads and forwards them as they are, so a codec that changes the payload encoding survives the hop and the handler's worker never needs the key. Supersession records are -not transported: the caller's reader re-synthesizes them from the attempts -it observes, which is the policy module's job on every provider. Cursors -pass through opaque, so the caller cannot tell which store produced them. - -The handler keeps one parked read per ``(workflow, stream, topic)`` and -serves consecutive calls from it, so an idle caller does not leave one -abandoned long poll on the store per call. A call whose token does not -match the parked position replaces the subscription, and an idle one is -released after a minute; that is the residual cost. - -Two stated prototype limits. Append deduplication lives in handler memory, -by batch index per producer attempt, so a handler that has no state for a -producer attempt refuses to continue it rather than starting a fresh -delegate whose numbering the store would drop as a repeat; the caller opens -a new attempt, which readers report as a supersession. The store-level fix, -a sequence-explicit append in the contract, is listed in the design doc's -work table. And any caller the endpoint admits may touch any workflow's -streams in the namespace; the endpoint's own authorization is the boundary. +not transported: the caller's reader re-synthesizes them from the attempts it +observes, which is the policy module's job on every provider. Cursors pass +through opaque, so the caller cannot tell which store produced them; a +foreign one is refused by the store behind the endpoint and reaches the +caller on the first read. + +The handler keeps one parked read per ``(workflow, run, topic)`` and serves +consecutive calls from it, so an idle caller does not leave one abandoned +long poll on the store per call. A call whose token does not match the parked +position replaces the subscription, and an idle one is released after a +minute; that is the residual cost. + +Two stated prototype limits. Append deduplication lives in handler memory, by +batch index per producer attempt, so a handler that has no state for a +producer attempt refuses to continue it rather than starting a fresh delegate +whose numbering the store would drop as a repeat; the caller opens a new +attempt, which readers report as a supersession. And any caller the endpoint +admits may touch any workflow's streams in the namespace; the endpoint's own +authorization is the boundary. + +A failure from the endpoint reaches the caller as the +:class:`temporalio.streams.StreamError` the store raised, when the handler +named one, and as :class:`temporalio.service.RPCError` otherwise, never as an +HTTP or urllib exception. """ from __future__ import annotations import asyncio +import json import logging import time import urllib.error @@ -50,7 +61,7 @@ from collections.abc import AsyncGenerator, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Any, TypeVar +from typing import Any, NoReturn, TypeVar, cast import nexusrpc import nexusrpc.handler @@ -59,11 +70,24 @@ import temporalio.converter from temporalio.api.common.v1 import Payload from temporalio.api.operatorservice.v1 import ListNexusEndpointsRequest +from temporalio.client import Client from temporalio.common import RawValue -from temporalio.streams import _frame, _provider -from temporalio.streams._handles import ReadSource, WriteSink -from temporalio.streams._policy import AttemptTracker +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams._errors import ( + StreamCursorError, + StreamError, + StreamNotFoundError, + StreamProducerError, + StreamUnsupportedError, +) +from temporalio.streams._provider import StreamHandle, StreamProducer, StreamProvider from temporalio.streams._record import BEGINNING, Cursor, RecordKind, StreamRecord +from temporalio.streams._wire import ( + RecordDecoder, + WireRecord, + producer_identity, + to_wire, +) from temporalio.streams.providers._nexus_generated import ( AppendInput, AppendOutput, @@ -73,9 +97,16 @@ TemporalStreams, ) +__all__ = [ + "NexusProducer", + "NexusStreamHandle", + "NexusStreams", + "TemporalStreamsHandler", +] + _WORKFLOW_SIDE_ERROR = ( "the nexus provider is an outside transport; a worker publishes and reads " - "through a storage provider, so configure one of those on the worker" + "through a storage provider, so give the worker one of those" ) # The contract leaves these unset, so the handler is the one place that says @@ -83,12 +114,13 @@ # shortens it to the request deadline anyway, and a shorter default only # means more round trips on an idle stream. _DEFAULT_MAX_RECORDS = 100 -_DEFAULT_WAIT_MS = 30_000 +_DEFAULT_READ_WAIT = timedelta(seconds=30) _DEADLINE_MARGIN = timedelta(milliseconds=500) _DEFAULT_MAX_PRODUCERS = 10_000 _DEFAULT_SUBSCRIPTION_IDLE = timedelta(seconds=60) _QUEUE_DEPTH = 1000 -_APPEND_TIMEOUT_MS = 30_000 +_APPEND_TIMEOUT = timedelta(seconds=30) +_ROUND_TRIP_MARGIN = timedelta(seconds=5) _definition = nexusrpc.get_service_definition(TemporalStreams) if _definition is None: # pragma: no cover @@ -104,16 +136,58 @@ _APPEND_OPERATION = _OPERATION_NAMES["append"] _READ_OPERATION = _OPERATION_NAMES["read"] +# The stream conditions that cross the endpoint under their own name, so the +# caller raises the class the store raised. +_STREAM_ERRORS: dict[str, type[StreamError]] = { + cls.__name__: cls + for cls in ( + StreamError, + StreamNotFoundError, + StreamCursorError, + StreamProducerError, + StreamUnsupportedError, + ) +} +_HTTP_TO_RPC = { + 400: RPCStatusCode.INVALID_ARGUMENT, + 401: RPCStatusCode.UNAUTHENTICATED, + 403: RPCStatusCode.PERMISSION_DENIED, + 404: RPCStatusCode.NOT_FOUND, + 408: RPCStatusCode.DEADLINE_EXCEEDED, + 409: RPCStatusCode.ALREADY_EXISTS, + 412: RPCStatusCode.FAILED_PRECONDITION, + 429: RPCStatusCode.RESOURCE_EXHAUSTED, + 500: RPCStatusCode.INTERNAL, + 501: RPCStatusCode.UNIMPLEMENTED, + 503: RPCStatusCode.UNAVAILABLE, + 504: RPCStatusCode.DEADLINE_EXCEEDED, +} + _OutputT = TypeVar("_OutputT") logger = logging.getLogger(__name__) +def _require_topic(topic: str) -> None: + if not topic: + raise ValueError("topic must not be empty") + + +def _handler_error(error: StreamError) -> nexusrpc.HandlerError: + """Carry a stream condition across the endpoint under its own class name.""" + kind = ( + nexusrpc.HandlerErrorType.NOT_FOUND + if isinstance(error, StreamNotFoundError) + else nexusrpc.HandlerErrorType.BAD_REQUEST + ) + return nexusrpc.HandlerError( + f"{type(error).__name__}: {error}", type=kind, retryable_override=False + ) + + def _bad_request(message: str) -> nexusrpc.HandlerError: return nexusrpc.HandlerError( - message, - type=nexusrpc.HandlerErrorType.BAD_REQUEST, - retryable_override=False, + message, type=nexusrpc.HandlerErrorType.BAD_REQUEST, retryable_override=False ) @@ -131,29 +205,30 @@ class _Subscription: @dataclass class _ProducerState: - delegate: Any + delegate: StreamProducer batch_index: int + sequence: int + last_cursor: str | None @nexusrpc.handler.service_handler(service=TemporalStreams) class TemporalStreamsHandler: - """Serves the stream endpoint, delegating to one storage provider. + """Serves the stream endpoint by fronting one storage provider's handles. - The delegate is an explicit instance rather than the process default, so - a caller and a handler can coexist in one process, and so an operator can + The provider is an explicit instance the application constructed, so a + caller and a handler can coexist in one process, and so an operator can run handlers for a new store next to handlers for the old one. """ def __init__( self, - client: Any, - provider: str, + provider: StreamProvider, + client: Client | None, *, max_producers: int = _DEFAULT_MAX_PRODUCERS, subscription_idle: timedelta = _DEFAULT_SUBSCRIPTION_IDLE, - **provider_options: Any, ) -> None: - """Serve the endpoint out of the provider named by ``provider``. + """Serve the endpoint out of ``provider``'s handles, opened with ``client``. ``max_producers`` bounds the dedupe state kept per producer attempt; the oldest is evicted, and a producer evicted mid-life gets a clear @@ -161,9 +236,9 @@ def __init__( ``subscription_idle`` is how long a parked read outlives its last caller. """ + self._provider = provider self._client = client - self._delegate = _provider.instance(provider, **provider_options) - self._producers: OrderedDict[tuple[str, str, str, int], _ProducerState] = ( + self._producers: OrderedDict[tuple[str, str, str, str, int], _ProducerState] = ( OrderedDict() ) self._max_producers = max_producers @@ -171,60 +246,82 @@ def __init__( self._read_locks: dict[tuple[str, str, str], asyncio.Lock] = {} self._subscription_idle = subscription_idle.total_seconds() + def _stream(self, workflow_id: str, run_id: str | None) -> StreamHandle: + # A storage provider wants a client; the memory provider, which the + # tests front, accepts none, so the cast only lies where it is unread. + return self._provider.get_stream_handle( + cast(Client, self._client), workflow_id, run_id=run_id or None + ) + @nexusrpc.handler.sync_operation async def append( self, _ctx: nexusrpc.handler.StartOperationContext, input: AppendInput ) -> AppendOutput: - """Append on the caller's account, dropping a repeated batch. + """Append on the caller's account, writing a repeated batch once. Raises: - nexusrpc.HandlerError: ``BAD_REQUEST`` when the batch index skips - ahead, when it continues a producer attempt this handler has - no state for, or when a payload is not a serialized - ``Payload``. + nexusrpc.HandlerError: ``BAD_REQUEST`` when the batch index or + sequence does not continue the producer attempt, when the + attempt is one this handler has no state for, or when a + payload is not a serialized ``Payload``; ``NOT_FOUND`` when + the store has no such workflow. """ - topic = input.topic or "" + try: + return await self._append(input) + except StreamError as error: + raise _handler_error(error) from error + except ValueError as error: + raise _bad_request(str(error)) from error + + async def _append(self, input: AppendInput) -> AppendOutput: + _require_topic(input.topic) key = ( input.workflow_id, - input.stream or f"@{topic}", + input.run_id or "", + input.topic, input.producer_id, input.attempt, ) state = self._producers.get(key) if state is None: - if input.batch_index != 1: + if input.batch_index != 1 or input.sequence != 0: # A fresh delegate would restart its numbering at zero, and # the store would drop that as a repeat of the first batch. # Refusing here turns a silent loss into a failure the caller # can act on by opening a new attempt. - raise _bad_request( + raise StreamProducerError( f"producer {input.producer_id!r} attempt {input.attempt} resumed " f"at batch {input.batch_index} on a handler that has no state for " "it; open a new attempt" ) - delegate = await self._delegate.producer( - self._client, - workflow_id=input.workflow_id, - stream=input.stream, - topic=topic, - producer_id=input.producer_id, - attempt=input.attempt, + delegate = self._stream(input.workflow_id, input.run_id).producer( + topic=input.topic, producer_id=input.producer_id, attempt=input.attempt ) - state = _ProducerState(delegate, 0) - elif input.batch_index <= state.batch_index: - # A repeat, or a producer that restarted its numbering: either - # way the store already holds the batch. + state = _ProducerState(delegate, 0, 0, None) + elif input.batch_index == state.batch_index: + # The last batch again: the store already holds it, and where it + # landed is the answer the original got. self._producers.move_to_end(key) - return AppendOutput() + return AppendOutput(cursor=state.last_cursor) + elif input.batch_index < state.batch_index: + raise StreamProducerError( + f"batch {input.batch_index} was written and is no longer the last " + f"for producer {input.producer_id!r} attempt {input.attempt}" + ) elif input.batch_index != state.batch_index + 1: - raise _bad_request( + raise StreamProducerError( f"batch {input.batch_index} skips ahead of {state.batch_index} for " f"producer {input.producer_id!r} attempt {input.attempt}" ) + if input.sequence != state.sequence: + raise StreamProducerError( + f"sequence {input.sequence} does not continue at {state.sequence} for " + f"producer {input.producer_id!r} attempt {input.attempt}" + ) payloads = self._payloads(input.payloads) cursor: str | None = None if payloads: - appended = await state.delegate.append(*payloads) + appended = await state.delegate.append(*(RawValue(p) for p in payloads)) cursor = appended.token if appended is not None else None if input.finish: await state.delegate.finish() @@ -233,6 +330,8 @@ async def append( # Recorded only once the store accepted the batch, so a failed append # is not mistaken for a repeat when the caller retries it. state.batch_index = input.batch_index + state.sequence += len(payloads) + state.last_cursor = cursor self._producers[key] = state self._producers.move_to_end(key) while len(self._producers) > self._max_producers: @@ -243,32 +342,46 @@ async def append( def _payloads(encoded: list[bytes] | None) -> list[Payload]: payloads = [] for index, raw in enumerate(encoded or []): - payload = Payload() try: - payload.ParseFromString(raw) + payloads.append(Payload.FromString(raw)) except DecodeError as error: raise _bad_request( f"payloads[{index}] is not a serialized Temporal Payload: {error}" ) from error - payloads.append(payload) return payloads @nexusrpc.handler.sync_operation async def read( self, ctx: nexusrpc.handler.StartOperationContext, input: ReadInput ) -> ReadOutput: - """Answer with the records after the caller's token, or time out.""" - stream = input.stream or "" - topic = input.topic or None + """Answer with the records after the caller's token, or time out. + + Raises: + nexusrpc.HandlerError: ``BAD_REQUEST`` when the store refuses the + token, ``NOT_FOUND`` when it has no such workflow. + """ + try: + return await self._read(ctx, input) + except StreamError as error: + raise _handler_error(error) from error + except ValueError as error: + raise _bad_request(str(error)) from error + + async def _read( + self, ctx: nexusrpc.handler.StartOperationContext, input: ReadInput + ) -> ReadOutput: + _require_topic(input.topic) + stream = self._stream(input.workflow_id, input.run_id) if input.latest_only: - delegate = await self._delegate.consumer( - self._client, workflow_id=input.workflow_id, stream=stream - ) - return ReadOutput(next_token=(await delegate.latest(topic=topic)).token) + return ReadOutput(next_token=(await stream.latest(topic=input.topic)).token) max_records = ( _DEFAULT_MAX_RECORDS if input.max_records is None else input.max_records ) - wait = (_DEFAULT_WAIT_MS if input.wait_ms is None else input.wait_ms) / 1000 + wait = ( + _DEFAULT_READ_WAIT.total_seconds() + if input.wait_ms is None + else input.wait_ms / 1000 + ) if ctx.request_deadline is not None: # The server answers the caller with a timeout at the deadline # whatever this call does, so collecting past it only holds the @@ -279,7 +392,7 @@ async def read( left = deadline - datetime.now(timezone.utc) - _DEADLINE_MARGIN wait = max(0.0, min(wait, left.total_seconds())) after = input.after_token or "" - key = (input.workflow_id, stream, topic or "") + key = (input.workflow_id, input.run_id or "", input.topic) await self._expire_subscriptions() lock = self._read_locks.setdefault(key, asyncio.Lock()) async with lock: @@ -287,34 +400,31 @@ async def read( if subscription is None or subscription.position != after: if subscription is not None: await self._drop(key) - subscription = await self._subscribe( - key, input.workflow_id, stream, topic, after - ) + subscription = self._subscribe(key, stream, input.topic, after) records, next_token = await self._drain(subscription, max_records, wait) subscription.position = next_token subscription.last_used = time.monotonic() - if subscription.done and subscription.records.empty(): - # The store ended the read, as the Workflow Streams delegate - # does when the run closes. Nothing more will arrive on it. + done = ( + subscription.done + and subscription.records.empty() + and subscription.failure is None + ) + if done: + # The store ended the read: the run or chain is closed and the + # tail has been handed over. Nothing more will arrive on it. await self._drop(key) - return ReadOutput(records=records, next_token=next_token) + return ReadOutput(records=records, next_token=next_token, done=done) - async def _subscribe( - self, - key: tuple[str, str, str], - workflow_id: str, - stream: str, - topic: str | None, - after: str, + def _subscribe( + self, key: tuple[str, str, str], stream: StreamHandle, topic: str, after: str ) -> _Subscription: - delegate = await self._delegate.consumer( - self._client, workflow_id=workflow_id, stream=stream - ) - # Raw payloads: the handler frames what the store holds without + # Raw payloads: the handler forwards what the store holds without # decoding it, so an encoding only the caller's codec understands - # passes through untouched. - source = delegate.read( - after=Cursor(after) if after else BEGINNING, topic=topic, type=RawValue + # passes through untouched. A foreign token is refused right here. + source = stream.read( + topic=topic, + after=Cursor(after) if after else BEGINNING, + result_type=RawValue, ) subscription = _Subscription( records=asyncio.Queue(maxsize=_QUEUE_DEPTH), @@ -334,8 +444,7 @@ async def pump() -> None: subscription.failure = error finally: subscription.done = True - if isinstance(source, AsyncGenerator): - await source.aclose() + await source.aclose() subscription.pump = asyncio.create_task(pump()) self._subscriptions[key] = subscription @@ -354,8 +463,10 @@ async def _drain( if records or remaining <= 0: break if subscription.done: + if subscription.failure is None: + break # Honour the wait even though nothing can arrive, so a - # caller polling a closed stream does not spin on the + # caller polling a failed stream does not spin on the # endpoint. await asyncio.sleep(remaining) break @@ -374,20 +485,19 @@ async def _drain( @staticmethod def _wire(record: StreamRecord[Any]) -> RecordWire: - body = ( - record.value.payload.SerializeToString() - if isinstance(record.value, RawValue) - else b"" - ) - frame = _frame.encode( + # The record read as RawValue carries the stored body untouched; the + # default converter passes a RawValue through, so no encoding happens + # here. + wire = to_wire( + temporalio.converter.DataConverter.default.payload_converter, topic=record.topic, kind=record.kind, - producer=record.producer, + value=record.value, + producer_id=record.producer_id, attempt=record.attempt, sequence=record.sequence, - body=body, ) - return RecordWire(token=record.cursor.token, frame=frame) + return RecordWire(token=record.cursor.token, record=wire.SerializeToString()) async def close(self) -> None: """Release every parked read. @@ -419,20 +529,44 @@ async def _expire_subscriptions(self) -> None: self._read_locks.pop(key, None) -class StreamEndpointError(RuntimeError): - """The stream endpoint refused a call or could not be reached. +class _EndpointFailure(Exception): + """What the transport saw: the endpoint's answer, or the reason it gave none.""" - ``status`` is the HTTP status when the endpoint answered, and ``None`` - when the connection itself failed. - """ - - def __init__(self, message: str, *, status: int | None = None) -> None: - """Carry the endpoint's answer, or the reason it gave none.""" - super().__init__(message) + def __init__(self, detail: str, status: int | None) -> None: + super().__init__(detail) + self.detail = detail self.status = status -def _post(url: str, body: bytes, headers: Mapping[str, str], timeout_ms: int) -> bytes: +def _translate(failure: _EndpointFailure) -> Exception: + """The exception a caller raises for an endpoint failure. + + The handler names a stream condition by its class in the failure message, + so the same class is raised here; anything else is an ``RPCError`` with + the status the HTTP answer maps to, or ``UNAVAILABLE`` when there was none. + """ + message = failure.detail + try: + parsed = json.loads(message) + if isinstance(parsed, dict) and isinstance(parsed.get("message"), str): + message = parsed["message"] + except ValueError: + pass + if failure.status is not None: + for name, cls in _STREAM_ERRORS.items(): + prefix = f"{name}: " + if message.startswith(prefix): + return cls(message[len(prefix) :]) + status = _HTTP_TO_RPC.get(failure.status, RPCStatusCode.UNKNOWN) + else: + status = RPCStatusCode.UNAVAILABLE + return RPCError(message, status, b"") + + +def _post( + url: str, body: bytes, headers: Mapping[str, str], timeout: timedelta +) -> bytes: + timeout_ms = int(timeout.total_seconds() * 1000) request = urllib.request.Request( url, data=body, @@ -446,57 +580,74 @@ def _post(url: str, body: bytes, headers: Mapping[str, str], timeout_ms: int) -> method="POST", ) try: - with urllib.request.urlopen(request, timeout=timeout_ms / 1000 + 5) as response: + with urllib.request.urlopen( + request, timeout=(timeout + _ROUND_TRIP_MARGIN).total_seconds() + ) as response: return response.read() except urllib.error.HTTPError as error: - detail = error.read().decode(errors="replace") - raise StreamEndpointError( - f"stream endpoint call failed ({error.code}): {detail}", status=error.code + raise _EndpointFailure( + error.read().decode(errors="replace"), error.code ) from error except urllib.error.URLError as error: - raise StreamEndpointError( - f"stream endpoint unreachable at {url}: {error.reason}" + raise _EndpointFailure( + f"stream endpoint unreachable at {url}: {error.reason}", None ) from error class _Front: - """Everything the caller half needs to reach one endpoint.""" + """Everything a handle needs to reach one endpoint.""" - def __init__( - self, - base_url: str, - data_converter: temporalio.converter.DataConverter, - headers: Mapping[str, str], - wait_ms: int, - max_records: int, - ) -> None: - self.base_url = base_url + def __init__(self, streams: NexusStreams, client: Client | None) -> None: + self._streams = streams + self._client = client + data_converter = streams._data_converter + if data_converter is None: + data_converter = ( + client.data_converter + if client is not None + else temporalio.converter.DataConverter.default + ) self.converter = data_converter.payload_converter self.codec = data_converter.payload_codec - self.headers = dict(headers) - self.wait_ms = wait_ms - self.max_records = max_records + self.read_wait = streams._read_wait + self.max_records = streams._max_records + + async def _base_url(self) -> str: + endpoint_id = await self._streams._endpoint_id(self._client) + return ( + f"{self._streams._http_address}/nexus/endpoints/{endpoint_id}" + f"/services/{_SERVICE_NAME}" + ) async def invoke( - self, operation: str, request: Any, output: type[_OutputT], *, timeout_ms: int + self, + operation: str, + request: Any, + output: type[_OutputT], + *, + timeout: timedelta, ) -> _OutputT: # The contract types carry their own JSON encoding, so the raw caller # and the worker serving the operation agree on the body without # either of them spelling the fields out. contract = temporalio.converter.DataConverter.default.payload_converter - raw = await asyncio.to_thread( - _post, - f"{self.base_url}/{operation}", - contract.to_payloads([request])[0].data, - self.headers, - timeout_ms, - ) + url = f"{await self._base_url()}/{operation}" + try: + raw = await asyncio.to_thread( + _post, + url, + contract.to_payloads([request])[0].data, + self._streams._headers, + timeout, + ) + except _EndpointFailure as failure: + raise _translate(failure) from failure payload = Payload(metadata={"encoding": b"json/plain"}, data=raw or b"{}") return contract.from_payloads([payload], [output])[0] class NexusProducer: - """Appends through the stream endpoint; framing happens behind it. + """Appends through the stream endpoint; the store behind it does the rest. Calls are serialized and the batch index is committed only once the endpoint answered. A batch whose call raised stays pending and is sent @@ -509,42 +660,50 @@ def __init__( self, front: _Front, workflow_id: str, - stream: str, + run_id: str | None, topic: str, producer_id: str, attempt: int, ) -> None: - """Bind this producer to one stream or topic on the endpoint.""" + """Bind this producer to ``topic`` on the workflow behind the endpoint.""" self._front = front self._workflow_id = workflow_id - self._stream = stream + self._run_id = run_id self._topic = topic self._producer_id = producer_id self._attempt = attempt self._batch_index = 0 + self._sequence = 0 + self._last: Cursor | None = BEGINNING self._pending: tuple[AppendInput, tuple[bytes, ...], bool] | None = None self._lock = asyncio.Lock() + @property + def producer_id(self) -> str: + """Who this producer writes as.""" + return self._producer_id + @property def attempt(self) -> int: """The generation this producer is writing.""" return self._attempt async def append(self, *values: Any) -> Cursor | None: - """Append ``values`` through the endpoint.""" + """Append ``values`` through the endpoint and return where the last one landed. + + A repeat answers with the original's position and an empty call with + the last one; ``None`` when the store behind the endpoint learns + positions only at read time. + """ if not values: - return None - payloads = [ - value - if isinstance(value, Payload) - else self._front.converter.to_payloads([value])[0] - for value in values - ] + return self._last + payloads = self._front.converter.to_payloads(list(values)) answer = await self._call(payloads, finish=False) - return Cursor(answer.cursor) if answer.cursor else None + self._last = Cursor(answer.cursor) if answer.cursor else None + return self._last async def finish(self) -> None: - """Mark this producer done, so a reader stops waiting on it.""" + """Write ``FINISH`` for this producer on this topic.""" await self._call([], finish=True) async def _call(self, payloads: list[Payload], *, finish: bool) -> AppendOutput: @@ -566,238 +725,202 @@ async def _call(self, payloads: list[Payload], *, finish: bool) -> AppendOutput: await self._send(*self._pending) elif self._pending is not None: return await self._send(*self._pending) - request = self._request(self._batch_index + 1, encoded, finish) + request = AppendInput( + workflow_id=self._workflow_id, + run_id=self._run_id, + topic=self._topic, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence, + batch_index=self._batch_index + 1, + payloads=encoded, + finish=finish, + ) return await self._send(request, identity, finish) - def _request( - self, batch_index: int, payloads: list[bytes], finish: bool - ) -> AppendInput: - return AppendInput( - workflow_id=self._workflow_id, - stream=self._stream, - producer_id=self._producer_id, - attempt=self._attempt, - batch_index=batch_index, - topic=self._topic, - payloads=payloads, - finish=finish, - ) - async def _send( self, request: AppendInput, identity: tuple[bytes, ...], finish: bool ) -> AppendOutput: self._pending = (request, identity, finish) answer = await self._front.invoke( - _APPEND_OPERATION, request, AppendOutput, timeout_ms=_APPEND_TIMEOUT_MS + _APPEND_OPERATION, request, AppendOutput, timeout=_APPEND_TIMEOUT ) self._batch_index = request.batch_index + self._sequence = request.sequence + len(request.payloads or []) + int(finish) self._pending = None return answer -class NexusConsumer: - """Reads batches from the stream endpoint, re-synthesizing supersession.""" +class NexusStreamHandle: + """One workflow's stream through the endpoint, re-synthesizing supersession.""" - def __init__(self, front: _Front, workflow_id: str, stream: str) -> None: - """Read what ``workflow_id`` publishes, through the endpoint.""" + def __init__(self, front: _Front, workflow_id: str, run_id: str | None) -> None: + """Address ``workflow_id``'s stream, pinned to ``run_id`` when one is given.""" self._front = front self._workflow_id = workflow_id - self._stream = stream + self._run_id = run_id - async def read( + def read( self, *, + topic: str, after: Cursor = BEGINNING, - topic: str | None = None, - type: type | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield records on ``topic`` after ``after``, one endpoint batch at a time. + + The token is opaque here, so a cursor from another store is refused by + the store behind the endpoint and raises + :class:`temporalio.streams.StreamCursorError` on the first iteration. + """ + _require_topic(topic) + return self._read(topic, after, result_type) + + async def _read( + self, topic: str, after: Cursor, result_type: type | None ) -> AsyncGenerator[StreamRecord[Any], None]: - """Yield records after ``after``, one endpoint batch at a time.""" - _provider.check_topic(self._stream, topic) - attempts = AttemptTracker() + decoder = RecordDecoder( + self._front.converter, result_type, after=after, warn=logger.warning + ) token = after.token while True: answer = await self._front.invoke( _READ_OPERATION, ReadInput( workflow_id=self._workflow_id, - stream=self._stream, - topic=topic or "", + run_id=self._run_id, + topic=topic, after_token=token, max_records=self._front.max_records, - wait_ms=self._front.wait_ms, + wait_ms=int(self._front.read_wait.total_seconds() * 1000), ), ReadOutput, - timeout_ms=self._front.wait_ms + 5000, + timeout=self._front.read_wait + _ROUND_TRIP_MARGIN, ) for wire in answer.records or []: cursor = Cursor(wire.token) try: - kind, frame_topic, source, attempt, sequence, body = _frame.decode( - wire.frame - ) - except ValueError as error: + record = WireRecord.FromString(wire.record) + except DecodeError as error: # Same answer as every other reader: skip and say so. logger.warning("skipping stream record at %s: %s", cursor, error) continue - if topic is not None and frame_topic != topic: - continue - superseded = attempts.note(source, attempt, cursor) - if superseded is not None: - yield superseded - value = ( - await self._decode(body, type) if kind is RecordKind.DATA else None - ) - yield StreamRecord( - value=value, - cursor=cursor, - kind=kind, - topic=frame_topic, - producer=source, - attempt=attempt, - sequence=sequence, - ) + if self._front.codec is not None and record.HasField("body"): + record.body.CopyFrom( + (await self._front.codec.decode([record.body]))[0] + ) + for out in decoder.decode(cursor, record): + yield out token = answer.next_token or token + if answer.done: + return - async def latest(self, *, topic: str | None = None) -> Cursor: - """The cursor of the last record written, for following from now.""" + async def latest(self, *, topic: str) -> Cursor: + """The newest position on ``topic`` behind the endpoint, for following from now.""" + _require_topic(topic) answer = await self._front.invoke( _READ_OPERATION, ReadInput( workflow_id=self._workflow_id, - stream=self._stream, - topic=topic or "", + run_id=self._run_id, + topic=topic, latest_only=True, ), ReadOutput, - timeout_ms=_APPEND_TIMEOUT_MS, + timeout=_APPEND_TIMEOUT, ) token = answer.next_token or "" return Cursor(token) if token else BEGINNING - async def _decode(self, body: bytes, as_type: type | None) -> Any: - payload = Payload() - payload.ParseFromString(body) - if self._front.codec is not None: - payload = (await self._front.codec.decode([payload]))[0] - if as_type is None: - return self._front.converter.from_payloads([payload])[0] - return self._front.converter.from_payloads([payload], [as_type])[0] - - -class _NexusProvider: - name = "nexus" - - def __init__(self) -> None: - self._endpoint: str | None = None - self._endpoint_id: str | None = None - self._http_address = "http://127.0.0.1:7243" - self._service = _SERVICE_NAME - self._data_converter: temporalio.converter.DataConverter | None = None - self._client: Any = None - self._headers: dict[str, str] = {} - self._wait_ms = _DEFAULT_WAIT_MS - self._max_records = _DEFAULT_MAX_RECORDS - - def configure(self, **options: Any) -> None: - """Point the caller half at one endpoint. - - ``endpoint`` is the endpoint's name when ``client`` is given, and is - resolved to its id through the operator service on first use; without - a client it has to be the id, because the HTTP ingress dispatches by - id. ``headers`` go on every request, which is where an authorization - header belongs. ``wait_ms`` and ``max_records`` are the read bounds - the caller asks the handler for. - """ - endpoint = options.pop("endpoint", None) - self._http_address = options.pop("http_address", self._http_address) - self._service = options.pop("service", _SERVICE_NAME) - self._data_converter = options.pop("data_converter", None) - self._client = options.pop("client", None) - self._headers = dict(options.pop("headers", None) or {}) - self._wait_ms = int(options.pop("wait_ms", _DEFAULT_WAIT_MS)) - self._max_records = int(options.pop("max_records", _DEFAULT_MAX_RECORDS)) - if options: - raise TypeError( - "the nexus provider takes endpoint, http_address, service, " - f"data_converter, client, headers, wait_ms and max_records, got " - f"{sorted(options)}" - ) - if not endpoint: - raise TypeError( - "the nexus provider needs endpoint= together with " - "client=, or endpoint= on its own" - ) - self._endpoint = endpoint - self._endpoint_id = None - - def worker_options(self) -> dict[str, Any]: - raise RuntimeError(_WORKFLOW_SIDE_ERROR) - - def open_read( - self, - stream: str, - *, - after: Cursor = BEGINNING, - idle_timeout: timedelta | None = None, - ) -> ReadSource: - del stream, after, idle_timeout - raise RuntimeError(_WORKFLOW_SIDE_ERROR) - - def open_write(self, topic: str) -> WriteSink: - del topic - raise RuntimeError(_WORKFLOW_SIDE_ERROR) - - async def _front(self, client: Any) -> _Front: - if self._endpoint is None: - raise RuntimeError("configure the nexus provider before opening handles") - if self._endpoint_id is None: - resolver = self._client if self._client is not None else client - if resolver is None: - self._endpoint_id = self._endpoint - else: - found = await resolver.operator_service.list_nexus_endpoints( - ListNexusEndpointsRequest(name=self._endpoint) - ) - if not found.endpoints: - raise RuntimeError(f"no nexus endpoint is named {self._endpoint!r}") - self._endpoint_id = found.endpoints[0].id - base_url = ( - f"{self._http_address}/nexus/endpoints/{self._endpoint_id}" - f"/services/{self._service}" - ) - return _Front( - base_url, - self._converter(client), - self._headers, - self._wait_ms, - self._max_records, - ) - - async def producer( - self, - client: Any, - *, - workflow_id: str, - stream: str = "", - topic: str = "", - producer_id: str = "", - attempt: int = 0, + def producer( + self, *, topic: str, producer_id: str = "", attempt: int = 0 ) -> NexusProducer: + """A producer on ``topic``; inside an activity its identity is the activity's.""" + _require_topic(topic) + producer_id, attempt = producer_identity(producer_id, attempt) return NexusProducer( - await self._front(client), workflow_id, stream, topic, producer_id, attempt + self._front, self._workflow_id, self._run_id, topic, producer_id, attempt ) - async def consumer( - self, client: Any, *, workflow_id: str, stream: str = "" - ) -> NexusConsumer: - return NexusConsumer(await self._front(client), workflow_id, stream) - def _converter(self, client: Any) -> temporalio.converter.DataConverter: - if self._data_converter is not None: - return self._data_converter - if client is None: - return temporalio.converter.DataConverter.default - return client.data_converter +class NexusStreams(StreamProvider): + """The outside half of a provider, over one Nexus endpoint. + Not a worker plugin: a workflow publishes and reads through the storage + provider its worker was given, and this front only serves code outside a + workflow. The endpoint hides which store that is. + """ -_provider.register("nexus", _NexusProvider) + def __init__( + self, + *, + endpoint: str, + http_address: str = "http://127.0.0.1:7243", + headers: Mapping[str, str] | None = None, + read_wait: timedelta = _DEFAULT_READ_WAIT, + max_records: int = _DEFAULT_MAX_RECORDS, + data_converter: temporalio.converter.DataConverter | None = None, + ) -> None: + """Point the front at one endpoint. + + Args: + endpoint: The Nexus endpoint's name, resolved to its id through + the operator service of the client a handle is opened with. + When a handle is opened without a client it has to be the id, + because the HTTP ingress dispatches by id. + http_address: The server's Nexus HTTP ingress. + headers: Sent on every request; where an authorization header + belongs. + read_wait: How long the handler may park a read before answering + with what it has. + max_records: The most records one read answer carries. + data_converter: Overrides the client's converter for record + bodies. A payload codec configured here runs on this side of + the endpoint, so records are encoded before they leave the + process and the handler's worker never holds the key. + """ + if not endpoint: + raise ValueError("endpoint must not be empty") + self._endpoint = endpoint + self._http_address = http_address.rstrip("/") + self._headers = dict(headers or {}) + self._read_wait = read_wait + self._max_records = max_records + self._data_converter = data_converter + self._resolved: str | None = None + self._resolving = asyncio.Lock() + + def workflow_provider(self) -> NoReturn: + """Raise: this front has no workflow half.""" + raise StreamUnsupportedError(_WORKFLOW_SIDE_ERROR) + + def get_stream_handle( + self, client: Client | None, workflow_id: str, *, run_id: str | None = None + ) -> NexusStreamHandle: + """A handle on ``workflow_id``'s stream through the endpoint. + + ``client`` is not used for transport: it resolves the endpoint's name + and supplies the data converter when none was configured. + """ + return NexusStreamHandle(_Front(self, client), workflow_id, run_id) + + async def close(self) -> None: + """Nothing to release: each call opens and closes its own connection.""" + + async def _endpoint_id(self, client: Client | None) -> str: + if self._resolved is not None: + return self._resolved + async with self._resolving: + if self._resolved is None: + if client is None: + self._resolved = self._endpoint + else: + found = await client.operator_service.list_nexus_endpoints( + ListNexusEndpointsRequest(name=self._endpoint) + ) + if not found.endpoints: + raise ValueError( + f"no nexus endpoint is named {self._endpoint!r}" + ) + self._resolved = found.endpoints[0].id + return self._resolved From 95be1ae140f34d7d062ba0ae3d9ddbdcc194aba0 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 21 Sep 2026 02:15:21 -0700 Subject: [PATCH 16/34] Covered the Nexus front on the new surface. --- CHANGELOG.md | 15 +- tests/streams/test_nexus_provider.py | 311 +++++++++++++++++---------- 2 files changed, 204 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ad3811ebe..605626802e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,12 +43,15 @@ to include examples, links to docs, or any other relevant information. stream interface over the shipped Workflow Streams transport, so a workflow reads and publishes through `temporalio.contrib.workflow_streams` without naming it. -- **Experimental**: `temporalio.streams.providers.nexus` puts one Nexus - endpoint in front of a storage provider, so a caller reaches a stream - through the endpoint and never names the store. Its contract is defined in - `temporal_streams.nexusrpc.yaml` and the bindings are generated from it. - Configure it with `data_converter=` to run a payload codec on the caller - side, so records are encoded before they leave the process. +- **Experimental**: `temporalio.streams.providers.nexus.NexusStreams` puts one + Nexus endpoint in front of a storage provider, so a caller reaches a stream + through the endpoint and never names the store, and + `TemporalStreamsHandler` serves that endpoint by fronting the provider's own + handles. Its contract is defined in `temporal_streams.nexusrpc.yaml` and the + bindings are generated from it; a record crosses as the serialized + `StreamRecord` proto. Configure the front with `data_converter=` to run a + payload codec on the caller side, so records are encoded before they leave + the process. ### Changed diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 77cec6f4eb..ecb0e7cf31 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -1,7 +1,7 @@ """Conformance for the Nexus front. -The caller talks only to the stream endpoint; the handler delegates to a -storage provider, so these tests are the provider-hiding demonstration: +The caller talks only to the stream endpoint; the handler fronts a storage +provider's own handles, so these tests are the provider-hiding demonstration: nothing on the caller side names or could name the store. The loop test runs over the server's Nexus HTTP ingress and is gated behind @@ -32,20 +32,27 @@ import pytest import temporalio.converter -from temporalio import streams from temporalio.api.common.v1 import Payload from temporalio.client import Client -from temporalio.streams import RecordKind -from temporalio.streams import _frame as frame -from temporalio.streams._provider import instance -from temporalio.streams.providers import memory, nexus -from temporalio.streams.providers._nexus_generated import AppendInput, ReadInput -from temporalio.streams.providers.nexus import ( - StreamEndpointError, - TemporalStreamsHandler, +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import ( + BEGINNING, + Cursor, + RecordKind, + StreamCursorError, + StreamProducerError, + StreamUnsupportedError, ) +from temporalio.streams._wire import WireRecord +from temporalio.streams.providers import nexus +from temporalio.streams.providers._nexus_generated import AppendInput, ReadInput +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.streams.providers.nexus import NexusStreams, TemporalStreamsHandler +from temporalio.streams.providers.workflow_streams import WorkflowStreamsProvider from temporalio.worker import Worker +from tests.helpers import new_worker from tests.streams.test_workflow_streams_provider import EchoLoop, take +from tests.streams.test_workflow_streams_provider import _feed as feed_echo_loop live_only = pytest.mark.skipif( os.environ.get("STREAMS_LIVE") != "nexus", @@ -56,6 +63,8 @@ HANDLER_TQ = "streams-handlers-e2e" APPEND_OPERATION = nexus._APPEND_OPERATION # pyright: ignore[reportPrivateUsage] READ_OPERATION = nexus._READ_OPERATION # pyright: ignore[reportPrivateUsage] +INPUTS = "inputs" +DECISIONS = "decisions" @live_only @@ -63,42 +72,34 @@ async def test_interface_loop_through_the_nexus_front(): # The workflow worker and the handler worker both use the storage # provider; only the caller goes through the front, and it names the # endpoint the way an operator does, by name. - streams.configure(provider="workflow_streams") + store = WorkflowStreamsProvider() client = await Client.connect(os.environ.get("TEMPORAL_ADDRESS", "localhost:7233")) - front = instance( - "nexus", + front = NexusStreams( endpoint=ENDPOINT, - client=client, http_address=os.environ.get("TEMPORAL_HTTP", "http://127.0.0.1:7243"), - wait_ms=5000, + read_wait=timedelta(seconds=5), ) workflow_id = f"streams-nexus-live-{uuid.uuid4().hex}" - handler = TemporalStreamsHandler(client, provider="workflow_streams") + handler = TemporalStreamsHandler(store, client) async with Worker(client, task_queue=HANDLER_TQ, nexus_service_handlers=[handler]): async with Worker( client, task_queue=f"tq-{workflow_id}", workflows=[EchoLoop], - **streams.worker_options(), + plugins=[store], ): handle = await client.start_workflow( EchoLoop.run, id=workflow_id, task_queue=f"tq-{workflow_id}" ) - producer = await front.producer( - None, - workflow_id=workflow_id, - stream="inputs", - producer_id="model", - attempt=1, - ) + stream = front.get_stream_handle(client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) await producer.append({"n": 1}, {"n": 2}) await producer.append({"n": 3}) await producer.finish() - consumer = await front.consumer(None, workflow_id=workflow_id) - records = await take(consumer.read(type=dict), 4, timeout=60) + records = await take(stream.read(topic=DECISIONS, result_type=dict), 4, 60) assert [r.kind for r in records] == [ RecordKind.DATA, RecordKind.DATA, @@ -110,8 +111,9 @@ async def test_interface_loop_through_the_nexus_front(): # An opaque cursor from behind the front resumes a fresh reader # just past the record it names. checkpoint = records[0].cursor - resumed = await front.consumer(None, workflow_id=workflow_id) - again = await take(resumed.read(type=dict, after=checkpoint), 2, timeout=60) + again = await take( + stream.read(topic=DECISIONS, result_type=dict, after=checkpoint), 2, 60 + ) assert [r.value["echo"] for r in again[:2]] == [2, 3] await handle.signal(EchoLoop.release) @@ -136,12 +138,7 @@ async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: """Recover the payloads :meth:`encode` wrapped.""" - out: list[Payload] = [] - for payload in payloads: - inner = Payload() - inner.ParseFromString(self._flip(payload.data)) - out.append(inner) - return out + return [Payload.FromString(self._flip(payload.data)) for payload in payloads] @classmethod def _flip(cls, data: bytes) -> bytes: @@ -202,9 +199,9 @@ def _in_process_endpoint( failing = set(fail_after_applying or []) def post( - url: str, body: bytes, headers: Mapping[str, str], timeout_ms: int + url: str, body: bytes, headers: Mapping[str, str], timeout: timedelta ) -> bytes: - del headers, timeout_ms + del headers, timeout posted.append(body) operation = url.rsplit("/", 1)[1] request_type = AppendInput if operation == APPEND_OPERATION else ReadInput @@ -216,24 +213,30 @@ def post( _dispatch(handler, operation, request), loop ).result() except nexusrpc.HandlerError as error: - raise StreamEndpointError(str(error), status=400) from error + # The ingress answers a handler error with a failure body and the + # status its type maps to. + status = 404 if error.type is nexusrpc.HandlerErrorType.NOT_FOUND else 400 + raise nexus._EndpointFailure( # pyright: ignore[reportPrivateUsage] + json.dumps({"message": str(error)}), status + ) from error raw = contract.to_payloads([answer])[0].data answered.append(raw) if operation in failing: failing.discard(operation) - raise StreamEndpointError("connection reset after the handler answered") + raise nexus._EndpointFailure( # pyright: ignore[reportPrivateUsage] + "connection reset after the handler answered", None + ) return raw return post -def _front(codec: temporalio.converter.PayloadCodec | None) -> Any: - return instance( - "nexus", +def _front(codec: temporalio.converter.PayloadCodec | None) -> NexusStreams: + return NexusStreams( endpoint="in-process", # This endpoint has no other traffic to wait for, so park briefly # rather than for the contract's default. - wait_ms=200, + read_wait=timedelta(milliseconds=200), data_converter=dataclasses.replace( temporalio.converter.DataConverter.default, payload_codec=codec ), @@ -244,28 +247,19 @@ async def _loop_through_an_endpoint( monkeypatch: pytest.MonkeyPatch, codec: temporalio.converter.PayloadCodec | None, ) -> tuple[list[Any], list[bytes], list[bytes]]: - memory.reset() - streams.configure(provider="memory") + store = MemoryStreams() posted: list[bytes] = [] answered: list[bytes] = [] - handler = TemporalStreamsHandler(None, provider="memory") + handler = TemporalStreamsHandler(store, None) monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, posted, answered)) - front = _front(codec) - workflow_id = "wf-codec" + stream = _front(codec).get_stream_handle(None, "wf-codec") - producer = await front.producer( - None, - workflow_id=workflow_id, - stream="inputs", - producer_id="model", - attempt=1, - ) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) await producer.append({"secret": "tuna"}) await producer.finish() - consumer = await front.consumer(None, workflow_id=workflow_id, stream="inputs") - records = await take(consumer.read(type=dict), 2, timeout=30) - memory.reset() + records = await take(stream.read(topic=INPUTS, result_type=dict), 2, timeout=30) + await handler.close() return records, posted, answered @@ -281,7 +275,8 @@ def _answered_bodies(answered: list[bytes]) -> bytes: out = b"" for body in answered: for record in json.loads(body).get("records", []): - out += frame.decode(base64.b64decode(record["frame"]))[5] + wire = WireRecord.FromString(base64.b64decode(record["record"])) + out += wire.body.SerializeToString() return out @@ -315,14 +310,21 @@ async def test_without_a_codec_the_same_records_go_out_in_the_clear( def _append( - workflow_id: str, batch_index: int, *values: Any, finish: bool = False + workflow_id: str, + batch_index: int, + *values: Any, + sequence: int | None = None, + finish: bool = False, ) -> AppendInput: converter = temporalio.converter.DataConverter.default.payload_converter return AppendInput( workflow_id=workflow_id, - stream="inputs", + topic=INPUTS, producer_id="model", attempt=1, + # Batches of one, numbered in step with the batch index unless a + # case wants them out of step. + sequence=batch_index - 1 if sequence is None else sequence, batch_index=batch_index, payloads=[ converter.to_payloads([value])[0].SerializeToString() for value in values @@ -331,57 +333,62 @@ def _append( ) -async def _stored(workflow_id: str) -> list[Any]: - consumer = await streams.consumer(None, workflow_id=workflow_id, stream="inputs") +async def _stored(store: MemoryStreams, workflow_id: str) -> list[Any]: + stream = store.get_stream_handle(None, workflow_id) + end = await stream.latest(topic=INPUTS) + if end == BEGINNING: + return [] out: list[Any] = [] - async for record in consumer.read(type=dict): + async for record in stream.read(topic=INPUTS, result_type=dict): if record.kind is RecordKind.DATA: out.append(record.value) - if await consumer.latest() == record.cursor: + if record.cursor == end: break return out -@pytest.fixture -def _memory_store(): # pyright: ignore[reportUnusedFunction] - memory.reset() - streams.configure(provider="memory") - yield - memory.reset() - - -@pytest.mark.usefixtures("_memory_store") async def test_a_handler_without_state_for_a_producer_refuses_to_continue_it(): # A second handler instance stands in for a restarted or load-balanced # handler worker: it shares the store but not the dedupe state. - first = TemporalStreamsHandler(None, provider="memory") - second = TemporalStreamsHandler(None, provider="memory") + store = MemoryStreams() + first = TemporalStreamsHandler(store, None) + second = TemporalStreamsHandler(store, None) await _dispatch(first, APPEND_OPERATION, _append("wf", 1, {"n": 1})) with pytest.raises(nexusrpc.HandlerError) as failed: await _dispatch(second, APPEND_OPERATION, _append("wf", 2, {"n": 2})) assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST assert failed.value.retryable_override is False + # Named so the caller can raise the same class. + assert str(failed.value).startswith("StreamProducerError: ") # The store holds the first batch and nothing was silently lost or doubled. - assert await _stored("wf") == [{"n": 1}] + assert await _stored(store, "wf") == [{"n": 1}] -@pytest.mark.usefixtures("_memory_store") -async def test_the_handler_drops_repeats_and_rejects_gaps(): - handler = TemporalStreamsHandler(None, provider="memory") +async def test_the_handler_answers_repeats_and_rejects_gaps(): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})) - await _dispatch(handler, APPEND_OPERATION, _append("wf", 2, {"n": 2})) - # An exact repeat is dropped with a success answer and no position. + second = await _dispatch(handler, APPEND_OPERATION, _append("wf", 2, {"n": 2})) + # A repeat of the last batch is written once and answers with where the + # original landed, so a retrying caller checkpoints the same position. repeat = await _dispatch(handler, APPEND_OPERATION, _append("wf", 2, {"n": 2})) - assert repeat.cursor is None - with pytest.raises(nexusrpc.HandlerError) as failed: + assert repeat.cursor == second.cursor + with pytest.raises(nexusrpc.HandlerError) as skipped: await _dispatch(handler, APPEND_OPERATION, _append("wf", 4, {"n": 4})) - assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST - assert await _stored("wf") == [{"n": 1}, {"n": 2}] + assert skipped.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + with pytest.raises(nexusrpc.HandlerError) as behind: + await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})) + assert str(behind.value).startswith("StreamProducerError: ") + with pytest.raises(nexusrpc.HandlerError) as renumbered: + await _dispatch( + handler, APPEND_OPERATION, _append("wf", 3, {"n": 3}, sequence=7) + ) + assert "sequence 7 does not continue at 2" in str(renumbered.value) + assert await _stored(store, "wf") == [{"n": 1}, {"n": 2}] -@pytest.mark.usefixtures("_memory_store") async def test_a_malformed_payload_is_the_callers_fault(): - handler = TemporalStreamsHandler(None, provider="memory") + handler = TemporalStreamsHandler(MemoryStreams(), None) request = _append("wf", 1) request.payloads = [b"\xff\xfe not a payload"] with pytest.raises(nexusrpc.HandlerError) as failed: @@ -389,12 +396,12 @@ async def test_a_malformed_payload_is_the_callers_fault(): assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST -@pytest.mark.usefixtures("_memory_store") async def test_the_caller_retries_an_ambiguous_append_under_the_same_index( monkeypatch: pytest.MonkeyPatch, ): posted: list[bytes] = [] - handler = TemporalStreamsHandler(None, provider="memory") + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) monkeypatch.setattr( nexus, "_post", @@ -402,28 +409,41 @@ async def test_the_caller_retries_an_ambiguous_append_under_the_same_index( handler, posted, [], fail_after_applying=[APPEND_OPERATION] ), ) - front = _front(None) - producer = await front.producer( - None, workflow_id="wf", stream="inputs", producer_id="model", attempt=1 - ) - with pytest.raises(StreamEndpointError): + stream = _front(None).get_stream_handle(None, "wf") + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + # A transport failure is an RPCError, never the transport's own type. + with pytest.raises(RPCError) as failed: await producer.append({"n": 1}) - await producer.append({"n": 1}) + assert failed.value.status == RPCStatusCode.UNAVAILABLE + landed = await producer.append({"n": 1}) await producer.append({"n": 2}) await producer.finish() indexes = [json.loads(body)["batch_index"] for body in posted] - # The retry re-sent index 1, which the handler dropped as a repeat, so - # the store holds each record once. + sequences = [json.loads(body)["sequence"] for body in posted] + # The retry re-sent index 1, which the handler answered as a repeat with + # the original's position, so the store holds each record once. assert indexes == [1, 1, 2, 3] - assert await _stored("wf") == [{"n": 1}, {"n": 2}] + assert sequences == [0, 0, 1, 2] + assert landed == await _memory_position(store, "wf", 0) + assert await _stored(store, "wf") == [{"n": 1}, {"n": 2}] + + +async def _memory_position( + store: MemoryStreams, workflow_id: str, index: int +) -> Cursor: + records = await take( + store.get_stream_handle(None, workflow_id).read(topic=INPUTS, result_type=dict), + index + 1, + ) + return records[index].cursor -@pytest.mark.usefixtures("_memory_store") async def test_a_failed_append_goes_out_before_the_next_one( monkeypatch: pytest.MonkeyPatch, ): posted: list[bytes] = [] - handler = TemporalStreamsHandler(None, provider="memory") + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) monkeypatch.setattr( nexus, "_post", @@ -431,28 +451,52 @@ async def test_a_failed_append_goes_out_before_the_next_one( handler, posted, [], fail_after_applying=[APPEND_OPERATION] ), ) - front = _front(None) - producer = await front.producer( - None, workflow_id="wf", stream="inputs", producer_id="model", attempt=1 - ) - with pytest.raises(StreamEndpointError): + stream = _front(None).get_stream_handle(None, "wf") + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + with pytest.raises(RPCError): await producer.append({"n": 1}) # The caller moves on without retrying; the pending batch is replayed # first under its own index and the new one takes the next. await producer.append({"n": 2}) indexes = [json.loads(body)["batch_index"] for body in posted] assert indexes == [1, 1, 2] - assert await _stored("wf") == [{"n": 1}, {"n": 2}] + assert await _stored(store, "wf") == [{"n": 1}, {"n": 2}] + + +async def test_a_stream_condition_crosses_the_endpoint_under_its_own_class( + monkeypatch: pytest.MonkeyPatch, +): + handler = TemporalStreamsHandler(MemoryStreams(), None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + stream = _front(None).get_stream_handle(None, "wf") + # The token is opaque on this side, so the store behind the endpoint is + # what refuses it, and the refusal arrives on the first read as the class + # the store raised. + with pytest.raises(StreamCursorError): + await take(stream.read(topic=INPUTS, after=Cursor("elsewhere:1")), 1) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}) + # A second handler has no state for the attempt; the caller learns that + # as a producer conflict. + fresh = TemporalStreamsHandler(MemoryStreams(), None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(fresh, [], [])) + with pytest.raises(StreamProducerError): + await producer.append({"n": 2}) + await handler.close() + + +def test_the_front_has_no_workflow_half(): + with pytest.raises(StreamUnsupportedError): + _front(None).workflow_provider() -@pytest.mark.usefixtures("_memory_store") async def test_consecutive_reads_share_one_parked_subscription(): - handler = TemporalStreamsHandler(None, provider="memory") + handler = TemporalStreamsHandler(MemoryStreams(), None) await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1}, {"n": 2})) first = await _dispatch( handler, READ_OPERATION, - ReadInput(workflow_id="wf", stream="inputs", max_records=1, wait_ms=200), + ReadInput(workflow_id="wf", topic=INPUTS, max_records=1, wait_ms=200), ) assert len(first.records) == 1 parked = handler._subscriptions # pyright: ignore[reportPrivateUsage] @@ -464,7 +508,7 @@ async def test_consecutive_reads_share_one_parked_subscription(): READ_OPERATION, ReadInput( workflow_id="wf", - stream="inputs", + topic=INPUTS, after_token=first.next_token, max_records=1, wait_ms=200, @@ -477,21 +521,56 @@ async def test_consecutive_reads_share_one_parked_subscription(): await _dispatch( handler, READ_OPERATION, - ReadInput(workflow_id="wf", stream="inputs", wait_ms=200), + ReadInput(workflow_id="wf", topic=INPUTS, wait_ms=200), ) assert len(parked) == 1 and next(iter(parked.values())).pump is not pump await handler.close() assert not parked -@pytest.mark.usefixtures("_memory_store") async def test_the_read_wait_is_cut_to_the_request_deadline(): - handler = TemporalStreamsHandler(None, provider="memory") + handler = TemporalStreamsHandler(MemoryStreams(), None) started = asyncio.get_running_loop().time() answer = await handler.read( _context(READ_OPERATION, datetime.now(timezone.utc) + timedelta(seconds=1)), - ReadInput(workflow_id="wf", stream="inputs", wait_ms=60000), + ReadInput(workflow_id="wf", topic=INPUTS, wait_ms=60000), ) - assert answer.records == [] and answer.next_token == "" + assert answer.records == [] and answer.next_token == "" and not answer.done assert asyncio.get_running_loop().time() - started < 5 await handler.close() + + +async def test_the_read_ends_when_the_store_ends_it( + client: Client, monkeypatch: pytest.MonkeyPatch +): + # Behind the endpoint sits the Workflow Streams store, whose read ends + # once the run has closed and its tail is served. The front carries that + # end to the caller, whose loop leaves by itself. + store = WorkflowStreamsProvider(poll_cooldown=timedelta(milliseconds=20)) + handler = TemporalStreamsHandler(store, client) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + workflow_id = f"streams-nexus-{uuid.uuid4().hex}" + async with new_worker(client, EchoLoop, plugins=[store]) as worker: + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + await feed_echo_loop(store, client, workflow_id) + await handle.signal(EchoLoop.release) + assert await handle.result() == 3 + + stream = _front(None).get_stream_handle(None, workflow_id) + + async def read_everything() -> list[Any]: + return [ + (r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + + records = await asyncio.wait_for(read_everything(), 30) + await handler.close() + assert records == [ + (RecordKind.DATA, {"echo": 1}), + (RecordKind.DATA, {"echo": 2}), + (RecordKind.DATA, {"echo": 3}), + (RecordKind.FINISH, None), + ] From 8996d414062540519d52614633e3f4aaa5f7d6fd Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 21 Sep 2026 10:32:58 -0700 Subject: [PATCH 17/34] Let the Nexus front register on a client like any provider. --- temporalio/streams/providers/nexus.py | 31 +++++++++++++++++++++------ tests/streams/test_nexus_provider.py | 15 ++++++++++++- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 07da0318b8..372f50785b 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -58,7 +58,7 @@ import urllib.error import urllib.request from collections import OrderedDict -from collections.abc import AsyncGenerator, Mapping +from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Any, NoReturn, TypeVar, cast @@ -67,12 +67,13 @@ import nexusrpc.handler from google.protobuf.message import DecodeError +import temporalio.client import temporalio.converter from temporalio.api.common.v1 import Payload from temporalio.api.operatorservice.v1 import ListNexusEndpointsRequest -from temporalio.client import Client +from temporalio.client import Client, ClientConfig from temporalio.common import RawValue -from temporalio.service import RPCError, RPCStatusCode +from temporalio.service import ConnectConfig, RPCError, RPCStatusCode, ServiceClient from temporalio.streams._errors import ( StreamCursorError, StreamError, @@ -843,12 +844,15 @@ def producer( ) -class NexusStreams(StreamProvider): +class NexusStreams(StreamProvider, temporalio.client.Plugin): """The outside half of a provider, over one Nexus endpoint. - Not a worker plugin: a workflow publishes and reads through the storage - provider its worker was given, and this front only serves code outside a - workflow. The endpoint hides which store that is. + A client plugin but not a worker plugin: a workflow publishes and reads + through the storage provider its worker was given, and this front only + serves code outside a workflow, so ``Client.connect(plugins=[front])`` + makes ``client.get_stream_handle()`` go through the endpoint while a + worker built from that client is left without a provider. The endpoint + hides which store sits behind it. """ def __init__( @@ -907,6 +911,19 @@ def get_stream_handle( async def close(self) -> None: """Nothing to release: each call opens and closes its own connection.""" + def configure_client(self, config: ClientConfig) -> ClientConfig: + """Set this front as the client's ``stream_provider``.""" + config["stream_provider"] = self + return config + + async def connect_service_client( + self, + config: ConnectConfig, + next: Callable[[ConnectConfig], Awaitable[ServiceClient]], + ) -> ServiceClient: + """Connect unchanged.""" + return await next(config) + async def _endpoint_id(self, client: Client | None) -> str: if self._resolved is not None: return self._resolved diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index ecb0e7cf31..b09b8d2e24 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -47,7 +47,11 @@ from temporalio.streams.providers import nexus from temporalio.streams.providers._nexus_generated import AppendInput, ReadInput from temporalio.streams.providers.memory import MemoryStreams -from temporalio.streams.providers.nexus import NexusStreams, TemporalStreamsHandler +from temporalio.streams.providers.nexus import ( + NexusStreamHandle, + NexusStreams, + TemporalStreamsHandler, +) from temporalio.streams.providers.workflow_streams import WorkflowStreamsProvider from temporalio.worker import Worker from tests.helpers import new_worker @@ -490,6 +494,15 @@ def test_the_front_has_no_workflow_half(): _front(None).workflow_provider() +async def test_the_front_registers_on_a_client(client: Client): + # The same accessor as any provider, so code outside a workflow does not + # change when the store moves behind an endpoint. + config = client.config() + config["plugins"] = [_front(None)] + registered = Client(**config) + assert isinstance(registered.get_stream_handle("wf"), NexusStreamHandle) + + async def test_consecutive_reads_share_one_parked_subscription(): handler = TemporalStreamsHandler(MemoryStreams(), None) await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1}, {"n": 2})) From 40348b9a9107b086e8c27ed3bfaff3bb3bc5e0ab Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 21 Sep 2026 10:55:54 -0700 Subject: [PATCH 18/34] Took topic definitions on the Nexus front's handle. --- temporalio/streams/providers/nexus.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 372f50785b..09e00cf438 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -61,7 +61,7 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Any, NoReturn, TypeVar, cast +from typing import Any, Generic, NoReturn, TypeVar, cast import nexusrpc import nexusrpc.handler @@ -83,6 +83,7 @@ ) from temporalio.streams._provider import StreamHandle, StreamProducer, StreamProvider from temporalio.streams._record import BEGINNING, Cursor, RecordKind, StreamRecord +from temporalio.streams._topic import StreamTopic, resolve_topic from temporalio.streams._wire import ( RecordDecoder, WireRecord, @@ -105,6 +106,8 @@ "TemporalStreamsHandler", ] +T = TypeVar("T") + _WORKFLOW_SIDE_ERROR = ( "the nexus provider is an outside transport; a worker publishes and reads " "through a storage provider, so give the worker one of those" @@ -647,7 +650,7 @@ async def invoke( return contract.from_payloads([payload], [output])[0] -class NexusProducer: +class NexusProducer(Generic[T]): """Appends through the stream endpoint; the store behind it does the rest. Calls are serialized and the batch index is committed only once the @@ -689,7 +692,7 @@ def attempt(self) -> int: """The generation this producer is writing.""" return self._attempt - async def append(self, *values: Any) -> Cursor | None: + async def append(self, *values: T) -> Cursor | None: """Append ``values`` through the endpoint and return where the last one landed. A repeat answers with the original's position and an empty call with @@ -764,7 +767,7 @@ def __init__(self, front: _Front, workflow_id: str, run_id: str | None) -> None: def read( self, *, - topic: str, + topic: str | StreamTopic[Any], after: Cursor = BEGINNING, result_type: type | None = None, ) -> AsyncGenerator[StreamRecord[Any], None]: @@ -774,7 +777,7 @@ def read( the store behind the endpoint and raises :class:`temporalio.streams.StreamCursorError` on the first iteration. """ - _require_topic(topic) + topic, result_type = resolve_topic(topic, result_type) return self._read(topic, after, result_type) async def _read( @@ -816,9 +819,9 @@ async def _read( if answer.done: return - async def latest(self, *, topic: str) -> Cursor: + async def latest(self, *, topic: str | StreamTopic[Any]) -> Cursor: """The newest position on ``topic`` behind the endpoint, for following from now.""" - _require_topic(topic) + topic, _ = resolve_topic(topic) answer = await self._front.invoke( _READ_OPERATION, ReadInput( @@ -834,10 +837,14 @@ async def latest(self, *, topic: str) -> Cursor: return Cursor(token) if token else BEGINNING def producer( - self, *, topic: str, producer_id: str = "", attempt: int = 0 - ) -> NexusProducer: + self, + *, + topic: str | StreamTopic[Any], + producer_id: str = "", + attempt: int = 0, + ) -> NexusProducer[Any]: """A producer on ``topic``; inside an activity its identity is the activity's.""" - _require_topic(topic) + topic, _ = resolve_topic(topic) producer_id, attempt = producer_identity(producer_id, attempt) return NexusProducer( self._front, self._workflow_id, self._run_id, topic, producer_id, attempt From 2d7f0c9fc776352e24205ceecb40d32d93537ed6 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 25 Sep 2026 14:19:15 -0700 Subject: [PATCH 19/34] Made the stream handler hold one batch and one read at a time. An append's repeat check and its commit have an await between them, so two copies of one batch both reached the store. A failed read left its subscription in place, where the caller's retry was answered end of stream, and a finish took its state away so the retry had nowhere to land. --- temporalio/streams/providers/nexus.py | 58 ++++++++-- tests/streams/test_nexus_provider.py | 153 +++++++++++++++++++++++++- 2 files changed, 199 insertions(+), 12 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 09e00cf438..7d17aa5225 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -57,6 +57,7 @@ import time import urllib.error import urllib.request +import weakref from collections import OrderedDict from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping from dataclasses import dataclass @@ -125,7 +126,6 @@ _QUEUE_DEPTH = 1000 _APPEND_TIMEOUT = timedelta(seconds=30) _ROUND_TRIP_MARGIN = timedelta(seconds=5) - _definition = nexusrpc.get_service_definition(TemporalStreams) if _definition is None: # pragma: no cover raise RuntimeError("the generated stream service carries no nexus definition") @@ -213,6 +213,7 @@ class _ProducerState: batch_index: int sequence: int last_cursor: str | None + finished: bool = False @nexusrpc.handler.service_handler(service=TemporalStreams) @@ -247,7 +248,15 @@ def __init__( ) self._max_producers = max_producers self._subscriptions: dict[tuple[str, str, str], _Subscription] = {} - self._read_locks: dict[tuple[str, str, str], asyncio.Lock] = {} + # Held weakly, and by the call that is using one for as long as it + # runs. A strong map would keep an entry per address ever read or + # appended to, and nothing would ever reach it again. + self._read_locks: weakref.WeakValueDictionary[ + tuple[str, str, str], asyncio.Lock + ] = weakref.WeakValueDictionary() + self._append_locks: weakref.WeakValueDictionary[ + tuple[str, str, str, str, int], asyncio.Lock + ] = weakref.WeakValueDictionary() self._subscription_idle = subscription_idle.total_seconds() def _stream(self, workflow_id: str, run_id: str | None) -> StreamHandle: @@ -286,6 +295,17 @@ async def _append(self, input: AppendInput) -> AppendOutput: input.producer_id, input.attempt, ) + # The repeat check and the commit have an await between them, so two + # in-flight copies of one batch would both pass the check and both + # reach the store. The read path serialises per address for the same + # reason. + lock = self._append_locks.setdefault(key, asyncio.Lock()) + async with lock: + return await self._append_locked(input, key) + + async def _append_locked( + self, input: AppendInput, key: tuple[str, str, str, str, int] + ) -> AppendOutput: state = self._producers.get(key) if state is None: if input.batch_index != 1 or input.sequence != 0: @@ -317,6 +337,11 @@ async def _append(self, input: AppendInput) -> AppendOutput: f"batch {input.batch_index} skips ahead of {state.batch_index} for " f"producer {input.producer_id!r} attempt {input.attempt}" ) + elif state.finished: + raise StreamProducerError( + f"producer {input.producer_id!r} attempt {input.attempt} already " + "finished this topic; open a new attempt" + ) if input.sequence != state.sequence: raise StreamProducerError( f"sequence {input.sequence} does not continue at {state.sequence} for " @@ -329,13 +354,15 @@ async def _append(self, input: AppendInput) -> AppendOutput: cursor = appended.token if appended is not None else None if input.finish: await state.delegate.finish() - self._producers.pop(key, None) - return AppendOutput(cursor=cursor) # Recorded only once the store accepted the batch, so a failed append - # is not mistaken for a repeat when the caller retries it. + # is not mistaken for a repeat when the caller retries it. A finish is + # recorded the same way rather than dropping the state: a lost + # response would otherwise leave the caller with a batch it can + # neither repeat nor abandon. state.batch_index = input.batch_index state.sequence += len(payloads) state.last_cursor = cursor + state.finished = state.finished or bool(input.finish) self._producers[key] = state self._producers.move_to_end(key) while len(self._producers) > self._max_producers: @@ -405,7 +432,14 @@ async def _read( if subscription is not None: await self._drop(key) subscription = self._subscribe(key, stream, input.topic, after) - records, next_token = await self._drain(subscription, max_records, wait) + try: + records, next_token = await self._drain(subscription, max_records, wait) + except Exception: + # The pump failed. Keeping the subscription would answer the + # caller's retry from the same token with end of stream, so a + # transient read failure would read as the stream ending. + await self._drop(key) + raise subscription.position = next_token subscription.last_used = time.monotonic() done = ( @@ -483,8 +517,10 @@ async def _drain( records.append(self._wire(record)) next_token = record.cursor.token if not records and subscription.failure is not None: - failure, subscription.failure = subscription.failure, None - raise failure + # Left on the subscription rather than cleared: the caller is + # answered with the failure and the subscription is dropped, so + # there is nothing for a second reader of it to be misled by. + raise subscription.failure return records, next_token @staticmethod @@ -511,7 +547,6 @@ async def close(self) -> None: """ for key in list(self._subscriptions): await self._drop(key) - self._read_locks.clear() async def _drop(self, key: tuple[str, str, str]) -> None: subscription = self._subscriptions.pop(key, None) @@ -530,7 +565,6 @@ async def _expire_subscriptions(self) -> None: if lock is not None and lock.locked(): continue await self._drop(key) - self._read_locks.pop(key, None) class _EndpointFailure(Exception): @@ -892,6 +926,10 @@ def __init__( """ if not endpoint: raise ValueError("endpoint must not be empty") + # Checked here rather than on the first read, because the contract + # refuses an out-of-range value with a payload validation error that + # is neither a StreamError nor an RPCError, a long way from the line + # that got it wrong. self._endpoint = endpoint self._http_address = http_address.rstrip("/") self._headers = dict(headers or {}) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index b09b8d2e24..726a5b7b9c 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -20,12 +20,13 @@ import asyncio import base64 import dataclasses +import gc import json import os import uuid -from collections.abc import Mapping, Sequence +from collections.abc import AsyncIterator, Mapping, Sequence from datetime import datetime, timedelta, timezone -from typing import Any +from typing import Any, cast import nexusrpc import nexusrpc.handler @@ -587,3 +588,151 @@ async def read_everything() -> list[Any]: (RecordKind.DATA, {"echo": 3}), (RecordKind.FINISH, None), ] + + +class _SlowStore(MemoryStreams): + """The memory store with a real await inside ``append``. + + The handler's repeat check and its commit have an await between them. + Without a gate there, two in-flight copies of one batch both finish the + check before either commits and both reach the store. + """ + + def __init__(self) -> None: + super().__init__() + self.gate = asyncio.Event() + self.appends = 0 + + def get_stream_handle( + self, client: Any, workflow_id: str, *, run_id: str | None = None + ) -> Any: + handle = super().get_stream_handle(client, workflow_id, run_id=run_id) + outer = self + make_producer = handle.producer + + def producer(**kwargs: Any) -> Any: + delegate = make_producer(**kwargs) + inner_append = delegate.append + + async def append(*values: Any) -> Any: + outer.appends += 1 + await outer.gate.wait() + return await inner_append(*values) + + delegate.append = append # type: ignore[method-assign] + return delegate + + handle.producer = producer # type: ignore[method-assign] + return handle + + +async def test_two_copies_of_one_batch_reach_the_store_once(): + store = _SlowStore() + handler = TemporalStreamsHandler(store, None) + both = asyncio.gather( + _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})), + _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})), + ) + await asyncio.sleep(0.1) + store.gate.set() + first, second = await asyncio.wait_for(both, 10) + # One of them wrote and the other was answered from the state it left. + # Counted at the store because a store that dedupes by itself would hide + # a second commit the handler should never have made. + assert store.appends == 1 + assert first.cursor == second.cursor + assert await _stored(store, "wf") == [{"n": 1}] + + +async def test_a_finished_batch_can_be_retried(): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})) + finished = await _dispatch( + handler, APPEND_OPERATION, _append("wf", 2, {"n": 2}, finish=True) + ) + # The caller never saw the answer. The byte-identical retry has to be + # answerable, because a finish the caller cannot repeat is a batch it can + # neither complete nor abandon. + again = await _dispatch( + handler, APPEND_OPERATION, _append("wf", 2, {"n": 2}, finish=True) + ) + assert again.cursor == finished.cursor + assert await _stored(store, "wf") == [{"n": 1}, {"n": 2}] + # And the attempt is over: a batch after the finish is refused rather + # than landing behind the marker. + with pytest.raises(nexusrpc.HandlerError) as after: + await _dispatch(handler, APPEND_OPERATION, _append("wf", 3, {"n": 3})) + assert "already finished" in str(after.value) + + +class _FailingReadStore(MemoryStreams): + """A store whose read fails once, the way a transient store failure does.""" + + def __init__(self) -> None: + super().__init__() + self.reads = 0 + + def get_stream_handle( + self, client: Any, workflow_id: str, *, run_id: str | None = None + ) -> Any: + handle = super().get_stream_handle(client, workflow_id, run_id=run_id) + outer = self + + def read(**kwargs: Any) -> Any: + outer.reads += 1 + if outer.reads == 1: + + async def failing() -> AsyncIterator[Any]: + for record in cast("list[Any]", []): + yield record + raise RuntimeError("the store went away") + + return failing() + return MemoryStreams.get_stream_handle( + outer, client, workflow_id, run_id=run_id + ).read(**kwargs) + + handle.read = read # type: ignore[method-assign] + return handle + + +async def test_a_failed_read_is_not_reported_as_the_end_of_the_stream(): + store = _FailingReadStore() + handler = TemporalStreamsHandler(store, None) + await ( + store.get_stream_handle(None, "wf") + .producer(topic=INPUTS, producer_id="model", attempt=1) + .append({"n": 1}) + ) + + request = ReadInput(workflow_id="wf", topic=INPUTS, wait_ms=200, max_records=10) + with pytest.raises(RuntimeError, match="the store went away"): + await _dispatch(handler, READ_OPERATION, request) + # The retry from the same token has to re-subscribe rather than be + # answered done=True off the subscription the failed pump left behind. + answer = await _dispatch(handler, READ_OPERATION, request) + assert [WireRecord.FromString(r.record).sequence for r in answer.records] == [0] + assert answer.done is False + await handler.close() + + +async def test_the_handler_does_not_grow_a_lock_per_address(): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + for index in range(25): + workflow_id = f"wf-{index}" + await _dispatch( + handler, APPEND_OPERATION, _append(workflow_id, 1, {"n": index}) + ) + await _dispatch( + handler, + READ_OPERATION, + ReadInput(workflow_id=workflow_id, topic=INPUTS, wait_ms=0, max_records=1), + ) + gc.collect() + # Held weakly, so an address nobody is reading or appending to leaves + # nothing behind. A strong map would hold one entry per address forever. + assert len(handler._read_locks) == 0 # pyright: ignore[reportPrivateUsage] + assert len(handler._append_locks) == 0 # pyright: ignore[reportPrivateUsage] + await handler.close() From f6ef1a7b9522bade073c7df3be3a4b2d12b44b3b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 25 Sep 2026 14:19:37 -0700 Subject: [PATCH 20/34] Kept the endpoint's transport failures off the caller. urllib wraps only the request, so a socket timeout left the call as builtins.TimeoutError, which on 3.11 and later cannot be told from the caller's own deadline. The read bounds are now refused where they are set rather than by the contract on the first read. --- temporalio/streams/providers/nexus.py | 63 ++++++++++++++++++++++----- tests/streams/test_nexus_provider.py | 46 +++++++++++++++++++ 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 7d17aa5225..324c3c3f0c 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -52,6 +52,7 @@ from __future__ import annotations import asyncio +import http.client import json import logging import time @@ -126,6 +127,13 @@ _QUEUE_DEPTH = 1000 _APPEND_TIMEOUT = timedelta(seconds=30) _ROUND_TRIP_MARGIN = timedelta(seconds=5) +# What ReadInput accepts in temporal_streams.nexusrpc.yaml. Repeated here so +# a caller's mistake is refused where it was made rather than on the wire. +_MIN_READ_WAIT = timedelta(0) +_MAX_READ_WAIT = timedelta(milliseconds=60_000) +_MIN_MAX_RECORDS = 1 +_MAX_MAX_RECORDS = 1000 + _definition = nexusrpc.get_service_definition(TemporalStreams) if _definition is None: # pragma: no cover raise RuntimeError("the generated stream service carries no nexus definition") @@ -605,19 +613,25 @@ def _post( url: str, body: bytes, headers: Mapping[str, str], timeout: timedelta ) -> bytes: timeout_ms = int(timeout.total_seconds() * 1000) - request = urllib.request.Request( - url, - data=body, - headers={ - "Content-Type": "application/json", - # Tells the server how long the handler may park, so it does not - # time the call out ahead of a wait the caller asked for. - "Request-Timeout": f"{timeout_ms}ms", - **headers, - }, - method="POST", - ) + # Everything is inside the try, because urllib wraps only the request in + # URLError: a bad url raises from Request(), and a socket timeout or a + # dropped connection raises from getresponse() and read() as + # builtins.TimeoutError or an http.client exception. On 3.11 and later + # TimeoutError is asyncio.TimeoutError, so letting one out would be + # indistinguishable from the caller's own wait_for expiring. try: + request = urllib.request.Request( + url, + data=body, + headers={ + "Content-Type": "application/json", + # Tells the server how long the handler may park, so it does + # not time the call out ahead of a wait the caller asked for. + "Request-Timeout": f"{timeout_ms}ms", + **headers, + }, + method="POST", + ) with urllib.request.urlopen( request, timeout=(timeout + _ROUND_TRIP_MARGIN).total_seconds() ) as response: @@ -630,6 +644,10 @@ def _post( raise _EndpointFailure( f"stream endpoint unreachable at {url}: {error.reason}", None ) from error + except (TimeoutError, http.client.HTTPException, OSError, ValueError) as error: + raise _EndpointFailure( + f"stream endpoint at {url} did not answer: {error!r}", None + ) from error class _Front: @@ -810,6 +828,13 @@ def read( The token is opaque here, so a cursor from another store is refused by the store behind the endpoint and raises :class:`temporalio.streams.StreamCursorError` on the first iteration. + + ``aclose()`` on the result releases nothing at the endpoint at once. + The contract carries no unsubscribe operation, so the handler cannot + be told; it reclaims the parked read when it has gone idle, which is + a minute by default. Until then the subscription and its long poll on + the store stay, and a caller that stops and starts many reads on one + topic should expect that lag rather than an immediate release. """ topic, result_type = resolve_topic(topic, result_type) return self._read(topic, after, result_type) @@ -930,6 +955,20 @@ def __init__( # refuses an out-of-range value with a payload validation error that # is neither a StreamError nor an RPCError, a long way from the line # that got it wrong. + if not _MIN_READ_WAIT <= read_wait <= _MAX_READ_WAIT: + raise ValueError( + f"read_wait must be between {_MIN_READ_WAIT} and {_MAX_READ_WAIT}, " + f"got {read_wait}" + ) + if read_wait.microseconds % 1000: + raise ValueError( + f"read_wait is carried in whole milliseconds, got {read_wait}" + ) + if not _MIN_MAX_RECORDS <= max_records <= _MAX_MAX_RECORDS: + raise ValueError( + f"max_records must be between {_MIN_MAX_RECORDS} and " + f"{_MAX_MAX_RECORDS}, got {max_records}" + ) self._endpoint = endpoint self._http_address = http_address.rstrip("/") self._headers = dict(headers or {}) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 726a5b7b9c..ed2ccdb6e5 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -21,12 +21,14 @@ import base64 import dataclasses import gc +import http.client import json import os import uuid from collections.abc import AsyncIterator, Mapping, Sequence from datetime import datetime, timedelta, timezone from typing import Any, cast +from unittest import mock import nexusrpc import nexusrpc.handler @@ -736,3 +738,47 @@ async def test_the_handler_does_not_grow_a_lock_per_address(): assert len(handler._read_locks) == 0 # pyright: ignore[reportPrivateUsage] assert len(handler._append_locks) == 0 # pyright: ignore[reportPrivateUsage] await handler.close() + + +def test_the_read_bounds_are_checked_where_they_are_set(): + # The contract accepts a wait up to a minute and a batch up to a + # thousand; past that the endpoint answers with a payload validation + # error, which is neither a stream condition nor an RPC failure. + with pytest.raises(ValueError, match="read_wait"): + NexusStreams(endpoint="e", read_wait=timedelta(minutes=2)) + with pytest.raises(ValueError, match="read_wait"): + NexusStreams(endpoint="e", read_wait=timedelta(seconds=-1)) + with pytest.raises(ValueError, match="max_records"): + NexusStreams(endpoint="e", max_records=0) + with pytest.raises(ValueError, match="max_records"): + NexusStreams(endpoint="e", max_records=1001) + NexusStreams(endpoint="e", read_wait=timedelta(seconds=60), max_records=1000) + + +def test_a_socket_failure_never_reaches_the_caller_as_a_timeout(): + # urllib wraps only the request in URLError, so a read timeout comes out + # of getresponse() as builtins.TimeoutError, which on 3.11 and later is + # asyncio.TimeoutError and would be taken for the caller's own deadline. + def _raise(*args: Any, **kwargs: Any) -> Any: + del args, kwargs + raise TimeoutError("the socket read timed out") + + with mock.patch("urllib.request.urlopen", _raise): + with pytest.raises(nexus._EndpointFailure): # pyright: ignore[reportPrivateUsage] + nexus._post( # pyright: ignore[reportPrivateUsage] + "http://127.0.0.1:1/x", b"{}", {}, timedelta(seconds=1) + ) + + def _disconnect(*args: Any, **kwargs: Any) -> Any: + del args, kwargs + raise http.client.RemoteDisconnected("the endpoint hung up") + + with mock.patch("urllib.request.urlopen", _disconnect): + with pytest.raises(nexus._EndpointFailure): # pyright: ignore[reportPrivateUsage] + nexus._post( # pyright: ignore[reportPrivateUsage] + "http://127.0.0.1:1/x", b"{}", {}, timedelta(seconds=1) + ) + + # A url urllib cannot even build a request from does not escape either. + with pytest.raises(nexus._EndpointFailure): # pyright: ignore[reportPrivateUsage] + nexus._post("not a url", b"{}", {}, timedelta(seconds=1)) # pyright: ignore[reportPrivateUsage] From e629ae11f00b11e91786b62ae2a3c89fc8e9a072 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 25 Sep 2026 20:06:24 -0700 Subject: [PATCH 21/34] Encoded the Nexus contract with the internal payload converter. The public default converter no longer applies transfer type hooks, and the generated contract types carry their JSON shape as one. The worker already decodes with the internal converter. --- temporalio/streams/providers/nexus.py | 7 +++++-- tests/streams/test_nexus_provider.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 324c3c3f0c..84e94c551c 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -685,8 +685,11 @@ async def invoke( ) -> _OutputT: # The contract types carry their own JSON encoding, so the raw caller # and the worker serving the operation agree on the body without - # either of them spelling the fields out. - contract = temporalio.converter.DataConverter.default.payload_converter + # either of them spelling the fields out. That encoding is a transfer + # type hook, which only the internal converter applies. + contract = ( + temporalio.converter.DataConverter.default._get_internal_payload_converter() + ) url = f"{await self._base_url()}/{operation}" try: raw = await asyncio.to_thread( diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index ed2ccdb6e5..19fb0a0c52 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -201,7 +201,9 @@ def _in_process_endpoint( then reported as a transport failure, the ambiguous case a retry has to survive. """ - contract = temporalio.converter.DataConverter.default.payload_converter + contract = ( + temporalio.converter.DataConverter.default._get_internal_payload_converter() + ) loop = asyncio.get_running_loop() failing = set(fail_after_applying or []) From 526532b49e5c52d0a3756e138f29bbfaff0d4235 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 25 Sep 2026 20:06:24 -0700 Subject: [PATCH 22/34] Expected the one-based sequence from the memory producer. --- tests/streams/test_nexus_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 19fb0a0c52..051ed13067 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -716,7 +716,7 @@ async def test_a_failed_read_is_not_reported_as_the_end_of_the_stream(): # The retry from the same token has to re-subscribe rather than be # answered done=True off the subscription the failed pump left behind. answer = await _dispatch(handler, READ_OPERATION, request) - assert [WireRecord.FromString(r.record).sequence for r in answer.records] == [0] + assert [WireRecord.FromString(r.record).sequence for r in answer.records] == [1] assert answer.done is False await handler.close() From 65635589edbce661e298022ba64878eb8c9e90ad Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 29 Sep 2026 10:12:52 -0700 Subject: [PATCH 23/34] Absorbed the current stream contract on the Nexus front. The front defaults the topic, carries last_n on the contract so the store behind the endpoint resolves where the newest records start, maps END onto the endpoint's newest position, and refuses a stream an activity owns, since the operations address a stream by workflow only. --- .../providers/_nexus_generated/models.py | 39 ++++++++ temporalio/streams/providers/nexus.py | 92 ++++++++++++++++--- .../providers/temporal_streams.nexusrpc.yaml | 8 ++ tests/streams/test_nexus_provider.py | 56 +++++++++++ tests/streams/test_streams_conformance.py | 5 + 5 files changed, 186 insertions(+), 14 deletions(-) diff --git a/temporalio/streams/providers/_nexus_generated/models.py b/temporalio/streams/providers/_nexus_generated/models.py index 2677843efa..cab83100b0 100644 --- a/temporalio/streams/providers/_nexus_generated/models.py +++ b/temporalio/streams/providers/_nexus_generated/models.py @@ -525,6 +525,27 @@ def from_transfer_type( else: latest_only_value = latest_only_value_raw + last_n_value: int | None = None + if "last_n" in raw: + last_n_value_raw = raw["last_n"] + if last_n_value_raw is None: + violations.append( + Violation(path="last_n", reason="explicit null not allowed") + ) + else: + last_n_value_parsed = _parse_spec_integer( + last_n_value_raw, "last_n", violations + ) + if last_n_value_parsed is not None: + last_n_value = last_n_value_parsed + if last_n_value < 1: + violations.append( + Violation( + path="last_n", + reason=f"must be >= 1, got {last_n_value}", + ) + ) + for key in raw: if ( key != "workflow_id" @@ -534,6 +555,7 @@ def from_transfer_type( and key != "max_records" and key != "wait_ms" and key != "latest_only" + and key != "last_n" ): violations.append(Violation(path=key, reason="unknown field")) if violations: @@ -546,6 +568,7 @@ def from_transfer_type( max_records=max_records_value, wait_ms=wait_ms_value, latest_only=latest_only_value, + last_n=last_n_value, ) @typing_extensions.override @@ -600,6 +623,16 @@ def to_transfer_type(self, value: "ReadInput") -> typing.Any: out["wait_ms"] = value.wait_ms if value.latest_only is not None: out["latest_only"] = value.latest_only + if value.last_n is not None: + if abs(value.last_n) > 9007199254740991: + violations.append( + Violation(path="last_n", reason="exceeds ±(2^53-1) integer cap") + ) + if value.last_n < 1: + violations.append( + Violation(path="last_n", reason=f"must be >= 1, got {value.last_n}") + ) + out["last_n"] = value.last_n if violations: raise temporalio.converter.create_payload_validation_error(violations) return out @@ -642,6 +675,12 @@ class ReadInput: from now. """ + last_n: int | None = None + """With no after_token, start at the newest this many records, or at all of them when + the stream holds fewer. Records of every kind count. Refused alongside an + after_token, which is how a read resumes. + """ + class _ReadOutputTransferTypeConverter( temporalio.converter.TransferTypeConverter["ReadOutput", typing.Any] diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 84e94c551c..dec565ea1f 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -84,7 +84,14 @@ StreamUnsupportedError, ) from temporalio.streams._provider import StreamHandle, StreamProducer, StreamProvider -from temporalio.streams._record import BEGINNING, Cursor, RecordKind, StreamRecord +from temporalio.streams._record import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamRecord, + check_read_start, +) from temporalio.streams._topic import StreamTopic, resolve_topic from temporalio.streams._wire import ( RecordDecoder, @@ -210,6 +217,8 @@ class _Subscription: records: asyncio.Queue[StreamRecord[Any]] position: str last_used: float + last_n: int | None = None + """The newest-N start this subscription was opened with, when it had no token.""" pump: asyncio.Task[None] | None = None failure: BaseException | None = None done: bool = False @@ -431,15 +440,26 @@ async def _read( left = deadline - datetime.now(timezone.utc) - _DEADLINE_MARGIN wait = max(0.0, min(wait, left.total_seconds())) after = input.after_token or "" + if after and input.last_n is not None: + raise ValueError( + "pass either after_token or last_n, not both: a token resumes a " + "read and last_n starts one" + ) key = (input.workflow_id, input.run_id or "", input.topic) await self._expire_subscriptions() lock = self._read_locks.setdefault(key, asyncio.Lock()) async with lock: subscription = self._subscriptions.get(key) - if subscription is None or subscription.position != after: + if ( + subscription is None + or subscription.position != after + or (not after and subscription.last_n != input.last_n) + ): if subscription is not None: await self._drop(key) - subscription = self._subscribe(key, stream, input.topic, after) + subscription = self._subscribe( + key, stream, input.topic, after, input.last_n + ) try: records, next_token = await self._drain(subscription, max_records, wait) except Exception: @@ -462,20 +482,30 @@ async def _read( return ReadOutput(records=records, next_token=next_token, done=done) def _subscribe( - self, key: tuple[str, str, str], stream: StreamHandle, topic: str, after: str + self, + key: tuple[str, str, str], + stream: StreamHandle, + topic: str, + after: str, + last_n: int | None, ) -> _Subscription: # Raw payloads: the handler forwards what the store holds without # decoding it, so an encoding only the caller's codec understands # passes through untouched. A foreign token is refused right here. - source = stream.read( - topic=topic, - after=Cursor(after) if after else BEGINNING, - result_type=RawValue, + source = ( + stream.read(topic=topic, last=last_n, result_type=RawValue) + if last_n is not None + else stream.read( + topic=topic, + after=Cursor(after) if after else BEGINNING, + result_type=RawValue, + ) ) subscription = _Subscription( records=asyncio.Queue(maxsize=_QUEUE_DEPTH), position=after, last_used=time.monotonic(), + last_n=last_n, ) async def pump() -> None: @@ -822,11 +852,17 @@ def __init__(self, front: _Front, workflow_id: str, run_id: str | None) -> None: def read( self, *, - topic: str | StreamTopic[Any], + topic: str | StreamTopic[Any] | None = None, after: Cursor = BEGINNING, + last: int | None = None, result_type: type | None = None, ) -> AsyncGenerator[StreamRecord[Any], None]: - """Yield records on ``topic`` after ``after``, one endpoint batch at a time. + """Yield records on ``topic`` from where the read starts, one endpoint batch at a time. + + ``BEGINNING`` and ``last=`` are resolved by the store behind the + endpoint on the first call. ``END`` is the endpoint's newest position, + asked for with a ``latest_only`` call when the read starts, and the + read resumes after it, so it yields only what is appended from then on. The token is opaque here, so a cursor from another store is refused by the store behind the endpoint and raises @@ -839,16 +875,23 @@ def read( the store stay, and a caller that stops and starts many reads on one topic should expect that lag rather than an immediate release. """ + check_read_start(after, last) topic, result_type = resolve_topic(topic, result_type) - return self._read(topic, after, result_type) + return self._read(topic, after, last, result_type) async def _read( - self, topic: str, after: Cursor, result_type: type | None + self, topic: str, after: Cursor, last: int | None, result_type: type | None ) -> AsyncGenerator[StreamRecord[Any], None]: + if after == END: + after = await self.latest(topic=topic) + # Where last= lands is the store's to say, so BEGINNING stands in for + # the position before it: a resume from it may repeat a record, where + # anything later could skip one. decoder = RecordDecoder( self._front.converter, result_type, after=after, warn=logger.warning ) token = after.token + last_n = last while True: answer = await self._front.invoke( _READ_OPERATION, @@ -857,6 +900,7 @@ async def _read( run_id=self._run_id, topic=topic, after_token=token, + last_n=last_n, max_records=self._front.max_records, wait_ms=int(self._front.read_wait.total_seconds() * 1000), ), @@ -878,10 +922,12 @@ async def _read( for out in decoder.decode(cursor, record): yield out token = answer.next_token or token + if token: + last_n = None if answer.done: return - async def latest(self, *, topic: str | StreamTopic[Any]) -> Cursor: + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: """The newest position on ``topic`` behind the endpoint, for following from now.""" topic, _ = resolve_topic(topic) answer = await self._front.invoke( @@ -901,7 +947,7 @@ async def latest(self, *, topic: str | StreamTopic[Any]) -> Cursor: def producer( self, *, - topic: str | StreamTopic[Any], + topic: str | StreamTopic[Any] | None = None, producer_id: str = "", attempt: int = 0, ) -> NexusProducer[Any]: @@ -995,6 +1041,24 @@ def get_stream_handle( """ return NexusStreamHandle(_Front(self, client), workflow_id, run_id) + def get_activity_stream_handle( + self, + client: Client | None, + activity_id: str, + *, + workflow_id: str | None = None, + run_id: str | None = None, + ) -> NoReturn: + """Refused: the endpoint's operations address a stream by workflow only. + + Raises: + StreamUnsupportedError: Always. + """ + raise StreamUnsupportedError( + "the nexus provider cannot reach a stream an activity owns: the endpoint's " + "operations address a stream by workflow only" + ) + async def close(self) -> None: """Nothing to release: each call opens and closes its own connection.""" diff --git a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml index bf97209b35..482335ed8d 100644 --- a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml +++ b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml @@ -157,6 +157,14 @@ $defs: description: >- Answer with the newest position and no records, for a reader that wants to follow from now. + last_n: + type: integer + minimum: 1 + description: >- + With no after_token, start at the newest this many records, or at + all of them when the stream holds fewer. Records of every kind + count. Refused alongside an after_token, which is how a read + resumes. required: - workflow_id - topic diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 051ed13067..660aa6e3b7 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -40,6 +40,7 @@ from temporalio.service import RPCError, RPCStatusCode from temporalio.streams import ( BEGINNING, + END, Cursor, RecordKind, StreamCursorError, @@ -784,3 +785,58 @@ def _disconnect(*args: Any, **kwargs: Any) -> Any: # A url urllib cannot even build a request from does not escape either. with pytest.raises(nexus._EndpointFailure): # pyright: ignore[reportPrivateUsage] nexus._post("not a url", b"{}", {}, timedelta(seconds=1)) # pyright: ignore[reportPrivateUsage] + + +async def test_the_front_carries_every_read_start(monkeypatch: pytest.MonkeyPatch): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + stream = _front(None).get_stream_handle(None, "wf-starts") + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}) + + # last= rides the contract to the store behind the endpoint. + newest = await take( + stream.read(topic=INPUTS, result_type=dict, last=2), 2, timeout=30 + ) + assert [record.value for record in newest] == [{"n": 2}, {"n": 3}] + + # END is the endpoint's newest position when the read starts. + at_end = stream.read(topic=INPUTS, result_type=dict, after=END) + first = asyncio.ensure_future(at_end.__anext__()) + try: + for _ in range(100): + await producer.append({"n": "new"}) + done, _ = await asyncio.wait({first}, timeout=0.1) + if done: + break + record = await asyncio.wait_for(first, 30) + finally: + await at_end.aclose() + assert record.value == {"n": "new"} + + # A token resumes a read and last_n starts one, so the two are refused. + with pytest.raises(nexusrpc.HandlerError) as both: + await _dispatch( + handler, + READ_OPERATION, + ReadInput( + workflow_id="wf-starts", + topic=INPUTS, + after_token=newest[0].cursor.token, + last_n=1, + ), + ) + assert both.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + await handler.close() + + +async def test_a_stream_an_activity_owns_is_refused(client: Client): + # The endpoint's operations address a stream by workflow, so the front has + # no way to reach one an activity owns. The refusal is the documented + # error, not an AttributeError or a stream silently put somewhere else. + front = _front(None) + with pytest.raises(StreamUnsupportedError, match="activity"): + front.get_activity_stream_handle(client, "act") + with pytest.raises(StreamUnsupportedError, match="activity"): + front.get_activity_stream_handle(client, "act", workflow_id="wf") diff --git a/tests/streams/test_streams_conformance.py b/tests/streams/test_streams_conformance.py index 942595170b..496efdb3f4 100644 --- a/tests/streams/test_streams_conformance.py +++ b/tests/streams/test_streams_conformance.py @@ -11,6 +11,11 @@ skipped with a reason on a provider whose ``append()`` learns positions at read time. +``NexusStreams`` is deliberately absent from ``SETUPS``. It is a front with no +workflow half (its ``workflow_provider()`` raises), so there is no store for a +host workflow to own; ``test_nexus_provider`` covers it through an endpoint +that fronts a storage provider. + What this file pins down is what a provider owes: producer identity, retry deduplication, positions, supersession, topic addressing, cursor resumption, cursor ownership, and releasing a read the caller stopped early. Every case From fa7c9947e89bf3dea0087f499b63df1ed28bf918 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 30 Sep 2026 02:29:20 -0700 Subject: [PATCH 24/34] Addressed a stream by reference on the Nexus front. The two operations took a workflow id, run id and topic, so an operation of the application's own had no standard value to return for a stream and the front could not reach a stream an activity owns. A StreamRef naming the owner kind, its ids and the topic replaces the triple, and the handler maps it onto the store's accessor for that owner. --- CHANGELOG.md | 8 +- pyproject.toml | 5 + .../providers/_nexus_generated/__init__.py | 2 + .../providers/_nexus_generated/models.py | 335 ++++++++++++------ temporalio/streams/providers/nexus.py | 212 +++++++---- .../providers/temporal_streams.nexusrpc.yaml | 61 ++-- tests/streams/test_nexus_provider.py | 167 +++++++-- 7 files changed, 556 insertions(+), 234 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7fb8e3fbb..fff14568e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,9 +67,11 @@ to include examples, links to docs, or any other relevant information. `TemporalStreamsHandler` serves that endpoint by fronting the provider's own handles. Its contract is defined in `temporal_streams.nexusrpc.yaml` and the bindings are generated from it; a record crosses as the serialized - `StreamRecord` proto. Configure the front with `data_converter=` to run a - payload codec on the caller side, so records are encoded before they leave - the process. + `StreamRecord` proto, and both operations address a stream by a `StreamRef` + naming its owner (a workflow, an activity or a standalone stream) and topic, + which the handler maps onto the store's accessor for that owner. Configure + the front with `data_converter=` to run a payload codec on the caller side, + so records are encoded before they leave the process. ### Changed diff --git a/pyproject.toml b/pyproject.toml index 227a6a2cc2..fcf82b5266 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -201,12 +201,17 @@ exclude = [ 'temporalio/api', 'temporalio/bridge/proto', 'temporalio/nexus/system/workflow_service', + 'temporalio/streams/providers/_nexus_generated', ] [[tool.mypy.overrides]] module = "temporalio.nexus.system.workflow_service.*" ignore_errors = true +[[tool.mypy.overrides]] +module = "temporalio.streams.providers._nexus_generated.*" +ignore_errors = true + [tool.pydocstyle] convention = "google" # https://github.com/PyCQA/pydocstyle/issues/363#issuecomment-625563088 diff --git a/temporalio/streams/providers/_nexus_generated/__init__.py b/temporalio/streams/providers/_nexus_generated/__init__.py index c62eaa6dd3..d855a63814 100644 --- a/temporalio/streams/providers/_nexus_generated/__init__.py +++ b/temporalio/streams/providers/_nexus_generated/__init__.py @@ -9,6 +9,7 @@ ReadInput, ReadOutput, RecordWire, + StreamRef, ) from .services import TemporalStreams @@ -19,5 +20,6 @@ "ReadInput", "ReadOutput", "RecordWire", + "StreamRef", "TemporalStreams", ] diff --git a/temporalio/streams/providers/_nexus_generated/models.py b/temporalio/streams/providers/_nexus_generated/models.py index cab83100b0..033cded90d 100644 --- a/temporalio/streams/providers/_nexus_generated/models.py +++ b/temporalio/streams/providers/_nexus_generated/models.py @@ -16,6 +16,7 @@ _format_base64, _parse_base64, _parse_spec_integer, + _quote, _transfer_type_convertible, ) @@ -42,42 +43,17 @@ def from_transfer_type( ) raw = typing.cast("dict[str, typing.Any]", value) - workflow_id_value: str = typing.cast("typing.Any", None) - if "workflow_id" not in raw or raw["workflow_id"] is None: - violations.append(Violation(path="workflow_id", reason="required")) + stream_value: StreamRef = typing.cast("typing.Any", None) + if "stream" not in raw or raw["stream"] is None: + violations.append(Violation(path="stream", reason="required")) else: - workflow_id_value_raw = raw["workflow_id"] - if not isinstance(workflow_id_value_raw, str): - violations.append( - Violation(path="workflow_id", reason="expected string") - ) - else: - workflow_id_value = workflow_id_value_raw - - run_id_value: str | None = None - if "run_id" in raw: - run_id_value_raw = raw["run_id"] - if run_id_value_raw is None: - violations.append( - Violation(path="run_id", reason="explicit null not allowed") + stream_value_raw = raw["stream"] + try: + stream_value = _StreamRefTransferTypeConverter().from_transfer_type( + stream_value_raw, StreamRef ) - else: - if not isinstance(run_id_value_raw, str): - violations.append( - Violation(path="run_id", reason="expected string") - ) - else: - run_id_value = run_id_value_raw - - topic_value: str = typing.cast("typing.Any", None) - if "topic" not in raw or raw["topic"] is None: - violations.append(Violation(path="topic", reason="required")) - else: - topic_value_raw = raw["topic"] - if not isinstance(topic_value_raw, str): - violations.append(Violation(path="topic", reason="expected string")) - else: - topic_value = topic_value_raw + except temporalio.exceptions.ApplicationError as error: + _collect(violations, "stream", error) producer_id_value: str = typing.cast("typing.Any", None) if "producer_id" not in raw or raw["producer_id"] is None: @@ -194,9 +170,7 @@ def from_transfer_type( for key in raw: if ( - key != "workflow_id" - and key != "run_id" - and key != "topic" + key != "stream" and key != "producer_id" and key != "attempt" and key != "sequence" @@ -208,9 +182,7 @@ def from_transfer_type( if violations: raise temporalio.converter.create_payload_validation_error(violations) return AppendInput( - workflow_id=workflow_id_value, - run_id=run_id_value, - topic=topic_value, + stream=stream_value, producer_id=producer_id_value, attempt=attempt_value, sequence=sequence_value, @@ -223,10 +195,12 @@ def from_transfer_type( def to_transfer_type(self, value: "AppendInput") -> typing.Any: violations: list[Violation] = [] out: dict[str, typing.Any] = {} - out["workflow_id"] = value.workflow_id - if value.run_id is not None: - out["run_id"] = value.run_id - out["topic"] = value.topic + try: + out["stream"] = _StreamRefTransferTypeConverter().to_transfer_type( + value.stream + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, "stream", error) out["producer_id"] = value.producer_id if abs(value.attempt) > 9007199254740991: violations.append( @@ -267,16 +241,7 @@ def to_transfer_type(self, value: "AppendInput") -> typing.Any: class AppendInput: """One append call: who is writing, where, and what.""" - workflow_id: str - """The workflow whose stream is being written.""" - - run_id: str | None = None - """The run to address. Empty addresses the workflow's execution chain, which is what a - producer normally wants. - """ - - topic: str - """The topic of the workflow's stream to append to.""" + stream: StreamRef producer_id: str """Identifies the writer across its retries, so its attempts can be ordered. Never @@ -402,42 +367,17 @@ def from_transfer_type( ) raw = typing.cast("dict[str, typing.Any]", value) - workflow_id_value: str = typing.cast("typing.Any", None) - if "workflow_id" not in raw or raw["workflow_id"] is None: - violations.append(Violation(path="workflow_id", reason="required")) + stream_value: StreamRef = typing.cast("typing.Any", None) + if "stream" not in raw or raw["stream"] is None: + violations.append(Violation(path="stream", reason="required")) else: - workflow_id_value_raw = raw["workflow_id"] - if not isinstance(workflow_id_value_raw, str): - violations.append( - Violation(path="workflow_id", reason="expected string") - ) - else: - workflow_id_value = workflow_id_value_raw - - run_id_value: str | None = None - if "run_id" in raw: - run_id_value_raw = raw["run_id"] - if run_id_value_raw is None: - violations.append( - Violation(path="run_id", reason="explicit null not allowed") + stream_value_raw = raw["stream"] + try: + stream_value = _StreamRefTransferTypeConverter().from_transfer_type( + stream_value_raw, StreamRef ) - else: - if not isinstance(run_id_value_raw, str): - violations.append( - Violation(path="run_id", reason="expected string") - ) - else: - run_id_value = run_id_value_raw - - topic_value: str = typing.cast("typing.Any", None) - if "topic" not in raw or raw["topic"] is None: - violations.append(Violation(path="topic", reason="required")) - else: - topic_value_raw = raw["topic"] - if not isinstance(topic_value_raw, str): - violations.append(Violation(path="topic", reason="expected string")) - else: - topic_value = topic_value_raw + except temporalio.exceptions.ApplicationError as error: + _collect(violations, "stream", error) after_token_value: str | None = None if "after_token" in raw: @@ -548,9 +488,7 @@ def from_transfer_type( for key in raw: if ( - key != "workflow_id" - and key != "run_id" - and key != "topic" + key != "stream" and key != "after_token" and key != "max_records" and key != "wait_ms" @@ -561,9 +499,7 @@ def from_transfer_type( if violations: raise temporalio.converter.create_payload_validation_error(violations) return ReadInput( - workflow_id=workflow_id_value, - run_id=run_id_value, - topic=topic_value, + stream=stream_value, after_token=after_token_value, max_records=max_records_value, wait_ms=wait_ms_value, @@ -575,10 +511,12 @@ def from_transfer_type( def to_transfer_type(self, value: "ReadInput") -> typing.Any: violations: list[Violation] = [] out: dict[str, typing.Any] = {} - out["workflow_id"] = value.workflow_id - if value.run_id is not None: - out["run_id"] = value.run_id - out["topic"] = value.topic + try: + out["stream"] = _StreamRefTransferTypeConverter().to_transfer_type( + value.stream + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, "stream", error) if value.after_token is not None: out["after_token"] = value.after_token if value.max_records is not None: @@ -643,16 +581,7 @@ def to_transfer_type(self, value: "ReadInput") -> typing.Any: class ReadInput: """One read call: where to resume from and how long to wait.""" - workflow_id: str - """The workflow whose stream is being read.""" - - run_id: str | None = None - """Pin the read to this run. Empty follows the workflow's execution chain across - continue-as-new. - """ - - topic: str - """The topic of the workflow's stream to read.""" + stream: StreamRef after_token: str | None = None """Opaque cursor from an earlier record or append. The read resumes strictly after the @@ -915,3 +844,193 @@ class RecordWire: additional_properties: dict[str, typing.Any] = dataclasses.field( default_factory=dict ) + + +class _StreamRefTransferTypeConverter( + temporalio.converter.TransferTypeConverter["StreamRef", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["StreamRef"] + ) -> "StreamRef": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + kind_value: typing.Literal["workflow", "activity", "standalone"] = typing.cast( + "typing.Any", None + ) + if "kind" not in raw or raw["kind"] is None: + violations.append(Violation(path="kind", reason="required")) + else: + kind_value_raw = raw["kind"] + if not isinstance(kind_value_raw, str): + violations.append(Violation(path="kind", reason="expected string")) + elif kind_value_raw not in ("workflow", "activity", "standalone"): + violations.append( + Violation( + path="kind", + reason=f'must be one of ["workflow", "activity", "standalone"], got {_quote(kind_value_raw)}', + ) + ) + else: + kind_value = kind_value_raw + + workflow_id_value: str | None = None + if "workflow_id" in raw: + workflow_id_value_raw = raw["workflow_id"] + if workflow_id_value_raw is None: + violations.append( + Violation(path="workflow_id", reason="explicit null not allowed") + ) + else: + if not isinstance(workflow_id_value_raw, str): + violations.append( + Violation(path="workflow_id", reason="expected string") + ) + else: + workflow_id_value = workflow_id_value_raw + + run_id_value: str | None = None + if "run_id" in raw: + run_id_value_raw = raw["run_id"] + if run_id_value_raw is None: + violations.append( + Violation(path="run_id", reason="explicit null not allowed") + ) + else: + if not isinstance(run_id_value_raw, str): + violations.append( + Violation(path="run_id", reason="expected string") + ) + else: + run_id_value = run_id_value_raw + + activity_id_value: str | None = None + if "activity_id" in raw: + activity_id_value_raw = raw["activity_id"] + if activity_id_value_raw is None: + violations.append( + Violation(path="activity_id", reason="explicit null not allowed") + ) + else: + if not isinstance(activity_id_value_raw, str): + violations.append( + Violation(path="activity_id", reason="expected string") + ) + else: + activity_id_value = activity_id_value_raw + + stream_id_value: str | None = None + if "stream_id" in raw: + stream_id_value_raw = raw["stream_id"] + if stream_id_value_raw is None: + violations.append( + Violation(path="stream_id", reason="explicit null not allowed") + ) + else: + if not isinstance(stream_id_value_raw, str): + violations.append( + Violation(path="stream_id", reason="expected string") + ) + else: + stream_id_value = stream_id_value_raw + + topic_value: str = typing.cast("typing.Any", None) + if "topic" not in raw or raw["topic"] is None: + violations.append(Violation(path="topic", reason="required")) + else: + topic_value_raw = raw["topic"] + if not isinstance(topic_value_raw, str): + violations.append(Violation(path="topic", reason="expected string")) + else: + topic_value = topic_value_raw + + for key in raw: + if ( + key != "kind" + and key != "workflow_id" + and key != "run_id" + and key != "activity_id" + and key != "stream_id" + and key != "topic" + ): + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return StreamRef( + kind=kind_value, + workflow_id=workflow_id_value, + run_id=run_id_value, + activity_id=activity_id_value, + stream_id=stream_id_value, + topic=topic_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "StreamRef") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + if typing.cast("object", value.kind) not in ( + "workflow", + "activity", + "standalone", + ): + violations.append( + Violation( + path="kind", + reason=f'must be one of ["workflow", "activity", "standalone"], got {_quote(value.kind)}', + ) + ) + out["kind"] = value.kind + if value.workflow_id is not None: + out["workflow_id"] = value.workflow_id + if value.run_id is not None: + out["run_id"] = value.run_id + if value.activity_id is not None: + out["activity_id"] = value.activity_id + if value.stream_id is not None: + out["stream_id"] = value.stream_id + out["topic"] = value.topic + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_StreamRefTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class StreamRef: + """A stream, named by its owner and a topic: what an operation returns to hand a stream + to its caller, and what read and append take in place of an owner spelled out. It + names no cursor and no store, so the same reference is good behind any endpoint that + serves the owner. + """ + + kind: typing.Literal["workflow", "activity", "standalone"] + """What owns the stream. A workflow's streams are keyed by workflow_id and, when + pinned, run_id. An activity's own streams are keyed by activity_id and, when a + workflow scheduled it, that workflow's ids. A standalone stream has its own + stream_id and no execution behind it. + """ + + workflow_id: str | None = None + """The owning workflow, or the workflow that scheduled the owning activity. Required + for a workflow owner. + """ + + run_id: str | None = None + """Pin the owner to this run. Absent follows the execution chain across + continue-as-new, which is what a producer normally wants. + """ + + activity_id: str | None = None + """The owning activity. Required for an activity owner.""" + + stream_id: str | None = None + """The stream's own id. Required for a standalone owner.""" + + topic: str + """The topic on the owner's stream.""" diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index dec565ea1f..c22118f544 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -29,7 +29,17 @@ foreign one is refused by the store behind the endpoint and reaches the caller on the first read. -The handler keeps one parked read per ``(workflow, run, topic)`` and serves +Both operations address a stream by a reference, ``StreamRef`` on the wire: +the owner (a workflow, an activity, or a standalone stream), the ids that +name it, and the topic. The handler maps the reference onto the store's own +accessor for that owner and refuses an owner the store cannot host with +:class:`temporalio.streams.StreamUnsupportedError`, so a reference is good +behind any endpoint that serves the owner. The same reference is what an +operation of the application's own returns to hand a stream to its caller; +:meth:`NexusStreamHandle.ref` makes one and :meth:`NexusStreams.open` opens +it. + +The handler keeps one parked read per stream reference and serves consecutive calls from it, so an idle caller does not leave one abandoned long poll on the store per call. A call whose token does not match the parked position replaces the subscription, and an idle one is released after a @@ -40,7 +50,7 @@ producer attempt refuses to continue it rather than starting a fresh delegate whose numbering the store would drop as a repeat; the caller opens a new attempt, which readers report as a supersession. And any caller the endpoint -admits may touch any workflow's streams in the namespace; the endpoint's own +admits may touch any owner's streams in the namespace; the endpoint's own authorization is the boundary. A failure from the endpoint reaches the caller as the @@ -63,7 +73,7 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Any, Generic, NoReturn, TypeVar, cast +from typing import Any, Generic, Literal, NoReturn, TypeVar, cast import nexusrpc import nexusrpc.handler @@ -107,16 +117,24 @@ RecordWire, TemporalStreams, ) +from temporalio.streams.providers._nexus_generated import StreamRef as WireStreamRef __all__ = [ "NexusProducer", "NexusStreamHandle", "NexusStreams", "TemporalStreamsHandler", + "WireStreamRef", ] T = TypeVar("T") +_Kind = Literal["workflow", "activity", "standalone"] +# The reference's identity on the handler, topic included, so one parked read +# and one producer state key on exactly what the wire names. +_StreamKey = tuple[str, str, str, str, str, str] +_ProducerKey = tuple[_StreamKey, str, int] + _WORKFLOW_SIDE_ERROR = ( "the nexus provider is an outside transport; a worker publishes and reads " "through a storage provider, so give the worker one of those" @@ -192,6 +210,49 @@ def _require_topic(topic: str) -> None: raise ValueError("topic must not be empty") +def _check_ref(ref: WireStreamRef) -> None: + """Refuse a reference whose owner lacks the id that names it.""" + _require_topic(ref.topic) + if ref.kind == "workflow" and not ref.workflow_id: + raise ValueError("a stream reference with a workflow owner needs workflow_id") + if ref.kind == "activity" and not ref.activity_id: + raise ValueError("a stream reference with an activity owner needs activity_id") + if ref.kind == "standalone" and not ref.stream_id: + raise ValueError("a stream reference with a standalone owner needs stream_id") + + +def _stream_key(ref: WireStreamRef) -> _StreamKey: + return ( + ref.kind, + ref.workflow_id or "", + ref.run_id or "", + ref.activity_id or "", + ref.stream_id or "", + ref.topic, + ) + + +@dataclass(frozen=True) +class _Address: + """The owner a handle is bound to; a topic completes it into a reference.""" + + kind: _Kind + workflow_id: str | None = None + run_id: str | None = None + activity_id: str | None = None + stream_id: str | None = None + + def ref(self, topic: str) -> WireStreamRef: + return WireStreamRef( + kind=self.kind, + workflow_id=self.workflow_id, + run_id=self.run_id, + activity_id=self.activity_id, + stream_id=self.stream_id, + topic=topic, + ) + + def _handler_error(error: StreamError) -> nexusrpc.HandlerError: """Carry a stream condition across the endpoint under its own class name.""" kind = ( @@ -260,27 +321,46 @@ def __init__( """ self._provider = provider self._client = client - self._producers: OrderedDict[tuple[str, str, str, str, int], _ProducerState] = ( - OrderedDict() - ) + self._producers: OrderedDict[_ProducerKey, _ProducerState] = OrderedDict() self._max_producers = max_producers - self._subscriptions: dict[tuple[str, str, str], _Subscription] = {} + self._subscriptions: dict[_StreamKey, _Subscription] = {} # Held weakly, and by the call that is using one for as long as it # runs. A strong map would keep an entry per address ever read or # appended to, and nothing would ever reach it again. - self._read_locks: weakref.WeakValueDictionary[ - tuple[str, str, str], asyncio.Lock - ] = weakref.WeakValueDictionary() - self._append_locks: weakref.WeakValueDictionary[ - tuple[str, str, str, str, int], asyncio.Lock - ] = weakref.WeakValueDictionary() + self._read_locks: weakref.WeakValueDictionary[_StreamKey, asyncio.Lock] = ( + weakref.WeakValueDictionary() + ) + self._append_locks: weakref.WeakValueDictionary[_ProducerKey, asyncio.Lock] = ( + weakref.WeakValueDictionary() + ) self._subscription_idle = subscription_idle.total_seconds() - def _stream(self, workflow_id: str, run_id: str | None) -> StreamHandle: + def _stream(self, ref: WireStreamRef) -> StreamHandle: + """The store's handle for the owner ``ref`` names. + + Raises: + StreamUnsupportedError: The store has no accessor for that owner. + """ # A storage provider wants a client; the memory provider, which the # tests front, accepts none, so the cast only lies where it is unread. - return self._provider.get_stream_handle( - cast(Client, self._client), workflow_id, run_id=run_id or None + client = cast(Client, self._client) + if ref.kind == "workflow": + return self._provider.get_stream_handle( + client, cast(str, ref.workflow_id), run_id=ref.run_id or None + ) + if ref.kind == "activity": + return self._provider.get_activity_stream_handle( + client, + cast(str, ref.activity_id), + workflow_id=ref.workflow_id or None, + run_id=ref.run_id or None, + ) + # The provider protocol reaches a stream through its owning execution; + # a standalone stream has none, so no store behind the endpoint can be + # asked for it yet. + raise StreamUnsupportedError( + "the store behind the endpoint cannot host a standalone stream: a " + "provider addresses a stream by its owning workflow or activity" ) @nexusrpc.handler.sync_operation @@ -292,9 +372,10 @@ async def append( Raises: nexusrpc.HandlerError: ``BAD_REQUEST`` when the batch index or sequence does not continue the producer attempt, when the - attempt is one this handler has no state for, or when a - payload is not a serialized ``Payload``; ``NOT_FOUND`` when - the store has no such workflow. + attempt is one this handler has no state for, when a payload + is not a serialized ``Payload``, or when the store cannot host + the reference's owner; ``NOT_FOUND`` when the store has no + such owner. """ try: return await self._append(input) @@ -304,11 +385,9 @@ async def append( raise _bad_request(str(error)) from error async def _append(self, input: AppendInput) -> AppendOutput: - _require_topic(input.topic) - key = ( - input.workflow_id, - input.run_id or "", - input.topic, + _check_ref(input.stream) + key: _ProducerKey = ( + _stream_key(input.stream), input.producer_id, input.attempt, ) @@ -321,7 +400,7 @@ async def _append(self, input: AppendInput) -> AppendOutput: return await self._append_locked(input, key) async def _append_locked( - self, input: AppendInput, key: tuple[str, str, str, str, int] + self, input: AppendInput, key: _ProducerKey ) -> AppendOutput: state = self._producers.get(key) if state is None: @@ -335,8 +414,10 @@ async def _append_locked( f"at batch {input.batch_index} on a handler that has no state for " "it; open a new attempt" ) - delegate = self._stream(input.workflow_id, input.run_id).producer( - topic=input.topic, producer_id=input.producer_id, attempt=input.attempt + delegate = self._stream(input.stream).producer( + topic=input.stream.topic, + producer_id=input.producer_id, + attempt=input.attempt, ) state = _ProducerState(delegate, 0, 0, None) elif input.batch_index == state.batch_index: @@ -406,7 +487,8 @@ async def read( Raises: nexusrpc.HandlerError: ``BAD_REQUEST`` when the store refuses the - token, ``NOT_FOUND`` when it has no such workflow. + token or cannot host the reference's owner, ``NOT_FOUND`` when + it has no such owner. """ try: return await self._read(ctx, input) @@ -418,10 +500,11 @@ async def read( async def _read( self, ctx: nexusrpc.handler.StartOperationContext, input: ReadInput ) -> ReadOutput: - _require_topic(input.topic) - stream = self._stream(input.workflow_id, input.run_id) + _check_ref(input.stream) + topic = input.stream.topic + stream = self._stream(input.stream) if input.latest_only: - return ReadOutput(next_token=(await stream.latest(topic=input.topic)).token) + return ReadOutput(next_token=(await stream.latest(topic=topic)).token) max_records = ( _DEFAULT_MAX_RECORDS if input.max_records is None else input.max_records ) @@ -445,7 +528,7 @@ async def _read( "pass either after_token or last_n, not both: a token resumes a " "read and last_n starts one" ) - key = (input.workflow_id, input.run_id or "", input.topic) + key = _stream_key(input.stream) await self._expire_subscriptions() lock = self._read_locks.setdefault(key, asyncio.Lock()) async with lock: @@ -457,9 +540,7 @@ async def _read( ): if subscription is not None: await self._drop(key) - subscription = self._subscribe( - key, stream, input.topic, after, input.last_n - ) + subscription = self._subscribe(key, stream, topic, after, input.last_n) try: records, next_token = await self._drain(subscription, max_records, wait) except Exception: @@ -483,7 +564,7 @@ async def _read( def _subscribe( self, - key: tuple[str, str, str], + key: _StreamKey, stream: StreamHandle, topic: str, after: str, @@ -586,7 +667,7 @@ async def close(self) -> None: for key in list(self._subscriptions): await self._drop(key) - async def _drop(self, key: tuple[str, str, str]) -> None: + async def _drop(self, key: _StreamKey) -> None: subscription = self._subscriptions.pop(key, None) if subscription is None or subscription.pump is None: return @@ -748,17 +829,13 @@ class NexusProducer(Generic[T]): def __init__( self, front: _Front, - workflow_id: str, - run_id: str | None, - topic: str, + stream: WireStreamRef, producer_id: str, attempt: int, ) -> None: - """Bind this producer to ``topic`` on the workflow behind the endpoint.""" + """Bind this producer to the stream ``stream`` names behind the endpoint.""" self._front = front - self._workflow_id = workflow_id - self._run_id = run_id - self._topic = topic + self._stream = stream self._producer_id = producer_id self._attempt = attempt self._batch_index = 0 @@ -815,9 +892,7 @@ async def _call(self, payloads: list[Payload], *, finish: bool) -> AppendOutput: elif self._pending is not None: return await self._send(*self._pending) request = AppendInput( - workflow_id=self._workflow_id, - run_id=self._run_id, - topic=self._topic, + stream=self._stream, producer_id=self._producer_id, attempt=self._attempt, sequence=self._sequence, @@ -841,13 +916,12 @@ async def _send( class NexusStreamHandle: - """One workflow's stream through the endpoint, re-synthesizing supersession.""" + """One owner's stream through the endpoint, re-synthesizing supersession.""" - def __init__(self, front: _Front, workflow_id: str, run_id: str | None) -> None: - """Address ``workflow_id``'s stream, pinned to ``run_id`` when one is given.""" + def __init__(self, front: _Front, address: _Address) -> None: + """Address the owner ``address`` names; each call completes it with a topic.""" self._front = front - self._workflow_id = workflow_id - self._run_id = run_id + self._address = address def read( self, @@ -892,13 +966,12 @@ async def _read( ) token = after.token last_n = last + stream = self._address.ref(topic) while True: answer = await self._front.invoke( _READ_OPERATION, ReadInput( - workflow_id=self._workflow_id, - run_id=self._run_id, - topic=topic, + stream=stream, after_token=token, last_n=last_n, max_records=self._front.max_records, @@ -932,12 +1005,7 @@ async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor topic, _ = resolve_topic(topic) answer = await self._front.invoke( _READ_OPERATION, - ReadInput( - workflow_id=self._workflow_id, - run_id=self._run_id, - topic=topic, - latest_only=True, - ), + ReadInput(stream=self._address.ref(topic), latest_only=True), ReadOutput, timeout=_APPEND_TIMEOUT, ) @@ -955,7 +1023,7 @@ def producer( topic, _ = resolve_topic(topic) producer_id, attempt = producer_identity(producer_id, attempt) return NexusProducer( - self._front, self._workflow_id, self._run_id, topic, producer_id, attempt + self._front, self._address.ref(topic), producer_id, attempt ) @@ -1039,7 +1107,9 @@ def get_stream_handle( ``client`` is not used for transport: it resolves the endpoint's name and supplies the data converter when none was configured. """ - return NexusStreamHandle(_Front(self, client), workflow_id, run_id) + return NexusStreamHandle( + _Front(self, client), _Address("workflow", workflow_id, run_id) + ) def get_activity_stream_handle( self, @@ -1048,15 +1118,17 @@ def get_activity_stream_handle( *, workflow_id: str | None = None, run_id: str | None = None, - ) -> NoReturn: - """Refused: the endpoint's operations address a stream by workflow only. + ) -> NexusStreamHandle: + """A handle on the streams ``activity_id`` owns, through the endpoint. - Raises: - StreamUnsupportedError: Always. + The reference carries the activity owner across; whether the store + behind the endpoint can host it is the store's answer, and a refusal + reaches the caller as :class:`temporalio.streams.StreamUnsupportedError` + on the first call. """ - raise StreamUnsupportedError( - "the nexus provider cannot reach a stream an activity owns: the endpoint's " - "operations address a stream by workflow only" + return NexusStreamHandle( + _Front(self, client), + _Address("activity", workflow_id, run_id, activity_id), ) async def close(self) -> None: diff --git a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml index 482335ed8d..61660c8613 100644 --- a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml +++ b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml @@ -38,21 +38,52 @@ services: output: { $ref: "#/$defs/ReadOutput" } $defs: - AppendInput: + StreamRef: type: object - description: "One append call: who is writing, where, and what." + description: >- + A stream, named by its owner and a topic: what an operation returns to + hand a stream to its caller, and what read and append take in place of + an owner spelled out. It names no cursor and no store, so the same + reference is good behind any endpoint that serves the owner. properties: + kind: + type: string + enum: [workflow, activity, standalone] + description: >- + What owns the stream. A workflow's streams are keyed by workflow_id + and, when pinned, run_id. An activity's own streams are keyed by + activity_id and, when a workflow scheduled it, that workflow's ids. + A standalone stream has its own stream_id and no execution behind + it. workflow_id: type: string - description: "The workflow whose stream is being written." + description: >- + The owning workflow, or the workflow that scheduled the owning + activity. Required for a workflow owner. run_id: type: string description: >- - The run to address. Empty addresses the workflow's execution chain, - which is what a producer normally wants. + Pin the owner to this run. Absent follows the execution chain + across continue-as-new, which is what a producer normally wants. + activity_id: + type: string + description: "The owning activity. Required for an activity owner." + stream_id: + type: string + description: "The stream's own id. Required for a standalone owner." topic: type: string - description: "The topic of the workflow's stream to append to." + description: "The topic on the owner's stream." + required: + - kind + - topic + additionalProperties: false + + AppendInput: + type: object + description: "One append call: who is writing, where, and what." + properties: + stream: { $ref: "#/$defs/StreamRef" } producer_id: type: string description: >- @@ -95,8 +126,7 @@ $defs: Write FINISH for this producer after the batch: it will write nothing more on the topic. Says nothing about the producer's outcome. required: - - workflow_id - - topic + - stream - producer_id - attempt - sequence @@ -120,17 +150,7 @@ $defs: type: object description: "One read call: where to resume from and how long to wait." properties: - workflow_id: - type: string - description: "The workflow whose stream is being read." - run_id: - type: string - description: >- - Pin the read to this run. Empty follows the workflow's execution - chain across continue-as-new. - topic: - type: string - description: "The topic of the workflow's stream to read." + stream: { $ref: "#/$defs/StreamRef" } after_token: type: string description: >- @@ -166,8 +186,7 @@ $defs: count. Refused alongside an after_token, which is how a read resumes. required: - - workflow_id - - topic + - stream additionalProperties: false RecordWire: diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 660aa6e3b7..0d5a0f43b6 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -4,12 +4,12 @@ provider's own handles, so these tests are the provider-hiding demonstration: nothing on the caller side names or could name the store. -The loop test runs over the server's Nexus HTTP ingress and is gated behind -``STREAMS_LIVE=nexus`` because it needs a dev server with an HTTP port and a -registered Nexus endpoint. Environment: ``TEMPORAL_ADDRESS`` (default -``localhost:7233``), ``TEMPORAL_HTTP`` (default ``http://127.0.0.1:7243``), -and an endpoint named ``streams-e2e`` targeting task queue -``streams-handlers-e2e``. +The live tests run over the server's Nexus HTTP ingress and are gated behind +``STREAMS_LIVE=nexus`` because they need a dev server with an HTTP port. +Environment: ``TEMPORAL_ADDRESS`` (default ``localhost:7233``) and +``TEMPORAL_HTTP`` (default ``http://127.0.0.1:7243``). Each live test +registers its own Nexus endpoint through the operator service and deletes it +at the end. Everything else stands the endpoint up in this process, because what it checks is the bytes the caller puts on the wire and the handler's own rules. @@ -19,6 +19,7 @@ import asyncio import base64 +import contextlib import dataclasses import gc import http.client @@ -36,6 +37,11 @@ import temporalio.converter from temporalio.api.common.v1 import Payload +from temporalio.api.nexus.v1 import EndpointSpec, EndpointTarget +from temporalio.api.operatorservice.v1 import ( + CreateNexusEndpointRequest, + DeleteNexusEndpointRequest, +) from temporalio.client import Client from temporalio.service import RPCError, RPCStatusCode from temporalio.streams import ( @@ -50,6 +56,7 @@ from temporalio.streams._wire import WireRecord from temporalio.streams.providers import nexus from temporalio.streams.providers._nexus_generated import AppendInput, ReadInput +from temporalio.streams.providers._nexus_generated import StreamRef as WireStreamRef from temporalio.streams.providers.memory import MemoryStreams from temporalio.streams.providers.nexus import ( NexusStreamHandle, @@ -67,30 +74,67 @@ reason="needs a live server and nexus endpoint; run with STREAMS_LIVE=nexus", ) -ENDPOINT = "streams-e2e" -HANDLER_TQ = "streams-handlers-e2e" APPEND_OPERATION = nexus._APPEND_OPERATION # pyright: ignore[reportPrivateUsage] READ_OPERATION = nexus._READ_OPERATION # pyright: ignore[reportPrivateUsage] INPUTS = "inputs" DECISIONS = "decisions" +async def _live_client() -> Client: + return await Client.connect(os.environ.get("TEMPORAL_ADDRESS", "localhost:7233")) + + +def _live_front(endpoint: str) -> NexusStreams: + return NexusStreams( + endpoint=endpoint, + http_address=os.environ.get("TEMPORAL_HTTP", "http://127.0.0.1:7243"), + read_wait=timedelta(seconds=5), + ) + + +@contextlib.asynccontextmanager +async def _own_endpoint( + client: Client, name: str, task_queue: str +) -> AsyncIterator[str]: + """Register a Nexus endpoint routed to ``task_queue`` for one test's life.""" + created = await client.operator_service.create_nexus_endpoint( + CreateNexusEndpointRequest( + spec=EndpointSpec( + name=name, + target=EndpointTarget( + worker=EndpointTarget.Worker( + namespace=client.namespace, task_queue=task_queue + ) + ), + ) + ) + ) + try: + yield name + finally: + await client.operator_service.delete_nexus_endpoint( + DeleteNexusEndpointRequest( + id=created.endpoint.id, version=created.endpoint.version + ) + ) + + @live_only async def test_interface_loop_through_the_nexus_front(): # The workflow worker and the handler worker both use the storage # provider; only the caller goes through the front, and it names the # endpoint the way an operator does, by name. store = WorkflowStreamsProvider() - client = await Client.connect(os.environ.get("TEMPORAL_ADDRESS", "localhost:7233")) - front = NexusStreams( - endpoint=ENDPOINT, - http_address=os.environ.get("TEMPORAL_HTTP", "http://127.0.0.1:7243"), - read_wait=timedelta(seconds=5), - ) + client = await _live_client() workflow_id = f"streams-nexus-live-{uuid.uuid4().hex}" handler = TemporalStreamsHandler(store, client) + handler_tq = f"handlers-{workflow_id}" - async with Worker(client, task_queue=HANDLER_TQ, nexus_service_handlers=[handler]): + async with ( + _own_endpoint(client, workflow_id, handler_tq) as endpoint, + Worker(client, task_queue=handler_tq, nexus_service_handlers=[handler]), + ): + front = _live_front(endpoint) async with Worker( client, task_queue=f"tq-{workflow_id}", @@ -319,6 +363,10 @@ async def test_without_a_codec_the_same_records_go_out_in_the_clear( assert b"tuna" in _answered_bodies(answered) +def _ref(workflow_id: str, topic: str = INPUTS) -> WireStreamRef: + return WireStreamRef(kind="workflow", workflow_id=workflow_id, topic=topic) + + def _append( workflow_id: str, batch_index: int, @@ -328,8 +376,7 @@ def _append( ) -> AppendInput: converter = temporalio.converter.DataConverter.default.payload_converter return AppendInput( - workflow_id=workflow_id, - topic=INPUTS, + stream=_ref(workflow_id), producer_id="model", attempt=1, # Batches of one, numbered in step with the batch index unless a @@ -515,7 +562,7 @@ async def test_consecutive_reads_share_one_parked_subscription(): first = await _dispatch( handler, READ_OPERATION, - ReadInput(workflow_id="wf", topic=INPUTS, max_records=1, wait_ms=200), + ReadInput(stream=_ref("wf"), max_records=1, wait_ms=200), ) assert len(first.records) == 1 parked = handler._subscriptions # pyright: ignore[reportPrivateUsage] @@ -526,8 +573,7 @@ async def test_consecutive_reads_share_one_parked_subscription(): handler, READ_OPERATION, ReadInput( - workflow_id="wf", - topic=INPUTS, + stream=_ref("wf"), after_token=first.next_token, max_records=1, wait_ms=200, @@ -540,7 +586,7 @@ async def test_consecutive_reads_share_one_parked_subscription(): await _dispatch( handler, READ_OPERATION, - ReadInput(workflow_id="wf", topic=INPUTS, wait_ms=200), + ReadInput(stream=_ref("wf"), wait_ms=200), ) assert len(parked) == 1 and next(iter(parked.values())).pump is not pump await handler.close() @@ -552,7 +598,7 @@ async def test_the_read_wait_is_cut_to_the_request_deadline(): started = asyncio.get_running_loop().time() answer = await handler.read( _context(READ_OPERATION, datetime.now(timezone.utc) + timedelta(seconds=1)), - ReadInput(workflow_id="wf", topic=INPUTS, wait_ms=60000), + ReadInput(stream=_ref("wf"), wait_ms=60000), ) assert answer.records == [] and answer.next_token == "" and not answer.done assert asyncio.get_running_loop().time() - started < 5 @@ -711,7 +757,7 @@ async def test_a_failed_read_is_not_reported_as_the_end_of_the_stream(): .append({"n": 1}) ) - request = ReadInput(workflow_id="wf", topic=INPUTS, wait_ms=200, max_records=10) + request = ReadInput(stream=_ref("wf"), wait_ms=200, max_records=10) with pytest.raises(RuntimeError, match="the store went away"): await _dispatch(handler, READ_OPERATION, request) # The retry from the same token has to re-subscribe rather than be @@ -733,7 +779,7 @@ async def test_the_handler_does_not_grow_a_lock_per_address(): await _dispatch( handler, READ_OPERATION, - ReadInput(workflow_id=workflow_id, topic=INPUTS, wait_ms=0, max_records=1), + ReadInput(stream=_ref(workflow_id), wait_ms=0, max_records=1), ) gc.collect() # Held weakly, so an address nobody is reading or appending to leaves @@ -821,8 +867,7 @@ async def test_the_front_carries_every_read_start(monkeypatch: pytest.MonkeyPatc handler, READ_OPERATION, ReadInput( - workflow_id="wf-starts", - topic=INPUTS, + stream=_ref("wf-starts"), after_token=newest[0].cursor.token, last_n=1, ), @@ -831,12 +876,70 @@ async def test_the_front_carries_every_read_start(monkeypatch: pytest.MonkeyPatc await handler.close() -async def test_a_stream_an_activity_owns_is_refused(client: Client): - # The endpoint's operations address a stream by workflow, so the front has - # no way to reach one an activity owns. The refusal is the documented - # error, not an AttributeError or a stream silently put somewhere else. +async def test_an_activity_owner_crosses_the_endpoint(monkeypatch: pytest.MonkeyPatch): + # The reference carries the owner, so an activity's own stream is reached + # through the same two operations and lands on the store's activity + # accessor, apart from the workflow's topics of the same name. + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) front = _front(None) + own = front.get_activity_stream_handle(None, "act", workflow_id="wf") + await own.producer(topic=INPUTS, producer_id="model", attempt=1).append({"n": 1}) + records = await take(own.read(topic=INPUTS, result_type=dict), 1, timeout=30) + assert records[0].value == {"n": 1} + assert await _stored(store, "wf") == [] + stored = store.get_activity_stream_handle(None, "act", workflow_id="wf") + assert [r.value async for r in _take_data(stored, 1)] == [{"n": 1}] + await handler.close() + + +async def _take_data(stream: Any, count: int) -> AsyncIterator[Any]: + async for record in stream.read(topic=INPUTS, result_type=dict): + if record.kind is RecordKind.DATA: + yield record + count -= 1 + if count == 0: + return + + +async def test_an_owner_the_store_cannot_host_is_refused_under_its_own_class( + client: Client, monkeypatch: pytest.MonkeyPatch +): + # Whether an owner is supported is the store's answer, not the front's: + # the Workflow Streams store has no activity accessor, and its refusal + # reaches the caller as the class it raised. + handler = TemporalStreamsHandler(WorkflowStreamsProvider(), client) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + own = _front(None).get_activity_stream_handle(None, "act", workflow_id="wf") with pytest.raises(StreamUnsupportedError, match="activity"): - front.get_activity_stream_handle(client, "act") + await own.producer(topic=INPUTS, producer_id="model", attempt=1).append( + {"n": 1} + ) with pytest.raises(StreamUnsupportedError, match="activity"): - front.get_activity_stream_handle(client, "act", workflow_id="wf") + await take(own.read(topic=INPUTS), 1) + # No provider addresses a standalone stream yet, so that owner is refused + # for every store. + standalone = WireStreamRef(kind="standalone", stream_id="s-1", topic=INPUTS) + with pytest.raises(nexusrpc.HandlerError) as refused: + await _dispatch(handler, READ_OPERATION, ReadInput(stream=standalone)) + assert str(refused.value).startswith("StreamUnsupportedError: ") + await handler.close() + + +async def test_a_reference_missing_its_owner_id_is_the_callers_fault(): + handler = TemporalStreamsHandler(MemoryStreams(), None) + for ref in ( + WireStreamRef(kind="workflow", topic=INPUTS), + WireStreamRef(kind="activity", workflow_id="wf", topic=INPUTS), + WireStreamRef(kind="standalone", topic=INPUTS), + WireStreamRef(kind="workflow", workflow_id="wf", topic=""), + ): + with pytest.raises(nexusrpc.HandlerError) as failed: + await _dispatch(handler, READ_OPERATION, ReadInput(stream=ref)) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + with pytest.raises(nexusrpc.HandlerError) as failed: + request = _append("wf", 1, {"n": 1}) + request.stream = ref + await _dispatch(handler, APPEND_OPERATION, request) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST From b8ca5b8ac4b36f88ea19a44169aba953d1b301f4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 30 Sep 2026 02:40:12 -0700 Subject: [PATCH 25/34] Handed a stream across a Nexus operation as a StreamRef. The front's handle names its stream as the SDK's StreamRef, the front implements the three owner accessors so client.get_stream_handle(ref) opens a ref through the endpoint, and the handler asks the store for a standalone stream by its own accessor. The live test is June scenario s8 (b) without the emulation: an operation returns a ref, another appends to one. --- temporalio/streams/providers/nexus.py | 102 +++++++++-- tests/streams/test_nexus_provider.py | 253 ++++++++++++++++++++++++++ 2 files changed, 342 insertions(+), 13 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index c22118f544..27ac56da3e 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -73,7 +73,7 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Any, Generic, Literal, NoReturn, TypeVar, cast +from typing import Any, Generic, NoReturn, TypeVar, cast import nexusrpc import nexusrpc.handler @@ -87,6 +87,7 @@ from temporalio.common import RawValue from temporalio.service import ConnectConfig, RPCError, RPCStatusCode, ServiceClient from temporalio.streams._errors import ( + StreamClosedError, StreamCursorError, StreamError, StreamNotFoundError, @@ -102,6 +103,7 @@ StreamRecord, check_read_start, ) +from temporalio.streams._ref import StreamOwnerKind, StreamRef from temporalio.streams._topic import StreamTopic, resolve_topic from temporalio.streams._wire import ( RecordDecoder, @@ -129,7 +131,6 @@ T = TypeVar("T") -_Kind = Literal["workflow", "activity", "standalone"] # The reference's identity on the handler, topic included, so one parked read # and one producer state key on exactly what the wire names. _StreamKey = tuple[str, str, str, str, str, str] @@ -182,6 +183,7 @@ StreamNotFoundError, StreamCursorError, StreamProducerError, + StreamClosedError, StreamUnsupportedError, ) } @@ -236,13 +238,13 @@ def _stream_key(ref: WireStreamRef) -> _StreamKey: class _Address: """The owner a handle is bound to; a topic completes it into a reference.""" - kind: _Kind + kind: StreamOwnerKind workflow_id: str | None = None run_id: str | None = None activity_id: str | None = None stream_id: str | None = None - def ref(self, topic: str) -> WireStreamRef: + def wire(self, topic: str) -> WireStreamRef: return WireStreamRef( kind=self.kind, workflow_id=self.workflow_id, @@ -252,6 +254,18 @@ def ref(self, topic: str) -> WireStreamRef: topic=topic, ) + def ref(self, topic: str) -> StreamRef: + # The same members under the SDK's own name, so what an operation + # returns is what client.get_stream_handle(ref) opens. + return StreamRef( + self.kind, + topic, + workflow_id=self.workflow_id, + run_id=self.run_id, + activity_id=self.activity_id, + stream_id=self.stream_id, + ) + def _handler_error(error: StreamError) -> nexusrpc.HandlerError: """Carry a stream condition across the endpoint under its own class name.""" @@ -355,12 +369,8 @@ def _stream(self, ref: WireStreamRef) -> StreamHandle: workflow_id=ref.workflow_id or None, run_id=ref.run_id or None, ) - # The provider protocol reaches a stream through its owning execution; - # a standalone stream has none, so no store behind the endpoint can be - # asked for it yet. - raise StreamUnsupportedError( - "the store behind the endpoint cannot host a standalone stream: a " - "provider addresses a stream by its owning workflow or activity" + return self._provider.get_standalone_stream_handle( + client, cast(str, ref.stream_id) ) @nexusrpc.handler.sync_operation @@ -966,7 +976,7 @@ async def _read( ) token = after.token last_n = last - stream = self._address.ref(topic) + stream = self._address.wire(topic) while True: answer = await self._front.invoke( _READ_OPERATION, @@ -1005,7 +1015,7 @@ async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor topic, _ = resolve_topic(topic) answer = await self._front.invoke( _READ_OPERATION, - ReadInput(stream=self._address.ref(topic), latest_only=True), + ReadInput(stream=self._address.wire(topic), latest_only=True), ReadOutput, timeout=_APPEND_TIMEOUT, ) @@ -1023,7 +1033,37 @@ def producer( topic, _ = resolve_topic(topic) producer_id, attempt = producer_identity(producer_id, attempt) return NexusProducer( - self._front, self._address.ref(topic), producer_id, attempt + self._front, self._address.wire(topic), producer_id, attempt + ) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A :class:`temporalio.streams.StreamRef` to ``topic`` of this owner. + + Plain data naming the owner as this handle addresses it and the + topic, with no cursor and no endpoint, so an operation can return it + and whoever receives it opens it with ``client.get_stream_handle(ref)`` + on the front or on the store itself. + """ + name, _ = resolve_topic(topic) + return self._address.ref(name) + + async def close(self) -> None: + """Refused: the contract carries no operation that seals a stream. + + Raises: + ValueError: The handle is on a workflow's or an activity's stream, + which ends with its owner. + StreamUnsupportedError: The handle is on a standalone stream; seal + it on the store's own handle. + """ + if self._address.kind != "standalone": + raise ValueError( + "only a standalone stream can be closed; this handle is on an owner's " + "stream, which ends when the owner does" + ) + raise StreamUnsupportedError( + "the stream endpoint has no operation that seals a standalone stream; " + "close it on the store's own handle" ) @@ -1131,6 +1171,42 @@ def get_activity_stream_handle( _Address("activity", workflow_id, run_id, activity_id), ) + def get_standalone_stream_handle( + self, client: Client | None, stream_id: str + ) -> NexusStreamHandle: + """A handle on the standalone stream ``stream_id``, through the endpoint. + + The reference carries the standalone owner across; the store behind + the endpoint answers whether it hosts one, and a refusal reaches the + caller as :class:`temporalio.streams.StreamUnsupportedError` on the + first call. + """ + if not stream_id: + raise ValueError("stream_id must not be empty") + return NexusStreamHandle( + _Front(self, client), _Address("standalone", stream_id=stream_id) + ) + + async def create_standalone_stream( + self, + client: Client | None, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> NoReturn: + """Refused: the contract carries no operation that creates a stream. + + Raises: + StreamUnsupportedError: Always. Create the stream on the store + behind the endpoint and hand its reference to callers. + """ + raise StreamUnsupportedError( + "the stream endpoint has no operation that creates a standalone stream; " + "create it on the store behind the endpoint and pass its StreamRef" + ) + async def close(self) -> None: """Nothing to release: each call opens and closes its own connection.""" diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 0d5a0f43b6..8982441adf 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -27,6 +27,7 @@ import os import uuid from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Any, cast from unittest import mock @@ -36,6 +37,8 @@ import pytest import temporalio.converter +from temporalio import nexus as temporal_nexus +from temporalio import streams, workflow from temporalio.api.common.v1 import Payload from temporalio.api.nexus.v1 import EndpointSpec, EndpointTarget from temporalio.api.operatorservice.v1 import ( @@ -43,6 +46,7 @@ DeleteNexusEndpointRequest, ) from temporalio.client import Client +from temporalio.common import WorkflowIDConflictPolicy from temporalio.service import RPCError, RPCStatusCode from temporalio.streams import ( BEGINNING, @@ -51,8 +55,10 @@ RecordKind, StreamCursorError, StreamProducerError, + StreamRef, StreamUnsupportedError, ) +from temporalio.streams._ref import open_ref from temporalio.streams._wire import WireRecord from temporalio.streams.providers import nexus from temporalio.streams.providers._nexus_generated import AppendInput, ReadInput @@ -943,3 +949,250 @@ async def test_a_reference_missing_its_owner_id_is_the_callers_fault(): request.stream = ref await _dispatch(handler, APPEND_OPERATION, request) assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + + +async def test_a_handle_names_its_stream_as_a_ref_that_opens_on_the_front( + monkeypatch: pytest.MonkeyPatch, +): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + front = _front(None) + handle = front.get_stream_handle(None, "wf", run_id="r1") + ref = handle.ref(topic=INPUTS) + assert ref == StreamRef.for_workflow("wf", run_id="r1", topic=INPUTS) + # Plain data: the default converter carries it the way an operation + # result or a workflow argument is carried. + converter = temporalio.converter.DataConverter.default.payload_converter + carried = converter.from_payloads([converter.to_payloads([ref])[0]], [StreamRef]) + assert carried == [ref] + # Opened on the front, the ref reaches the same stream and its topic is + # the handle's default; the client accessor goes through open_ref. + opened = open_ref(front, cast(Client, None), ref) + await opened.producer(producer_id="model", attempt=1).append({"n": 1}) + records = await take(handle.read(topic=INPUTS, result_type=dict), 1, timeout=30) + assert records[0].value == {"n": 1} + assert opened.ref() == ref + # Every owner kind names itself. + assert front.get_activity_stream_handle( + None, "act", workflow_id="wf" + ).ref() == StreamRef.for_activity("act", workflow_id="wf") + assert front.get_standalone_stream_handle(None, "s-1").ref( + topic="t" + ) == StreamRef.for_standalone("s-1", topic="t") + await handler.close() + + +async def test_the_front_refuses_what_the_contract_cannot_carry(): + # No create and no seal operation: an owned stream cannot be closed by + # anyone, and a standalone stream is created and sealed on the store. + front = _front(None) + with pytest.raises(ValueError, match="only a standalone stream"): + await front.get_stream_handle(None, "wf").close() + with pytest.raises(StreamUnsupportedError, match="seals"): + await front.get_standalone_stream_handle(None, "s-1").close() + with pytest.raises(StreamUnsupportedError, match="creates"): + await front.create_standalone_stream(None, "s-1") + with pytest.raises(ValueError, match="stream_id"): + front.get_standalone_stream_handle(None, "") + + +# --------------------------------------------------------------------------- +# A stream as an operation result and as an operation input (June s8 b). +# --------------------------------------------------------------------------- + + +@dataclass +class ScoreUpdate: + """One score change.""" + + home: int + away: int + + +@dataclass +class GameRequest: + """Which game to start, and where its workflow runs.""" + + game_id: str + task_queue: str + + +@dataclass +class ScorePost: + """A score for the stream given, and whether it is the last one.""" + + stream: StreamRef + score: ScoreUpdate + last: bool = False + + +@dataclass +class CallerInput: + """The endpoint to call, the game to ask for, and the scores to post.""" + + endpoint: str + game_id: str + task_queue: str + scores: list[ScoreUpdate] + + +SCORES = streams.topic("scores", ScoreUpdate) +COMMANDS = streams.topic("commands", ScoreUpdate) + + +@workflow.defn +class ScoreboardGame: + """Reads posted scores on ``commands`` and publishes each on ``scores``.""" + + @workflow.run + async def run(self) -> int: + """Relay every posted score until the poster finishes, then finish.""" + posted = 0 + scores = workflow.stream_writer(SCORES) + async for record in workflow.stream_reader(COMMANDS): + if record.kind is RecordKind.FINISH: + break + if record.kind is RecordKind.DATA: + assert record.value is not None + scores.publish(record.value) + posted += 1 + scores.finish() + return posted + + +@nexusrpc.service +class ScoreboardService: + """Starts a game and hands back its stream; takes a stream to post to.""" + + start_game: nexusrpc.Operation[GameRequest, StreamRef] + post_score: nexusrpc.Operation[ScorePost, None] + + +@nexusrpc.handler.service_handler(service=ScoreboardService) +class ScoreboardHandler: + """Serves the two operations as a client of the stream front.""" + + @nexusrpc.handler.sync_operation + async def start_game( + self, _ctx: nexusrpc.handler.StartOperationContext, input: GameRequest + ) -> StreamRef: + """Start the game workflow and return a ref to its ``scores`` topic.""" + client = temporal_nexus.client() + # Reusing a running game makes a retried start hand back the same ref. + await client.start_workflow( + ScoreboardGame.run, + id=input.game_id, + task_queue=input.task_queue, + id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING, + ) + return client.get_stream_handle(input.game_id).ref(topic=SCORES) + + @nexusrpc.handler.sync_operation + async def post_score( + self, ctx: nexusrpc.handler.StartOperationContext, input: ScorePost + ) -> None: + """Append the score to the stream the caller named.""" + # The request id is the producer, so a retried post writes once. + producer = ( + temporal_nexus.client() + .get_stream_handle(input.stream) + .producer(producer_id=f"post-{ctx.request_id}", attempt=1) + ) + await producer.append(input.score) + if input.last: + await producer.finish() + + +@workflow.defn +class ScoreboardCaller: + """Calls the operations and passes the ref on; it never reads the stream.""" + + @workflow.run + async def run(self, input: CallerInput) -> StreamRef: + """Start a game, post the scores to it, and return where to read them.""" + scoreboard = workflow.create_nexus_client( + service=ScoreboardService, endpoint=input.endpoint + ) + ref = await scoreboard.execute_operation( + ScoreboardService.start_game, + GameRequest(input.game_id, input.task_queue), + ) + for index, score in enumerate(input.scores): + await scoreboard.execute_operation( + ScoreboardService.post_score, + ScorePost( + ref.with_topic(COMMANDS), score, last=index == len(input.scores) - 1 + ), + ) + return ref + + +@live_only +async def test_an_operation_returns_a_stream_ref_and_another_takes_one(): + # June s8 (b) without the emulation: the operation's result type is the + # SDK's StreamRef, the calling workflow passes it on as data, the client + # opens it on the front it carries and long-polls it, and a second + # operation appends to a stream it was given as input. The handlers are + # clients of the front like any process; only the game workflow's worker + # holds the store. + store = WorkflowStreamsProvider() + plain = await _live_client() + game_id = f"streams-nexus-ref-{uuid.uuid4().hex}" + handler_tq = f"handlers-{game_id}" + stream_handler = TemporalStreamsHandler(store, plain) + + async with ( + _own_endpoint(plain, game_id, handler_tq) as endpoint, + Worker( + plain, + task_queue=handler_tq, + nexus_service_handlers=[stream_handler], + ), + ): + front = _live_front(endpoint) + config = plain.config() + config["plugins"] = [front] + fronted = Client(**config) + async with ( + Worker( + fronted, + task_queue=f"scoreboard-{game_id}", + nexus_service_handlers=[ScoreboardHandler()], + ), + Worker( + plain, + task_queue=f"tq-{game_id}", + workflows=[ScoreboardGame, ScoreboardCaller], + plugins=[store], + ), + ): + # The scoreboard service sits behind its own endpoint. + async with _own_endpoint( + plain, f"scoreboard-{game_id}", f"scoreboard-{game_id}" + ) as scoreboard_endpoint: + posted = [ScoreUpdate(1, 0), ScoreUpdate(1, 1), ScoreUpdate(2, 1)] + ref = await plain.execute_workflow( + ScoreboardCaller.run, + CallerInput(scoreboard_endpoint, game_id, f"tq-{game_id}", posted), + id=f"{game_id}-caller", + task_queue=f"tq-{game_id}", + ) + assert ref == StreamRef.for_workflow(game_id, topic=SCORES) + + # The ref is plain data the client opens on its own provider, + # here the front, and reads with the endpoint's long poll. It + # names the topic, not the record type, so the reader says + # what to decode as, the way a runtime topic name does. + records = await take( + fronted.get_stream_handle(ref).read(result_type=ScoreUpdate), 4, 60 + ) + assert [record.kind for record in records] == [ + RecordKind.DATA, + RecordKind.DATA, + RecordKind.DATA, + RecordKind.FINISH, + ] + assert [record.value for record in records[:3]] == posted + assert await plain.get_workflow_handle(game_id).result() == 3 + await stream_handler.close() From f7fcf03a0e89990564e54734f31736c631b8587c Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 30 Sep 2026 02:45:13 -0700 Subject: [PATCH 26/34] Used a standalone activity as the owner Workflow Streams refuses. The store now hosts a workflow's activity streams as reserved topics, so the case that checks a store's refusal reaching the caller under its own class needs an owner it still refuses. --- tests/streams/test_nexus_provider.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 8982441adf..89c2bb8c68 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -913,11 +913,12 @@ async def test_an_owner_the_store_cannot_host_is_refused_under_its_own_class( client: Client, monkeypatch: pytest.MonkeyPatch ): # Whether an owner is supported is the store's answer, not the front's: - # the Workflow Streams store has no activity accessor, and its refusal + # the Workflow Streams store hosts only the streams of a workflow's + # activities, so a standalone activity's are refused, and its refusal # reaches the caller as the class it raised. handler = TemporalStreamsHandler(WorkflowStreamsProvider(), client) monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) - own = _front(None).get_activity_stream_handle(None, "act", workflow_id="wf") + own = _front(None).get_activity_stream_handle(None, "act") with pytest.raises(StreamUnsupportedError, match="activity"): await own.producer(topic=INPUTS, producer_id="model", attempt=1).append( {"n": 1} From 85126ae8346caaa0f1e46b6c4d7161eedd3f4266 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 30 Sep 2026 02:51:16 -0700 Subject: [PATCH 27/34] Typed the absent client in the ref-opening test as Any. basedpyright refuses a cast from None to Client, and the in-process endpoint is reached by id, so the test says outright that it opens without one. --- tests/streams/test_nexus_provider.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 89c2bb8c68..1a61b5728e 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -968,8 +968,10 @@ async def test_a_handle_names_its_stream_as_a_ref_that_opens_on_the_front( carried = converter.from_payloads([converter.to_payloads([ref])[0]], [StreamRef]) assert carried == [ref] # Opened on the front, the ref reaches the same stream and its topic is - # the handle's default; the client accessor goes through open_ref. - opened = open_ref(front, cast(Client, None), ref) + # the handle's default; the client accessor goes through open_ref. The + # in-process endpoint is reached by id, so no client is needed to resolve it. + no_client: Any = None + opened = open_ref(front, no_client, ref) await opened.producer(producer_id="model", attempt=1).append({"n": 1}) records = await take(handle.read(topic=INPUTS, result_type=dict), 1, timeout=30) assert records[0].value == {"n": 1} From a7804c250bb13f0a7f42e5a9b37f0053daacc115 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 1 Oct 2026 09:42:16 -0700 Subject: [PATCH 28/34] Declared the reference's optional members nullable on the wire. The SDK writes null for every member a StreamRef does not use, and a generated caller refuses an explicit null on a member the contract did not make nullable, so a ref an operation returns as plain data could not cross a generated converter. --- .../providers/_nexus_generated/models.py | 19 ++++------- .../providers/temporal_streams.nexusrpc.yaml | 12 ++++--- tests/streams/test_nexus_provider.py | 33 +++++++++++++++++++ 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/temporalio/streams/providers/_nexus_generated/models.py b/temporalio/streams/providers/_nexus_generated/models.py index 033cded90d..fd4c65fb67 100644 --- a/temporalio/streams/providers/_nexus_generated/models.py +++ b/temporalio/streams/providers/_nexus_generated/models.py @@ -883,9 +883,7 @@ def from_transfer_type( if "workflow_id" in raw: workflow_id_value_raw = raw["workflow_id"] if workflow_id_value_raw is None: - violations.append( - Violation(path="workflow_id", reason="explicit null not allowed") - ) + workflow_id_value = None else: if not isinstance(workflow_id_value_raw, str): violations.append( @@ -898,9 +896,7 @@ def from_transfer_type( if "run_id" in raw: run_id_value_raw = raw["run_id"] if run_id_value_raw is None: - violations.append( - Violation(path="run_id", reason="explicit null not allowed") - ) + run_id_value = None else: if not isinstance(run_id_value_raw, str): violations.append( @@ -913,9 +909,7 @@ def from_transfer_type( if "activity_id" in raw: activity_id_value_raw = raw["activity_id"] if activity_id_value_raw is None: - violations.append( - Violation(path="activity_id", reason="explicit null not allowed") - ) + activity_id_value = None else: if not isinstance(activity_id_value_raw, str): violations.append( @@ -928,9 +922,7 @@ def from_transfer_type( if "stream_id" in raw: stream_id_value_raw = raw["stream_id"] if stream_id_value_raw is None: - violations.append( - Violation(path="stream_id", reason="explicit null not allowed") - ) + stream_id_value = None else: if not isinstance(stream_id_value_raw, str): violations.append( @@ -1006,7 +998,8 @@ class StreamRef: """A stream, named by its owner and a topic: what an operation returns to hand a stream to its caller, and what read and append take in place of an owner spelled out. It names no cursor and no store, so the same reference is good behind any endpoint that - serves the owner. + serves the owner. A member the owner kind does not use is absent or null; null is + how the SDK writes an unset member of its own StreamRef. """ kind: typing.Literal["workflow", "activity", "standalone"] diff --git a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml index 61660c8613..1b027cfebc 100644 --- a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml +++ b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml @@ -44,7 +44,9 @@ $defs: A stream, named by its owner and a topic: what an operation returns to hand a stream to its caller, and what read and append take in place of an owner spelled out. It names no cursor and no store, so the same - reference is good behind any endpoint that serves the owner. + reference is good behind any endpoint that serves the owner. A member + the owner kind does not use is absent or null; null is how the SDK + writes an unset member of its own StreamRef. properties: kind: type: string @@ -56,20 +58,20 @@ $defs: A standalone stream has its own stream_id and no execution behind it. workflow_id: - type: string + oneOf: [{ type: string }, { type: "null" }] description: >- The owning workflow, or the workflow that scheduled the owning activity. Required for a workflow owner. run_id: - type: string + oneOf: [{ type: string }, { type: "null" }] description: >- Pin the owner to this run. Absent follows the execution chain across continue-as-new, which is what a producer normally wants. activity_id: - type: string + oneOf: [{ type: string }, { type: "null" }] description: "The owning activity. Required for an activity owner." stream_id: - type: string + oneOf: [{ type: string }, { type: "null" }] description: "The stream's own id. Required for a standalone owner." topic: type: string diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py index 1a61b5728e..efec762c89 100644 --- a/tests/streams/test_nexus_provider.py +++ b/tests/streams/test_nexus_provider.py @@ -986,6 +986,39 @@ async def test_a_handle_names_its_stream_as_a_ref_that_opens_on_the_front( await handler.close() +def test_an_sdk_stream_ref_crosses_the_wire_model_with_its_nulls(): + # The SDK's default converter writes null for every member a ref does not + # use. The wire model declares those members nullable, so a ref an + # operation returned as plain data parses into the same wire reference + # the front builds, and the wire form of that reference reads back as + # the SDK type with nothing lost. + converter = temporalio.converter.DataConverter.default.payload_converter + contract = ( + temporalio.converter.DataConverter.default._get_internal_payload_converter() + ) + for ref in ( + StreamRef.for_workflow("wf", topic=INPUTS), + StreamRef.for_workflow("wf", run_id="r1", topic=INPUTS), + StreamRef.for_activity("act", workflow_id="wf", topic=INPUTS), + StreamRef.for_standalone("s-1", topic=INPUTS), + ): + as_sdk = converter.to_payloads([ref])[0] + assert b"null" in as_sdk.data, as_sdk.data + wire = contract.from_payloads([as_sdk], [WireStreamRef])[0] + assert wire == WireStreamRef( + kind=ref.kind, + topic=ref.topic, + workflow_id=ref.workflow_id, + run_id=ref.run_id, + activity_id=ref.activity_id, + stream_id=ref.stream_id, + ) + back = converter.from_payloads([contract.to_payloads([wire])[0]], [StreamRef])[ + 0 + ] + assert back == ref + + async def test_the_front_refuses_what_the_contract_cannot_carry(): # No create and no seal operation: an owned stream cannot be closed by # anyone, and a standalone stream is created and sealed on the store. From 0b9bf5790075f8e8d88f73a3ebe09f473e5a8865 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 1 Oct 2026 12:19:40 -0700 Subject: [PATCH 29/34] Pointed the Nexus provider docstring at the call that opens a reference. NexusStreams has no open method, so pydoctor found no link target and the docs build failed. --- temporalio/streams/providers/nexus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 27ac56da3e..1cc60d2364 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -36,8 +36,8 @@ :class:`temporalio.streams.StreamUnsupportedError`, so a reference is good behind any endpoint that serves the owner. The same reference is what an operation of the application's own returns to hand a stream to its caller; -:meth:`NexusStreamHandle.ref` makes one and :meth:`NexusStreams.open` opens -it. +:meth:`NexusStreamHandle.ref` makes one and ``client.get_stream_handle(ref)`` +opens it. The handler keeps one parked read per stream reference and serves consecutive calls from it, so an idle caller does not leave one abandoned From c051962c08f090c44d3302b1b97b2206edfeb2dd Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 2 Oct 2026 03:12:35 -0700 Subject: [PATCH 30/34] Added the Nexus operation that consumes a stream through its channel. The handler registers a callback listener on the stream's channel and reads from its cursor on each delivery, so the Nexus and channel designs share one path instead of the handler polling the front. The aiohttp service is the standalone host the PoC demonstrates. --- temporalio/streams/providers/nexus.py | 617 +++++++++++++++++- .../providers/nexus_consumer_service.py | 509 +++++++++++++++ 2 files changed, 1124 insertions(+), 2 deletions(-) create mode 100644 temporalio/streams/providers/nexus_consumer_service.py diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 1cc60d2364..531c1d5155 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -57,33 +57,56 @@ :class:`temporalio.streams.StreamError` the store raised, when the handler named one, and as :class:`temporalio.service.RPCError` otherwise, never as an HTTP or urllib exception. + +The third piece is the consumer side of a Nexus operation. +:class:`StreamConsumerOperation`, built with :func:`stream_consumer_operation`, +is an asynchronous operation handler whose input is a ``StreamRef`` and whose +result is what a consume function folded the stream's records into. It does +not poll. On start it registers a callback listener on the stream's +notification channel, with a URL the hosting process serves and a header that +names the operation, and reads the stream from its start through the client's +provider, the front when the client carries one. The server posts every +notification on the channel to that URL, the handler reads from its cursor to +the head, and the close completes the operation through the caller's +completion callback. The delivery is what the server's channel library posts: +``POST`` to the listener's URL, the ``Notification`` as protobuf JSON with +``Content-Type: application/json``, the listener's own headers and the +channel's name in ``Temporal-Notification-Channel``. A process that hosts the +handler feeds each such request to :meth:`StreamConsumerOperation.deliver`; +``nexus_consumer_service`` in this package is the standalone one. """ from __future__ import annotations import asyncio +import email.utils import http.client +import inspect import json import logging import time import urllib.error import urllib.request +import uuid import weakref from collections import OrderedDict from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any, Generic, NoReturn, TypeVar, cast import nexusrpc import nexusrpc.handler +from google.protobuf import json_format from google.protobuf.message import DecodeError +import temporalio.api.notification.v1 import temporalio.client import temporalio.converter +import temporalio.nexus from temporalio.api.common.v1 import Payload from temporalio.api.operatorservice.v1 import ListNexusEndpointsRequest -from temporalio.client import Client, ClientConfig +from temporalio.client import Callback, Client, ClientConfig from temporalio.common import RawValue from temporalio.service import ConnectConfig, RPCError, RPCStatusCode, ServiceClient from temporalio.streams._errors import ( @@ -120,16 +143,25 @@ TemporalStreams, ) from temporalio.streams.providers._nexus_generated import StreamRef as WireStreamRef +from temporalio.workflow import Notification __all__ = [ + "NOTIFICATION_CHANNEL_HEADER", + "STREAM_CONSUMER_TOKEN_HEADER", + "ConsumerState", + "Delivery", "NexusProducer", "NexusStreamHandle", "NexusStreams", + "StreamConsumerOperation", "TemporalStreamsHandler", "WireStreamRef", + "stream_channel_name", + "stream_consumer_operation", ] T = TypeVar("T") +S = TypeVar("S") # The reference's identity on the handler, topic included, so one parked read # and one producer state key on exactly what the wire names. @@ -1240,3 +1272,584 @@ async def _endpoint_id(self, client: Client | None) -> str: ) self._resolved = found.endpoints[0].id return self._resolved + + +# --------------------------------------------------------------------------- +# A Nexus operation as a stream consumer. +# --------------------------------------------------------------------------- + +NOTIFICATION_CHANNEL_HEADER = "Temporal-Notification-Channel" +"""The header the server adds to a channel's callback delivery, naming the channel.""" + +STREAM_CONSUMER_TOKEN_HEADER = "Temporal-Stream-Consumer-Token" +"""The header a consumer's listener registration carries, naming the operation. + +The server sends a listener's own headers back on every delivery, so this is +how one delivery URL serves any number of operations. +""" + +_COMPLETION_TIMEOUT = timedelta(seconds=30) +_CLOSED_KEY = "closed" +_FAILURE_CONTENT_TYPE = "application/json" +_CONTENT_TYPE_HEADER = "Content-Type" +_STATE_HEADER = "Nexus-Operation-State" +_TOKEN_HEADER = "Nexus-Operation-Token" +_START_TIME_HEADER = "Nexus-Operation-Start-Time" + + +def stream_channel_name(ref: StreamRef) -> str: + """The notification channel a stream's writers notify, from the stream's identity. + + The name folds the owner the ref names and the topic into one string, so + a producer and a consumer that hold the same ref meet on the same + channel. A ref pinned to a run names a different channel from one that + follows the chain, because the two name different streams. + """ + parts = ["stream", ref.kind] + if ref.kind == "workflow": + parts += [ref.workflow_id or "", ref.run_id or ""] + elif ref.kind == "activity": + parts += [ref.workflow_id or "", ref.run_id or "", ref.activity_id or ""] + else: + parts.append(ref.stream_id or "") + parts.append(ref.topic) + return ":".join(parts) + + +ConsumeFunction = Callable[[StreamRecord[Any], S], "S | Awaitable[S]"] + + +@dataclass(frozen=True) +class ConsumerState(Generic[S]): + """What a consumer operation holds for one token, read-only. + + .. warning:: + This API is experimental and unstable. + """ + + ref: StreamRef + """The stream being consumed.""" + + channel: str + """The channel the listener is registered on.""" + + listener_id: str + """The listener id the server assigned.""" + + cursor: Cursor + """Where the next read starts. :data:`BEGINNING` until a record was consumed.""" + + value: S + """What the consume function has folded the records into so far.""" + + reads: int + """How many read passes reached the store, the start's included.""" + + deliveries: int + """How many notifications were delivered for this operation.""" + + records: int + """How many records the consume function has been handed.""" + + +@dataclass(frozen=True) +class Delivery: + """What one delivered notification led to. + + .. warning:: + This API is experimental and unstable. + """ + + token: str | None + """The operation the delivery named, when the headers carried one.""" + + known: bool + """Whether this process holds that operation. A delivery for one it does not is ignored.""" + + read: bool + """Whether the delivery led to a read. One that brought nothing new does not.""" + + records: int + """How many records that read handed to the consume function.""" + + completed: bool + """Whether the delivery closed the operation.""" + + +@dataclass +class _Consumption(Generic[S]): + ref: StreamRef + channel: str + listener_id: str + callback_url: str | None + callback_headers: dict[str, str] + started: datetime + value: S + cursor: Cursor = BEGINNING + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + reads: int = 0 + deliveries: int = 0 + records: int = 0 + finished: bool = False + done: bool = False + opening: asyncio.Task[None] | None = None + + +def _header(headers: Mapping[str, str], name: str) -> str | None: + wanted = name.lower() + for key, value in headers.items(): + if key.lower() == wanted: + return value + return None + + +def _parse_notification(body: bytes | str | Mapping[str, Any]) -> Notification: + """The notification a delivery carries, from the JSON the server posts.""" + text = json.dumps(body) if isinstance(body, Mapping) else body + if isinstance(text, bytes): + text = text.decode() + proto = json_format.Parse( + text, temporalio.api.notification.v1.Notification(), ignore_unknown_fields=True + ) + return Notification._from_proto(proto) + + +def _payload_content(payload: Payload) -> tuple[dict[str, str], bytes]: + """The HTTP content headers and body that carry ``payload`` to the server. + + The mapping is the server's: a plain JSON payload travels as JSON, a null + one as an empty body with no type, and anything else, a codec's output + included, as the serialized payload itself. + """ + metadata = {key: value.decode() for key, value in payload.metadata.items()} + encoding = metadata.get("encoding") + if set(metadata) == {"encoding"}: + if encoding == "json/plain": + return {_CONTENT_TYPE_HEADER: "application/json"}, payload.data + if encoding == "binary/plain": + return {_CONTENT_TYPE_HEADER: "application/octet-stream"}, payload.data + if encoding == "binary/null": + return {}, b"" + return ( + {_CONTENT_TYPE_HEADER: "application/x-temporal-payload"}, + payload.SerializeToString(), + ) + + +def _post_completion( + url: str, body: bytes, headers: Mapping[str, str], timeout: timedelta +) -> None: + """Post an operation completion to the caller's callback URL. + + Unlike :func:`_post` this sets no content type of its own, because a null + result travels without one. + + Raises: + _EndpointFailure: The callback answered with an error or not at all. + """ + try: + request = urllib.request.Request( + url, data=body or None, headers=dict(headers), method="POST" + ) + with urllib.request.urlopen(request, timeout=timeout.total_seconds()): + return + except urllib.error.HTTPError as error: + raise _EndpointFailure( + error.read().decode(errors="replace"), error.code + ) from error + except urllib.error.URLError as error: + raise _EndpointFailure( + f"completion callback unreachable at {url}: {error.reason}", None + ) from error + except (TimeoutError, http.client.HTTPException, OSError, ValueError) as error: + raise _EndpointFailure( + f"completion callback at {url} did not answer: {error!r}", None + ) from error + + +class StreamConsumerOperation(nexusrpc.handler.OperationHandler[StreamRef, S]): + """An asynchronous operation that consumes the stream its input names. + + The caller starts it with a :class:`temporalio.streams.StreamRef` and + awaits its result. On start the handler registers a callback listener on + the stream's notification channel, with the URL the hosting process + serves and this operation's token in a header, and reads the stream from + its start through the client's provider. Every notification the server + delivers leads to one read from the cursor to the head, however many + writes the channel folded into it, and a delivery that brings nothing new + reads nothing. The operation completes through the caller's completion + callback when the notification says the stream closed, when the read + reaches ``FINISH`` or when the store ends the read, with what the consume + function folded the records into. Cancel unregisters the listener and + reports the operation canceled. + + The consume function is a reducer: it takes a record and the value so far + and returns the new value, awaitable or not. It is handed every record + kind and narrows on ``kind`` itself. An exception from it fails the + operation. The cursor and the value live in this process keyed by the + operation token, so a process that loses them loses the operation; a + durable cursor is a follow-up. + + .. warning:: + This API is experimental and unstable. + """ + + def __init__( + self, + consume: ConsumeFunction[S], + *, + initial: Callable[[], S], + listener_url: str, + client: Client | None = None, + channel_for: Callable[[StreamRef], str] = stream_channel_name, + result_type: type | None = None, + token_header: str = STREAM_CONSUMER_TOKEN_HEADER, + ) -> None: + """Consume with ``consume``, served at ``listener_url``. + + Args: + consume: The reducer each record is handed with the value so far. + initial: Makes the value an operation starts with. + listener_url: Where the server posts the channel's notifications. + The hosting process serves it and feeds each request to + :meth:`deliver`. + client: Registers the listener and opens the stream. Leave it + unset in a handler hosted by a Temporal worker, where the + operation context's client is used. + channel_for: Names the channel for a ref. The default is + :func:`stream_channel_name`; a store that names channels + its own way passes its rule. + result_type: What record values are decoded as. + token_header: The header carrying the operation token on the + registration and so on every delivery. + """ + if not listener_url: + raise ValueError("listener_url must not be empty") + self._consume = consume + self._initial = initial + self._listener_url = listener_url + self._given_client = client + self._channel_for = channel_for + self._result_type = result_type + self._token_header = token_header + self._states: dict[str, _Consumption[S]] = {} + + @property + def tokens(self) -> list[str]: + """The operations this process holds, oldest first.""" + return list(self._states) + + def state(self, token: str) -> ConsumerState[S] | None: + """What is held for ``token``, or ``None`` once it completed or never was.""" + held = self._states.get(token) + if held is None: + return None + return ConsumerState( + ref=held.ref, + channel=held.channel, + listener_id=held.listener_id, + cursor=held.cursor, + value=held.value, + reads=held.reads, + deliveries=held.deliveries, + records=held.records, + ) + + def _client(self) -> Client: + if self._given_client is not None: + return self._given_client + return temporalio.nexus.client() + + async def start( + self, ctx: nexusrpc.handler.StartOperationContext, input: StreamRef + ) -> nexusrpc.handler.StartOperationResultAsync: + """Register as the stream's listener and start reading it. + + The read runs after the start answers, so a long stream does not hold + the start request, and the first delivery waits on it. + """ + client = self._client() + token = uuid.uuid4().hex + channel = self._channel_for(input) + # Registered before the first read: a record appended between the + # read and the registration would otherwise be missed, where one + # appended between the registration and the read is read twice at + # worst, once by the read and once by the delivery it provokes, and + # the cursor makes the second a no-op. + listener_id = await client.register_channel_listener( + channel, + Callback(url=self._listener_url, headers={self._token_header: token}), + ) + state: _Consumption[S] = _Consumption( + ref=input, + channel=channel, + listener_id=listener_id, + callback_url=ctx.callback_url, + callback_headers=dict(ctx.callback_headers), + started=datetime.now(timezone.utc), + value=self._initial(), + ) + self._states[token] = state + state.opening = asyncio.create_task(self._open(token, state, client)) + return nexusrpc.handler.StartOperationResultAsync(token) + + async def _open(self, token: str, state: _Consumption[S], client: Client) -> None: + try: + async with state.lock: + if state.done: + return + await self._catch_up(token, state, client) + if state.finished and not state.done: + await self._complete(token, state, client) + except Exception: + # The next delivery reads again from the cursor, so a failure here + # costs nothing but the records it would have consumed early. + logger.exception("the first read of %s failed", state.ref) + + async def deliver( + self, headers: Mapping[str, str], body: bytes | str | Mapping[str, Any] + ) -> Delivery: + """Act on one notification the server posted to the listener URL. + + ``headers`` are the request's, in any case; ``body`` is the + ``Notification`` as protobuf JSON, raw or already parsed. The delivery + is matched to its operation by the token header. Reading from the + cursor to the head happens under the operation's lock, so a delivery + that arrives during the first read waits for it and then finds the + cursor at the head. + + A failure to read raises, so the hosting process answers the server + with an error and the server retries the delivery; a failure in the + consume function fails the operation instead. + """ + token = _header(headers, self._token_header) + state = self._states.get(token) if token is not None else None + if token is None or state is None: + logger.warning( + "ignoring a channel delivery for an operation this process does not hold" + ) + return Delivery(token, known=False, read=False, records=0, completed=False) + client = self._client() + notification = _parse_notification(body) + closed = await self._closed(notification, client) + async with state.lock: + if state.done: + return Delivery(token, True, read=False, records=0, completed=True) + state.deliveries += 1 + reads_before = state.reads + records = await self._catch_up(token, state, client) + completed = False + if (closed or state.finished) and not state.done: + await self._complete(token, state, client) + completed = True + return Delivery( + token, + True, + read=state.reads > reads_before, + records=records, + completed=completed, + ) + + async def _closed(self, notification: Notification, client: Client) -> bool: + payload = notification.metadata.get(_CLOSED_KEY) + if payload is None: + return False + try: + converter = client.data_converter + if converter.payload_codec is not None: + [payload] = await converter.payload_codec.decode([payload]) + return bool(converter.payload_converter.from_payload(payload, bool)) + except Exception: + logger.warning( + "could not decode the notification's closed flag", exc_info=True + ) + return False + + async def _catch_up( + self, token: str, state: _Consumption[S], client: Client + ) -> int: + """Read from the cursor to the head, consuming, and return how many records.""" + handle = client.get_stream_handle(state.ref) + head = await handle.latest() + if head == state.cursor: + return 0 + state.reads += 1 + source = handle.read(after=state.cursor, result_type=self._result_type) + count = 0 + exhausted = True + try: + async for record in source: + try: + value = self._consume(record, state.value) + if inspect.isawaitable(value): + value = await value + except Exception as error: + await self._complete(token, state, client, error=error) + return count + state.value = cast(S, value) + state.cursor = record.cursor + state.records += 1 + count += 1 + if record.kind is RecordKind.FINISH: + state.finished = True + exhausted = False + break + if record.cursor == head: + exhausted = False + break + finally: + await source.aclose() + if exhausted: + # The store ended the read: nothing more will arrive on it. + state.finished = True + return count + + async def _complete( + self, + token: str, + state: _Consumption[S], + client: Client, + *, + error: BaseException | None = None, + ) -> None: + state.done = True + if error is None: + await self._post_result(token, state, client) + else: + await self._post_failure( + token, state, "failed", f"{type(error).__name__}: {error}" + ) + await self._unregister(client, state) + self._states.pop(token, None) + + async def cancel( + self, ctx: nexusrpc.handler.CancelOperationContext, token: str + ) -> None: + """Unregister the listener and report the operation canceled. + + Raises: + nexusrpc.HandlerError: ``NOT_FOUND`` when this process holds no + operation under ``token``. + """ + del ctx + state = self._states.get(token) + if state is None: + raise nexusrpc.HandlerError( + "no stream consumer operation is held under this token", + type=nexusrpc.HandlerErrorType.NOT_FOUND, + retryable_override=False, + ) + client = self._client() + if state.opening is not None and not state.opening.done(): + state.opening.cancel() + async with state.lock: + if state.done: + return + state.done = True + await self._unregister(client, state) + await self._post_failure( + token, state, "canceled", "the caller canceled the stream consumer" + ) + self._states.pop(token, None) + + async def close(self) -> None: + """Unregister every listener this process still holds, completing nothing. + + Call it when the hosting process stops, so the server does not keep + posting to a URL nobody serves. + """ + client = self._given_client + for token, state in list(self._states.items()): + if state.opening is not None and not state.opening.done(): + state.opening.cancel() + state.done = True + if client is not None: + await self._unregister(client, state) + self._states.pop(token, None) + + async def _unregister(self, client: Client, state: _Consumption[S]) -> None: + try: + await client.unregister_channel_listener(state.channel, state.listener_id) + except RPCError: + logger.warning( + "could not unregister listener %s from channel %s", + state.listener_id, + state.channel, + exc_info=True, + ) + + async def _post_result( + self, token: str, state: _Consumption[S], client: Client + ) -> None: + converter = client.data_converter + [payload] = converter.payload_converter.to_payloads([state.value]) + if converter.payload_codec is not None: + [payload] = await converter.payload_codec.encode([payload]) + headers, body = _payload_content(payload) + await self._post(token, state, "succeeded", headers, body) + + async def _post_failure( + self, token: str, state: _Consumption[S], outcome: str, message: str + ) -> None: + await self._post( + token, + state, + outcome, + {_CONTENT_TYPE_HEADER: _FAILURE_CONTENT_TYPE}, + json.dumps({"message": message}).encode(), + ) + + async def _post( + self, + token: str, + state: _Consumption[S], + outcome: str, + content: Mapping[str, str], + body: bytes, + ) -> None: + if not state.callback_url: + logger.warning( + "operation %s %s with no completion callback to tell", token, outcome + ) + return + headers = { + **state.callback_headers, + _TOKEN_HEADER: token, + _STATE_HEADER: outcome, + _START_TIME_HEADER: email.utils.format_datetime(state.started, usegmt=True), + **content, + } + try: + await asyncio.to_thread( + _post_completion, state.callback_url, body, headers, _COMPLETION_TIMEOUT + ) + except _EndpointFailure: + logger.exception("could not complete operation %s as %s", token, outcome) + + +def stream_consumer_operation( + consume: ConsumeFunction[S], + *, + initial: Callable[[], S], + listener_url: str, + client: Client | None = None, + channel_for: Callable[[StreamRef], str] = stream_channel_name, + result_type: type | None = None, + token_header: str = STREAM_CONSUMER_TOKEN_HEADER, +) -> StreamConsumerOperation[S]: + """Turn ``consume`` into an asynchronous operation handler that consumes a stream. + + A service handler returns the result from an operation handler factory, + and the hosting process feeds the deliveries it receives at + ``listener_url`` to the same object's :meth:`StreamConsumerOperation.deliver`. + See :class:`StreamConsumerOperation` for the arguments and the contract. + """ + return StreamConsumerOperation( + consume, + initial=initial, + listener_url=listener_url, + client=client, + channel_for=channel_for, + result_type=result_type, + token_header=token_header, + ) diff --git a/temporalio/streams/providers/nexus_consumer_service.py b/temporalio/streams/providers/nexus_consumer_service.py new file mode 100644 index 0000000000..0aea10d466 --- /dev/null +++ b/temporalio/streams/providers/nexus_consumer_service.py @@ -0,0 +1,509 @@ +r"""A standalone Nexus handler process that consumes a stream through its channel. + +The Temporal frontend forwards a caller's ``StartOperation`` to an endpoint +whose target is an external URL, over the Nexus HTTP protocol. This module +serves that protocol with aiohttp for one :class:`nexusrpc.handler.Handler`, +start and cancel, and one more route the server's notification channels post +to, which it hands to the :class:`StreamConsumerOperation` instances it was +given. It is the shape the PoC demonstrates: a process with a URL of its own, +no worker, consuming a stream the server tells it about. + +Run it with one command, against a server that serves channels and a stream +front endpoint named ``streams``:: + + uv run python -m temporalio.streams.providers.nexus_consumer_service \\ + --address 127.0.0.1:7813 --http http://127.0.0.1:7823 \\ + --endpoint streams --port 8813 --register-endpoint consumers + +``--register-endpoint`` creates a Nexus endpoint whose target is this process, +so a workflow calls ``CollectStream.collect`` on it with a ``StreamRef`` and +gets the stream's values back when the stream closes. The server has to allow +this host in ``callback.allowedAddresses``. + +The wire details are the Nexus SDK's: a start is ``POST /{service}/{operation}`` +with the input as the body, the caller's completion callback in the ``callback`` +query parameter and its headers under the ``Nexus-Callback-`` prefix; an +asynchronous start answers ``201`` with the operation token; a cancel is +``POST /{service}/{operation}/cancel`` with the token in +``Nexus-Operation-Token`` and answers ``202``; a handler error answers the +status its type maps to with a JSON failure body. The body of a start is +turned into a payload the way the server does it, by content type, and a +synchronous result goes back the same way. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import re +import uuid +from collections.abc import Mapping, Sequence +from datetime import datetime, timedelta, timezone +from typing import Any + +import nexusrpc +import nexusrpc.handler +from aiohttp import web + +import temporalio.converter +from temporalio.api.common.v1 import Payload +from temporalio.api.nexus.v1 import EndpointSpec, EndpointTarget +from temporalio.api.operatorservice.v1 import CreateNexusEndpointRequest +from temporalio.client import Client +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams._record import RecordKind, StreamRecord +from temporalio.streams._ref import StreamRef +from temporalio.streams.providers.nexus import ( + NexusStreams, + StreamConsumerOperation, + _payload_content, + stream_consumer_operation, +) + +__all__ = [ + "CollectStream", + "CollectStreamHandler", + "NexusHttpService", + "collect_values", + "main", +] + +logger = logging.getLogger(__name__) + +_CALLBACK_PREFIX = "nexus-callback-" +_CONTENT_PREFIX = "content-" +_REQUEST_ID_HEADER = "nexus-request-id" +_REQUEST_TIMEOUT_HEADER = "request-timeout" +_OPERATION_TOKEN_HEADER = "nexus-operation-token" +_OPERATION_STATE_HEADER = "nexus-operation-state" +_RETRYABLE_HEADER = "nexus-request-retryable" +_CALLBACK_QUERY = "callback" +_TOKEN_QUERY = "token" +_DEFAULT_DELIVERIES_PATH = "/deliveries" + +# The status each handler error type answers with, per the Nexus HTTP spec. +_ERROR_STATUS = { + nexusrpc.HandlerErrorType.BAD_REQUEST: 400, + nexusrpc.HandlerErrorType.UNAUTHENTICATED: 401, + nexusrpc.HandlerErrorType.UNAUTHORIZED: 403, + nexusrpc.HandlerErrorType.NOT_FOUND: 404, + nexusrpc.HandlerErrorType.REQUEST_TIMEOUT: 408, + nexusrpc.HandlerErrorType.CONFLICT: 409, + nexusrpc.HandlerErrorType.RESOURCE_EXHAUSTED: 429, + nexusrpc.HandlerErrorType.INTERNAL: 500, + nexusrpc.HandlerErrorType.NOT_IMPLEMENTED: 501, + nexusrpc.HandlerErrorType.UNAVAILABLE: 503, + nexusrpc.HandlerErrorType.UPSTREAM_TIMEOUT: 520, +} +_OPERATION_FAILED_STATUS = 424 + +_DURATION = re.compile(r"(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)") +_UNIT_SECONDS = { + "ns": 1e-9, + "us": 1e-6, + "µs": 1e-6, + "ms": 1e-3, + "s": 1.0, + "m": 60.0, + "h": 3600.0, +} + + +class _NeverCancelled(nexusrpc.handler.OperationTaskCancellation): + """The task cancellation of a request no worker is going to cancel.""" + + def is_cancelled(self) -> bool: + return False + + def cancellation_reason(self) -> str | None: + return None + + def wait_until_cancelled_sync(self, timeout: float | None = None) -> bool: + del timeout + return False + + async def wait_until_cancelled(self) -> None: + await asyncio.Event().wait() + + +def _deadline(timeout: str | None) -> datetime | None: + """The request deadline a Go duration string in ``Request-Timeout`` names.""" + if not timeout: + return None + seconds = 0.0 + matched = False + for amount, unit in _DURATION.findall(timeout): + seconds += float(amount) * _UNIT_SECONDS[unit] + matched = True + if not matched: + return None + return datetime.now(timezone.utc) + timedelta(seconds=seconds) + + +def _media_type(content_type: str) -> tuple[str, dict[str, str]]: + media, _, rest = content_type.partition(";") + params: dict[str, str] = {} + for part in rest.split(";"): + key, _, value = part.strip().partition("=") + if key: + params[key.strip().lower()] = value.strip().strip('"') + return media.strip().lower(), params + + +def _payload_from_content(headers: Mapping[str, str], body: bytes) -> Payload: + """The payload a start body is, mapped the way the server maps Nexus content.""" + content = { + key[len(_CONTENT_PREFIX) :]: value + for key, value in headers.items() + if key.startswith(_CONTENT_PREFIX) + } + content_type = content.pop("type", "") + content.pop("length", None) + if not content_type: + if not content and not body: + return Payload(metadata={"encoding": b"binary/null"}) + return _unknown(content, body) + media, params = _media_type(content_type) + if media == "application/x-temporal-payload": + return Payload.FromString(body) + if media == "application/json": + if params.get("format") == "protobuf" and params.get("message-type"): + return Payload( + metadata={ + "encoding": b"json/protobuf", + "messageType": params["message-type"].encode(), + }, + data=body, + ) + return Payload(metadata={"encoding": b"json/plain"}, data=body) + if media == "application/x-protobuf" and params.get("message-type"): + return Payload( + metadata={ + "encoding": b"binary/protobuf", + "messageType": params["message-type"].encode(), + }, + data=body, + ) + if media == "application/octet-stream": + return Payload(metadata={"encoding": b"binary/plain"}, data=body) + content["type"] = content_type + return _unknown(content, body) + + +def _unknown(content: Mapping[str, str], body: bytes) -> Payload: + metadata = {key: value.encode() for key, value in content.items()} + metadata["encoding"] = b"unknown/nexus-content" + return Payload(metadata=metadata, data=body) + + +class _PayloadSerializer: + """Hands a start's payload to the handler through the data converter.""" + + def __init__( + self, converter: temporalio.converter.DataConverter, payload: Payload + ) -> None: + self._converter = converter + self._payload = payload + + async def serialize(self, value: Any) -> nexusrpc.Content: + del value + raise NotImplementedError("the service serializes results itself") + + async def deserialize( + self, content: nexusrpc.Content, as_type: type[Any] | None = None + ) -> Any: + del content + payload = self._payload + if self._converter.payload_codec is not None: + [payload] = await self._converter.payload_codec.decode([payload]) + try: + [value] = self._converter.payload_converter.from_payloads( + [payload], [as_type] if as_type is not None else None + ) + except Exception as error: + raise nexusrpc.HandlerError( + f"invalid operation input: {error}", + type=nexusrpc.HandlerErrorType.BAD_REQUEST, + retryable_override=False, + ) from error + return value + + +def _failure(error: nexusrpc.HandlerError) -> web.Response: + headers = {} + if error.retryable_override is not None: + headers[_RETRYABLE_HEADER] = "true" if error.retryable_override else "false" + error_type = ( + error.type + if isinstance(error.type, nexusrpc.HandlerErrorType) + else nexusrpc.HandlerErrorType.UNKNOWN + ) + return web.json_response( + {"message": error.message}, + status=_ERROR_STATUS.get(error_type, 500), + headers=headers, + ) + + +class NexusHttpService: + """Serves one Nexus handler over HTTP, plus the route channel deliveries reach. + + ``handler`` is the Nexus SDK's dispatcher over the service handlers the + process hosts. ``consumers`` are the stream consumer operations among + them; a delivery is offered to each until one holds the operation it + names. ``data_converter`` turns start bodies into inputs and results + into bodies; the caller's converter when a client is at hand. + + .. warning:: + This API is experimental and unstable. + """ + + def __init__( + self, + handler: nexusrpc.handler.Handler, + consumers: Sequence[StreamConsumerOperation[Any]], + *, + data_converter: temporalio.converter.DataConverter | None = None, + deliveries_path: str = _DEFAULT_DELIVERIES_PATH, + ) -> None: + """Serve ``handler`` and route deliveries at ``deliveries_path`` to ``consumers``.""" + if deliveries_path.count("/") != 1 or not deliveries_path.startswith("/"): + raise ValueError( + "deliveries_path must be one path segment, so it cannot be mistaken " + "for a start request" + ) + self._handler = handler + self._consumers = list(consumers) + self._converter = data_converter or temporalio.converter.DataConverter.default + self._deliveries_path = deliveries_path + + @property + def deliveries_path(self) -> str: + """The route the channel's deliveries are served at.""" + return self._deliveries_path + + def application(self) -> web.Application: + """The aiohttp application serving the routes; run it with the usual runner.""" + app = web.Application() + app.add_routes( + [ + web.post(self._deliveries_path, self._deliver), + web.post("/{service}/{operation}", self._start), + web.post("/{service}/{operation}/cancel", self._cancel), + ] + ) + return app + + async def _start(self, request: web.Request) -> web.Response: + headers = {key.lower(): value for key, value in request.headers.items()} + body = await request.read() + ctx = nexusrpc.handler.StartOperationContext( + service=request.match_info["service"], + operation=request.match_info["operation"], + headers={ + key: value + for key, value in headers.items() + if not key.startswith((_CONTENT_PREFIX, _CALLBACK_PREFIX)) + }, + request_id=headers.get(_REQUEST_ID_HEADER) or uuid.uuid4().hex, + callback_url=request.query.get(_CALLBACK_QUERY), + callback_headers={ + key[len(_CALLBACK_PREFIX) :]: value + for key, value in headers.items() + if key.startswith(_CALLBACK_PREFIX) + }, + task_cancellation=_NeverCancelled(), + request_deadline=_deadline(headers.get(_REQUEST_TIMEOUT_HEADER)), + ) + input = nexusrpc.LazyValue( + serializer=_PayloadSerializer( + self._converter, _payload_from_content(headers, body) + ), + headers={}, + stream=None, + ) + try: + result = await self._handler.start_operation(ctx, input) + except nexusrpc.HandlerError as error: + return _failure(error) + except nexusrpc.OperationError as error: + return web.json_response( + {"message": str(error)}, + status=_OPERATION_FAILED_STATUS, + headers={_OPERATION_STATE_HEADER: error.state.value}, + ) + if isinstance(result, nexusrpc.handler.StartOperationResultAsync): + return web.json_response( + {"token": result.token, "state": "running"}, status=201 + ) + [payload] = self._converter.payload_converter.to_payloads([result.value]) + if self._converter.payload_codec is not None: + [payload] = await self._converter.payload_codec.encode([payload]) + content, data = _payload_content(payload) + return web.Response(status=200, body=data, headers=content) + + async def _cancel(self, request: web.Request) -> web.Response: + headers = {key.lower(): value for key, value in request.headers.items()} + token = headers.get(_OPERATION_TOKEN_HEADER) or request.query.get(_TOKEN_QUERY) + if not token: + return _failure( + nexusrpc.HandlerError( + "missing operation token", + type=nexusrpc.HandlerErrorType.BAD_REQUEST, + ) + ) + ctx = nexusrpc.handler.CancelOperationContext( + service=request.match_info["service"], + operation=request.match_info["operation"], + headers={ + key: value + for key, value in headers.items() + if key != _OPERATION_TOKEN_HEADER + }, + task_cancellation=_NeverCancelled(), + ) + try: + await self._handler.cancel_operation(ctx, token) + except nexusrpc.HandlerError as error: + return _failure(error) + return web.Response(status=202) + + async def _deliver(self, request: web.Request) -> web.Response: + body = await request.read() + headers = dict(request.headers) + try: + for consumer in self._consumers: + if (await consumer.deliver(headers, body)).known: + break + except Exception as error: + # A 5xx makes the server retry the delivery, which is what a + # failed read needs. + logger.exception("a channel delivery failed") + return web.json_response({"message": str(error)}, status=500) + return web.Response(status=200) + + +# --------------------------------------------------------------------------- +# The demo service: collect a stream's values into the operation's result. +# --------------------------------------------------------------------------- + + +def collect_values(record: StreamRecord[Any], values: list[Any]) -> list[Any]: + """The demo's consume function: keep every published value, in order.""" + if record.kind is RecordKind.DATA: + values.append(record.value) + return values + + +@nexusrpc.service +class CollectStream: + """Consumes the stream a ref names and answers with its values when it closes.""" + + collect: nexusrpc.Operation[StreamRef, list[Any]] + + +@nexusrpc.handler.service_handler(service=CollectStream) +class CollectStreamHandler: + """Serves :class:`CollectStream` out of one consumer operation.""" + + def __init__(self, consumer: StreamConsumerOperation[list[Any]]) -> None: + """Serve ``collect`` with ``consumer``, which the deliveries also reach.""" + self._consumer = consumer + + @nexusrpc.handler.operation_handler + def collect(self) -> nexusrpc.handler.OperationHandler[StreamRef, list[Any]]: + """The consumer operation, as the operation handler factory hands it out.""" + return self._consumer + + +async def _register_endpoint(client: Client, name: str, url: str) -> None: + try: + await client.operator_service.create_nexus_endpoint( + CreateNexusEndpointRequest( + spec=EndpointSpec( + name=name, + target=EndpointTarget(external=EndpointTarget.External(url=url)), + ) + ) + ) + logger.info("registered nexus endpoint %s -> %s", name, url) + except RPCError as error: + if error.status is not RPCStatusCode.ALREADY_EXISTS: + raise + logger.info("nexus endpoint %s exists; leaving it as it is", name) + + +async def main(argv: Sequence[str] | None = None) -> None: + """Serve the demo consumer until interrupted; see the module docstring.""" + parser = argparse.ArgumentParser(description=(__doc__ or "").split("\n\n")[0]) + parser.add_argument( + "--address", default="127.0.0.1:7233", help="the server's gRPC address" + ) + parser.add_argument("--namespace", default="default") + parser.add_argument( + "--http", + default="http://127.0.0.1:7243", + help="the server's Nexus HTTP ingress, where the stream front is read", + ) + parser.add_argument( + "--endpoint", required=True, help="the stream front's Nexus endpoint name" + ) + parser.add_argument("--host", default="127.0.0.1", help="the interface to serve on") + parser.add_argument("--port", type=int, default=8813, help="the port to serve on") + parser.add_argument( + "--listener-url", + default=None, + help="the URL the server posts deliveries to; defaults to this process's route", + ) + parser.add_argument( + "--register-endpoint", + default=None, + metavar="NAME", + help="create a Nexus endpoint of this name whose target is this process", + ) + args = parser.parse_args(argv) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" + ) + + base_url = f"http://{args.host}:{args.port}" + client = await Client.connect( + args.address, + namespace=args.namespace, + plugins=[NexusStreams(endpoint=args.endpoint, http_address=args.http)], + ) + consumer = stream_consumer_operation( + collect_values, + initial=list, + listener_url=args.listener_url or base_url + _DEFAULT_DELIVERIES_PATH, + client=client, + ) + service = NexusHttpService( + nexusrpc.handler.Handler([CollectStreamHandler(consumer)]), + [consumer], + data_converter=client.data_converter, + ) + runner = web.AppRunner(service.application()) + await runner.setup() + try: + await web.TCPSite(runner, args.host, args.port).start() + if args.register_endpoint: + await _register_endpoint(client, args.register_endpoint, base_url) + logger.info( + "serving %s at %s, deliveries at %s", + CollectStream.__name__, + base_url, + consumer._listener_url, # pyright: ignore[reportPrivateUsage] + ) + await asyncio.Event().wait() + finally: + await consumer.close() + await runner.cleanup() + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + pass From 560e278bad84dba9b19ef72b5999dcb19c1c1e38 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 2 Oct 2026 03:12:35 -0700 Subject: [PATCH 31/34] Tested the stream consumer against a channel server. Unit cases drive the state machine with a client stand-in over the memory store. Live cases put the memory store behind the front and notify the channel by hand, since this layer has no store that notifies on its own. The worker-hosted variant stops before the close: a worker's start context names temporal://system as the callback, which an HTTP post cannot reach. --- tests/streams/conftest.py | 24 + tests/streams/test_nexus_consumer.py | 977 +++++++++++++++++++++++++++ 2 files changed, 1001 insertions(+) create mode 100644 tests/streams/test_nexus_consumer.py diff --git a/tests/streams/conftest.py b/tests/streams/conftest.py index cf16360dea..afdb1bcf7f 100644 --- a/tests/streams/conftest.py +++ b/tests/streams/conftest.py @@ -14,6 +14,12 @@ # only talks to the server from the client runs on every layer. PINNED_CORE_HANDLES_LINKED_CHANNEL = False +# A native stream's writes notify a channel named after the stream only on a +# server that has the stream component do so. The channel servers this chain +# runs against do not yet, so a case that consumes a native stream through +# its channel waits for one that does. +SERVER_NAMES_STREAM_CHANNELS = False + def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( @@ -64,6 +70,11 @@ def pytest_configure(config: pytest.Config) -> None: "needs_linked_core: the case needs the pinned Core to hand a linked " "channel's notifications to workflow code", ) + config.addinivalue_line( + "markers", + "needs_stream_channel_server: the case needs a server whose native streams " + "notify a channel named after them, named with -E host:port", + ) def pytest_collection_modifyitems( @@ -113,6 +124,19 @@ def pytest_collection_modifyitems( ), ) ) + if ( + config.getoption("--workflow-environment") in _ENVIRONMENTS_WITHOUT_CHANNELS + or not SERVER_NAMES_STREAM_CHANNELS + ): + skips.append( + ( + "needs_stream_channel_server", + pytest.mark.skip( + reason="needs a server whose native streams notify a channel " + "named after them; name one with -E once a layer has it" + ), + ) + ) for item in items: for marker, skip in skips: if item.get_closest_marker(marker): diff --git a/tests/streams/test_nexus_consumer.py b/tests/streams/test_nexus_consumer.py new file mode 100644 index 0000000000..7156744258 --- /dev/null +++ b/tests/streams/test_nexus_consumer.py @@ -0,0 +1,977 @@ +"""The Nexus operation that consumes a stream through its notification channel. + +The unit cases drive :class:`StreamConsumerOperation` with a client stand-in +over the memory store and the deliveries a server would post, and catch what +it posts back as completions. The live cases need a server with notification +channels and a Nexus HTTP ingress, named with ``-E host:port`` and +``TEMPORAL_HTTP``; they put the memory store behind the Nexus front, because +this layer carries no store whose producers notify a channel themselves, so +the producer notifies the stream's channel by hand the way such a store +would. The consumer's HTTP port is ``STREAM_CONSUMER_PORT``, 8813 by default. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import email.utils +import json +import os +import uuid +from collections.abc import AsyncIterator, Callable, Mapping +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, cast + +import nexusrpc +import nexusrpc.handler +import pytest +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer +from google.protobuf import json_format + +import temporalio.api.notification.v1 +import temporalio.converter +from temporalio import workflow +from temporalio.api.nexus.v1 import EndpointSpec, EndpointTarget +from temporalio.api.operatorservice.v1 import ( + CreateNexusEndpointRequest, + DeleteNexusEndpointRequest, +) +from temporalio.client import Callback, Client +from temporalio.exceptions import NexusOperationError +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import BEGINNING, StreamProvider, StreamRecord, StreamRef +from temporalio.streams._ref import open_ref +from temporalio.streams.providers import nexus +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.streams.providers.nexus import ( + STREAM_CONSUMER_TOKEN_HEADER, + NexusStreams, + StreamConsumerOperation, + TemporalStreamsHandler, + stream_channel_name, + stream_consumer_operation, +) +from temporalio.streams.providers.nexus_consumer_service import ( + CollectStream, + CollectStreamHandler, + NexusHttpService, + _NeverCancelled, + collect_values, +) +from temporalio.worker import Worker +from tests.helpers import assert_eventually +from tests.streams.test_nexus_provider import _own_endpoint + +VALUES = "values" +LISTENER_URL = "http://127.0.0.1:8813/deliveries" +CALLBACK_URL = "http://caller.invalid/namespaces/default/nexus/callback" +CALLBACK_HEADERS = {"temporal-callback-token": "caller-token"} + +_HTTP = os.environ.get("TEMPORAL_HTTP", "http://127.0.0.1:7243") +_PORT = int(os.environ.get("STREAM_CONSUMER_PORT", "8813")) + + +# --------------------------------------------------------------------------- +# Stand-ins. +# --------------------------------------------------------------------------- + + +def _as_client(stand_in: object) -> Client: + """Pass a stand-in where a client is typed; the consumer uses only what both have.""" + return cast(Client, stand_in) + + +class _ClientStandIn: + """What the consumer asks of a client: a provider, a converter, the channel calls.""" + + def __init__(self, store: MemoryStreams) -> None: + self._store = store + self.data_converter = temporalio.converter.DataConverter.default + self.registered: list[tuple[str, Callback]] = [] + self.unregistered: list[tuple[str, str]] = [] + + def get_stream_handle(self, ref: StreamRef) -> Any: + # The memory store opens a handle without a client. + return open_ref(self._store, _as_client(None), ref) + + async def register_channel_listener(self, channel: str, callback: Callback) -> str: + self.registered.append((channel, callback)) + return f"listener-{len(self.registered)}" + + async def unregister_channel_listener(self, channel: str, listener_id: str) -> None: + self.unregistered.append((channel, listener_id)) + + +@dataclass +class _Posted: + url: str + headers: dict[str, str] + body: bytes + + @property + def state(self) -> str: + return self.headers["Nexus-Operation-State"] + + +@pytest.fixture +def posted(monkeypatch: pytest.MonkeyPatch) -> list[_Posted]: + """Catch the completions the consumer posts to the caller's callback.""" + caught: list[_Posted] = [] + + def record(url: str, body: bytes, headers: Mapping[str, str], timeout: Any) -> None: + del timeout + caught.append(_Posted(url, dict(headers), body)) + + monkeypatch.setattr(nexus, "_post_completion", record) + return caught + + +def _start_context( + callback_url: str | None = CALLBACK_URL, +) -> nexusrpc.handler.StartOperationContext: + return nexusrpc.handler.StartOperationContext( + service="CollectStream", + operation="collect", + headers={}, + request_id=uuid.uuid4().hex, + callback_url=callback_url, + callback_headers=dict(CALLBACK_HEADERS), + task_cancellation=_NeverCancelled(), + ) + + +def _cancel_context() -> nexusrpc.handler.CancelOperationContext: + return nexusrpc.handler.CancelOperationContext( + service="CollectStream", + operation="collect", + headers={}, + task_cancellation=_NeverCancelled(), + ) + + +def _notification( + channel: str, counter: int, *, position: bytes = b"", closed: bool = False +) -> bytes: + """The body the server posts: the notification as protobuf JSON.""" + proto = temporalio.api.notification.v1.Notification( + channel=channel, counter=counter, position=position + ) + if closed: + proto.metadata["closed"].CopyFrom( + temporalio.converter.DataConverter.default.payload_converter.to_payload( + True + ) + ) + return json_format.MessageToJson(proto).encode() + + +@dataclass +class _Rig: + store: MemoryStreams + client: _ClientStandIn + operation: StreamConsumerOperation[list[Any]] + ref: StreamRef + channel: str + producer: Any + + async def start(self) -> str: + result = await self.operation.start(_start_context(), self.ref) + assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) + await self.settled(result.token) + return result.token + + async def settled(self, token: str) -> None: + """Wait for the read the start kicked off.""" + opening = self.operation._states[token].opening + assert opening is not None + await asyncio.wait_for(opening, 10) + + def headers(self, token: str) -> dict[str, str]: + return {STREAM_CONSUMER_TOKEN_HEADER: token} + + async def deliver(self, token: str, counter: int, *, closed: bool = False) -> Any: + return await self.operation.deliver( + self.headers(token), _notification(self.channel, counter, closed=closed) + ) + + +async def _rig( + consume: Callable[[StreamRecord[Any], list[Any]], list[Any]] = collect_values, +) -> _Rig: + store = MemoryStreams() + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await store.create_standalone_stream(None, stream_id) + ref = handle.ref(topic=VALUES) + client = _ClientStandIn(store) + operation = stream_consumer_operation( + consume, + initial=list, + listener_url=LISTENER_URL, + client=_as_client(client), + ) + return _Rig( + store, + client, + operation, + ref, + stream_channel_name(ref), + handle.producer(topic=VALUES, producer_id="writer", attempt=1), + ) + + +# --------------------------------------------------------------------------- +# The helper's state machine. +# --------------------------------------------------------------------------- + + +def test_the_channel_name_folds_the_owner_and_the_topic(): + assert ( + stream_channel_name(StreamRef.for_standalone("s-1", topic="values")) + == "stream:standalone:s-1:values" + ) + assert ( + stream_channel_name(StreamRef.for_workflow("wf", topic="t")) + == "stream:workflow:wf::t" + ) + assert ( + stream_channel_name(StreamRef.for_workflow("wf", run_id="r", topic="t")) + == "stream:workflow:wf:r:t" + ) + assert ( + stream_channel_name(StreamRef.for_activity("act", workflow_id="wf", topic="t")) + == "stream:activity:wf::act:t" + ) + + +async def test_start_registers_the_listener_and_reads_what_was_there( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a", "b") + token = await rig.start() + + assert rig.operation.tokens == [token] + assert rig.client.registered == [ + ( + rig.channel, + Callback(url=LISTENER_URL, headers={STREAM_CONSUMER_TOKEN_HEADER: token}), + ) + ] + state = rig.operation.state(token) + assert state is not None + assert state.value == ["a", "b"] + assert (state.reads, state.deliveries, state.records) == (1, 0, 2) + assert state.cursor != BEGINNING + assert state.listener_id == "listener-1" + assert posted == [] + + +async def test_a_delivery_reads_from_the_cursor_once_and_in_order( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a") + token = await rig.start() + + # Records after the registration arrive on the next delivery, each once. + await rig.producer.append("b", "c") + delivery = await rig.deliver(token, 1) + assert (delivery.known, delivery.read, delivery.records, delivery.completed) == ( + True, + True, + 2, + False, + ) + state = rig.operation.state(token) + assert state is not None and state.value == ["a", "b", "c"] + + # A delivery that brings nothing new reads nothing. + delivery = await rig.deliver(token, 1) + assert (delivery.read, delivery.records) == (False, 0) + state = rig.operation.state(token) + assert state is not None and (state.reads, state.deliveries) == (2, 2) + + # A burst the channel folded into one delivery costs one read. + await rig.producer.append("d", "e", "f") + delivery = await rig.operation.deliver( + {STREAM_CONSUMER_TOKEN_HEADER.lower(): token}, _notification(rig.channel, 4) + ) + assert (delivery.read, delivery.records) == (True, 3) + state = rig.operation.state(token) + assert state is not None + assert state.value == ["a", "b", "c", "d", "e", "f"] + assert (state.reads, state.deliveries, state.records) == (3, 3, 6) + assert posted == [] + + +async def test_the_close_completes_through_the_callers_callback( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a") + token = await rig.start() + await rig.producer.append("b") + + delivery = await rig.deliver(token, 2, closed=True) + assert (delivery.read, delivery.records, delivery.completed) == (True, 1, True) + [completion] = posted + assert completion.url == CALLBACK_URL + assert completion.state == "succeeded" + assert completion.headers["temporal-callback-token"] == "caller-token" + assert completion.headers["Nexus-Operation-Token"] == token + assert completion.headers["Content-Type"] == "application/json" + assert email.utils.parsedate_to_datetime( + completion.headers["Nexus-Operation-Start-Time"] + ) + assert json.loads(completion.body) == ["a", "b"] + assert rig.client.unregistered == [(rig.channel, "listener-1")] + assert rig.operation.state(token) is None + assert rig.operation.tokens == [] + + # Once complete, the token is unknown and a late delivery is ignored. + delivery = await rig.deliver(token, 3) + assert (delivery.known, delivery.read) == (False, False) + assert len(posted) == 1 + + +async def test_a_finish_record_closes_the_operation(posted: list[_Posted]): + rig = await _rig() + token = await rig.start() + await rig.producer.append("only") + await rig.producer.finish() + + delivery = await rig.deliver(token, 1) + assert delivery.completed + [completion] = posted + assert completion.state == "succeeded" + assert json.loads(completion.body) == ["only"] + assert rig.client.unregistered == [(rig.channel, "listener-1")] + + +async def test_a_stream_finished_before_the_start_completes_at_once( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a") + await rig.producer.finish() + token = await rig.start() + assert rig.operation.state(token) is None + [completion] = posted + assert completion.state == "succeeded" + assert json.loads(completion.body) == ["a"] + + +async def test_cancel_unregisters_and_reports_canceled(posted: list[_Posted]): + rig = await _rig() + token = await rig.start() + + await rig.operation.cancel(_cancel_context(), token) + assert rig.client.unregistered == [(rig.channel, "listener-1")] + [completion] = posted + assert completion.state == "canceled" + assert completion.headers["Nexus-Operation-Token"] == token + assert completion.headers["Content-Type"] == "application/json" + assert "canceled" in json.loads(completion.body)["message"] + assert rig.operation.state(token) is None + + with pytest.raises(nexusrpc.HandlerError) as unknown: + await rig.operation.cancel(_cancel_context(), token) + assert unknown.value.type is nexusrpc.HandlerErrorType.NOT_FOUND + + +async def test_a_delivery_for_an_operation_not_held_here_is_ignored( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a") + delivery = await rig.operation.deliver({}, _notification(rig.channel, 1)) + assert (delivery.token, delivery.known) == (None, False) + delivery = await rig.deliver("nobody", 1) + assert (delivery.token, delivery.known, delivery.read) == ("nobody", False, False) + assert posted == [] + + +async def test_a_consume_failure_fails_the_operation(posted: list[_Posted]): + def fussy(record: StreamRecord[Any], values: list[Any]) -> list[Any]: + if record.value == "bad": + raise ValueError("cannot take bad") + return collect_values(record, values) + + rig = await _rig(fussy) + token = await rig.start() + await rig.producer.append("good", "bad", "later") + delivery = await rig.deliver(token, 1) + assert (delivery.records, delivery.completed) == (1, False) + [completion] = posted + assert completion.state == "failed" + assert json.loads(completion.body)["message"] == "ValueError: cannot take bad" + assert rig.client.unregistered == [(rig.channel, "listener-1")] + assert rig.operation.state(token) is None + + +async def test_a_read_failure_leaves_the_operation_for_the_retry( + posted: list[_Posted], monkeypatch: pytest.MonkeyPatch +): + rig = await _rig() + token = await rig.start() + await rig.producer.append("a") + handle = rig.client.get_stream_handle + + def broken(ref: StreamRef) -> Any: + del ref + raise RuntimeError("the store is away") + + monkeypatch.setattr(rig.client, "get_stream_handle", broken) + with pytest.raises(RuntimeError, match="away"): + await rig.deliver(token, 1) + monkeypatch.setattr(rig.client, "get_stream_handle", handle) + delivery = await rig.deliver(token, 1) + assert (delivery.read, delivery.records) == (True, 1) + assert posted == [] + + +async def test_close_lets_go_of_every_listener_without_completing( + posted: list[_Posted], +): + rig = await _rig() + first = await rig.start() + second = await rig.start() + assert rig.operation.tokens == [first, second] + await rig.operation.close() + assert rig.operation.tokens == [] + assert rig.client.unregistered == [ + (rig.channel, "listener-1"), + (rig.channel, "listener-2"), + ] + assert posted == [] + + +async def test_a_null_result_travels_without_a_content_type(posted: list[_Posted]): + def nothing(record: StreamRecord[Any], value: None) -> None: + del record + return value + + store = MemoryStreams() + handle = await store.create_standalone_stream(None, "silent") + ref = handle.ref(topic=VALUES) + client = _ClientStandIn(store) + operation = stream_consumer_operation( + nothing, + initial=lambda: None, + listener_url=LISTENER_URL, + client=_as_client(client), + ) + result = await operation.start(_start_context(), ref) + assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) + await handle.producer(topic=VALUES, producer_id="w", attempt=1).finish() + await operation.deliver( + {STREAM_CONSUMER_TOKEN_HEADER: result.token}, + _notification(stream_channel_name(ref), 1), + ) + [completion] = posted + assert completion.state == "succeeded" + assert "Content-Type" not in completion.headers + assert completion.body == b"" + + +async def test_the_listener_url_must_be_given(): + with pytest.raises(ValueError, match="listener_url"): + stream_consumer_operation(collect_values, initial=list, listener_url="") + + +# --------------------------------------------------------------------------- +# The standalone service speaks the Nexus HTTP protocol. +# --------------------------------------------------------------------------- + + +async def test_the_service_speaks_the_nexus_http_protocol(posted: list[_Posted]): + rig = await _rig() + service = NexusHttpService( + nexusrpc.handler.Handler([CollectStreamHandler(rig.operation)]), [rig.operation] + ) + converter = temporalio.converter.DataConverter.default.payload_converter + ref_body = converter.to_payload(rig.ref).data + async with TestClient(TestServer(service.application())) as http: + # A start carries the completion callback in the query and its + # headers under the callback prefix; an asynchronous start is a 201. + answer = await http.post( + "/CollectStream/collect", + params={"callback": CALLBACK_URL}, + data=ref_body, + headers={ + "Content-Type": "application/json", + "Nexus-Request-Id": "req-1", + "Nexus-Callback-Temporal-Callback-Token": "caller-token", + "Request-Timeout": "9.5s", + }, + ) + assert answer.status == 201 + token = (await answer.json())["token"] + assert rig.operation.tokens == [token] + held = rig.operation._states[token] + assert held.callback_url == CALLBACK_URL + assert held.callback_headers == {"temporal-callback-token": "caller-token"} + await rig.settled(token) + + # The delivery route reaches the operation the token names. + await rig.producer.append("a") + answer = await http.post( + service.deliveries_path, + data=_notification(rig.channel, 1), + headers={ + "Content-Type": "application/json", + "Temporal-Notification-Channel": rig.channel, + STREAM_CONSUMER_TOKEN_HEADER: token, + }, + ) + assert answer.status == 200 + state = rig.operation.state(token) + assert state is not None and state.value == ["a"] + + # A cancel names the token in its header and is accepted; a second one + # finds nothing and says so. + answer = await http.post( + "/CollectStream/collect/cancel", headers={"Nexus-Operation-Token": token} + ) + assert answer.status == 202 + assert [p.state for p in posted] == ["canceled"] + answer = await http.post( + "/CollectStream/collect/cancel", headers={"Nexus-Operation-Token": token} + ) + assert answer.status == 404 + assert "token" in (await answer.json())["message"] + + # What is not a stream reference is the caller's fault. + answer = await http.post( + "/CollectStream/collect", + data=b'{"kind": "nowhere"}', + headers={"Content-Type": "application/json"}, + ) + assert answer.status == 400 + answer = await http.post("/Other/op", data=b"{}") + assert answer.status == 404 + + +# --------------------------------------------------------------------------- +# Live: an external stream whose producer notifies the stream's channel. +# --------------------------------------------------------------------------- + + +@dataclass +class ConsumeInput: + """Which endpoint and service to call, which stream to hand it, and whether to cancel.""" + + endpoint: str + service: str + ref: StreamRef + cancel: bool = False + + +@workflow.defn +class ConsumeCaller: + """Starts the consumer operation with a ref and awaits its result.""" + + def __init__(self) -> None: + self._go = False + + @workflow.signal + def go(self) -> None: + """Let a cancelling run cancel now.""" + self._go = True + + @workflow.run + async def run(self, input: ConsumeInput) -> Any: + """Return the operation's result, or ``"canceled"`` after a cancel.""" + client = workflow.create_nexus_client( + service=input.service, endpoint=input.endpoint + ) + handle = await client.start_operation("collect", input.ref, output_type=list) + if not input.cancel: + return await handle + await workflow.wait_condition(lambda: self._go) + handle.cancel() + try: + await handle + except NexusOperationError: + return "canceled" + return "completed" + + +@contextlib.asynccontextmanager +async def _own_external_endpoint( + client: Client, name: str, url: str +) -> AsyncIterator[str]: + """Register a Nexus endpoint whose target is ``url`` for one test's life.""" + created = await client.operator_service.create_nexus_endpoint( + CreateNexusEndpointRequest( + spec=EndpointSpec( + name=name, + target=EndpointTarget(external=EndpointTarget.External(url=url)), + ) + ) + ) + try: + yield name + finally: + await client.operator_service.delete_nexus_endpoint( + DeleteNexusEndpointRequest( + id=created.endpoint.id, version=created.endpoint.version + ) + ) + + +@dataclass +class _Live: + store: StreamProvider + fronted: Client + consumer: StreamConsumerOperation[list[Any]] + endpoint: str + service: str + + +@contextlib.asynccontextmanager +async def _consumer_service( + client: Client, store: StreamProvider | None = None +) -> AsyncIterator[_Live]: + """Stand up the front over ``store`` and the standalone consumer behind an endpoint.""" + store = store or MemoryStreams() + uid = uuid.uuid4().hex + stream_handler = TemporalStreamsHandler(store, client) + handler_tq = f"streams-{uid}" + async with ( + _own_endpoint(client, f"streams-{uid}", handler_tq) as front_endpoint, + Worker(client, task_queue=handler_tq, nexus_service_handlers=[stream_handler]), + ): + front = NexusStreams( + endpoint=front_endpoint, http_address=_HTTP, read_wait=timedelta(seconds=2) + ) + config = client.config() + config["plugins"] = [front] + fronted = Client(**config) + consumer = stream_consumer_operation( + collect_values, + initial=list, + listener_url=f"http://127.0.0.1:{_PORT}/deliveries", + client=fronted, + ) + service = NexusHttpService( + nexusrpc.handler.Handler([CollectStreamHandler(consumer)]), + [consumer], + data_converter=fronted.data_converter, + ) + runner = web.AppRunner(service.application()) + await runner.setup() + await web.TCPSite(runner, "127.0.0.1", _PORT).start() + try: + async with _own_external_endpoint( + client, f"consumers-{uid}", f"http://127.0.0.1:{_PORT}" + ) as endpoint: + yield _Live(store, fronted, consumer, endpoint, CollectStream.__name__) + finally: + await consumer.close() + await runner.cleanup() + await stream_handler.close() + + +async def _listener_token(client: Client, channel: str) -> str: + """The operation token the one callback listener on ``channel`` carries.""" + + async def registered() -> str: + # The channel comes into being with the registration, so a describe + # that races it is answered with not found. + try: + description = await client.describe_channel(channel) + except RPCError as err: + assert err.status != RPCStatusCode.NOT_FOUND, "channel not created yet" + raise + assert len(description.listeners) == 1, description.listeners + [listener] = description.listeners + assert listener.callback is not None + # The server reports the registration's headers in lower case. + headers = { + key.lower(): value for key, value in listener.callback.headers.items() + } + return headers[STREAM_CONSUMER_TOKEN_HEADER.lower()] + + return await assert_eventually(registered, timeout=timedelta(seconds=30)) + + +async def _consumed( + consumer: StreamConsumerOperation[list[Any]], token: str, count: int +) -> None: + async def reached() -> None: + state = consumer.state(token) + assert state is not None and state.records == count, state + + await assert_eventually(reached, timeout=timedelta(seconds=30)) + + +@pytest.mark.needs_channel_server +async def test_a_standalone_service_consumes_an_external_stream_through_its_channel( + client: Client, +): + store = MemoryStreams() + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await store.create_standalone_stream(None, stream_id) + ref = handle.ref(topic=VALUES) + channel = stream_channel_name(ref) + producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) + + async with _consumer_service(client, store) as live: + # Written before anyone listens: the channel keeps the notification, + # and the registration is handed it once. + await producer.append("a", "b") + assert await client.notify_channel(channel, position=b"2", counter=2) == 0 + + async with Worker( + client, task_queue=f"callers-{stream_id}", workflows=[ConsumeCaller] + ) as worker: + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(live.endpoint, live.service, ref), + id=f"caller-{stream_id}", + task_queue=worker.task_queue, + ) + token = await _listener_token(client, channel) + await _consumed(live.consumer, token, 2) + + # Written after the registration: each notification is delivered + # and read from the cursor; the retained one brought nothing new. + await producer.append("c") + assert await client.notify_channel(channel, position=b"3", counter=3) == 1 + await _consumed(live.consumer, token, 3) + await producer.append("d", "e", "f") + for counter in (4, 5, 6): + await client.notify_channel( + channel, position=b"%d" % counter, counter=counter + ) + await _consumed(live.consumer, token, 6) + state = live.consumer.state(token) + assert state is not None + assert state.value == ["a", "b", "c", "d", "e", "f"] + # Every read moved the cursor, none repeated a record, and a burst + # of three cost at most three reads. + assert state.reads <= 1 + 1 + 3 + assert state.deliveries >= 2 + + # The close completes the operation with what was consumed. + await producer.append("g") + await producer.finish() + await client.notify_channel( + channel, position=b"7", counter=7, metadata={"closed": True} + ) + assert await asyncio.wait_for(run.result(), 60) == [ + "a", + "b", + "c", + "d", + "e", + "f", + "g", + ] + assert live.consumer.state(token) is None + assert (await client.describe_channel(channel)).listeners == [] + + +@pytest.mark.needs_channel_server +async def test_cancel_unregisters_the_listener(client: Client): + store = MemoryStreams() + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await store.create_standalone_stream(None, stream_id) + ref = handle.ref(topic=VALUES) + channel = stream_channel_name(ref) + + async with ( + _consumer_service(client, store) as live, + Worker( + client, task_queue=f"callers-{stream_id}", workflows=[ConsumeCaller] + ) as worker, + ): + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(live.endpoint, live.service, ref, cancel=True), + id=f"caller-{stream_id}", + task_queue=worker.task_queue, + ) + token = await _listener_token(client, channel) + await run.signal(ConsumeCaller.go) + assert await asyncio.wait_for(run.result(), 60) == "canceled" + assert live.consumer.state(token) is None + assert (await client.describe_channel(channel)).listeners == [] + + +# --------------------------------------------------------------------------- +# Live: the handler hosted in a Temporal worker, deliveries through the +# frontend's Nexus route of a companion operation. +# --------------------------------------------------------------------------- + + +@nexusrpc.service +class CollectInWorker: + """The consumer next to the synchronous operation the channel posts to.""" + + collect: nexusrpc.Operation[StreamRef, list[Any]] + deliver: nexusrpc.Operation[dict[str, Any], None] + + +@nexusrpc.handler.service_handler(service=CollectInWorker) +class CollectInWorkerHandler: + """Hosts the consumer in a worker; ``deliver`` is the listener's URL target.""" + + def __init__(self, consumer: StreamConsumerOperation[list[Any]]) -> None: + self._consumer = consumer + + @nexusrpc.handler.operation_handler + def collect(self) -> nexusrpc.handler.OperationHandler[StreamRef, list[Any]]: + return self._consumer + + @nexusrpc.handler.sync_operation + async def deliver( + self, ctx: nexusrpc.handler.StartOperationContext, input: dict[str, Any] + ) -> None: + # The server's delivery arrives as a start of this operation: the + # notification is the input and the listener's headers are the + # request's. + await self._consumer.deliver(ctx.headers, input) + + +@pytest.mark.needs_channel_server +async def test_a_worker_hosted_consumer_is_reached_through_the_frontend( + client: Client, +): + """The frontend takes the channel's post as a start of the companion operation. + + What the worker-hosted shape still lacks is the completion: the start + context a worker hands the handler names ``temporal://system`` as the + callback, which only a workflow or activity run the server completes + can reach, not an HTTP post. So this case checks the deliveries and + stops before the close; the completion through a run is a follow-up. + """ + store = MemoryStreams() + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await store.create_standalone_stream(None, stream_id) + ref = handle.ref(topic=VALUES) + channel = stream_channel_name(ref) + producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) + uid = uuid.uuid4().hex + stream_handler = TemporalStreamsHandler(store, client) + + async with ( + _own_endpoint(client, f"streams-{uid}", f"streams-{uid}") as front_endpoint, + Worker( + client, task_queue=f"streams-{uid}", nexus_service_handlers=[stream_handler] + ), + ): + front = NexusStreams( + endpoint=front_endpoint, http_address=_HTTP, read_wait=timedelta(seconds=2) + ) + config = client.config() + config["plugins"] = [front] + fronted = Client(**config) + # The listener's URL is the frontend's route to the companion + # operation, so the endpoint has to exist before the consumer does. + created = await client.operator_service.create_nexus_endpoint( + CreateNexusEndpointRequest( + spec=EndpointSpec( + name=f"hosted-{uid}", + target=EndpointTarget( + worker=EndpointTarget.Worker( + namespace=client.namespace, task_queue=f"hosted-{uid}" + ) + ), + ) + ) + ) + consumer = stream_consumer_operation( + collect_values, + initial=list, + listener_url=( + f"{_HTTP}/nexus/endpoints/{created.endpoint.id}/services/" + f"{CollectInWorker.__name__}/deliver" + ), + ) + try: + async with ( + Worker( + fronted, + task_queue=f"hosted-{uid}", + nexus_service_handlers=[CollectInWorkerHandler(consumer)], + ), + Worker( + client, task_queue=f"callers-{uid}", workflows=[ConsumeCaller] + ) as callers, + ): + await producer.append("a") + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(f"hosted-{uid}", CollectInWorker.__name__, ref), + id=f"caller-{uid}", + task_queue=callers.task_queue, + ) + token = await _listener_token(client, channel) + await _consumed(consumer, token, 1) + await producer.append("b", "c") + assert ( + await client.notify_channel(channel, position=b"3", counter=3) == 1 + ) + await _consumed(consumer, token, 3) + state = consumer.state(token) + assert state is not None + assert state.value == ["a", "b", "c"] + assert state.deliveries >= 1 + held = consumer._states[token] + assert held.callback_url is not None + assert held.callback_url.startswith("temporal://"), held.callback_url + await run.terminate() + finally: + await consumer.close() + await stream_handler.close() + await client.operator_service.delete_nexus_endpoint( + DeleteNexusEndpointRequest( + id=created.endpoint.id, version=created.endpoint.version + ) + ) + + +# --------------------------------------------------------------------------- +# Live, waiting for a server: a native standalone stream notifies its own +# channel, so nobody notifies by hand. +# --------------------------------------------------------------------------- + + +@pytest.mark.needs_stream_channel_server +async def test_a_native_standalone_stream_is_consumed_through_its_channel( + client: Client, +): + provider: StreamProvider | None = client.config().get("stream_provider") + assert provider is not None, "the client needs the native provider registered" + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await client.create_stream(stream_id) + ref = handle.ref(topic=VALUES) + producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) + + async with _consumer_service(client, provider) as live: + await producer.append("a", "b") + async with Worker( + client, task_queue=f"callers-{stream_id}", workflows=[ConsumeCaller] + ) as worker: + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(live.endpoint, live.service, ref), + id=f"caller-{stream_id}", + task_queue=worker.task_queue, + ) + token = await _listener_token(client, stream_channel_name(ref)) + await _consumed(live.consumer, token, 2) + # The server notifies the stream's channel on every append and on + # the close, so the records reach the consumer without a notify. + await producer.append("c") + await _consumed(live.consumer, token, 3) + await producer.finish() + await handle.close() + assert await asyncio.wait_for(run.result(), 60) == ["a", "b", "c"] + assert ( + await client.describe_channel(stream_channel_name(ref)) + ).listeners == [] From 65df92f00116c72ff5585d0d2a7a9b78754674df Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 2 Oct 2026 03:21:05 -0700 Subject: [PATCH 32/34] Named the consumer's channel the way the server names a stream's. A standalone stream notifies stream/, an owned one stream/ linked to the owning workflow, so the registration carries the owner. The rule is the channel_for default and a store with its own passes it. --- temporalio/streams/providers/nexus.py | 58 +++++++++------ tests/streams/test_nexus_consumer.py | 100 +++++++++++++++++--------- 2 files changed, 102 insertions(+), 56 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 531c1d5155..798a375e55 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -156,7 +156,7 @@ "StreamConsumerOperation", "TemporalStreamsHandler", "WireStreamRef", - "stream_channel_name", + "stream_channel", "stream_consumer_operation", ] @@ -1297,26 +1297,27 @@ async def _endpoint_id(self, client: Client | None) -> str: _START_TIME_HEADER = "Nexus-Operation-Start-Time" -def stream_channel_name(ref: StreamRef) -> str: - """The notification channel a stream's writers notify, from the stream's identity. +def stream_channel(ref: StreamRef) -> tuple[str, str | None]: + """The channel a native stream notifies, and the owner a linked one belongs to. - The name folds the owner the ref names and the topic into one string, so - a producer and a consumer that hold the same ref meet on the same - channel. A ref pinned to a run names a different channel from one that - follows the chain, because the two name different streams. + The server's rule: a standalone stream notifies the independent channel + ``stream/``; a workflow's stream notifies ``stream/`` + linked to the owning workflow; an activity's stream notifies + ``stream//``, linked to the workflow that scheduled + the activity and independent for a standalone activity. The second + member is the workflow id a registration addresses the linked channel + with, ``None`` for an independent channel. A store that names channels + its own way passes its rule as ``channel_for`` instead. """ - parts = ["stream", ref.kind] - if ref.kind == "workflow": - parts += [ref.workflow_id or "", ref.run_id or ""] - elif ref.kind == "activity": - parts += [ref.workflow_id or "", ref.run_id or "", ref.activity_id or ""] - else: - parts.append(ref.stream_id or "") - parts.append(ref.topic) - return ":".join(parts) + if ref.kind == "standalone": + return f"stream/{ref.stream_id}", None + if ref.kind == "activity": + return f"stream/{ref.activity_id}/{ref.topic}", ref.workflow_id + return f"stream/{ref.topic}", ref.workflow_id ConsumeFunction = Callable[[StreamRecord[Any], S], "S | Awaitable[S]"] +ChannelRule = Callable[[StreamRef], tuple[str, "str | None"]] @dataclass(frozen=True) @@ -1333,6 +1334,9 @@ class ConsumerState(Generic[S]): channel: str """The channel the listener is registered on.""" + owner: str | None + """The workflow a linked channel belongs to, ``None`` for an independent one.""" + listener_id: str """The listener id the server assigned.""" @@ -1380,6 +1384,7 @@ class Delivery: class _Consumption(Generic[S]): ref: StreamRef channel: str + owner: str | None listener_id: str callback_url: str | None callback_headers: dict[str, str] @@ -1501,7 +1506,7 @@ def __init__( initial: Callable[[], S], listener_url: str, client: Client | None = None, - channel_for: Callable[[StreamRef], str] = stream_channel_name, + channel_for: ChannelRule = stream_channel, result_type: type | None = None, token_header: str = STREAM_CONSUMER_TOKEN_HEADER, ) -> None: @@ -1516,9 +1521,11 @@ def __init__( client: Registers the listener and opens the stream. Leave it unset in a handler hosted by a Temporal worker, where the operation context's client is used. - channel_for: Names the channel for a ref. The default is - :func:`stream_channel_name`; a store that names channels - its own way passes its rule. + channel_for: Names the channel for a ref, and the workflow a + linked channel belongs to. The default is + :func:`stream_channel`, the server's rule for native + streams; a store that names channels its own way passes + its rule. result_type: What record values are decoded as. token_header: The header carrying the operation token on the registration and so on every delivery. @@ -1547,6 +1554,7 @@ def state(self, token: str) -> ConsumerState[S] | None: return ConsumerState( ref=held.ref, channel=held.channel, + owner=held.owner, listener_id=held.listener_id, cursor=held.cursor, value=held.value, @@ -1570,7 +1578,7 @@ async def start( """ client = self._client() token = uuid.uuid4().hex - channel = self._channel_for(input) + channel, owner = self._channel_for(input) # Registered before the first read: a record appended between the # read and the registration would otherwise be missed, where one # appended between the registration and the read is read twice at @@ -1579,10 +1587,12 @@ async def start( listener_id = await client.register_channel_listener( channel, Callback(url=self._listener_url, headers={self._token_header: token}), + workflow_id=owner, ) state: _Consumption[S] = _Consumption( ref=input, channel=channel, + owner=owner, listener_id=listener_id, callback_url=ctx.callback_url, callback_headers=dict(ctx.callback_headers), @@ -1769,7 +1779,9 @@ async def close(self) -> None: async def _unregister(self, client: Client, state: _Consumption[S]) -> None: try: - await client.unregister_channel_listener(state.channel, state.listener_id) + await client.unregister_channel_listener( + state.channel, state.listener_id, workflow_id=state.owner + ) except RPCError: logger.warning( "could not unregister listener %s from channel %s", @@ -1833,7 +1845,7 @@ def stream_consumer_operation( initial: Callable[[], S], listener_url: str, client: Client | None = None, - channel_for: Callable[[StreamRef], str] = stream_channel_name, + channel_for: ChannelRule = stream_channel, result_type: type | None = None, token_header: str = STREAM_CONSUMER_TOKEN_HEADER, ) -> StreamConsumerOperation[S]: diff --git a/tests/streams/test_nexus_consumer.py b/tests/streams/test_nexus_consumer.py index 7156744258..c512bd6261 100644 --- a/tests/streams/test_nexus_consumer.py +++ b/tests/streams/test_nexus_consumer.py @@ -50,7 +50,7 @@ NexusStreams, StreamConsumerOperation, TemporalStreamsHandler, - stream_channel_name, + stream_channel, stream_consumer_operation, ) from temporalio.streams.providers.nexus_consumer_service import ( @@ -89,19 +89,23 @@ class _ClientStandIn: def __init__(self, store: MemoryStreams) -> None: self._store = store self.data_converter = temporalio.converter.DataConverter.default - self.registered: list[tuple[str, Callback]] = [] - self.unregistered: list[tuple[str, str]] = [] + self.registered: list[tuple[str, Callback, str | None]] = [] + self.unregistered: list[tuple[str, str, str | None]] = [] def get_stream_handle(self, ref: StreamRef) -> Any: # The memory store opens a handle without a client. return open_ref(self._store, _as_client(None), ref) - async def register_channel_listener(self, channel: str, callback: Callback) -> str: - self.registered.append((channel, callback)) + async def register_channel_listener( + self, channel: str, callback: Callback, *, workflow_id: str | None = None + ) -> str: + self.registered.append((channel, callback, workflow_id)) return f"listener-{len(self.registered)}" - async def unregister_channel_listener(self, channel: str, listener_id: str) -> None: - self.unregistered.append((channel, listener_id)) + async def unregister_channel_listener( + self, channel: str, listener_id: str, *, workflow_id: str | None = None + ) -> None: + self.unregistered.append((channel, listener_id, workflow_id)) @dataclass @@ -216,7 +220,7 @@ async def _rig( client, operation, ref, - stream_channel_name(ref), + stream_channel(ref)[0], handle.producer(topic=VALUES, producer_id="writer", attempt=1), ) @@ -226,23 +230,52 @@ async def _rig( # --------------------------------------------------------------------------- -def test_the_channel_name_folds_the_owner_and_the_topic(): - assert ( - stream_channel_name(StreamRef.for_standalone("s-1", topic="values")) - == "stream:standalone:s-1:values" +def test_the_channel_follows_the_servers_names(): + # A standalone stream's channel is independent; an owned stream's is + # linked to the owning workflow, which the second member names. + assert stream_channel(StreamRef.for_standalone("s-1", topic="values")) == ( + "stream/s-1", + None, + ) + assert stream_channel(StreamRef.for_workflow("wf", topic="t")) == ("stream/t", "wf") + assert stream_channel(StreamRef.for_workflow("wf", run_id="r", topic="t")) == ( + "stream/t", + "wf", ) - assert ( - stream_channel_name(StreamRef.for_workflow("wf", topic="t")) - == "stream:workflow:wf::t" + assert stream_channel( + StreamRef.for_activity("act", workflow_id="wf", topic="t") + ) == ("stream/act/t", "wf") + assert stream_channel(StreamRef.for_activity("act", topic="t")) == ( + "stream/act/t", + None, ) - assert ( - stream_channel_name(StreamRef.for_workflow("wf", run_id="r", topic="t")) - == "stream:workflow:wf:r:t" + + +async def test_an_owned_stream_registers_on_the_owners_linked_channel( + posted: list[_Posted], +): + store = MemoryStreams() + ref = store.get_stream_handle(None, "wf-1").ref(topic=VALUES) + client = _ClientStandIn(store) + operation = stream_consumer_operation( + collect_values, + initial=list, + listener_url=LISTENER_URL, + client=_as_client(client), ) - assert ( - stream_channel_name(StreamRef.for_activity("act", workflow_id="wf", topic="t")) - == "stream:activity:wf::act:t" + result = await operation.start(_start_context(), ref) + assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) + state = operation.state(result.token) + assert state is not None and (state.channel, state.owner) == ( + "stream/values", + "wf-1", ) + assert [(channel, owner) for channel, _, owner in client.registered] == [ + ("stream/values", "wf-1") + ] + await operation.close() + assert client.unregistered == [("stream/values", "listener-1", "wf-1")] + assert posted == [] async def test_start_registers_the_listener_and_reads_what_was_there( @@ -257,6 +290,7 @@ async def test_start_registers_the_listener_and_reads_what_was_there( ( rig.channel, Callback(url=LISTENER_URL, headers={STREAM_CONSUMER_TOKEN_HEADER: token}), + None, ) ] state = rig.operation.state(token) @@ -326,7 +360,7 @@ async def test_the_close_completes_through_the_callers_callback( completion.headers["Nexus-Operation-Start-Time"] ) assert json.loads(completion.body) == ["a", "b"] - assert rig.client.unregistered == [(rig.channel, "listener-1")] + assert rig.client.unregistered == [(rig.channel, "listener-1", None)] assert rig.operation.state(token) is None assert rig.operation.tokens == [] @@ -347,7 +381,7 @@ async def test_a_finish_record_closes_the_operation(posted: list[_Posted]): [completion] = posted assert completion.state == "succeeded" assert json.loads(completion.body) == ["only"] - assert rig.client.unregistered == [(rig.channel, "listener-1")] + assert rig.client.unregistered == [(rig.channel, "listener-1", None)] async def test_a_stream_finished_before_the_start_completes_at_once( @@ -368,7 +402,7 @@ async def test_cancel_unregisters_and_reports_canceled(posted: list[_Posted]): token = await rig.start() await rig.operation.cancel(_cancel_context(), token) - assert rig.client.unregistered == [(rig.channel, "listener-1")] + assert rig.client.unregistered == [(rig.channel, "listener-1", None)] [completion] = posted assert completion.state == "canceled" assert completion.headers["Nexus-Operation-Token"] == token @@ -407,7 +441,7 @@ def fussy(record: StreamRecord[Any], values: list[Any]) -> list[Any]: [completion] = posted assert completion.state == "failed" assert json.loads(completion.body)["message"] == "ValueError: cannot take bad" - assert rig.client.unregistered == [(rig.channel, "listener-1")] + assert rig.client.unregistered == [(rig.channel, "listener-1", None)] assert rig.operation.state(token) is None @@ -442,8 +476,8 @@ async def test_close_lets_go_of_every_listener_without_completing( await rig.operation.close() assert rig.operation.tokens == [] assert rig.client.unregistered == [ - (rig.channel, "listener-1"), - (rig.channel, "listener-2"), + (rig.channel, "listener-1", None), + (rig.channel, "listener-2", None), ] assert posted == [] @@ -468,7 +502,7 @@ def nothing(record: StreamRecord[Any], value: None) -> None: await handle.producer(topic=VALUES, producer_id="w", attempt=1).finish() await operation.deliver( {STREAM_CONSUMER_TOKEN_HEADER: result.token}, - _notification(stream_channel_name(ref), 1), + _notification(stream_channel(ref)[0], 1), ) [completion] = posted assert completion.state == "succeeded" @@ -716,7 +750,7 @@ async def test_a_standalone_service_consumes_an_external_stream_through_its_chan stream_id = f"consumed-{uuid.uuid4().hex}" handle = await store.create_standalone_stream(None, stream_id) ref = handle.ref(topic=VALUES) - channel = stream_channel_name(ref) + channel = stream_channel(ref)[0] producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) async with _consumer_service(client, store) as live: @@ -781,7 +815,7 @@ async def test_cancel_unregisters_the_listener(client: Client): stream_id = f"consumed-{uuid.uuid4().hex}" handle = await store.create_standalone_stream(None, stream_id) ref = handle.ref(topic=VALUES) - channel = stream_channel_name(ref) + channel = stream_channel(ref)[0] async with ( _consumer_service(client, store) as live, @@ -853,7 +887,7 @@ async def test_a_worker_hosted_consumer_is_reached_through_the_frontend( stream_id = f"consumed-{uuid.uuid4().hex}" handle = await store.create_standalone_stream(None, stream_id) ref = handle.ref(topic=VALUES) - channel = stream_channel_name(ref) + channel = stream_channel(ref)[0] producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) uid = uuid.uuid4().hex stream_handler = TemporalStreamsHandler(store, client) @@ -963,7 +997,7 @@ async def test_a_native_standalone_stream_is_consumed_through_its_channel( id=f"caller-{stream_id}", task_queue=worker.task_queue, ) - token = await _listener_token(client, stream_channel_name(ref)) + token = await _listener_token(client, stream_channel(ref)[0]) await _consumed(live.consumer, token, 2) # The server notifies the stream's channel on every append and on # the close, so the records reach the consumer without a notify. @@ -973,5 +1007,5 @@ async def test_a_native_standalone_stream_is_consumed_through_its_channel( await handle.close() assert await asyncio.wait_for(run.result(), 60) == ["a", "b", "c"] assert ( - await client.describe_channel(stream_channel_name(ref)) + await client.describe_channel(stream_channel(ref)[0]) ).listeners == [] From 22db1792b26a24d034fe101030c661ede4a6c9eb Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 2 Oct 2026 03:46:29 -0700 Subject: [PATCH 33/34] Took the channel address from the client's stream_channel rule. channel_for answers with a ChannelAddress or a (channel, workflow_id) pair and defaults to the main chain's rule, so the consumer and the server agree on a native stream's channel. The native case also waits for a layer that carries a native provider, which this one does not. --- temporalio/streams/providers/nexus.py | 49 +++++++++------------ tests/streams/conftest.py | 21 +++++++++ tests/streams/test_nexus_consumer.py | 61 ++++++++++++++++++--------- 3 files changed, 82 insertions(+), 49 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 798a375e55..356f8fc639 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -156,7 +156,6 @@ "StreamConsumerOperation", "TemporalStreamsHandler", "WireStreamRef", - "stream_channel", "stream_consumer_operation", ] @@ -1297,27 +1296,19 @@ async def _endpoint_id(self, client: Client | None) -> str: _START_TIME_HEADER = "Nexus-Operation-Start-Time" -def stream_channel(ref: StreamRef) -> tuple[str, str | None]: - """The channel a native stream notifies, and the owner a linked one belongs to. - - The server's rule: a standalone stream notifies the independent channel - ``stream/``; a workflow's stream notifies ``stream/`` - linked to the owning workflow; an activity's stream notifies - ``stream//``, linked to the workflow that scheduled - the activity and independent for a standalone activity. The second - member is the workflow id a registration addresses the linked channel - with, ``None`` for an independent channel. A store that names channels - its own way passes its rule as ``channel_for`` instead. - """ - if ref.kind == "standalone": - return f"stream/{ref.stream_id}", None - if ref.kind == "activity": - return f"stream/{ref.activity_id}/{ref.topic}", ref.workflow_id - return f"stream/{ref.topic}", ref.workflow_id +ConsumeFunction = Callable[[StreamRecord[Any], S], "S | Awaitable[S]"] +ChannelRule = Callable[ + [StreamRef], "temporalio.client.ChannelAddress | tuple[str, str | None]" +] -ConsumeFunction = Callable[[StreamRecord[Any], S], "S | Awaitable[S]"] -ChannelRule = Callable[[StreamRef], tuple[str, "str | None"]] +def _address(where: Any) -> tuple[str, str | None]: + """The channel name and the owner a rule answered with, as an address or a pair.""" + channel = getattr(where, "channel", None) + if isinstance(channel, str): + return channel, getattr(where, "workflow_id", None) + channel, owner = where + return channel, owner @dataclass(frozen=True) @@ -1506,7 +1497,7 @@ def __init__( initial: Callable[[], S], listener_url: str, client: Client | None = None, - channel_for: ChannelRule = stream_channel, + channel_for: ChannelRule = temporalio.client.stream_channel, result_type: type | None = None, token_header: str = STREAM_CONSUMER_TOKEN_HEADER, ) -> None: @@ -1521,11 +1512,13 @@ def __init__( client: Registers the listener and opens the stream. Leave it unset in a handler hosted by a Temporal worker, where the operation context's client is used. - channel_for: Names the channel for a ref, and the workflow a - linked channel belongs to. The default is - :func:`stream_channel`, the server's rule for native - streams; a store that names channels its own way passes - its rule. + channel_for: Names the channel for a ref and the workflow a + linked channel belongs to, as a + :class:`temporalio.client.ChannelAddress` or a + ``(channel, workflow_id)`` pair. The default is + :func:`temporalio.client.stream_channel`, the server's rule + for native streams; a store that names channels its own + way passes its rule. result_type: What record values are decoded as. token_header: The header carrying the operation token on the registration and so on every delivery. @@ -1578,7 +1571,7 @@ async def start( """ client = self._client() token = uuid.uuid4().hex - channel, owner = self._channel_for(input) + channel, owner = _address(self._channel_for(input)) # Registered before the first read: a record appended between the # read and the registration would otherwise be missed, where one # appended between the registration and the read is read twice at @@ -1845,7 +1838,7 @@ def stream_consumer_operation( initial: Callable[[], S], listener_url: str, client: Client | None = None, - channel_for: ChannelRule = stream_channel, + channel_for: ChannelRule = temporalio.client.stream_channel, result_type: type | None = None, token_header: str = STREAM_CONSUMER_TOKEN_HEADER, ) -> StreamConsumerOperation[S]: diff --git a/tests/streams/conftest.py b/tests/streams/conftest.py index a1dbe53592..bfa934ddfb 100644 --- a/tests/streams/conftest.py +++ b/tests/streams/conftest.py @@ -18,6 +18,12 @@ # pin and matched against its event by the delivery pin. PINNED_CORE_HANDLES_UNSUBSCRIBE = False +# A native stream lives on the server, and only the native layers carry a +# provider that opens one; the providers here hold streams in memory or in a +# workflow's History. A case that produces to a native stream waits for the +# layer with that provider. +PINNED_LAYER_HOSTS_NATIVE_STREAMS = False + def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( @@ -88,6 +94,11 @@ def pytest_configure(config: pytest.Config) -> None: "needs_stream_channel_server: the case needs a server on which a native " "stream notifies the channel named by the stream, named with -E host:port", ) + config.addinivalue_line( + "markers", + "needs_native_provider: the case needs a provider that opens native streams " + "on the server, which only the native layers carry", + ) def pytest_collection_modifyitems( @@ -161,6 +172,16 @@ def pytest_collection_modifyitems( ), ) ) + if not PINNED_LAYER_HOSTS_NATIVE_STREAMS: + skips.append( + ( + "needs_native_provider", + pytest.mark.skip( + reason="this layer carries no provider for native streams; the " + "native layers and the union do" + ), + ) + ) for item in items: for marker, skip in skips: if item.get_closest_marker(marker): diff --git a/tests/streams/test_nexus_consumer.py b/tests/streams/test_nexus_consumer.py index c512bd6261..61b8610c54 100644 --- a/tests/streams/test_nexus_consumer.py +++ b/tests/streams/test_nexus_consumer.py @@ -38,7 +38,7 @@ CreateNexusEndpointRequest, DeleteNexusEndpointRequest, ) -from temporalio.client import Callback, Client +from temporalio.client import Callback, ChannelAddress, Client, stream_channel from temporalio.exceptions import NexusOperationError from temporalio.service import RPCError, RPCStatusCode from temporalio.streams import BEGINNING, StreamProvider, StreamRecord, StreamRef @@ -50,7 +50,6 @@ NexusStreams, StreamConsumerOperation, TemporalStreamsHandler, - stream_channel, stream_consumer_operation, ) from temporalio.streams.providers.nexus_consumer_service import ( @@ -220,7 +219,7 @@ async def _rig( client, operation, ref, - stream_channel(ref)[0], + stream_channel(ref).channel, handle.producer(topic=VALUES, producer_id="writer", attempt=1), ) @@ -230,27 +229,46 @@ async def _rig( # --------------------------------------------------------------------------- -def test_the_channel_follows_the_servers_names(): - # A standalone stream's channel is independent; an owned stream's is - # linked to the owning workflow, which the second member names. +def test_the_default_rule_is_the_servers_naming_of_a_streams_channel(): + # The consumer registers where the server's stream component notifies: + # a standalone stream's channel is independent, an owned stream's is + # linked to the owning workflow. assert stream_channel(StreamRef.for_standalone("s-1", topic="values")) == ( - "stream/s-1", - None, + ChannelAddress("stream/s-1", None) ) - assert stream_channel(StreamRef.for_workflow("wf", topic="t")) == ("stream/t", "wf") - assert stream_channel(StreamRef.for_workflow("wf", run_id="r", topic="t")) == ( - "stream/t", - "wf", + assert stream_channel(StreamRef.for_workflow("wf", topic="t")) == ChannelAddress( + "stream/t", "wf" ) assert stream_channel( StreamRef.for_activity("act", workflow_id="wf", topic="t") - ) == ("stream/act/t", "wf") + ) == ChannelAddress("stream/act/t", "wf") assert stream_channel(StreamRef.for_activity("act", topic="t")) == ( - "stream/act/t", - None, + ChannelAddress("stream/act/t", None) ) +async def test_a_rule_may_answer_with_a_pair(posted: list[_Posted]): + # A store's rule that returns (channel, workflow_id) fits as well as an + # address does. + store = MemoryStreams() + handle = await store.create_standalone_stream(None, "paired") + client = _ClientStandIn(store) + operation = stream_consumer_operation( + collect_values, + initial=list, + listener_url=LISTENER_URL, + client=_as_client(client), + channel_for=lambda ref: (f"custom/{ref.stream_id}", None), + ) + result = await operation.start(_start_context(), handle.ref(topic=VALUES)) + assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) + state = operation.state(result.token) + assert state is not None and (state.channel, state.owner) == ("custom/paired", None) + await operation.close() + assert client.unregistered == [("custom/paired", "listener-1", None)] + assert posted == [] + + async def test_an_owned_stream_registers_on_the_owners_linked_channel( posted: list[_Posted], ): @@ -502,7 +520,7 @@ def nothing(record: StreamRecord[Any], value: None) -> None: await handle.producer(topic=VALUES, producer_id="w", attempt=1).finish() await operation.deliver( {STREAM_CONSUMER_TOKEN_HEADER: result.token}, - _notification(stream_channel(ref)[0], 1), + _notification(stream_channel(ref).channel, 1), ) [completion] = posted assert completion.state == "succeeded" @@ -750,7 +768,7 @@ async def test_a_standalone_service_consumes_an_external_stream_through_its_chan stream_id = f"consumed-{uuid.uuid4().hex}" handle = await store.create_standalone_stream(None, stream_id) ref = handle.ref(topic=VALUES) - channel = stream_channel(ref)[0] + channel = stream_channel(ref).channel producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) async with _consumer_service(client, store) as live: @@ -815,7 +833,7 @@ async def test_cancel_unregisters_the_listener(client: Client): stream_id = f"consumed-{uuid.uuid4().hex}" handle = await store.create_standalone_stream(None, stream_id) ref = handle.ref(topic=VALUES) - channel = stream_channel(ref)[0] + channel = stream_channel(ref).channel async with ( _consumer_service(client, store) as live, @@ -887,7 +905,7 @@ async def test_a_worker_hosted_consumer_is_reached_through_the_frontend( stream_id = f"consumed-{uuid.uuid4().hex}" handle = await store.create_standalone_stream(None, stream_id) ref = handle.ref(topic=VALUES) - channel = stream_channel(ref)[0] + channel = stream_channel(ref).channel producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) uid = uuid.uuid4().hex stream_handler = TemporalStreamsHandler(store, client) @@ -976,6 +994,7 @@ async def test_a_worker_hosted_consumer_is_reached_through_the_frontend( @pytest.mark.needs_stream_channel_server +@pytest.mark.needs_native_provider async def test_a_native_standalone_stream_is_consumed_through_its_channel( client: Client, ): @@ -997,7 +1016,7 @@ async def test_a_native_standalone_stream_is_consumed_through_its_channel( id=f"caller-{stream_id}", task_queue=worker.task_queue, ) - token = await _listener_token(client, stream_channel(ref)[0]) + token = await _listener_token(client, stream_channel(ref).channel) await _consumed(live.consumer, token, 2) # The server notifies the stream's channel on every append and on # the close, so the records reach the consumer without a notify. @@ -1007,5 +1026,5 @@ async def test_a_native_standalone_stream_is_consumed_through_its_channel( await handle.close() assert await asyncio.wait_for(run.result(), 60) == ["a", "b", "c"] assert ( - await client.describe_channel(stream_channel(ref)[0]) + await client.describe_channel(stream_channel(ref).channel) ).listeners == [] From c85be89dae2eb4acb59ea1a9700928ee6092f8d6 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 2 Oct 2026 18:04:39 -0700 Subject: [PATCH 34/34] Registered the stream consumer by execution. The helper hands the owner `stream_channel` names to the registration as `execution`, so a stream a standalone activity owns is consumed through the channel linked to the activity. A `channel_for` rule answers with an address or a `(channel, execution)` pair. --- temporalio/streams/providers/nexus.py | 27 ++++++++++++++------------ tests/streams/test_nexus_consumer.py | 28 +++++++++++++++------------ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py index 356f8fc639..f4ad1f9cea 100644 --- a/temporalio/streams/providers/nexus.py +++ b/temporalio/streams/providers/nexus.py @@ -102,6 +102,7 @@ import temporalio.api.notification.v1 import temporalio.client +import temporalio.common import temporalio.converter import temporalio.nexus from temporalio.api.common.v1 import Payload @@ -1298,15 +1299,16 @@ async def _endpoint_id(self, client: Client | None) -> str: ConsumeFunction = Callable[[StreamRecord[Any], S], "S | Awaitable[S]"] ChannelRule = Callable[ - [StreamRef], "temporalio.client.ChannelAddress | tuple[str, str | None]" + [StreamRef], + "temporalio.client.ChannelAddress | tuple[str, temporalio.common.Execution | None]", ] -def _address(where: Any) -> tuple[str, str | None]: +def _address(where: Any) -> tuple[str, temporalio.common.Execution | None]: """The channel name and the owner a rule answered with, as an address or a pair.""" channel = getattr(where, "channel", None) if isinstance(channel, str): - return channel, getattr(where, "workflow_id", None) + return channel, getattr(where, "execution", None) channel, owner = where return channel, owner @@ -1325,8 +1327,8 @@ class ConsumerState(Generic[S]): channel: str """The channel the listener is registered on.""" - owner: str | None - """The workflow a linked channel belongs to, ``None`` for an independent one.""" + owner: temporalio.common.Execution | None + """The execution a linked channel belongs to, ``None`` for an independent one.""" listener_id: str """The listener id the server assigned.""" @@ -1375,7 +1377,7 @@ class Delivery: class _Consumption(Generic[S]): ref: StreamRef channel: str - owner: str | None + owner: temporalio.common.Execution | None listener_id: str callback_url: str | None callback_headers: dict[str, str] @@ -1512,10 +1514,11 @@ def __init__( client: Registers the listener and opens the stream. Leave it unset in a handler hosted by a Temporal worker, where the operation context's client is used. - channel_for: Names the channel for a ref and the workflow a - linked channel belongs to, as a - :class:`temporalio.client.ChannelAddress` or a - ``(channel, workflow_id)`` pair. The default is + channel_for: Names the channel for a ref and the execution a + linked channel belongs to, a workflow or a standalone + activity, as a :class:`temporalio.client.ChannelAddress` or a + ``(channel, execution)`` pair with ``None`` for an + independent channel. The default is :func:`temporalio.client.stream_channel`, the server's rule for native streams; a store that names channels its own way passes its rule. @@ -1580,7 +1583,7 @@ async def start( listener_id = await client.register_channel_listener( channel, Callback(url=self._listener_url, headers={self._token_header: token}), - workflow_id=owner, + execution=owner, ) state: _Consumption[S] = _Consumption( ref=input, @@ -1773,7 +1776,7 @@ async def close(self) -> None: async def _unregister(self, client: Client, state: _Consumption[S]) -> None: try: await client.unregister_channel_listener( - state.channel, state.listener_id, workflow_id=state.owner + state.channel, state.listener_id, execution=state.owner ) except RPCError: logger.warning( diff --git a/tests/streams/test_nexus_consumer.py b/tests/streams/test_nexus_consumer.py index 61b8610c54..afe0819931 100644 --- a/tests/streams/test_nexus_consumer.py +++ b/tests/streams/test_nexus_consumer.py @@ -39,6 +39,7 @@ DeleteNexusEndpointRequest, ) from temporalio.client import Callback, ChannelAddress, Client, stream_channel +from temporalio.common import Execution from temporalio.exceptions import NexusOperationError from temporalio.service import RPCError, RPCStatusCode from temporalio.streams import BEGINNING, StreamProvider, StreamRecord, StreamRef @@ -88,23 +89,23 @@ class _ClientStandIn: def __init__(self, store: MemoryStreams) -> None: self._store = store self.data_converter = temporalio.converter.DataConverter.default - self.registered: list[tuple[str, Callback, str | None]] = [] - self.unregistered: list[tuple[str, str, str | None]] = [] + self.registered: list[tuple[str, Callback, Execution | None]] = [] + self.unregistered: list[tuple[str, str, Execution | None]] = [] def get_stream_handle(self, ref: StreamRef) -> Any: # The memory store opens a handle without a client. return open_ref(self._store, _as_client(None), ref) async def register_channel_listener( - self, channel: str, callback: Callback, *, workflow_id: str | None = None + self, channel: str, callback: Callback, *, execution: Execution | None = None ) -> str: - self.registered.append((channel, callback, workflow_id)) + self.registered.append((channel, callback, execution)) return f"listener-{len(self.registered)}" async def unregister_channel_listener( - self, channel: str, listener_id: str, *, workflow_id: str | None = None + self, channel: str, listener_id: str, *, execution: Execution | None = None ) -> None: - self.unregistered.append((channel, listener_id, workflow_id)) + self.unregistered.append((channel, listener_id, execution)) @dataclass @@ -237,13 +238,15 @@ def test_the_default_rule_is_the_servers_naming_of_a_streams_channel(): ChannelAddress("stream/s-1", None) ) assert stream_channel(StreamRef.for_workflow("wf", topic="t")) == ChannelAddress( - "stream/t", "wf" + "stream/t", Execution.workflow("wf") ) assert stream_channel( StreamRef.for_activity("act", workflow_id="wf", topic="t") - ) == ChannelAddress("stream/act/t", "wf") + ) == ChannelAddress("stream/act/t", Execution.workflow("wf")) + # A standalone activity is an execution of its own, so its stream's + # channel is linked to it. assert stream_channel(StreamRef.for_activity("act", topic="t")) == ( - ChannelAddress("stream/act/t", None) + ChannelAddress("stream/t", Execution.activity("act")) ) @@ -284,15 +287,16 @@ async def test_an_owned_stream_registers_on_the_owners_linked_channel( result = await operation.start(_start_context(), ref) assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) state = operation.state(result.token) + owner = Execution.workflow("wf-1") assert state is not None and (state.channel, state.owner) == ( "stream/values", - "wf-1", + owner, ) assert [(channel, owner) for channel, _, owner in client.registered] == [ - ("stream/values", "wf-1") + ("stream/values", owner) ] await operation.close() - assert client.unregistered == [("stream/values", "listener-1", "wf-1")] + assert client.unregistered == [("stream/values", "listener-1", owner)] assert posted == []