diff --git a/.gitmodules b/.gitmodules index a4c911631..1b51ec3d6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "sdk-core"] path = temporalio/bridge/sdk-core - url = https://github.com/temporalio/sdk-rust.git + url = https://github.com/moedash/sdk-rust.git diff --git a/CHANGELOG.md b/CHANGELOG.md index c74bdf72a..8c07338e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,27 @@ to include examples, links to docs, or any other relevant information. is `temporal.api.stream.v1.StreamRecord` on every provider. `temporalio.streams.providers.memory.MemoryStreams` is the in-memory reference provider the conformance tests run against. +- **Experimental**: server-side streams. A workflow publishes to a stream it + owns with a command the server applies in its Workflow Task's commit, and + reads the ranges the server delivers on its Workflow Tasks, through + `temporalio.workflow.append_stream_records`, `subscribe_stream` and + `read_stream_records`. `temporalio.client_stream` and + `temporalio.contrib.server_streams` reach the same stream from outside a + workflow, and `temporalio.streams.providers.native.NativeStreams` puts it + behind the shared stream interface with one owned stream per topic. Requires + a server that serves the stream service. `Replayer(stream_client=)` replays a + workflow that read such a stream while the server still holds it: History + records only the offsets each task consumed, so the replayer fetches the + records from the stream service and hands them to the replay with the + history. A range the stream no longer holds fails the replay with + `StreamNotFoundError`. A handle without a run id follows a workflow reset as + it follows a continue-as-new, reading the reset run from the floor its stream + reports, and the replayer fetches the ranges recorded before a reset point + from the run the workflow was reset from. For offline replay, + `Replayer.fetch_stream_slices(client, history)` attaches the records to a + `WorkflowHistory` while the stream is retained, `to_json()` and `from_json()` + carry them as `streamSlices` beside the events, and a history that carries + them replays with no server. ### Changed diff --git a/scripts/gen_stream_protos.py b/scripts/gen_stream_protos.py new file mode 100644 index 000000000..faf5453d5 --- /dev/null +++ b/scripts/gen_stream_protos.py @@ -0,0 +1,193 @@ +"""Regenerate the vendored server-side stream protos. + +The stream service is still defined in the server rather than in the public +API repo, so its protos are copied in and compiled here. Run this whenever the +server-side definitions change: + + uv run python scripts/gen_stream_protos.py /path/to/temporal + +Run it with an interpreter whose ``grpcio-tools`` matches the oldest protobuf +runtime this package has to load on. Generated code refuses to load on a +runtime older than the one it was built against, and these modules end up in +applications that pin protobuf themselves. The stubs come from +``mypy-protobuf`` rather than protoc's own ``--pyi_out``, which that +generation of protoc does not have. + +Two rewrites make the copies compile against this SDK. The server's own +routing and API-category options are dropped, because they mean nothing to a +client and their proto files are not vendored. Paths move under +``temporalio/`` so the generated modules import each other and the public API +messages by the names this package actually publishes. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +BASE = Path(__file__).parent.parent +OUT = BASE / "temporalio" / "api" / "streamservice" / "v1" +# The stream protos reference the public API messages. They normally come from +# the sdk-core submodule; TEMPORAL_API_PROTOS points at a `temporalio/api` +# checkout instead when the submodule is not initialized. +UPSTREAM_API = ( + BASE + / "temporalio" + / "bridge" + / "sdk-core" + / "crates" + / "protos" + / "protos" + / "api_upstream" +) + + +def api_proto_path() -> Path: + override = os.environ.get("TEMPORAL_API_PROTOS") + if override: + return Path(override).resolve() + if ( + UPSTREAM_API / "temporal" / "api" / "common" / "v1" / "message.proto" + ).is_file(): + return UPSTREAM_API + raise SystemExit( + "no api protos found: initialize the sdk-core submodule or set " + "TEMPORAL_API_PROTOS to a temporalio/api checkout" + ) + + +SERVER_PROTO_DIR = Path("chasm/lib/stream/proto/v1") +STAGE_PROTO_DIR = Path("temporalio/api/stream/v1") +FILES = [ + "message.proto", + "stream_state.proto", + "request_response.proto", + "service.proto", +] + +# Server-only options. They carry shard routing and rate-limit category, which +# a client neither reads nor can resolve, since the files defining them are not +# vendored. +DROP_IMPORT = re.compile(r'^import "temporal/server/.*?";\n', re.M) +DROP_OPTION = re.compile(r"^\s*option \(temporal\.server\.api\..*?\n", re.M) + +# protoc names a python module after the proto path. These put the generated +# modules where this package publishes them. +FIX_PY = [ + ( + re.compile(r"from temporalio\.api\.stream\.v1 import"), + "from temporalio.api.streamservice.v1 import", + ), + (re.compile(r"temporalio\.api\.stream\.v1\."), "temporalio.api.streamservice.v1."), + (re.compile(r"from temporal\.api\."), "from temporalio.api."), + (re.compile(r"import temporal\.api\."), "import temporalio.api."), + # mypy-protobuf also writes the public API types fully qualified in the + # stubs, `temporal.api.common.v1.message_pb2.Payload`, which the import + # rewrites above do not reach. The `_pb2` suffix keeps this off the proto + # package names inside the serialized descriptors, which stay `temporal.api`. + # After the streamservice rewrite on purpose, so the public + # `temporal.api.stream.v1` package is not sent to the vendored one. + ( + re.compile(r"\btemporal\.api\.(\w+)\.v1\.(\w+_pb2)\b"), + r"temporalio.api.\1.v1.\2", + ), +] + + +def stage(server: Path, into: Path) -> list[str]: + dest = into / STAGE_PROTO_DIR + dest.mkdir(parents=True) + for name in FILES: + text = (server / SERVER_PROTO_DIR / name).read_text() + text = text.replace(str(SERVER_PROTO_DIR) + "/", str(STAGE_PROTO_DIR) + "/") + text = DROP_IMPORT.sub("", text) + text = DROP_OPTION.sub("", text) + (dest / name).write_text(text) + return [str(STAGE_PROTO_DIR / name) for name in FILES] + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit("usage: gen_stream_protos.py /path/to/temporal-server-checkout") + server = Path(sys.argv[1]).resolve() + if not (server / SERVER_PROTO_DIR).is_dir(): + sys.exit(f"no stream protos under {server / SERVER_PROTO_DIR}") + + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + targets = stage(server, work) + out = work / "out" + out.mkdir() + subprocess.run( + [ + sys.executable, + "-m", + "grpc_tools.protoc", + f"--proto_path={work}", + f"--proto_path={api_proto_path()}", + f"--python_out={out}", + f"--grpc_python_out={out}", + f"--mypy_out={out}", + f"--mypy_grpc_out={out}", + *targets, + ], + check=True, + cwd=work, + ) + + generated = out / STAGE_PROTO_DIR + for path in sorted(generated.iterdir()): + if path.name == "__init__.py": + continue + text = path.read_text() + for pattern, repl in FIX_PY: + text = pattern.sub(repl, text) + (OUT / path.name).write_text(text) + print(f"wrote {OUT / path.name}") + + shutil.rmtree(OUT / "__pycache__", ignore_errors=True) + write_init() + + +MODULES = ["message_pb2", "stream_state_pb2", "request_response_pb2"] + +INIT_HEADER = """\ +# Generated by scripts/gen_stream_protos.py. Do not edit. +# +# Vendored rather than taken from the api submodule, because the stream service +# is still defined in the server. See temporalio/client_stream.py. +""" + + +def write_init() -> None: + """Re-export every generated message and enum, so the package surface + cannot drift from the protos it was built from.""" + import importlib + + sys.path.insert(0, str(BASE)) + blocks, exported = [], [] + for module in MODULES: + mod = importlib.import_module(f"temporalio.api.streamservice.v1.{module}") + names = list(mod.DESCRIPTOR.message_types_by_name) + for enum in mod.DESCRIPTOR.enum_types_by_name.values(): + names.append(enum.name) + names.extend(value.name for value in enum.values) + names.sort() + exported.extend(names) + joined = "".join(f" {name},\n" for name in names) + blocks.append(f"from .{module} import (\n{joined})\n") + + listed = "".join(f' "{name}",\n' for name in sorted(exported)) + (OUT / "__init__.py").write_text( + INIT_HEADER + "\n" + "\n".join(blocks) + f"\n__all__ = [\n{listed}]\n" + ) + print(f"wrote {OUT / '__init__.py'}") + + +if __name__ == "__main__": + main() diff --git a/temporalio/api/streamservice/__init__.py b/temporalio/api/streamservice/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/temporalio/api/streamservice/v1/__init__.py b/temporalio/api/streamservice/v1/__init__.py new file mode 100644 index 000000000..1e909c223 --- /dev/null +++ b/temporalio/api/streamservice/v1/__init__.py @@ -0,0 +1,146 @@ +# Generated by scripts/gen_stream_protos.py. Do not edit. +# +# Vendored rather than taken from the api submodule, because the stream service +# is still defined in the server. See temporalio/client_stream.py. + +from .message_pb2 import ( + StreamRecord, + StreamRecordBatch, +) +from .request_response_pb2 import ( + AddMessagesInput, + AddMessagesOutput, + AddMessagesRequest, + AddMessagesResponse, + AddWorkflowMessagesInput, + AddWorkflowMessagesRequest, + AddWorkflowMessagesResponse, + AdvanceConsumerHeadInput, + AdvanceConsumerHeadOutput, + AdvanceConsumerHeadRequest, + AdvanceConsumerHeadResponse, + CloseStreamInput, + CloseStreamOutput, + CloseStreamRequest, + CloseStreamResponse, + CreateStreamInput, + CreateStreamOutput, + CreateStreamRequest, + CreateStreamResponse, + DeleteStreamInput, + DeleteStreamOutput, + DeleteStreamRequest, + DeleteStreamResponse, + DescribeStreamInput, + DescribeStreamOutput, + DescribeStreamRequest, + DescribeStreamResponse, + DescribeWorkflowStreamInput, + DescribeWorkflowStreamRequest, + DescribeWorkflowStreamResponse, + FinishWritingInput, + FinishWritingOutput, + FinishWritingRequest, + FinishWritingResponse, + ListStreamsInput, + ListStreamsOutput, + ListStreamsRequest, + ListStreamsResponse, + PollMessagesInput, + PollMessagesOutput, + PollMessagesRequest, + PollMessagesResponse, + PollWorkflowMessagesInput, + PollWorkflowMessagesRequest, + PollWorkflowMessagesResponse, + RegisterStreamConsumerInput, + RegisterStreamConsumerOutput, + RegisterStreamConsumerRequest, + RegisterStreamConsumerResponse, + StreamListEntry, + SubscribeWorkflowInput, + SubscribeWorkflowOutput, + SubscribeWorkflowRequest, + SubscribeWorkflowResponse, + TruncateStreamInput, + TruncateStreamOutput, + TruncateStreamRequest, + TruncateStreamResponse, +) +from .stream_state_pb2 import ( + ConsumerCursor, + ProducerCursor, + StreamBudget, + StreamLifecycle, + StreamState, + WorkflowStreamCursor, +) + +__all__ = [ + "AddMessagesInput", + "AddMessagesOutput", + "AddMessagesRequest", + "AddMessagesResponse", + "AddWorkflowMessagesInput", + "AddWorkflowMessagesRequest", + "AddWorkflowMessagesResponse", + "AdvanceConsumerHeadInput", + "AdvanceConsumerHeadOutput", + "AdvanceConsumerHeadRequest", + "AdvanceConsumerHeadResponse", + "CloseStreamInput", + "CloseStreamOutput", + "CloseStreamRequest", + "CloseStreamResponse", + "ConsumerCursor", + "CreateStreamInput", + "CreateStreamOutput", + "CreateStreamRequest", + "CreateStreamResponse", + "DeleteStreamInput", + "DeleteStreamOutput", + "DeleteStreamRequest", + "DeleteStreamResponse", + "DescribeStreamInput", + "DescribeStreamOutput", + "DescribeStreamRequest", + "DescribeStreamResponse", + "DescribeWorkflowStreamInput", + "DescribeWorkflowStreamRequest", + "DescribeWorkflowStreamResponse", + "FinishWritingInput", + "FinishWritingOutput", + "FinishWritingRequest", + "FinishWritingResponse", + "ListStreamsInput", + "ListStreamsOutput", + "ListStreamsRequest", + "ListStreamsResponse", + "PollMessagesInput", + "PollMessagesOutput", + "PollMessagesRequest", + "PollMessagesResponse", + "PollWorkflowMessagesInput", + "PollWorkflowMessagesRequest", + "PollWorkflowMessagesResponse", + "ProducerCursor", + "RegisterStreamConsumerInput", + "RegisterStreamConsumerOutput", + "RegisterStreamConsumerRequest", + "RegisterStreamConsumerResponse", + "StreamBudget", + "StreamLifecycle", + "StreamListEntry", + "StreamRecord", + "StreamRecordBatch", + "StreamState", + "SubscribeWorkflowInput", + "SubscribeWorkflowOutput", + "SubscribeWorkflowRequest", + "SubscribeWorkflowResponse", + "TruncateStreamInput", + "TruncateStreamOutput", + "TruncateStreamRequest", + "TruncateStreamResponse", + "WorkflowStreamCursor", +] diff --git a/temporalio/api/streamservice/v1/message_pb2.py b/temporalio/api/streamservice/v1/message_pb2.py new file mode 100644 index 000000000..8876ac29b --- /dev/null +++ b/temporalio/api/streamservice/v1/message_pb2.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporalio/api/stream/v1/message.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n&temporalio/api/stream/v1/message.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto"\xf7\x02\n\x0cStreamRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12W\n\x08metadata\x18\x02 \x03(\x0b\x32\x45.temporal.server.chasm.lib.stream.proto.v1.StreamRecord.MetadataEntry\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x10\n\x08sequence\x18\x04 \x01(\x03\x12\x36\n\x04kind\x18\x05 \x01(\x0e\x32(.temporal.api.stream.v1.StreamRecordKind\x12\x0e\n\x06offset\x18\x06 \x01(\x03\x12\x13\n\x0bproducer_id\x18\x07 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x08 \x01(\x03\x1aP\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"]\n\x11StreamRecordBatch\x12H\n\x07records\x18\x01 \x03(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamRecordB>Z= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class StreamRecord(google.protobuf.message.Message): + """The stored shape of temporal.api.stream.v1.StreamRecord plus the offset a + read assigns. Field for field the public record, so one crosses the frontend + in either direction without translation. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class MetadataEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: temporalio.api.common.v1.message_pb2.Payload | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["key", b"key", "value", b"value"], + ) -> None: ... + + BODY_FIELD_NUMBER: builtins.int + METADATA_FIELD_NUMBER: builtins.int + TOPIC_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + KIND_FIELD_NUMBER: builtins.int + OFFSET_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + ATTEMPT_FIELD_NUMBER: builtins.int + @property + def body(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + @property + def metadata( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, temporalio.api.common.v1.message_pb2.Payload + ]: + """Producer-supplied provenance, stored as sent.""" + topic: builtins.str + sequence: builtins.int + """The producer's position within its attempt, or -1 when unnumbered. Stored + as sent; the global offset is what orders the stream. + """ + kind: temporalio.api.stream.v1.message_pb2.StreamRecordKind.ValueType + """Settled to DATA on append when left unspecified, so a retry hashes the + same bytes and a reader never sees the zero value. + """ + offset: builtins.int + """Position in the whole stream, set on read and never stored. A consumer + that resumes at record granularity needs it, and a topic-filtered read + leaves gaps that make it underivable from the response alone. + """ + producer_id: builtins.str + """Who wrote the record. Empty when the owning Workflow did: its publish + command clears the field, whatever the worker sent. + """ + attempt: builtins.int + """The producer's attempt, stored as sent. Readers treat a later attempt by + the same producer as superseding what the earlier one wrote. + """ + def __init__( + self, + *, + body: temporalio.api.common.v1.message_pb2.Payload | None = ..., + metadata: collections.abc.Mapping[ + builtins.str, temporalio.api.common.v1.message_pb2.Payload + ] + | None = ..., + topic: builtins.str = ..., + sequence: builtins.int = ..., + kind: temporalio.api.stream.v1.message_pb2.StreamRecordKind.ValueType = ..., + offset: builtins.int = ..., + producer_id: builtins.str = ..., + attempt: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["body", b"body"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "attempt", + b"attempt", + "body", + b"body", + "kind", + b"kind", + "metadata", + b"metadata", + "offset", + b"offset", + "producer_id", + b"producer_id", + "sequence", + b"sequence", + "topic", + b"topic", + ], + ) -> None: ... + +global___StreamRecord = StreamRecord + +class StreamRecordBatch(google.protobuf.message.Message): + """One append is one batch, and one batch is one data node. The server stores + this serialized and opaque; it decodes only to trim a partial first page or + to apply a topic filter. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RECORDS_FIELD_NUMBER: builtins.int + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamRecord + ]: ... + def __init__( + self, + *, + records: collections.abc.Iterable[global___StreamRecord] | None = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["records", b"records"] + ) -> None: ... + +global___StreamRecordBatch = StreamRecordBatch diff --git a/temporalio/api/streamservice/v1/message_pb2_grpc.py b/temporalio/api/streamservice/v1/message_pb2_grpc.py new file mode 100644 index 000000000..bf947056a --- /dev/null +++ b/temporalio/api/streamservice/v1/message_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc diff --git a/temporalio/api/streamservice/v1/message_pb2_grpc.pyi b/temporalio/api/streamservice/v1/message_pb2_grpc.pyi new file mode 100644 index 000000000..f3a5a087e --- /dev/null +++ b/temporalio/api/streamservice/v1/message_pb2_grpc.pyi @@ -0,0 +1,4 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" diff --git a/temporalio/api/streamservice/v1/request_response_pb2.py b/temporalio/api/streamservice/v1/request_response_pb2.py new file mode 100644 index 000000000..3df3e3c35 --- /dev/null +++ b/temporalio/api/streamservice/v1/request_response_pb2.py @@ -0,0 +1,881 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporalio/api/stream/v1/request_response.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) +from temporalio.api.streamservice.v1 import ( + message_pb2 as temporalio_dot_api_dot_stream_dot_v1_dot_message__pb2, +) +from temporalio.api.streamservice.v1 import ( + stream_state_pb2 as temporalio_dot_api_dot_stream_dot_v1_dot_stream__state__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n/temporalio/api/stream/v1/request_response.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a&temporalio/api/stream/v1/message.proto\x1a+temporalio/api/stream/v1/stream_state.proto\x1a$temporal/api/common/v1/message.proto"\x88\x01\n\x11\x43reateStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12M\n\tlifecycle\x18\x03 \x01(\x0b\x32:.temporal.server.chasm.lib.stream.proto.v1.StreamLifecycle"$\n\x12\x43reateStreamOutput\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xf5\x01\n\x10\x41\x64\x64MessagesInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\t \x01(\t\x12H\n\x07records\x18\x03 \x03(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamRecord\x12\x13\n\x0bproducer_id\x18\x04 \x01(\t\x12\x10\n\x08sequence\x18\x05 \x01(\x03\x12\x17\n\x0f\x65xpected_offset\x18\x06 \x01(\x03\x12\x1b\n\x13use_expected_offset\x18\x08 \x01(\x08J\x04\x08\x07\x10\x08"c\n\x11\x41\x64\x64MessagesOutput\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\x12\x14\n\x0c\x64\x65\x64uplicated\x18\x04 \x01(\x08"O\n\x12\x46inishWritingInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x13\n\x0bproducer_id\x18\x03 \x01(\t"\x15\n\x13\x46inishWritingOutput"\x94\x01\n\x16SubscribeWorkflowInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x06 \x01(\t\x12\x13\n\x0bstream_name\x18\x03 \x01(\t\x12\x11\n\tstream_id\x18\x05 \x01(\t\x12\x14\n\x0cstart_offset\x18\x04 \x01(\x03"/\n\x17SubscribeWorkflowOutput\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03"\x9f\x01\n\x11PollMessagesInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x07 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x03\x12\x14\n\x0cmax_messages\x18\x04 \x01(\x05\x12\x0e\n\x06topics\x18\x05 \x03(\t\x12\x19\n\x11wait_new_messages\x18\x06 \x01(\x08"\xdf\x01\n\x12PollMessagesOutput\x12H\n\x07records\x18\x01 \x03(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamRecord\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x13\n\x0bhead_offset\x18\x03 \x01(\x03\x12\x0e\n\x06\x63losed\x18\x04 \x01(\x08\x12\x35\n\x0c\x63lose_reason\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x0e\n\x06run_id\x18\x06 \x01(\t";\n\x13\x44\x65scribeStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t"\xc4\x01\n\x19PollWorkflowMessagesInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x08 \x01(\t\x12\x13\n\x0bstream_name\x18\x03 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x04 \x01(\x03\x12\x14\n\x0cmax_messages\x18\x05 \x01(\x05\x12\x0e\n\x06topics\x18\x06 \x03(\t\x12\x19\n\x11wait_new_messages\x18\x07 \x01(\x08"p\n\x1b\x44\x65scribeWorkflowStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x04 \x01(\t\x12\x13\n\x0bstream_name\x18\x03 \x01(\t"\xde\x01\n\x18\x41\x64\x64WorkflowMessagesInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x07 \x01(\t\x12\x13\n\x0bstream_name\x18\x03 \x01(\t\x12H\n\x07records\x18\x04 \x03(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamRecord\x12\x13\n\x0bproducer_id\x18\x05 \x01(\t\x12\x10\n\x08sequence\x18\x06 \x01(\x03"]\n\x14\x44\x65scribeStreamOutput\x12\x45\n\x05state\x18\x01 \x01(\x0b\x32\x36.temporal.server.chasm.lib.stream.proto.v1.StreamState"i\n\x10\x43loseStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12/\n\x06reason\x18\x03 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"\x13\n\x11\x43loseStreamOutput"T\n\x13TruncateStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x17\n\x0fnew_base_offset\x18\x03 \x01(\x03"\x16\n\x14TruncateStreamOutput"H\n\x11\x44\x65leteStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\r\n\x05\x66orce\x18\x03 \x01(\x08"\x14\n\x12\x44\x65leteStreamOutput"\x83\x01\n\x13\x43reateStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12V\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.CreateStreamInput"p\n\x14\x43reateStreamResponse\x12X\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.CreateStreamOutput"\x81\x01\n\x12\x41\x64\x64MessagesRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12U\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32;.temporal.server.chasm.lib.stream.proto.v1.AddMessagesInput"n\n\x13\x41\x64\x64MessagesResponse\x12W\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.AddMessagesOutput"\x85\x01\n\x14\x46inishWritingRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12W\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.FinishWritingInput"r\n\x15\x46inishWritingResponse\x12Y\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32>.temporal.server.chasm.lib.stream.proto.v1.FinishWritingOutput"\x8d\x01\n\x18SubscribeWorkflowRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12[\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x41.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowInput"z\n\x19SubscribeWorkflowResponse\x12]\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32\x42.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowOutput"\x83\x01\n\x13PollMessagesRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12V\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.PollMessagesInput"p\n\x14PollMessagesResponse\x12X\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutput"\x87\x01\n\x15\x44\x65scribeStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12X\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32>.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamInput"t\n\x16\x44\x65scribeStreamResponse\x12Z\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32?.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutput"\x93\x01\n\x1bPollWorkflowMessagesRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12^\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x44.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesInput"x\n\x1cPollWorkflowMessagesResponse\x12X\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutput"\x97\x01\n\x1d\x44\x65scribeWorkflowStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12`\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x46.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamInput"|\n\x1e\x44\x65scribeWorkflowStreamResponse\x12Z\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32?.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutput"\x90\x01\n\x1bRegisterStreamConsumerInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x1c\n\x14\x63onsumer_workflow_id\x18\x03 \x01(\t\x12\x17\n\x0f\x63onsumer_run_id\x18\x05 \x01(\t\x12\x14\n\x0cstart_offset\x18\x04 \x01(\x03"T\n\x1cRegisterStreamConsumerOutput\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x12\n\nknown_head\x18\x04 \x01(\x03J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04"\x80\x01\n\x18\x41\x64vanceConsumerHeadInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x05 \x01(\t\x12\x11\n\tstream_id\x18\x03 \x01(\t\x12\x13\n\x0bhead_offset\x18\x04 \x01(\x03"n\n\x19\x41\x64vanceConsumerHeadOutput\x12\x17\n\x0f\x63onsumer_closed\x18\x01 \x01(\x08\x12\x18\n\x10successor_run_id\x18\x02 \x01(\t\x12\x1e\n\x16successor_start_offset\x18\x03 \x01(\x03"\x91\x01\n\x1a\x41\x64\x64WorkflowMessagesRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12]\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x43.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesInput"v\n\x1b\x41\x64\x64WorkflowMessagesResponse\x12W\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.AddMessagesOutput"\x97\x01\n\x1dRegisterStreamConsumerRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12`\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x46.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerInput"\x84\x01\n\x1eRegisterStreamConsumerResponse\x12\x62\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32G.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerOutput"\x91\x01\n\x1a\x41\x64vanceConsumerHeadRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12]\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x43.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadInput"~\n\x1b\x41\x64vanceConsumerHeadResponse\x12_\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32\x44.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadOutput"\x81\x01\n\x12\x43loseStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12U\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32;.temporal.server.chasm.lib.stream.proto.v1.CloseStreamInput"n\n\x13\x43loseStreamResponse\x12W\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.CloseStreamOutput"\x87\x01\n\x15TruncateStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12X\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32>.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamInput"t\n\x16TruncateStreamResponse\x12Z\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32?.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamOutput"`\n\x10ListStreamsInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"4\n\x0fStreamListEntry\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t"y\n\x11ListStreamsOutput\x12K\n\x07streams\x18\x01 \x03(\x0b\x32:.temporal.server.chasm.lib.stream.proto.v1.StreamListEntry\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\x81\x01\n\x12ListStreamsRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12U\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32;.temporal.server.chasm.lib.stream.proto.v1.ListStreamsInput"n\n\x13ListStreamsResponse\x12W\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.ListStreamsOutput"\x83\x01\n\x13\x44\x65leteStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12V\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamInput"p\n\x14\x44\x65leteStreamResponse\x12X\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamOutputB>Z= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class CreateStreamInput(google.protobuf.message.Message): + """The frontend-facing shapes are defined here rather than in the public API + because streams have no public API yet. Keeping them in a nested + frontend_request mirrors the other CHASM libraries, so promoting them later + is a package move rather than a redesign. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + LIFECYCLE_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + @property + def lifecycle( + self, + ) -> temporalio.api.streamservice.v1.stream_state_pb2.StreamLifecycle: ... + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + lifecycle: temporalio.api.streamservice.v1.stream_state_pb2.StreamLifecycle + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["lifecycle", b"lifecycle"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "lifecycle", + b"lifecycle", + "namespace", + b"namespace", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___CreateStreamInput = CreateStreamInput + +class CreateStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RUN_ID_FIELD_NUMBER: builtins.int + run_id: builtins.str + def __init__( + self, + *, + run_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["run_id", b"run_id"] + ) -> None: ... + +global___CreateStreamOutput = CreateStreamOutput + +class AddMessagesInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + EXPECTED_OFFSET_FIELD_NUMBER: builtins.int + USE_EXPECTED_OFFSET_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + run_id: builtins.str + """Optional. Supplying it skips resolving the stream's current run, which is + a persistence lookup on every call. CreateStream returns it. + """ + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ]: ... + producer_id: builtins.str + """Idempotency, all optional. Supply a producer identity and sequence, or an + expected offset, or neither and accept at-least-once. + """ + sequence: builtins.int + expected_offset: builtins.int + """Guarded by use_expected_offset because proto3 optional is not supported + by this repo's helper generator. + """ + use_expected_offset: builtins.bool + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + run_id: builtins.str = ..., + records: collections.abc.Iterable[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ] + | None = ..., + producer_id: builtins.str = ..., + sequence: builtins.int = ..., + expected_offset: builtins.int = ..., + use_expected_offset: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "expected_offset", + b"expected_offset", + "namespace", + b"namespace", + "producer_id", + b"producer_id", + "records", + b"records", + "run_id", + b"run_id", + "sequence", + b"sequence", + "stream_id", + b"stream_id", + "use_expected_offset", + b"use_expected_offset", + ], + ) -> None: ... + +global___AddMessagesInput = AddMessagesInput + +class AddMessagesOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FIRST_OFFSET_FIELD_NUMBER: builtins.int + NEXT_OFFSET_FIELD_NUMBER: builtins.int + COUNT_FIELD_NUMBER: builtins.int + DEDUPLICATED_FIELD_NUMBER: builtins.int + first_offset: builtins.int + next_offset: builtins.int + count: builtins.int + deduplicated: builtins.bool + """True when a retry matched a recorded sequence and nothing was appended.""" + def __init__( + self, + *, + first_offset: builtins.int = ..., + next_offset: builtins.int = ..., + count: builtins.int = ..., + deduplicated: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "count", + b"count", + "deduplicated", + b"deduplicated", + "first_offset", + b"first_offset", + "next_offset", + b"next_offset", + ], + ) -> None: ... + +global___AddMessagesOutput = AddMessagesOutput + +class FinishWritingInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + producer_id: builtins.str + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + producer_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "producer_id", + b"producer_id", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___FinishWritingInput = FinishWritingInput + +class FinishWritingOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___FinishWritingOutput = FinishWritingOutput + +class SubscribeWorkflowInput(google.protobuf.message.Message): + """Registers a Workflow as a consumer of a stream it owns. The cursor lands in + the Workflow's own state, so from then on each of its Workflow Tasks carries + the next range and records what it consumed. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + namespace: builtins.str + workflow_id: builtins.str + owner_run_id: builtins.str + """Optional. Pins to one run, so a caller that has continued as new is not + silently redirected to the successor's stream, which starts empty and at + offset zero. Empty means whichever run is current. + """ + stream_name: builtins.str + """Name of the stream within the Workflow, for a stream it owns.""" + stream_id: builtins.str + """Id of a standalone stream in another execution. Exactly one of this and + stream_name is set. + """ + start_offset: builtins.int + """Where to start. Resolved here rather than at delivery, so the first + recorded range starts from a fact instead of a reading. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_name: builtins.str = ..., + stream_id: builtins.str = ..., + start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "owner_run_id", + b"owner_run_id", + "start_offset", + b"start_offset", + "stream_id", + b"stream_id", + "stream_name", + b"stream_name", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___SubscribeWorkflowInput = SubscribeWorkflowInput + +class SubscribeWorkflowOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + START_OFFSET_FIELD_NUMBER: builtins.int + start_offset: builtins.int + def __init__( + self, + *, + start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["start_offset", b"start_offset"] + ) -> None: ... + +global___SubscribeWorkflowOutput = SubscribeWorkflowOutput + +class PollMessagesInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + MAX_MESSAGES_FIELD_NUMBER: builtins.int + TOPICS_FIELD_NUMBER: builtins.int + WAIT_NEW_MESSAGES_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + run_id: builtins.str + """Optional, as on AddMessagesInput.""" + from_offset: builtins.int + max_messages: builtins.int + @property + def topics( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Filters by exact topic. Offsets are assigned over the unfiltered stream, so + next_offset advances past filtered-out records too. + """ + wait_new_messages: builtins.bool + """When set and the reader is caught up, block until something arrives, the + stream closes, or the server's long-poll timeout elapses. A timeout returns + an empty response rather than an error, so the caller simply polls again. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + run_id: builtins.str = ..., + from_offset: builtins.int = ..., + max_messages: builtins.int = ..., + topics: collections.abc.Iterable[builtins.str] | None = ..., + wait_new_messages: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "max_messages", + b"max_messages", + "namespace", + b"namespace", + "run_id", + b"run_id", + "stream_id", + b"stream_id", + "topics", + b"topics", + "wait_new_messages", + b"wait_new_messages", + ], + ) -> None: ... + +global___PollMessagesInput = PollMessagesInput + +class PollMessagesOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RECORDS_FIELD_NUMBER: builtins.int + NEXT_OFFSET_FIELD_NUMBER: builtins.int + HEAD_OFFSET_FIELD_NUMBER: builtins.int + CLOSED_FIELD_NUMBER: builtins.int + CLOSE_REASON_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ]: ... + next_offset: builtins.int + head_offset: builtins.int + closed: builtins.bool + @property + def close_reason(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + run_id: builtins.str + """The execution holding the stream. A workflow task slice built from this + read names the run it came from. + """ + def __init__( + self, + *, + records: collections.abc.Iterable[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ] + | None = ..., + next_offset: builtins.int = ..., + head_offset: builtins.int = ..., + closed: builtins.bool = ..., + close_reason: temporalio.api.common.v1.message_pb2.Payload | None = ..., + run_id: builtins.str = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["close_reason", b"close_reason"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "close_reason", + b"close_reason", + "closed", + b"closed", + "head_offset", + b"head_offset", + "next_offset", + b"next_offset", + "records", + b"records", + "run_id", + b"run_id", + ], + ) -> None: ... + +global___PollMessagesOutput = PollMessagesOutput + +class DescribeStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", b"namespace", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___DescribeStreamInput = DescribeStreamInput + +class PollWorkflowMessagesInput(google.protobuf.message.Message): + """A stream a workflow owns lives inside that workflow's execution, so it has + no standalone id to address it by. It is named by its owner and its name + instead, and routed on the owner. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + MAX_MESSAGES_FIELD_NUMBER: builtins.int + TOPICS_FIELD_NUMBER: builtins.int + WAIT_NEW_MESSAGES_FIELD_NUMBER: builtins.int + namespace: builtins.str + workflow_id: builtins.str + owner_run_id: builtins.str + """Optional. Pins to one run, so a caller that has continued as new is not + silently redirected to the successor's stream, which starts empty and at + offset zero. Empty means whichever run is current. + """ + stream_name: builtins.str + """Empty means the workflow's default output stream.""" + from_offset: builtins.int + max_messages: builtins.int + @property + def topics( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Filters as on PollMessagesInput.""" + wait_new_messages: builtins.bool + def __init__( + self, + *, + namespace: builtins.str = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_name: builtins.str = ..., + from_offset: builtins.int = ..., + max_messages: builtins.int = ..., + topics: collections.abc.Iterable[builtins.str] | None = ..., + wait_new_messages: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "max_messages", + b"max_messages", + "namespace", + b"namespace", + "owner_run_id", + b"owner_run_id", + "stream_name", + b"stream_name", + "topics", + b"topics", + "wait_new_messages", + b"wait_new_messages", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___PollWorkflowMessagesInput = PollWorkflowMessagesInput + +class DescribeWorkflowStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + namespace: builtins.str + workflow_id: builtins.str + owner_run_id: builtins.str + """Optional. Pins to one run, so a caller that has continued as new is not + silently redirected to the successor's stream, which starts empty and at + offset zero. Empty means whichever run is current. + """ + stream_name: builtins.str + def __init__( + self, + *, + namespace: builtins.str = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_name: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "owner_run_id", + b"owner_run_id", + "stream_name", + b"stream_name", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___DescribeWorkflowStreamInput = DescribeWorkflowStreamInput + +class AddWorkflowMessagesInput(google.protobuf.message.Message): + """Appending to a stream a workflow owns, from outside that workflow. The + workflow's own publishes ride its Workflow Task instead. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + namespace: builtins.str + workflow_id: builtins.str + owner_run_id: builtins.str + """Optional. Pins to one run, so a caller that has continued as new is not + silently redirected to the successor's stream, which starts empty and at + offset zero. Empty means whichever run is current. + """ + stream_name: builtins.str + """Empty means the workflow's default output stream.""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ]: ... + producer_id: builtins.str + """Optional idempotency, as on AddMessagesInput.""" + sequence: builtins.int + def __init__( + self, + *, + namespace: builtins.str = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_name: builtins.str = ..., + records: collections.abc.Iterable[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ] + | None = ..., + producer_id: builtins.str = ..., + sequence: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "owner_run_id", + b"owner_run_id", + "producer_id", + b"producer_id", + "records", + b"records", + "sequence", + b"sequence", + "stream_name", + b"stream_name", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___AddWorkflowMessagesInput = AddWorkflowMessagesInput + +class DescribeStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATE_FIELD_NUMBER: builtins.int + @property + def state(self) -> temporalio.api.streamservice.v1.stream_state_pb2.StreamState: ... + def __init__( + self, + *, + state: temporalio.api.streamservice.v1.stream_state_pb2.StreamState + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["state", b"state"] + ) -> builtins.bool: ... + def ClearField( + self, field_name: typing_extensions.Literal["state", b"state"] + ) -> None: ... + +global___DescribeStreamOutput = DescribeStreamOutput + +class CloseStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + REASON_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + @property + def reason(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + reason: temporalio.api.common.v1.message_pb2.Payload | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["reason", b"reason"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", b"namespace", "reason", b"reason", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___CloseStreamInput = CloseStreamInput + +class CloseStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___CloseStreamOutput = CloseStreamOutput + +class TruncateStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + NEW_BASE_OFFSET_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + new_base_offset: builtins.int + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + new_base_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "new_base_offset", + b"new_base_offset", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___TruncateStreamInput = TruncateStreamInput + +class TruncateStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___TruncateStreamOutput = TruncateStreamOutput + +class DeleteStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + FORCE_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + force: builtins.bool + """Delete even while a workflow consumer is active. Without it the call is + refused, because the consumer's History depends on ranges the deletion + takes with it. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + force: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "force", b"force", "namespace", b"namespace", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___DeleteStreamInput = DeleteStreamInput + +class DeleteStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___DeleteStreamOutput = DeleteStreamOutput + +class CreateStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___CreateStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___CreateStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___CreateStreamRequest = CreateStreamRequest + +class CreateStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___CreateStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___CreateStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___CreateStreamResponse = CreateStreamResponse + +class AddMessagesRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___AddMessagesInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___AddMessagesInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___AddMessagesRequest = AddMessagesRequest + +class AddMessagesResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___AddMessagesOutput: ... + def __init__( + self, + *, + frontend_response: global___AddMessagesOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___AddMessagesResponse = AddMessagesResponse + +class FinishWritingRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___FinishWritingInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___FinishWritingInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___FinishWritingRequest = FinishWritingRequest + +class FinishWritingResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___FinishWritingOutput: ... + def __init__( + self, + *, + frontend_response: global___FinishWritingOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___FinishWritingResponse = FinishWritingResponse + +class SubscribeWorkflowRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___SubscribeWorkflowInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___SubscribeWorkflowInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___SubscribeWorkflowRequest = SubscribeWorkflowRequest + +class SubscribeWorkflowResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___SubscribeWorkflowOutput: ... + def __init__( + self, + *, + frontend_response: global___SubscribeWorkflowOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___SubscribeWorkflowResponse = SubscribeWorkflowResponse + +class PollMessagesRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___PollMessagesInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___PollMessagesInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___PollMessagesRequest = PollMessagesRequest + +class PollMessagesResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___PollMessagesOutput: ... + def __init__( + self, + *, + frontend_response: global___PollMessagesOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___PollMessagesResponse = PollMessagesResponse + +class DescribeStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___DescribeStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___DescribeStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___DescribeStreamRequest = DescribeStreamRequest + +class DescribeStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___DescribeStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___DescribeStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___DescribeStreamResponse = DescribeStreamResponse + +class PollWorkflowMessagesRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___PollWorkflowMessagesInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___PollWorkflowMessagesInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___PollWorkflowMessagesRequest = PollWorkflowMessagesRequest + +class PollWorkflowMessagesResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___PollMessagesOutput: ... + def __init__( + self, + *, + frontend_response: global___PollMessagesOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___PollWorkflowMessagesResponse = PollWorkflowMessagesResponse + +class DescribeWorkflowStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___DescribeWorkflowStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___DescribeWorkflowStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___DescribeWorkflowStreamRequest = DescribeWorkflowStreamRequest + +class DescribeWorkflowStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___DescribeStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___DescribeStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___DescribeWorkflowStreamResponse = DescribeWorkflowStreamResponse + +class RegisterStreamConsumerInput(google.protobuf.message.Message): + """Registering a consumer on a stream in another execution. Split out from + SubscribeWorkflow because the two halves live on different shards: the pin + goes on the stream, the cursor goes on the consuming workflow, and a handler + can only reach the shard it was routed to. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + CONSUMER_WORKFLOW_ID_FIELD_NUMBER: builtins.int + CONSUMER_RUN_ID_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + consumer_workflow_id: builtins.str + """The workflow that will consume. Together with the run it names the pin, + so a later run of the same workflow id registers fresh rather than + inheriting a closed run's floor. + """ + consumer_run_id: builtins.str + start_offset: builtins.int + """Negative means from wherever the stream is when the pin is taken. Resolved + here, where the frontier is, and returned so the cursor records a fact. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + consumer_workflow_id: builtins.str = ..., + consumer_run_id: builtins.str = ..., + start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "consumer_run_id", + b"consumer_run_id", + "consumer_workflow_id", + b"consumer_workflow_id", + "namespace", + b"namespace", + "start_offset", + b"start_offset", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___RegisterStreamConsumerInput = RegisterStreamConsumerInput + +class RegisterStreamConsumerOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + START_OFFSET_FIELD_NUMBER: builtins.int + KNOWN_HEAD_FIELD_NUMBER: builtins.int + start_offset: builtins.int + known_head: builtins.int + """The frontier at registration, so the cursor starts with a known head + instead of waiting for the first push. + """ + def __init__( + self, + *, + start_offset: builtins.int = ..., + known_head: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "known_head", b"known_head", "start_offset", b"start_offset" + ], + ) -> None: ... + +global___RegisterStreamConsumerOutput = RegisterStreamConsumerOutput + +class AdvanceConsumerHeadInput(google.protobuf.message.Message): + """Telling one consumer that the frontier moved. Routed to the consumer, which + is not where the stream lives. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + HEAD_OFFSET_FIELD_NUMBER: builtins.int + namespace: builtins.str + workflow_id: builtins.str + owner_run_id: builtins.str + """The run that subscribed. A successor from continue-as-new holds no cursor + for this stream, so pushing the frontier at it would land nowhere. + """ + stream_id: builtins.str + head_offset: builtins.int + def __init__( + self, + *, + namespace: builtins.str = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_id: builtins.str = ..., + head_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "head_offset", + b"head_offset", + "namespace", + b"namespace", + "owner_run_id", + b"owner_run_id", + "stream_id", + b"stream_id", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___AdvanceConsumerHeadInput = AdvanceConsumerHeadInput + +class AdvanceConsumerHeadOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CONSUMER_CLOSED_FIELD_NUMBER: builtins.int + SUCCESSOR_RUN_ID_FIELD_NUMBER: builtins.int + SUCCESSOR_START_OFFSET_FIELD_NUMBER: builtins.int + consumer_closed: builtins.bool + """The run that held the pin is closed and no current run of the workflow + consumes the stream, so the stream can release the pin. + """ + successor_run_id: builtins.str + """The run that held the pin is closed but a successor carries the + subscription, so the stream re-keys the pin to it, with the floor the + successor's cursor started from. + """ + successor_start_offset: builtins.int + def __init__( + self, + *, + consumer_closed: builtins.bool = ..., + successor_run_id: builtins.str = ..., + successor_start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "consumer_closed", + b"consumer_closed", + "successor_run_id", + b"successor_run_id", + "successor_start_offset", + b"successor_start_offset", + ], + ) -> None: ... + +global___AdvanceConsumerHeadOutput = AdvanceConsumerHeadOutput + +class AddWorkflowMessagesRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___AddWorkflowMessagesInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___AddWorkflowMessagesInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___AddWorkflowMessagesRequest = AddWorkflowMessagesRequest + +class AddWorkflowMessagesResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___AddMessagesOutput: ... + def __init__( + self, + *, + frontend_response: global___AddMessagesOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___AddWorkflowMessagesResponse = AddWorkflowMessagesResponse + +class RegisterStreamConsumerRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___RegisterStreamConsumerInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___RegisterStreamConsumerInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___RegisterStreamConsumerRequest = RegisterStreamConsumerRequest + +class RegisterStreamConsumerResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___RegisterStreamConsumerOutput: ... + def __init__( + self, + *, + frontend_response: global___RegisterStreamConsumerOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___RegisterStreamConsumerResponse = RegisterStreamConsumerResponse + +class AdvanceConsumerHeadRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___AdvanceConsumerHeadInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___AdvanceConsumerHeadInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___AdvanceConsumerHeadRequest = AdvanceConsumerHeadRequest + +class AdvanceConsumerHeadResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___AdvanceConsumerHeadOutput: ... + def __init__( + self, + *, + frontend_response: global___AdvanceConsumerHeadOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___AdvanceConsumerHeadResponse = AdvanceConsumerHeadResponse + +class CloseStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___CloseStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___CloseStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___CloseStreamRequest = CloseStreamRequest + +class CloseStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___CloseStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___CloseStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___CloseStreamResponse = CloseStreamResponse + +class TruncateStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___TruncateStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___TruncateStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___TruncateStreamRequest = TruncateStreamRequest + +class TruncateStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___TruncateStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___TruncateStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___TruncateStreamResponse = TruncateStreamResponse + +class ListStreamsInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + PAGE_SIZE_FIELD_NUMBER: builtins.int + NEXT_PAGE_TOKEN_FIELD_NUMBER: builtins.int + QUERY_FIELD_NUMBER: builtins.int + namespace: builtins.str + page_size: builtins.int + next_page_token: builtins.bytes + query: builtins.str + def __init__( + self, + *, + namespace: builtins.str = ..., + page_size: builtins.int = ..., + next_page_token: builtins.bytes = ..., + query: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "next_page_token", + b"next_page_token", + "page_size", + b"page_size", + "query", + b"query", + ], + ) -> None: ... + +global___ListStreamsInput = ListStreamsInput + +class StreamListEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + stream_id: builtins.str + run_id: builtins.str + def __init__( + self, + *, + stream_id: builtins.str = ..., + run_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "run_id", b"run_id", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___StreamListEntry = StreamListEntry + +class ListStreamsOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAMS_FIELD_NUMBER: builtins.int + NEXT_PAGE_TOKEN_FIELD_NUMBER: builtins.int + @property + def streams( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamListEntry + ]: ... + next_page_token: builtins.bytes + def __init__( + self, + *, + streams: collections.abc.Iterable[global___StreamListEntry] | None = ..., + next_page_token: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "next_page_token", b"next_page_token", "streams", b"streams" + ], + ) -> None: ... + +global___ListStreamsOutput = ListStreamsOutput + +class ListStreamsRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___ListStreamsInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___ListStreamsInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___ListStreamsRequest = ListStreamsRequest + +class ListStreamsResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___ListStreamsOutput: ... + def __init__( + self, + *, + frontend_response: global___ListStreamsOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___ListStreamsResponse = ListStreamsResponse + +class DeleteStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___DeleteStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___DeleteStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___DeleteStreamRequest = DeleteStreamRequest + +class DeleteStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___DeleteStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___DeleteStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___DeleteStreamResponse = DeleteStreamResponse diff --git a/temporalio/api/streamservice/v1/request_response_pb2_grpc.py b/temporalio/api/streamservice/v1/request_response_pb2_grpc.py new file mode 100644 index 000000000..bf947056a --- /dev/null +++ b/temporalio/api/streamservice/v1/request_response_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc diff --git a/temporalio/api/streamservice/v1/request_response_pb2_grpc.pyi b/temporalio/api/streamservice/v1/request_response_pb2_grpc.pyi new file mode 100644 index 000000000..f3a5a087e --- /dev/null +++ b/temporalio/api/streamservice/v1/request_response_pb2_grpc.pyi @@ -0,0 +1,4 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" diff --git a/temporalio/api/streamservice/v1/service_pb2.py b/temporalio/api/streamservice/v1/service_pb2.py new file mode 100644 index 000000000..432abfc5c --- /dev/null +++ b/temporalio/api/streamservice/v1/service_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporalio/api/stream/v1/service.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.streamservice.v1 import ( + request_response_pb2 as temporalio_dot_api_dot_stream_dot_v1_dot_request__response__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n&temporalio/api/stream/v1/service.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a/temporalio/api/stream/v1/request_response.proto2\xce\x12\n\rStreamService\x12\x91\x01\n\x0c\x43reateStream\x12>.temporal.server.chasm.lib.stream.proto.v1.CreateStreamRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.CreateStreamResponse"\x00\x12\x8e\x01\n\x0b\x41\x64\x64Messages\x12=.temporal.server.chasm.lib.stream.proto.v1.AddMessagesRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.AddMessagesResponse"\x00\x12\x94\x01\n\rFinishWriting\x12?.temporal.server.chasm.lib.stream.proto.v1.FinishWritingRequest\x1a@.temporal.server.chasm.lib.stream.proto.v1.FinishWritingResponse"\x00\x12\xa0\x01\n\x11SubscribeWorkflow\x12\x43.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowRequest\x1a\x44.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowResponse"\x00\x12\x91\x01\n\x0cPollMessages\x12>.temporal.server.chasm.lib.stream.proto.v1.PollMessagesRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.PollMessagesResponse"\x00\x12\x97\x01\n\x0e\x44\x65scribeStream\x12@.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamRequest\x1a\x41.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamResponse"\x00\x12\xa9\x01\n\x14PollWorkflowMessages\x12\x46.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesRequest\x1aG.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesResponse"\x00\x12\xaf\x01\n\x16\x44\x65scribeWorkflowStream\x12H.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamRequest\x1aI.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamResponse"\x00\x12\xa6\x01\n\x13\x41\x64\x64WorkflowMessages\x12\x45.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesRequest\x1a\x46.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesResponse"\x00\x12\xaf\x01\n\x16RegisterStreamConsumer\x12H.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerRequest\x1aI.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerResponse"\x00\x12\xa6\x01\n\x13\x41\x64vanceConsumerHead\x12\x45.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadRequest\x1a\x46.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadResponse"\x00\x12\x8e\x01\n\x0b\x43loseStream\x12=.temporal.server.chasm.lib.stream.proto.v1.CloseStreamRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.CloseStreamResponse"\x00\x12\x97\x01\n\x0eTruncateStream\x12@.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamRequest\x1a\x41.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamResponse"\x00\x12\x8e\x01\n\x0bListStreams\x12=.temporal.server.chasm.lib.stream.proto.v1.ListStreamsRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.ListStreamsResponse"\x00\x12\x91\x01\n\x0c\x44\x65leteStream\x12>.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamResponse"\x00\x42>Z None: ... + CreateStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.CreateStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.CreateStreamResponse, + ] + AddMessages: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.AddMessagesRequest, + temporalio.api.streamservice.v1.request_response_pb2.AddMessagesResponse, + ] + FinishWriting: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.FinishWritingRequest, + temporalio.api.streamservice.v1.request_response_pb2.FinishWritingResponse, + ] + SubscribeWorkflow: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.SubscribeWorkflowRequest, + temporalio.api.streamservice.v1.request_response_pb2.SubscribeWorkflowResponse, + ] + PollMessages: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.PollMessagesRequest, + temporalio.api.streamservice.v1.request_response_pb2.PollMessagesResponse, + ] + DescribeStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.DescribeStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.DescribeStreamResponse, + ] + PollWorkflowMessages: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.PollWorkflowMessagesRequest, + temporalio.api.streamservice.v1.request_response_pb2.PollWorkflowMessagesResponse, + ] + """Routed on the owner, because the stream it reads has no id of its own.""" + DescribeWorkflowStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.DescribeWorkflowStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.DescribeWorkflowStreamResponse, + ] + AddWorkflowMessages: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.AddWorkflowMessagesRequest, + temporalio.api.streamservice.v1.request_response_pb2.AddWorkflowMessagesResponse, + ] + RegisterStreamConsumer: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.RegisterStreamConsumerRequest, + temporalio.api.streamservice.v1.request_response_pb2.RegisterStreamConsumerResponse, + ] + """Internal. History calls these on itself to reach a shard it does not own, + which is the only way a step that spans two executions can work on a + cluster with more than one history host. + """ + AdvanceConsumerHead: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.AdvanceConsumerHeadRequest, + temporalio.api.streamservice.v1.request_response_pb2.AdvanceConsumerHeadResponse, + ] + CloseStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.CloseStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.CloseStreamResponse, + ] + TruncateStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.TruncateStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.TruncateStreamResponse, + ] + ListStreams: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.ListStreamsRequest, + temporalio.api.streamservice.v1.request_response_pb2.ListStreamsResponse, + ] + """Served on the frontend only: it queries visibility rather than a stream, + so there is no business ID to route on and nothing for a shard to answer. + """ + DeleteStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.DeleteStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.DeleteStreamResponse, + ] + +class StreamServiceServicer(metaclass=abc.ABCMeta): + @abc.abstractmethod + def CreateStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.CreateStreamRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.CreateStreamResponse: ... + @abc.abstractmethod + def AddMessages( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.AddMessagesRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.AddMessagesResponse: ... + @abc.abstractmethod + def FinishWriting( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.FinishWritingRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.FinishWritingResponse: ... + @abc.abstractmethod + def SubscribeWorkflow( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.SubscribeWorkflowRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.SubscribeWorkflowResponse + ): ... + @abc.abstractmethod + def PollMessages( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.PollMessagesRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.PollMessagesResponse: ... + @abc.abstractmethod + def DescribeStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.DescribeStreamRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.DescribeStreamResponse + ): ... + @abc.abstractmethod + def PollWorkflowMessages( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.PollWorkflowMessagesRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.PollWorkflowMessagesResponse: + """Routed on the owner, because the stream it reads has no id of its own.""" + @abc.abstractmethod + def DescribeWorkflowStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.DescribeWorkflowStreamRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.DescribeWorkflowStreamResponse: ... + @abc.abstractmethod + def AddWorkflowMessages( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.AddWorkflowMessagesRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.AddWorkflowMessagesResponse + ): ... + @abc.abstractmethod + def RegisterStreamConsumer( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.RegisterStreamConsumerRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.RegisterStreamConsumerResponse: + """Internal. History calls these on itself to reach a shard it does not own, + which is the only way a step that spans two executions can work on a + cluster with more than one history host. + """ + @abc.abstractmethod + def AdvanceConsumerHead( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.AdvanceConsumerHeadRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.AdvanceConsumerHeadResponse + ): ... + @abc.abstractmethod + def CloseStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.CloseStreamRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.CloseStreamResponse: ... + @abc.abstractmethod + def TruncateStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.TruncateStreamRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.TruncateStreamResponse + ): ... + @abc.abstractmethod + def ListStreams( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.ListStreamsRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.ListStreamsResponse: + """Served on the frontend only: it queries visibility rather than a stream, + so there is no business ID to route on and nothing for a shard to answer. + """ + @abc.abstractmethod + def DeleteStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.DeleteStreamRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.DeleteStreamResponse: ... + +def add_StreamServiceServicer_to_server( + servicer: StreamServiceServicer, server: grpc.Server +) -> None: ... diff --git a/temporalio/api/streamservice/v1/stream_state_pb2.py b/temporalio/api/streamservice/v1/stream_state_pb2.py new file mode 100644 index 000000000..9f334a50f --- /dev/null +++ b/temporalio/api/streamservice/v1/stream_state_pb2.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporalio/api/stream/v1/stream_state.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n+temporalio/api/stream/v1/stream_state.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto"\xb5\x06\n\x0bStreamState\x12\x13\n\x0bhead_offset\x18\x01 \x01(\x03\x12\x13\n\x0b\x62\x61se_offset\x18\x02 \x01(\x03\x12\x0e\n\x06\x63losed\x18\x04 \x01(\x08\x12\x35\n\x0c\x63lose_reason\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\tproducers\x18\t \x03(\x0b\x32\x45.temporal.server.chasm.lib.stream.proto.v1.StreamState.ProducersEntry\x12X\n\tconsumers\x18\n \x03(\x0b\x32\x45.temporal.server.chasm.lib.stream.proto.v1.StreamState.ConsumersEntry\x12M\n\tlifecycle\x18\x0b \x01(\x0b\x32:.temporal.server.chasm.lib.stream.proto.v1.StreamLifecycle\x12\x17\n\x0fredirect_run_id\x18\x0c \x01(\t\x12.\n\nclose_time\x18\r \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12G\n\x06\x62udget\x18\x0e \x01(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamBudget\x12\x16\n\x0e\x61ppended_bytes\x18\x0f \x01(\x03\x12\x16\n\x0enotify_pending\x18\x10 \x01(\x08\x1ak\n\x0eProducersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12H\n\x05value\x18\x02 \x01(\x0b\x32\x39.temporal.server.chasm.lib.stream.proto.v1.ProducerCursor:\x02\x38\x01\x1ak\n\x0e\x43onsumersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12H\n\x05value\x18\x02 \x01(\x0b\x32\x39.temporal.server.chasm.lib.stream.proto.v1.ConsumerCursor:\x02\x38\x01J\x04\x08\x03\x10\x04J\x04\x08\x06\x10\x07J\x04\x08\x07\x10\x08J\x04\x08\x08\x10\t"4\n\x0cStreamBudget\x12\x11\n\tmax_items\x18\x01 \x01(\x03\x12\x11\n\tmax_bytes\x18\x02 \x01(\x03"h\n\x0eProducerCursor\x12\x0b\n\x03seq\x18\x01 \x01(\x03\x12\x14\n\x0c\x66irst_offset\x18\x02 \x01(\x03\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\x12\x14\n\x0c\x63ontent_hash\x18\x04 \x01(\x0c\x12\x0e\n\x06\x66\x65nced\x18\x05 \x01(\x08"}\n\x0e\x43onsumerCursor\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x03\x12\x0e\n\x06\x61\x63tive\x18\x04 \x01(\x08\x12\x10\n\x08\x65xternal\x18\x05 \x01(\x08\x12\x14\n\x0creplay_floor\x18\x06 \x01(\x03"\xc0\x01\n\x14WorkflowStreamCursor\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x04 \x01(\x03\x12\x12\n\nknown_head\x18\x08 \x01(\x03\x12\x10\n\x08\x65xternal\x18\t \x01(\x08\x12\x14\n\x0cpending_from\x18\x05 \x01(\x03\x12\x12\n\npending_to\x18\x06 \x01(\x03\x12\x13\n\x0bhas_pending\x18\x07 \x01(\x08\x12\x14\n\x0cstart_offset\x18\n \x01(\x03J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04"R\n\x0fStreamLifecycle\x12,\n\tretention\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x11\n\tmax_items\x18\x02 \x01(\x03\x42>Z= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class StreamState(google.protobuf.message.Message): + """Size is O(producers + consumers), never O(messages). Payload bytes live in + the component's data nodes, not here, which keeps this off the CHASM + partial-read path. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class ProducersEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___ProducerCursor: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___ProducerCursor | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["key", b"key", "value", b"value"], + ) -> None: ... + + class ConsumersEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___ConsumerCursor: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___ConsumerCursor | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["key", b"key", "value", b"value"], + ) -> None: ... + + HEAD_OFFSET_FIELD_NUMBER: builtins.int + BASE_OFFSET_FIELD_NUMBER: builtins.int + CLOSED_FIELD_NUMBER: builtins.int + CLOSE_REASON_FIELD_NUMBER: builtins.int + PRODUCERS_FIELD_NUMBER: builtins.int + CONSUMERS_FIELD_NUMBER: builtins.int + LIFECYCLE_FIELD_NUMBER: builtins.int + REDIRECT_RUN_ID_FIELD_NUMBER: builtins.int + CLOSE_TIME_FIELD_NUMBER: builtins.int + BUDGET_FIELD_NUMBER: builtins.int + APPENDED_BYTES_FIELD_NUMBER: builtins.int + NOTIFY_PENDING_FIELD_NUMBER: builtins.int + head_offset: builtins.int + """Visibility frontier. Readers never observe an offset at or past this.""" + base_offset: builtins.int + """Truncation floor. Offsets below this are gone.""" + closed: builtins.bool + @property + def close_reason(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + @property + def producers( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, global___ProducerCursor + ]: ... + @property + def consumers( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, global___ConsumerCursor + ]: ... + @property + def lifecycle(self) -> global___StreamLifecycle: ... + redirect_run_id: builtins.str + """Set when a successor run takes ownership, so an in-flight poll can follow + the chain instead of stalling on a superseded run. + """ + @property + def close_time(self) -> google.protobuf.timestamp_pb2.Timestamp: + """Wall-clock close time, used to schedule retention deletion.""" + @property + def budget(self) -> global___StreamBudget: + """Set on a stream a workflow owns. Its batches are the owning execution's + mutable state, so appends past the budget are refused rather than left to + the execution size limit, which terminates the workflow. + """ + appended_bytes: builtins.int + """Bytes appended over the stream's life, kept for the budget check. A stream + with a budget never reclaims, so this is also what it holds. + """ + notify_pending: builtins.bool + """A notify task is scheduled and has not run yet. Appends while it is set + schedule none of their own; the task reads the head when it runs, so it + carries every append that landed before it. + """ + def __init__( + self, + *, + head_offset: builtins.int = ..., + base_offset: builtins.int = ..., + closed: builtins.bool = ..., + close_reason: temporalio.api.common.v1.message_pb2.Payload | None = ..., + producers: collections.abc.Mapping[builtins.str, global___ProducerCursor] + | None = ..., + consumers: collections.abc.Mapping[builtins.str, global___ConsumerCursor] + | None = ..., + lifecycle: global___StreamLifecycle | None = ..., + redirect_run_id: builtins.str = ..., + close_time: google.protobuf.timestamp_pb2.Timestamp | None = ..., + budget: global___StreamBudget | None = ..., + appended_bytes: builtins.int = ..., + notify_pending: builtins.bool = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "budget", + b"budget", + "close_reason", + b"close_reason", + "close_time", + b"close_time", + "lifecycle", + b"lifecycle", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "appended_bytes", + b"appended_bytes", + "base_offset", + b"base_offset", + "budget", + b"budget", + "close_reason", + b"close_reason", + "close_time", + b"close_time", + "closed", + b"closed", + "consumers", + b"consumers", + "head_offset", + b"head_offset", + "lifecycle", + b"lifecycle", + "notify_pending", + b"notify_pending", + "producers", + b"producers", + "redirect_run_id", + b"redirect_run_id", + ], + ) -> None: ... + +global___StreamState = StreamState + +class StreamBudget(google.protobuf.message.Message): + """Hard bounds on what a stream may hold. Distinct from StreamLifecycle.max_items, + which reclaims the oldest messages: a budget refuses the newest. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + MAX_ITEMS_FIELD_NUMBER: builtins.int + MAX_BYTES_FIELD_NUMBER: builtins.int + max_items: builtins.int + max_bytes: builtins.int + def __init__( + self, + *, + max_items: builtins.int = ..., + max_bytes: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "max_bytes", b"max_bytes", "max_items", b"max_items" + ], + ) -> None: ... + +global___StreamBudget = StreamBudget + +class ProducerCursor(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + FIRST_OFFSET_FIELD_NUMBER: builtins.int + COUNT_FIELD_NUMBER: builtins.int + CONTENT_HASH_FIELD_NUMBER: builtins.int + FENCED_FIELD_NUMBER: builtins.int + seq: builtins.int + first_offset: builtins.int + count: builtins.int + content_hash: builtins.bytes + """Distinguishes a genuine retry from a client reusing a sequence with + different content, which must be rejected rather than deduplicated. + """ + fenced: builtins.bool + """Set by FinishWriting. Ends this producer's writes without closing the + stream for anyone else. + """ + def __init__( + self, + *, + seq: builtins.int = ..., + first_offset: builtins.int = ..., + count: builtins.int = ..., + content_hash: builtins.bytes = ..., + fenced: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "content_hash", + b"content_hash", + "count", + b"count", + "fenced", + b"fenced", + "first_offset", + b"first_offset", + "seq", + b"seq", + ], + ) -> None: ... + +global___ProducerCursor = ProducerCursor + +class ConsumerCursor(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WORKFLOW_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + OFFSET_FIELD_NUMBER: builtins.int + ACTIVE_FIELD_NUMBER: builtins.int + EXTERNAL_FIELD_NUMBER: builtins.int + REPLAY_FLOOR_FIELD_NUMBER: builtins.int + workflow_id: builtins.str + run_id: builtins.str + offset: builtins.int + """How far this consumer has read. Used to decide whether it needs waking, + not to decide what the stream may drop. + """ + active: builtins.bool + external: builtins.bool + """Set when the consumer is a workflow in another execution, which is the + only case that has to be told the frontier moved. A workflow consuming a + stream it owns sees that while closing its own transaction. + """ + replay_floor: builtins.int + """The oldest offset this consumer's History still depends on. + + It is where the subscription started, not where it has read to. A range + this consumer already consumed is recorded in its History and has to be + re-readable for the workflow to replay, so bytes below the read position + are exactly the ones a replay needs most. It stays put while the consumer + is active and is released when it deregisters. + """ + def __init__( + self, + *, + workflow_id: builtins.str = ..., + run_id: builtins.str = ..., + offset: builtins.int = ..., + active: builtins.bool = ..., + external: builtins.bool = ..., + replay_floor: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "active", + b"active", + "external", + b"external", + "offset", + b"offset", + "replay_floor", + b"replay_floor", + "run_id", + b"run_id", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___ConsumerCursor = ConsumerCursor + +class WorkflowStreamCursor(google.protobuf.message.Message): + """A consuming Workflow's position in a stream. This lives in the consuming + Workflow's own state rather than on the stream, so advancing it commits in + the same transaction as the WorkflowTaskCompleted event that records the + range. Keeping it on the stream would make the advance a cross-execution + write, and a crash between the two would either redeliver or skip. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + OFFSET_FIELD_NUMBER: builtins.int + KNOWN_HEAD_FIELD_NUMBER: builtins.int + EXTERNAL_FIELD_NUMBER: builtins.int + PENDING_FROM_FIELD_NUMBER: builtins.int + PENDING_TO_FIELD_NUMBER: builtins.int + HAS_PENDING_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + stream_id: builtins.str + offset: builtins.int + """Next offset to deliver.""" + known_head: builtins.int + """The stream's frontier as of the last delivery. A workflow consuming a + stream in another execution cannot read the real frontier while closing its + own transaction, so this is what tells it a capped slice left more behind. + """ + external: builtins.bool + """Set when the stream lives in another execution, which decides whether the + frontier is read locally or taken from known_head. + """ + pending_from: builtins.int + """The range attached to the Workflow Task currently in flight. Recorded on + the event that closes that task, then folded into offset. An empty range + is still recorded: a task where the subscription saw nothing is a fact + replay has to reproduce. + """ + pending_to: builtins.int + has_pending: builtins.bool + start_offset: builtins.int + """Where this subscription began reading. Retained separately from offset, + which advances, because replay has to tell "committed nothing yet" apart + from "the events recording what was committed are not in this page". + """ + def __init__( + self, + *, + stream_id: builtins.str = ..., + offset: builtins.int = ..., + known_head: builtins.int = ..., + external: builtins.bool = ..., + pending_from: builtins.int = ..., + pending_to: builtins.int = ..., + has_pending: builtins.bool = ..., + start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "external", + b"external", + "has_pending", + b"has_pending", + "known_head", + b"known_head", + "offset", + b"offset", + "pending_from", + b"pending_from", + "pending_to", + b"pending_to", + "start_offset", + b"start_offset", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___WorkflowStreamCursor = WorkflowStreamCursor + +class StreamLifecycle(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RETENTION_FIELD_NUMBER: builtins.int + MAX_ITEMS_FIELD_NUMBER: builtins.int + @property + def retention(self) -> google.protobuf.duration_pb2.Duration: + """How long a closed stream stays readable before it is deleted.""" + max_items: builtins.int + """Cap on readable messages. Whole batches are reclaimed once the floor + passes them, so a capped stream has bounded storage. + """ + def __init__( + self, + *, + retention: google.protobuf.duration_pb2.Duration | None = ..., + max_items: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["retention", b"retention"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "max_items", b"max_items", "retention", b"retention" + ], + ) -> None: ... + +global___StreamLifecycle = StreamLifecycle diff --git a/temporalio/api/streamservice/v1/stream_state_pb2_grpc.py b/temporalio/api/streamservice/v1/stream_state_pb2_grpc.py new file mode 100644 index 000000000..bf947056a --- /dev/null +++ b/temporalio/api/streamservice/v1/stream_state_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc diff --git a/temporalio/api/streamservice/v1/stream_state_pb2_grpc.pyi b/temporalio/api/streamservice/v1/stream_state_pb2_grpc.pyi new file mode 100644 index 000000000..f3a5a087e --- /dev/null +++ b/temporalio/api/streamservice/v1/stream_state_pb2_grpc.pyi @@ -0,0 +1,4 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index d8fa05edf..c04d1c60f 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -308,6 +308,20 @@ async def _visit_coresdk_workflow_activation_ResolveNexusOperation( if o.HasField("result"): await self._visit_coresdk_nexus_NexusOperationResult(fs, o.result) + async def _visit_temporal_api_stream_v1_StreamRecord( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("body"): + await self._visit_temporal_api_common_v1_Payload(fs, o.body) + for v in o.metadata.values(): + await self._visit_temporal_api_common_v1_Payload(fs, v) + + async def _visit_coresdk_workflow_activation_DeliverStreamRecords( + self, fs: VisitorFunctions, o: Any + ): + for v in o.records: + await self._visit_temporal_api_stream_v1_StreamRecord(fs, v) + async def _visit_coresdk_workflow_activation_WorkflowActivationJob( self, fs: VisitorFunctions, o: Any ): @@ -353,6 +367,10 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob( await self._visit_coresdk_workflow_activation_ResolveNexusOperation( fs, o.resolve_nexus_operation ) + elif o.HasField("deliver_stream_records"): + await self._visit_coresdk_workflow_activation_DeliverStreamRecords( + fs, o.deliver_stream_records + ) async def _visit_coresdk_workflow_activation_WorkflowActivation( self, fs: VisitorFunctions, o: Any @@ -492,6 +510,12 @@ async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( if o.HasField("input"): await self._visit_nexus_operation_input_payload(fs, o.input) + async def _visit_coresdk_workflow_commands_AppendStreamRecords( + self, fs: VisitorFunctions, o: Any + ): + for v in o.records: + await self._visit_temporal_api_stream_v1_StreamRecord(fs, v) + async def _visit_coresdk_workflow_commands_WorkflowCommand( self, fs: VisitorFunctions, o: Any ): @@ -551,6 +575,10 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand( await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( fs, o.schedule_nexus_operation ) + elif o.HasField("append_stream_records"): + await self._visit_coresdk_workflow_commands_AppendStreamRecords( + fs, o.append_stream_records + ) async def _visit_coresdk_workflow_completion_Success( self, fs: VisitorFunctions, o: Any diff --git a/temporalio/bridge/proto/workflow_activation/__init__.py b/temporalio/bridge/proto/workflow_activation/__init__.py index 5c178e218..f17880890 100644 --- a/temporalio/bridge/proto/workflow_activation/__init__.py +++ b/temporalio/bridge/proto/workflow_activation/__init__.py @@ -1,5 +1,6 @@ from .workflow_activation_pb2 import ( CancelWorkflow, + DeliverStreamRecords, DoUpdate, FireTimer, InitializeWorkflow, @@ -24,6 +25,7 @@ __all__ = [ "CancelWorkflow", + "DeliverStreamRecords", "DoUpdate", "FireTimer", "InitializeWorkflow", diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py index 0caca7d48..7f6d3e782 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py @@ -27,6 +27,9 @@ from temporalio.api.failure.v1 import ( message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.update.v1 import ( message_pb2 as temporal_dot_api_dot_update_dot_v1_dot_message__pb2, ) @@ -44,12 +47,13 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xe0\n\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xfc\n\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"_\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"f\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xcb\x02\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\nB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' + b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xb5\x0b\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12S\n\x16\x64\x65liver_stream_records\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_activation.DeliverStreamRecordsH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\x88\x01\n\x14\x44\x65liverStreamRecords\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x02 \x01(\x03\x12\x11\n\tto_offset\x18\x03 \x01(\x03\x12\x35\n\x07records\x18\x04 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord"\xfc\n\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"_\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"f\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xcb\x02\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\nB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' ) _WORKFLOWACTIVATION = DESCRIPTOR.message_types_by_name["WorkflowActivation"] _WORKFLOWACTIVATIONJOB = DESCRIPTOR.message_types_by_name["WorkflowActivationJob"] +_DELIVERSTREAMRECORDS = DESCRIPTOR.message_types_by_name["DeliverStreamRecords"] _INITIALIZEWORKFLOW = DESCRIPTOR.message_types_by_name["InitializeWorkflow"] _INITIALIZEWORKFLOW_HEADERSENTRY = _INITIALIZEWORKFLOW.nested_types_by_name[ "HeadersEntry" @@ -114,6 +118,17 @@ ) _sym_db.RegisterMessage(WorkflowActivationJob) +DeliverStreamRecords = _reflection.GeneratedProtocolMessageType( + "DeliverStreamRecords", + (_message.Message,), + { + "DESCRIPTOR": _DELIVERSTREAMRECORDS, + "__module__": "temporal.sdk.core.workflow_activation.workflow_activation_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_activation.DeliverStreamRecords) + }, +) +_sym_db.RegisterMessage(DeliverStreamRecords) + InitializeWorkflow = _reflection.GeneratedProtocolMessageType( "InitializeWorkflow", (_message.Message,), @@ -376,56 +391,58 @@ _SIGNALWORKFLOW_HEADERSENTRY._serialized_options = b"8\001" _DOUPDATE_HEADERSENTRY._options = None _DOUPDATE_HEADERSENTRY._serialized_options = b"8\001" - _WORKFLOWACTIVATION._serialized_start = 532 - _WORKFLOWACTIVATION._serialized_end = 1078 - _WORKFLOWACTIVATIONJOB._serialized_start = 1081 - _WORKFLOWACTIVATIONJOB._serialized_end = 2457 - _INITIALIZEWORKFLOW._serialized_start = 2460 - _INITIALIZEWORKFLOW._serialized_end = 3864 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 3785 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 3864 - _FIRETIMER._serialized_start = 3866 - _FIRETIMER._serialized_end = 3890 - _RESOLVEACTIVITY._serialized_start = 3892 - _RESOLVEACTIVITY._serialized_end = 4001 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4004 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4341 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4343 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4402 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4405 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 4571 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 4573 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 4669 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 4671 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 4776 - _UPDATERANDOMSEED._serialized_start = 4778 - _UPDATERANDOMSEED._serialized_end = 4821 - _QUERYWORKFLOW._serialized_start = 4824 - _QUERYWORKFLOW._serialized_end = 5084 - _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 3785 - _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 3864 - _CANCELWORKFLOW._serialized_start = 5086 - _CANCELWORKFLOW._serialized_end = 5118 - _SIGNALWORKFLOW._serialized_start = 5121 - _SIGNALWORKFLOW._serialized_end = 5410 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 3785 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 3864 - _NOTIFYHASPATCH._serialized_start = 5412 - _NOTIFYHASPATCH._serialized_end = 5446 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5448 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 5543 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 5545 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 5647 - _DOUPDATE._serialized_start = 5650 - _DOUPDATE._serialized_end = 5981 - _DOUPDATE_HEADERSENTRY._serialized_start = 3785 - _DOUPDATE_HEADERSENTRY._serialized_end = 3864 - _RESOLVENEXUSOPERATIONSTART._serialized_start = 5984 - _RESOLVENEXUSOPERATIONSTART._serialized_end = 6138 - _RESOLVENEXUSOPERATION._serialized_start = 6140 - _RESOLVENEXUSOPERATION._serialized_end = 6229 - _REMOVEFROMCACHE._serialized_start = 6232 - _REMOVEFROMCACHE._serialized_end = 6584 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 6346 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 6584 + _WORKFLOWACTIVATION._serialized_start = 570 + _WORKFLOWACTIVATION._serialized_end = 1116 + _WORKFLOWACTIVATIONJOB._serialized_start = 1119 + _WORKFLOWACTIVATIONJOB._serialized_end = 2580 + _DELIVERSTREAMRECORDS._serialized_start = 2583 + _DELIVERSTREAMRECORDS._serialized_end = 2719 + _INITIALIZEWORKFLOW._serialized_start = 2722 + _INITIALIZEWORKFLOW._serialized_end = 4126 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 4047 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4126 + _FIRETIMER._serialized_start = 4128 + _FIRETIMER._serialized_end = 4152 + _RESOLVEACTIVITY._serialized_start = 4154 + _RESOLVEACTIVITY._serialized_end = 4263 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4266 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4603 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4605 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4664 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4667 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 4833 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 4835 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 4931 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 4933 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 5038 + _UPDATERANDOMSEED._serialized_start = 5040 + _UPDATERANDOMSEED._serialized_end = 5083 + _QUERYWORKFLOW._serialized_start = 5086 + _QUERYWORKFLOW._serialized_end = 5346 + _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 4047 + _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4126 + _CANCELWORKFLOW._serialized_start = 5348 + _CANCELWORKFLOW._serialized_end = 5380 + _SIGNALWORKFLOW._serialized_start = 5383 + _SIGNALWORKFLOW._serialized_end = 5672 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 4047 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4126 + _NOTIFYHASPATCH._serialized_start = 5674 + _NOTIFYHASPATCH._serialized_end = 5708 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5710 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 5805 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 5807 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 5909 + _DOUPDATE._serialized_start = 5912 + _DOUPDATE._serialized_end = 6243 + _DOUPDATE_HEADERSENTRY._serialized_start = 4047 + _DOUPDATE_HEADERSENTRY._serialized_end = 4126 + _RESOLVENEXUSOPERATIONSTART._serialized_start = 6246 + _RESOLVENEXUSOPERATIONSTART._serialized_end = 6400 + _RESOLVENEXUSOPERATION._serialized_start = 6402 + _RESOLVENEXUSOPERATION._serialized_end = 6491 + _REMOVEFROMCACHE._serialized_start = 6494 + _REMOVEFROMCACHE._serialized_end = 6846 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 6608 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 6846 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi index afc7eb4ab..ac9ac13f7 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi @@ -21,6 +21,7 @@ import google.protobuf.timestamp_pb2 import temporalio.api.common.v1.message_pb2 import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.update.v1.message_pb2 import temporalio.bridge.proto.activity_result.activity_result_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 @@ -235,6 +236,7 @@ class WorkflowActivationJob(google.protobuf.message.Message): DO_UPDATE_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_START_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + DELIVER_STREAM_RECORDS_FIELD_NUMBER: builtins.int REMOVE_FROM_CACHE_FIELD_NUMBER: builtins.int @property def initialize_workflow(self) -> global___InitializeWorkflow: @@ -295,6 +297,14 @@ class WorkflowActivationJob(google.protobuf.message.Message): def resolve_nexus_operation(self) -> global___ResolveNexusOperation: """A nexus operation resolved.""" @property + def deliver_stream_records(self) -> global___DeliverStreamRecords: + """17 to 20 are taken by the external stream jobs, which are developed + alongside this one and share this message. The number below is fixed + with that family and must not be reused. + + A range of a stream the workflow subscribed to. + """ + @property def remove_from_cache(self) -> global___RemoveFromCache: """Remove the workflow identified by the [WorkflowActivation] containing this job from the cache after performing the activation. It is guaranteed that this will be the only job @@ -322,6 +332,7 @@ class WorkflowActivationJob(google.protobuf.message.Message): do_update: global___DoUpdate | None = ..., resolve_nexus_operation_start: global___ResolveNexusOperationStart | None = ..., resolve_nexus_operation: global___ResolveNexusOperation | None = ..., + deliver_stream_records: global___DeliverStreamRecords | None = ..., remove_from_cache: global___RemoveFromCache | None = ..., ) -> None: ... def HasField( @@ -329,6 +340,8 @@ class WorkflowActivationJob(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "cancel_workflow", b"cancel_workflow", + "deliver_stream_records", + b"deliver_stream_records", "do_update", b"do_update", "fire_timer", @@ -368,6 +381,8 @@ class WorkflowActivationJob(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "cancel_workflow", b"cancel_workflow", + "deliver_stream_records", + b"deliver_stream_records", "do_update", b"do_update", "fire_timer", @@ -421,6 +436,7 @@ class WorkflowActivationJob(google.protobuf.message.Message): "do_update", "resolve_nexus_operation_start", "resolve_nexus_operation", + "deliver_stream_records", "remove_from_cache", ] | None @@ -428,6 +444,63 @@ class WorkflowActivationJob(google.protobuf.message.Message): global___WorkflowActivationJob = WorkflowActivationJob +class DeliverStreamRecords(google.protobuf.message.Message): + """Hand a workflow the next range of a stream it subscribed to. + + The range is delivered once, on the task the server decided it belongs to, + and the offsets it covered are recorded in History rather than the payloads. + On replay the server re-supplies the same range by reading the stream again, + so this job appears at the same point with the same contents both times. + + An empty range is still delivered: a task where the subscription saw nothing + is a fact replay has to reproduce, not an absence of one. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + TO_OFFSET_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + stream_id: builtins.str + """Id of the stream this range came from.""" + from_offset: builtins.int + """Inclusive.""" + to_offset: builtins.int + """Exclusive. Equal to from_offset when the subscription saw nothing.""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: ... + def __init__( + self, + *, + stream_id: builtins.str = ..., + from_offset: builtins.int = ..., + to_offset: builtins.int = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "records", + b"records", + "stream_id", + b"stream_id", + "to_offset", + b"to_offset", + ], + ) -> None: ... + +global___DeliverStreamRecords = DeliverStreamRecords + class InitializeWorkflow(google.protobuf.message.Message): """Initialize a new workflow""" diff --git a/temporalio/bridge/proto/workflow_commands/__init__.py b/temporalio/bridge/proto/workflow_commands/__init__.py index f599917af..9fb1c8f08 100644 --- a/temporalio/bridge/proto/workflow_commands/__init__.py +++ b/temporalio/bridge/proto/workflow_commands/__init__.py @@ -1,5 +1,6 @@ from .workflow_commands_pb2 import ( ActivityCancellationType, + AppendStreamRecords, CancelChildWorkflowExecution, CancelSignalWorkflow, CancelTimer, @@ -21,6 +22,7 @@ SignalExternalWorkflowExecution, StartChildWorkflowExecution, StartTimer, + SubscribeStream, UpdateResponse, UpsertWorkflowSearchAttributes, WorkflowCommand, @@ -28,6 +30,7 @@ __all__ = [ "ActivityCancellationType", + "AppendStreamRecords", "CancelChildWorkflowExecution", "CancelSignalWorkflow", "CancelTimer", @@ -49,6 +52,7 @@ "SignalExternalWorkflowExecution", "StartChildWorkflowExecution", "StartTimer", + "SubscribeStream", "UpdateResponse", "UpsertWorkflowSearchAttributes", "WorkflowCommand", diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py index 0cdc02c6b..5af8f0744 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py @@ -34,6 +34,9 @@ from temporalio.api.sdk.v1 import ( user_metadata_pb2 as temporal_dot_api_dot_sdk_dot_v1_dot_user__metadata__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.bridge.proto.child_workflow import ( child_workflow_pb2 as temporal_dot_sdk_dot_core_dot_child__workflow_dot_child__workflow__pb2, ) @@ -45,7 +48,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xa9\x10\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x95\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12%\n\x1dinclude_arguments_into_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\x96\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' + b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a$temporal/api/stream/v1/message.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xc2\x11\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x12\x46\n\x10subscribe_stream\x18\x1d \x01(\x0b\x32*.coresdk.workflow_commands.SubscribeStreamH\x00\x12O\n\x15\x61ppend_stream_records\x18\x1e \x01(\x0b\x32..coresdk.workflow_commands.AppendStreamRecordsH\x00\x42\t\n\x07variant"_\n\x13\x41ppendStreamRecords\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x35\n\x07records\x18\x02 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord":\n\x0fSubscribeStream\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x14\n\x0cstart_offset\x18\x02 \x01(\x03"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x95\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12%\n\x1dinclude_arguments_into_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\x96\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' ) _ACTIVITYCANCELLATIONTYPE = DESCRIPTOR.enum_types_by_name["ActivityCancellationType"] @@ -56,6 +59,8 @@ _WORKFLOWCOMMAND = DESCRIPTOR.message_types_by_name["WorkflowCommand"] +_APPENDSTREAMRECORDS = DESCRIPTOR.message_types_by_name["AppendStreamRecords"] +_SUBSCRIBESTREAM = DESCRIPTOR.message_types_by_name["SubscribeStream"] _STARTTIMER = DESCRIPTOR.message_types_by_name["StartTimer"] _CANCELTIMER = DESCRIPTOR.message_types_by_name["CancelTimer"] _SCHEDULEACTIVITY = DESCRIPTOR.message_types_by_name["ScheduleActivity"] @@ -130,6 +135,28 @@ ) _sym_db.RegisterMessage(WorkflowCommand) +AppendStreamRecords = _reflection.GeneratedProtocolMessageType( + "AppendStreamRecords", + (_message.Message,), + { + "DESCRIPTOR": _APPENDSTREAMRECORDS, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.AppendStreamRecords) + }, +) +_sym_db.RegisterMessage(AppendStreamRecords) + +SubscribeStream = _reflection.GeneratedProtocolMessageType( + "SubscribeStream", + (_message.Message,), + { + "DESCRIPTOR": _SUBSCRIBESTREAM, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.SubscribeStream) + }, +) +_sym_db.RegisterMessage(SubscribeStream) + StartTimer = _reflection.GeneratedProtocolMessageType( "StartTimer", (_message.Message,), @@ -484,70 +511,74 @@ _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_options = b"8\001" _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._options = None _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_options = b"8\001" - _ACTIVITYCANCELLATIONTYPE._serialized_start = 8672 - _ACTIVITYCANCELLATIONTYPE._serialized_end = 8760 - _WORKFLOWCOMMAND._serialized_start = 518 - _WORKFLOWCOMMAND._serialized_end = 2607 - _STARTTIMER._serialized_start = 2609 - _STARTTIMER._serialized_end = 2692 - _CANCELTIMER._serialized_start = 2694 - _CANCELTIMER._serialized_end = 2720 - _SCHEDULEACTIVITY._serialized_start = 2723 - _SCHEDULEACTIVITY._serialized_end = 3547 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3468 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3547 - _SCHEDULELOCALACTIVITY._serialized_start = 3550 - _SCHEDULELOCALACTIVITY._serialized_end = 4339 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3468 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3547 - _REQUESTCANCELACTIVITY._serialized_start = 4341 - _REQUESTCANCELACTIVITY._serialized_end = 4377 - _REQUESTCANCELLOCALACTIVITY._serialized_start = 4379 - _REQUESTCANCELLOCALACTIVITY._serialized_end = 4420 - _QUERYRESULT._serialized_start = 4423 - _QUERYRESULT._serialized_end = 4579 - _QUERYSUCCESS._serialized_start = 4581 - _QUERYSUCCESS._serialized_end = 4646 - _COMPLETEWORKFLOWEXECUTION._serialized_start = 4648 - _COMPLETEWORKFLOWEXECUTION._serialized_end = 4724 - _FAILWORKFLOWEXECUTION._serialized_start = 4726 - _FAILWORKFLOWEXECUTION._serialized_end = 4800 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 4803 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 5717 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5560 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5636 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3468 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3547 - _CANCELWORKFLOWEXECUTION._serialized_start = 5719 - _CANCELWORKFLOWEXECUTION._serialized_end = 5795 - _SETPATCHMARKER._serialized_start = 5797 - _SETPATCHMARKER._serialized_end = 5851 - _STARTCHILDWORKFLOWEXECUTION._serialized_start = 5854 - _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7028 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3468 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3547 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5560 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5636 - _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7030 - _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7104 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7107 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7249 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7252 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 7651 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3468 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3547 - _CANCELSIGNALWORKFLOW._serialized_start = 7653 - _CANCELSIGNALWORKFLOW._serialized_end = 7688 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 7690 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 7791 - _MODIFYWORKFLOWPROPERTIES._serialized_start = 7793 - _MODIFYWORKFLOWPROPERTIES._serialized_end = 7872 - _UPDATERESPONSE._serialized_start = 7875 - _UPDATERESPONSE._serialized_end = 8085 - _SCHEDULENEXUSOPERATION._serialized_start = 8088 - _SCHEDULENEXUSOPERATION._serialized_end = 8626 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 8576 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 8626 - _REQUESTCANCELNEXUSOPERATION._serialized_start = 8628 - _REQUESTCANCELNEXUSOPERATION._serialized_end = 8670 + _ACTIVITYCANCELLATIONTYPE._serialized_start = 9020 + _ACTIVITYCANCELLATIONTYPE._serialized_end = 9108 + _WORKFLOWCOMMAND._serialized_start = 556 + _WORKFLOWCOMMAND._serialized_end = 2798 + _APPENDSTREAMRECORDS._serialized_start = 2800 + _APPENDSTREAMRECORDS._serialized_end = 2895 + _SUBSCRIBESTREAM._serialized_start = 2897 + _SUBSCRIBESTREAM._serialized_end = 2955 + _STARTTIMER._serialized_start = 2957 + _STARTTIMER._serialized_end = 3040 + _CANCELTIMER._serialized_start = 3042 + _CANCELTIMER._serialized_end = 3068 + _SCHEDULEACTIVITY._serialized_start = 3071 + _SCHEDULEACTIVITY._serialized_end = 3895 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3816 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3895 + _SCHEDULELOCALACTIVITY._serialized_start = 3898 + _SCHEDULELOCALACTIVITY._serialized_end = 4687 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3816 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3895 + _REQUESTCANCELACTIVITY._serialized_start = 4689 + _REQUESTCANCELACTIVITY._serialized_end = 4725 + _REQUESTCANCELLOCALACTIVITY._serialized_start = 4727 + _REQUESTCANCELLOCALACTIVITY._serialized_end = 4768 + _QUERYRESULT._serialized_start = 4771 + _QUERYRESULT._serialized_end = 4927 + _QUERYSUCCESS._serialized_start = 4929 + _QUERYSUCCESS._serialized_end = 4994 + _COMPLETEWORKFLOWEXECUTION._serialized_start = 4996 + _COMPLETEWORKFLOWEXECUTION._serialized_end = 5072 + _FAILWORKFLOWEXECUTION._serialized_start = 5074 + _FAILWORKFLOWEXECUTION._serialized_end = 5148 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 5151 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 6065 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5908 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5984 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3816 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3895 + _CANCELWORKFLOWEXECUTION._serialized_start = 6067 + _CANCELWORKFLOWEXECUTION._serialized_end = 6143 + _SETPATCHMARKER._serialized_start = 6145 + _SETPATCHMARKER._serialized_end = 6199 + _STARTCHILDWORKFLOWEXECUTION._serialized_start = 6202 + _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7376 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3816 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3895 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5908 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5984 + _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7378 + _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7452 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7455 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7597 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7600 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 7999 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3816 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3895 + _CANCELSIGNALWORKFLOW._serialized_start = 8001 + _CANCELSIGNALWORKFLOW._serialized_end = 8036 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 8038 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 8139 + _MODIFYWORKFLOWPROPERTIES._serialized_start = 8141 + _MODIFYWORKFLOWPROPERTIES._serialized_end = 8220 + _UPDATERESPONSE._serialized_start = 8223 + _UPDATERESPONSE._serialized_end = 8433 + _SCHEDULENEXUSOPERATION._serialized_start = 8436 + _SCHEDULENEXUSOPERATION._serialized_end = 8974 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 8924 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 8974 + _REQUESTCANCELNEXUSOPERATION._serialized_start = 8976 + _REQUESTCANCELNEXUSOPERATION._serialized_end = 9018 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi index ad4a5f487..b4848ddb1 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi @@ -25,6 +25,7 @@ import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.sdk.v1.event_group_marker_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 import temporalio.bridge.proto.common.common_pb2 import temporalio.bridge.proto.nexus.nexus_pb2 @@ -103,6 +104,8 @@ class WorkflowCommand(google.protobuf.message.Message): UPDATE_RESPONSE_FIELD_NUMBER: builtins.int SCHEDULE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int REQUEST_CANCEL_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + SUBSCRIBE_STREAM_FIELD_NUMBER: builtins.int + APPEND_STREAM_RECORDS_FIELD_NUMBER: builtins.int @property def user_metadata(self) -> temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata: """User metadata that may or may not be persisted into history depending on the command type. @@ -177,6 +180,14 @@ class WorkflowCommand(google.protobuf.message.Message): def request_cancel_nexus_operation( self, ) -> global___RequestCancelNexusOperation: ... + @property + def subscribe_stream(self) -> global___SubscribeStream: + """23 to 28 are taken by the external stream commands, which are developed + alongside these and share this message. The two numbers below are fixed + with that family and must not be reused. + """ + @property + def append_stream_records(self) -> global___AppendStreamRecords: ... def __init__( self, *, @@ -215,10 +226,14 @@ class WorkflowCommand(google.protobuf.message.Message): schedule_nexus_operation: global___ScheduleNexusOperation | None = ..., request_cancel_nexus_operation: global___RequestCancelNexusOperation | None = ..., + subscribe_stream: global___SubscribeStream | None = ..., + append_stream_records: global___AppendStreamRecords | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ + "append_stream_records", + b"append_stream_records", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -259,6 +274,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"start_child_workflow_execution", "start_timer", b"start_timer", + "subscribe_stream", + b"subscribe_stream", "update_response", b"update_response", "upsert_workflow_search_attributes", @@ -272,6 +289,8 @@ class WorkflowCommand(google.protobuf.message.Message): def ClearField( self, field_name: typing_extensions.Literal[ + "append_stream_records", + b"append_stream_records", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -314,6 +333,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"start_child_workflow_execution", "start_timer", b"start_timer", + "subscribe_stream", + b"subscribe_stream", "update_response", b"update_response", "upsert_workflow_search_attributes", @@ -350,12 +371,87 @@ class WorkflowCommand(google.protobuf.message.Message): "update_response", "schedule_nexus_operation", "request_cancel_nexus_operation", + "subscribe_stream", + "append_stream_records", ] | None ): ... global___WorkflowCommand = WorkflowCommand +class AppendStreamRecords(google.protobuf.message.Message): + """Append a batch of records to a stream this workflow owns. + + The bodies go to the stream's own log rather than into History, which gets + one fixed-size event naming the offset range. That is what makes the batch + size free: a thousand records cost the same in History as one. The server + stores each record with an empty producer id, because the workflow is the + producer here. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + stream_id: builtins.str + """Empty means the workflow's default output stream.""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: ... + def __init__( + self, + *, + stream_id: builtins.str = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "records", b"records", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___AppendStreamRecords = AppendStreamRecords + +class SubscribeStream(google.protobuf.message.Message): + """Subscribe this workflow to a stream, so later Workflow Tasks carry the + ranges it has not consumed yet. + + The stream's addressing is resolved by the server. A workflow cannot look it + up without doing I/O, and a value it carried would be a reading rather than a + fact, so it could differ on replay. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + stream_id: builtins.str + start_offset: builtins.int + """Negative means from wherever the stream is when the subscription is + registered. The server resolves that once and records it. + """ + def __init__( + self, + *, + stream_id: builtins.str = ..., + start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "start_offset", b"start_offset", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___SubscribeStream = SubscribeStream + class StartTimer(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 5209468cb..32c66693f 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 5209468cb61533f95b78c9f9d5561d55400e2d25 +Subproject commit 32c66693f07222832443a5acb1ddb92006c23ea7 diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index e74698e94..44e5a2005 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -14,6 +14,7 @@ use temporalio_common::protos::coresdk::{ nexus::NexusTaskCompletion, ActivityHeartbeat, ActivityTaskCompletion, }; use temporalio_common::protos::temporal::api::history::v1::History; +use temporalio_common::protos::temporal::api::stream::v1::StreamSlice; use temporalio_common::protos::temporal::api::worker::v1::{PluginInfo, StorageDriverInfo}; use temporalio_sdk_core::replay::{HistoryForReplay, ReplayWorkerInput}; use temporalio_sdk_core::{ @@ -950,14 +951,24 @@ impl HistoryPusher { #[pymethods] impl HistoryPusher { + /// Feed one history to the replay worker. `stream_slices` are serialized + /// `temporal.api.stream.v1.StreamSlice` messages carrying the records the + /// history's completed tasks consumed, which History itself never holds. + #[pyo3(signature = (workflow_id, history_proto, stream_slices = Vec::new()))] fn push_history<'p>( &self, py: Python<'p>, workflow_id: &str, history_proto: &Bound<'_, PyBytes>, + stream_slices: Vec>, ) -> PyResult> { let history = History::decode(history_proto.as_bytes()) .map_err(|err| PyValueError::new_err(format!("Invalid proto: {err}")))?; + let slices = stream_slices + .iter() + .map(|slice| StreamSlice::decode(slice.as_bytes())) + .collect::, _>>() + .map_err(|err| PyValueError::new_err(format!("Invalid stream slice proto: {err}")))?; let wfid = workflow_id.to_string(); let tx = if let Some(tx) = self.tx.as_ref() { tx.clone() @@ -968,7 +979,7 @@ impl HistoryPusher { }; // We accept this doesn't have logging/tracing self.runtime.future_into_py(py, async move { - tx.send(HistoryForReplay::new(history, wfid)) + tx.send(HistoryForReplay::new(history, wfid).with_stream_slices(slices)) .await .map_err(|_| { PyRuntimeError::new_err( diff --git a/temporalio/client/_workflow.py b/temporalio/client/_workflow.py index 0607ade0a..10a4853fd 100644 --- a/temporalio/client/_workflow.py +++ b/temporalio/client/_workflow.py @@ -4,6 +4,7 @@ import asyncio import functools +import json import warnings from asyncio import Future from collections.abc import ( @@ -31,6 +32,7 @@ import temporalio.api.common.v1 import temporalio.api.enums.v1 import temporalio.api.history.v1 +import temporalio.api.stream.v1 import temporalio.api.update.v1 import temporalio.api.workflow.v1 import temporalio.api.workflowservice.v1 @@ -1681,6 +1683,20 @@ class WorkflowHistory: events: Sequence[temporalio.api.history.v1.HistoryEvent] """History events for the workflow.""" + stream_slices: Sequence[temporalio.api.stream.v1.StreamSlice] = () + """The stream records the workflow's completed tasks consumed, if captured. + + History records only the offsets each Workflow Task consumed from a + server-side stream; the records live in the stream. A history whose tasks + consumed records and that carries none here cannot be replayed without the + server that still holds the stream. One slice per recorded range, tagged + with the ``WorkflowTaskCompleted`` event that recorded it, in the shape the + server puts on a poll response. + :py:meth:`temporalio.worker.Replayer.fetch_stream_slices` fills it while + the stream is retained, and :py:meth:`to_json` and :py:meth:`from_json` + carry it, so an exported history file is the whole replay input. + """ + @property def run_id(self) -> str: """Run ID extracted from the first event.""" @@ -1702,31 +1718,56 @@ def from_json(workflow_id: str, history: str | dict[str, Any]) -> WorkflowHistor Args: workflow_id: The workflow's ID history: A string or parsed-to-dict representation of workflow - history + history. A ``streamSlices`` list beside ``events``, as + :py:meth:`to_json` writes one, is read into + :py:attr:`stream_slices`. Returns: Workflow history """ + if isinstance(history, str): + history = json.loads(history) parsed = _history_from_json(history) - return WorkflowHistory(workflow_id, parsed.events) + slices: list[temporalio.api.stream.v1.StreamSlice] = [] + if isinstance(history, dict): + raw = history.get("streamSlices") or history.get("stream_slices") or [] + for entry in raw: + slices.append( + google.protobuf.json_format.ParseDict( + entry, + temporalio.api.stream.v1.StreamSlice(), + ignore_unknown_fields=True, + ) + ) + return WorkflowHistory(workflow_id, parsed.events, slices) def to_json(self) -> str: """Convert this history to JSON. - Note, this does not include the workflow ID. + Note, this does not include the workflow ID. The stream slices, when + there are any, are written as a ``streamSlices`` list beside the + events; without them the output is the history proto's JSON alone. """ - return google.protobuf.json_format.MessageToJson( - temporalio.api.history.v1.History(events=self.events) - ) + if not self.stream_slices: + return google.protobuf.json_format.MessageToJson( + temporalio.api.history.v1.History(events=self.events) + ) + return json.dumps(self.to_json_dict(), indent=2) def to_json_dict(self) -> dict[str, Any]: """Convert this history to JSON-compatible dict. - Note, this does not include the workflow ID. + Note, this does not include the workflow ID. See :py:meth:`to_json` + for how the stream slices are carried. """ - return google.protobuf.json_format.MessageToDict( + out = google.protobuf.json_format.MessageToDict( temporalio.api.history.v1.History(events=self.events) ) + if self.stream_slices: + out["streamSlices"] = [ + google.protobuf.json_format.MessageToDict(s) for s in self.stream_slices + ] + return out @dataclass diff --git a/temporalio/client_stream.py b/temporalio/client_stream.py new file mode 100644 index 000000000..b482941af --- /dev/null +++ b/temporalio/client_stream.py @@ -0,0 +1,614 @@ +"""Client for Temporal server-side streams. + +A stream is a durable, offset-addressed append-only sequence that lives beside +Workflow History rather than inside it. Appending schedules no Workflow Task, +and each reader holds its own cursor, so adding a reader costs nothing on the +write side. The record is ``temporal.api.stream.v1.StreamRecord`` on the wire +and in the store, so a reader in any language decodes the same bytes. + +Prototype support for AI-198. Four things about it are temporary and will +change before this is a real feature: + +- **Requires the ``grpc`` extra.** The rest of this SDK reaches the server + through sdk-core, which does not know about this service yet, so the client + here opens its own channel with ``grpcio``. +- **The protos are vendored** under ``temporalio.api.streamservice.v1`` instead of + coming from the api submodule, because the service is still defined in the + server. That is why the wire names read as server-internal. +- **This client is for use outside a Workflow.** Workflow code publishes and + consumes with ``workflow.append_stream_records`` and + ``workflow.read_stream_records`` instead. +- **No TLS or API-key support**, for the same reason: the channel is built + here rather than by the machinery that normally handles that. + +A failed call raises :class:`temporalio.streams.StreamNotFoundError` when the +server answers ``NOT_FOUND`` and :class:`temporalio.service.RPCError` +otherwise, never the transport's own exception type. +""" + +from __future__ import annotations + +import asyncio +import weakref +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from dataclasses import dataclass +from typing import Any, TypeVar + +import google.protobuf.duration_pb2 +import grpc +import grpc.aio + +import temporalio.api.streamservice.v1 as stream +from temporalio.api.stream.v1 import StreamRecord +from temporalio.api.streamservice.v1 import service_pb2_grpc +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import StreamNotFoundError + +__all__ = [ + "Appended", + "Page", + "StreamClient", + "StreamEntry", + "StreamHandle", + "WorkflowStreamHandle", + "close_shared_clients", + "shared_client", +] + +_T = TypeVar("_T") + + +@dataclass(frozen=True) +class Appended: + """Where one append landed. + + ``deduplicated`` says the server already held this producer's batch at + this sequence and wrote nothing; the offsets are then the original ones. + """ + + first_offset: int + next_offset: int + count: int + deduplicated: bool = False + + +@dataclass(frozen=True) +class StreamEntry: + """One record read from a stream, with where it sits. + + Offsets are assigned over the unfiltered stream, so a topic-filtered read + hands back entries whose offsets are not contiguous. A reader that resumes + between records takes ``offset`` rather than counting what it received. + """ + + record: StreamRecord + offset: int = 0 + + +@dataclass(frozen=True) +class Page: + """One read of a stream. + + ``closed`` with ``next_offset >= head_offset`` is the end: nothing more can + be added and this reader has everything. ``run_id`` names the execution + holding the stream, which is how a reader of a workflow's stream learns + which run it is on when it did not pin one. + """ + + entries: list[StreamEntry] + next_offset: int + head_offset: int + closed: bool + run_id: str = "" + + +def _to_service(record: StreamRecord) -> stream.StreamRecord: + # Field for field the public record; the stored shape only adds the offset + # a read assigns. An unset body stays unset so a FINISH record reads back + # as one. + out = stream.StreamRecord( + topic=record.topic, + kind=record.kind, + producer_id=record.producer_id, + attempt=record.attempt, + sequence=record.sequence, + ) + if record.HasField("body"): + out.body.CopyFrom(record.body) + for key, value in record.metadata.items(): + out.metadata[key].CopyFrom(value) + return out + + +def _to_public(record: stream.StreamRecord) -> StreamEntry: + out = StreamRecord( + topic=record.topic, + kind=record.kind, + producer_id=record.producer_id, + attempt=record.attempt, + sequence=record.sequence, + ) + if record.HasField("body"): + out.body.CopyFrom(record.body) + for key, value in record.metadata.items(): + out.metadata[key].CopyFrom(value) + return StreamEntry(record=out, offset=record.offset) + + +def _translate(error: grpc.aio.AioRpcError) -> Exception: + code = error.code() + details = error.details() or code.name + if code is grpc.StatusCode.NOT_FOUND: + return StreamNotFoundError(details) + raw = b"" + # The aio metadata iterates as (key, value) pairs at runtime, whatever + # shape the stubs give its items. + trailing: Any = error.trailing_metadata() + for item in trailing or (): + key, value = item[0], item[1] + if key == "grpc-status-details-bin" and isinstance(value, bytes): + raw = value + return RPCError(details, RPCStatusCode(code.value[0]), raw) + + +async def _call(method: Callable[[Any], Awaitable[_T]], request: Any) -> _T: + """Make one stub call, translating the transport's failure to the SDK's.""" + try: + return await method(request) + except grpc.aio.AioRpcError as error: + raise _translate(error) from error + + +class StreamClient: + """Creates and opens streams on a namespace.""" + + def __init__(self, channel: Any, namespace: str) -> None: + """Wrap an existing ``grpc.aio`` channel. Prefer :meth:`connect`.""" + self._channel = channel + self._namespace = namespace + # The generated stub is typed for a synchronous channel. This client + # drives it over ``grpc.aio``, where every call is awaited. + self._stub: Any = service_pb2_grpc.StreamServiceStub(channel) + + @staticmethod + def connect(target_host: str, namespace: str = "default") -> StreamClient: + """Open a channel to a frontend. + + Separate from ``Client.connect`` because this does not share the + connection the rest of the SDK uses. + """ + return StreamClient(grpc.aio.insecure_channel(target_host), namespace) + + async def close(self) -> None: + """Close the underlying channel.""" + await self._channel.close() + + async def create( + self, + stream_id: str, + *, + retention: float | None = None, + max_items: int | None = None, + ) -> StreamHandle: + """Create a stream and return a handle to it. + + ``retention`` is how long a closed stream stays readable, in seconds. + ``max_items`` caps how many records remain readable, dropping the + oldest, which bounds storage for a stream nobody truncates. + """ + lifecycle = stream.StreamLifecycle() + if retention is not None: + lifecycle.retention.CopyFrom( + google.protobuf.duration_pb2.Duration(seconds=int(retention)) + ) + if max_items is not None: + lifecycle.max_items = max_items + + response = await _call( + self._stub.CreateStream, + stream.CreateStreamRequest( + frontend_request=stream.CreateStreamInput( + namespace=self._namespace, + stream_id=stream_id, + lifecycle=lifecycle, + ) + ), + ) + return StreamHandle( + self._stub, + self._namespace, + stream_id, + run_id=response.frontend_response.run_id, + ) + + def get(self, stream_id: str) -> StreamHandle: + """Open an existing stream without a round trip.""" + return StreamHandle(self._stub, self._namespace, stream_id) + + def workflow_stream( + self, workflow_id: str, name: str = "", *, owner_run_id: str = "" + ) -> WorkflowStreamHandle: + """Open a stream a workflow owns. + + A stream a workflow owns lives inside that workflow's execution and has + no id of its own, so it is named by its owner and its name. An empty + name is the workflow's default output stream, which is what + ``workflow.append_stream_records`` writes to when it is given none. + + The stream is created by the workflow's first publish or subscription, + or by the first append from outside. Reading one that does not exist + yet is not an error: it reads as empty and a + :meth:`WorkflowStreamHandle.follow` parked on it wakes when something + is written. + + ``owner_run_id`` pins the handle to one run of the workflow; see + :meth:`WorkflowStreamHandle.pin` for why a follower wants that. + """ + return WorkflowStreamHandle( + self._stub, self._namespace, workflow_id, name, owner_run_id + ) + + +class StreamHandle: + """A handle to one standalone stream.""" + + def __init__( + self, stub: Any, namespace: str, stream_id: str, run_id: str = "" + ) -> None: + """Prefer :meth:`StreamClient.get` or :meth:`StreamClient.create`.""" + self._stub = stub + self._namespace = namespace + self._id = stream_id + # Passing this back saves the server resolving the current run on every + # call, which is otherwise a persistence lookup per request. + self._run_id = run_id + + @property + def id(self) -> str: + """Id of the stream this handle points at.""" + return self._id + + async def append( + self, + *records: StreamRecord, + producer_id: str = "", + sequence: int = 0, + ) -> Appended: + """Append records and return where they landed. + + Supplying ``producer_id`` and ``sequence`` makes the append idempotent: + a retry with the same pair returns the original offsets rather than + appending twice, and says so. Without them the append is + at-least-once, which is only the right trade when duplicates are + harmless. + """ + response = await _call( + self._stub.AddMessages, + stream.AddMessagesRequest( + frontend_request=stream.AddMessagesInput( + namespace=self._namespace, + stream_id=self._id, + run_id=self._run_id, + records=[_to_service(record) for record in records], + producer_id=producer_id, + sequence=sequence, + ) + ), + ) + return _appended(response.frontend_response) + + async def read( + self, + *, + from_offset: int = 0, + max_records: int = 0, + topics: Sequence[str] = (), + wait: bool = False, + ) -> tuple[list[StreamEntry], int]: + """Read once from ``from_offset``, returning the entries and the + offset to read from next. + + With ``wait`` set, blocks until something arrives, the stream closes, + or the server's long-poll window elapses. A window that elapses returns + an empty list rather than raising, so the caller just reads again. + """ + page = await self.poll( + from_offset=from_offset, max_records=max_records, topics=topics, wait=wait + ) + return page.entries, page.next_offset + + async def follow( + self, + *, + from_offset: int = 0, + topics: Sequence[str] = (), + ) -> AsyncIterator[StreamEntry]: + """Yield entries as they arrive, starting at ``from_offset``. + + Ends once the stream is closed and this reader has drained it. A closed + stream stays readable until its retention expires, so a reader that + starts late still sees everything instead of having to coordinate a + shutdown with the producer. + """ + offset = from_offset + while True: + page = await self.poll(from_offset=offset, topics=topics) + for entry in page.entries: + yield entry + offset = page.next_offset + if page.closed and offset >= page.head_offset: + return + + async def poll( + self, + *, + from_offset: int = 0, + topics: Sequence[str] = (), + max_records: int = 0, + wait: bool = True, + ) -> Page: + """One read, with everything the caller needs to decide what to do next.""" + response = await _call( + self._stub.PollMessages, + stream.PollMessagesRequest( + frontend_request=stream.PollMessagesInput( + namespace=self._namespace, + stream_id=self._id, + run_id=self._run_id, + from_offset=from_offset, + max_messages=max_records, + topics=list(topics), + wait_new_messages=wait, + ) + ), + ) + return _page(response.frontend_response) + + async def finish_writing(self, producer_id: str) -> None: + """Declare one producer done without ending the stream for others.""" + await _call( + self._stub.FinishWriting, + stream.FinishWritingRequest( + frontend_request=stream.FinishWritingInput( + namespace=self._namespace, + stream_id=self._id, + producer_id=producer_id, + ) + ), + ) + + async def close(self) -> None: + """Seal the stream. Readers can still drain what is already there.""" + await _call( + self._stub.CloseStream, + stream.CloseStreamRequest( + frontend_request=stream.CloseStreamInput( + namespace=self._namespace, stream_id=self._id + ) + ), + ) + + async def describe(self) -> stream.StreamState: + """Read the stream's current frontier, floor and closed state.""" + response = await _call( + self._stub.DescribeStream, + stream.DescribeStreamRequest( + frontend_request=stream.DescribeStreamInput( + namespace=self._namespace, stream_id=self._id + ) + ), + ) + return response.frontend_response.state + + +class WorkflowStreamHandle: + """A handle to a stream a workflow owns. + + The workflow writes to it from inside its Workflow Task, which costs it no + transition of its own. Anything else writes through :meth:`append`, which + costs one transition on the owning execution per batch. Both land in the + same log in the order the server accepted them. + """ + + def __init__( + self, + stub: Any, + namespace: str, + workflow_id: str, + name: str = "", + owner_run_id: str = "", + ) -> None: + """Prefer :meth:`StreamClient.workflow_stream`.""" + self._stub = stub + self._namespace = namespace + self._workflow_id = workflow_id + self._name = name + self._owner_run_id = owner_run_id + + @property + def workflow_id(self) -> str: + """Id of the workflow that owns this stream.""" + return self._workflow_id + + @property + def name(self) -> str: + """Name the owner publishes under, empty for its default stream.""" + return self._name + + @property + def owner_run_id(self) -> str: + """The run this handle is pinned to, empty while it follows the current run.""" + return self._owner_run_id + + def pin(self, run_id: str) -> None: + """Address one run of the workflow from now on. + + Unpinned, every call resolves to whichever run is current. A follower + holding an offset from one run would then be redirected when the + workflow continues as new: the successor's stream starts empty at + offset zero, so the server reads the follower's offset as "caught up", + parks it until the successor has published that many records, and + then skips exactly that many. Pinned, the run's end shows up as + ``closed`` on the page, and the caller decides whether to follow the + successor from the beginning. + """ + self._owner_run_id = run_id + + async def append( + self, + *records: StreamRecord, + producer_id: str = "", + sequence: int = 0, + ) -> Appended: + """Append from outside the workflow, as :meth:`StreamHandle.append`. + + Batch where you can. The append costs one transition on the owning + execution whatever its size, so a batch of a hundred costs what a batch + of one does. + """ + response = await _call( + self._stub.AddWorkflowMessages, + stream.AddWorkflowMessagesRequest( + frontend_request=stream.AddWorkflowMessagesInput( + namespace=self._namespace, + workflow_id=self._workflow_id, + owner_run_id=self._owner_run_id, + stream_name=self._name, + records=[_to_service(record) for record in records], + producer_id=producer_id, + sequence=sequence, + ) + ), + ) + return _appended(response.frontend_response) + + async def read( + self, + *, + from_offset: int = 0, + max_records: int = 0, + topics: Sequence[str] = (), + wait: bool = False, + ) -> tuple[list[StreamEntry], int]: + """Read once from ``from_offset``, as :meth:`StreamHandle.read`.""" + page = await self.poll( + from_offset=from_offset, max_records=max_records, topics=topics, wait=wait + ) + return page.entries, page.next_offset + + async def follow( + self, + *, + from_offset: int = 0, + topics: Sequence[str] = (), + ) -> AsyncIterator[StreamEntry]: + """Yield entries as they arrive, as :meth:`StreamHandle.follow`.""" + offset = from_offset + while True: + page = await self.poll(from_offset=offset, topics=topics) + for entry in page.entries: + yield entry + offset = page.next_offset + if page.closed and offset >= page.head_offset: + return + + async def poll( + self, + *, + from_offset: int = 0, + topics: Sequence[str] = (), + max_records: int = 0, + wait: bool = True, + ) -> Page: + """One read, with everything the caller needs to decide what to do next. + + :meth:`read` is the same call without the frontier and the closed flag. + A caller that has to tell "nothing yet" from "nothing ever" needs both. + On a pinned handle ``closed`` also says the run has ended. + """ + response = await _call( + self._stub.PollWorkflowMessages, + stream.PollWorkflowMessagesRequest( + frontend_request=stream.PollWorkflowMessagesInput( + namespace=self._namespace, + workflow_id=self._workflow_id, + owner_run_id=self._owner_run_id, + stream_name=self._name, + from_offset=from_offset, + max_messages=max_records, + topics=list(topics), + wait_new_messages=wait, + ) + ), + ) + return _page(response.frontend_response) + + async def describe(self) -> stream.StreamState: + """Read the stream's current frontier, floor and closed state. + + A reader that wants only what comes next starts from the head this + reports rather than from zero. + """ + response = await _call( + self._stub.DescribeWorkflowStream, + stream.DescribeWorkflowStreamRequest( + frontend_request=stream.DescribeWorkflowStreamInput( + namespace=self._namespace, + workflow_id=self._workflow_id, + owner_run_id=self._owner_run_id, + stream_name=self._name, + ) + ), + ) + return response.frontend_response.state + + +def _appended(out: stream.AddMessagesOutput) -> Appended: + return Appended( + first_offset=out.first_offset, + next_offset=out.next_offset, + count=out.count, + deduplicated=out.deduplicated, + ) + + +def _page(out: stream.PollMessagesOutput) -> Page: + return Page( + entries=[_to_public(record) for record in out.records], + next_offset=out.next_offset, + head_offset=out.head_offset, + closed=out.closed, + run_id=out.run_id, + ) + + +# One channel per loop, target and namespace, shared by every handle in the +# process. A channel is multiplexed and long lived, and callers open a handle +# per subscription, which would otherwise be a connection per subscription. The +# loop is the key because a grpc.aio channel belongs to the loop that made it, +# and it is held weakly so a loop that is gone cannot lend its channel to a +# successor that happens to reuse its id. +_shared: weakref.WeakKeyDictionary[ + asyncio.AbstractEventLoop, dict[tuple[str, str], StreamClient] +] = weakref.WeakKeyDictionary() + + +def shared_client(target_host: str, namespace: str) -> StreamClient: + """The process-wide client for ``target_host`` and ``namespace`` on this loop.""" + per_loop = _shared.setdefault(asyncio.get_running_loop(), {}) + key = (target_host, namespace) + existing = per_loop.get(key) + if existing is None: + existing = per_loop[key] = StreamClient.connect(target_host, namespace) + return existing + + +async def close_shared_clients() -> None: + """Close every shared client this loop opened. + + For a process that is done with streams, and for tests, which open a + loop per case and would otherwise leave a channel behind on each. + """ + per_loop = _shared.pop(asyncio.get_running_loop(), {}) + for client in per_loop.values(): + await client.close() diff --git a/temporalio/contrib/langgraph/_plugin.py b/temporalio/contrib/langgraph/_plugin.py index 03881ca2a..1819c646d 100644 --- a/temporalio/contrib/langgraph/_plugin.py +++ b/temporalio/contrib/langgraph/_plugin.py @@ -104,11 +104,13 @@ class LangGraphPlugin(SimplePlugin): inherited default of either form. streaming_topic: When set, ``langgraph.config.get_stream_writer()`` inside a node publishes to this topic on the workflow's - :class:`WorkflowStream`. The workflow must construct - ``WorkflowStream()`` in its ``@workflow.init`` (the plugin's + :class:`temporalio.contrib.workflow_streams.WorkflowStream`. The + workflow must construct ``WorkflowStream()`` in its + ``@workflow.init`` (the plugin's interceptor verifies this on workflow start). Nodes with ``execute_in='activity'`` publish through - :class:`WorkflowStreamClient` (signal); nodes with + :class:`temporalio.contrib.workflow_streams.WorkflowStreamClient` + (signal); nodes with ``execute_in='workflow'`` publish synchronously to the in-workflow stream (no signal). streaming_batch_interval: How often the activity-side stream diff --git a/temporalio/contrib/server_streams/__init__.py b/temporalio/contrib/server_streams/__init__.py new file mode 100644 index 000000000..95eb48d7e --- /dev/null +++ b/temporalio/contrib/server_streams/__init__.py @@ -0,0 +1,424 @@ +"""Server-side streams behind the Workflow Streams API. + +This is the same surface as :mod:`temporalio.contrib.workflow_streams`, backed +by a Temporal-owned log instead of by Signals and Updates. An application swaps +the import and keeps its code: publishing from a Workflow is still a plain +call, publishing from an Activity is still a buffered handle, and a consumer +still subscribes by topic from an offset. + +What changes is underneath. A publish is a Workflow Command whose payload never +enters History, so History gets one fixed-size event per Workflow Task rather +than a Signal per batch. A consumer reads the log directly rather than +long-polling an Update, so there is no per-Workflow limit on how many can read +at once, and a closed Workflow stays readable until its stream's retention +expires. + +As in the shipped feature, a topic here is a label on a record in the +Workflow's one default stream, which a consumer filters on. The provider in +:mod:`temporalio.streams.providers.native` keeps one stream per topic instead; +the two do not share a log. + +Prototype support for AI-198. It needs a server built from that branch, and it +opens its own gRPC channel because sdk-core does not know the stream service +yet, which is also why it does not support TLS or API keys. +""" + +from __future__ import annotations + +import asyncio +import builtins +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Generic, TypeVar, overload + +from temporalio import activity, workflow +from temporalio.api.stream.v1 import StreamRecord, StreamRecordKind +from temporalio.client import Client, WorkflowExecutionDescription +from temporalio.client_stream import StreamClient, WorkflowStreamHandle, shared_client +from temporalio.common import RawValue +from temporalio.converter import PayloadCodec, PayloadConverter + +__all__ = [ + "RawPage", + "TopicHandle", + "WorkflowStream", + "WorkflowStreamClient", + "WorkflowStreamItem", + "WorkflowTopicHandle", +] + +T = TypeVar("T") + +DEFAULT_BATCH_INTERVAL = timedelta(milliseconds=50) + + +@dataclass +class RawPage: + """One read whose items are still the stored records. + + ``closed`` with ``next_offset >= head_offset`` is the end of the stream. + The bodies are as the server holds them, codec included, for a caller + that forwards records rather than using them. + """ + + items: list[WorkflowStreamItem[StreamRecord]] + next_offset: int + head_offset: int + closed: bool + + +@dataclass +class WorkflowStreamItem(Generic[T]): + """One item read from a workflow's stream. + + ``offset`` is where the item sits in the whole stream, so it is what a + consumer hands back to resume. A topic filter leaves gaps in it. + """ + + topic: str + data: T + offset: int = 0 + + +def _record(converter: PayloadConverter, topic: str, value: Any) -> StreamRecord: + """The record one published value becomes. + + The body is the value's payload, so the encoding metadata the consumer + needs to decode into a type travels with it and a + :class:`temporalio.common.RawValue` passes through pre-encoded. + """ + record = StreamRecord(topic=topic, kind=StreamRecordKind.STREAM_RECORD_KIND_DATA) + record.body.CopyFrom(converter.to_payloads([value])[0]) + return record + + +def _decode( + converter: PayloadConverter, record: StreamRecord, as_type: type | None +) -> Any: + if as_type is None: + return converter.from_payloads([record.body])[0] + return converter.from_payloads([record.body], [as_type])[0] + + +class WorkflowTopicHandle(Generic[T]): + """A topic on the stream the running Workflow owns.""" + + def __init__(self, topic: str, value_type: type[T]) -> None: + """Prefer :meth:`WorkflowStream.topic`.""" + self._name = topic + self._type = value_type + + @property + def name(self) -> str: + """The topic name this handle is bound to.""" + return self._name + + @property + def type(self) -> type[T]: + """The value type this handle is bound to.""" + return self._type + + def publish(self, value: T | RawValue) -> None: + """Append ``value`` to the Workflow's stream on this topic. + + Returns at once. There is nothing to await: the Workflow Task's + publishes become one command the server applies in the task's own + commit, so it costs this Workflow no round trip and no extra + transition. The Worker's payload codec applies to the body as it does + to any other payload the Workflow sends. + """ + workflow.append_stream_records( + [_record(workflow.payload_converter(), self._name, value)] + ) + + +class WorkflowStream: + """The stream the running Workflow owns, from inside it. + + Construct in ``@workflow.init``. Unlike the Signals-and-Updates + implementation this holds no state of its own: the log lives on the server, + so there is nothing here for replay to reconstruct. + """ + + def __init__(self, prior_state: Any = None) -> None: + """Take the same argument as the Workflow Streams version and ignore it. + + That version carried the log across a continue-as-new, because the log + was Workflow state. Here it is not, so there is nothing to carry. + """ + self._prior_state = prior_state + + @overload + def topic(self, name: str) -> WorkflowTopicHandle[Any]: ... + + @overload + def topic(self, name: str, *, type: type[T]) -> WorkflowTopicHandle[T]: ... + + def topic(self, name: str, *, type: type = object) -> WorkflowTopicHandle[Any]: + """Bind a topic on this Workflow's stream.""" + return WorkflowTopicHandle(name, type) + + +class TopicHandle(Generic[T]): + """A topic on a Workflow's stream, from outside that Workflow.""" + + def __init__( + self, client: WorkflowStreamClient, topic: str, value_type: type[T] + ) -> None: + """Prefer :meth:`WorkflowStreamClient.topic`.""" + self._client = client + self._name = topic + self._type = value_type + + @property + def name(self) -> str: + """The topic name this handle is bound to.""" + return self._name + + @property + def type(self) -> type[T]: + """The value type this handle is bound to.""" + return self._type + + def publish(self, value: T | RawValue, *, force_flush: bool = False) -> None: + """Buffer ``value`` for the next flush. + + Buffered rather than sent, because an append costs one transition on + the owning execution whatever its size. A token at a time would pay + that per token. + """ + self._client._buffer(self._name, value) + if force_flush: + self._client._flush_soon() + + def subscribe( + self, + *, + from_offset: int = 0, + # Spelled out because `type` in this class body is the property below. + result_type: builtins.type | None = None, + poll_cooldown: timedelta | None = None, + ) -> AsyncIterator[WorkflowStreamItem[T]]: + """Read this topic from ``from_offset`` onwards.""" + return self._client.subscribe( + topics=[self._name], + from_offset=from_offset, + result_type=result_type or self._type, + poll_cooldown=poll_cooldown, + ) + + +class WorkflowStreamClient: + """Publishes to and reads from a Workflow's stream, from outside it.""" + + def __init__( + self, + handle: WorkflowStreamHandle, + converter: PayloadConverter, + batch_interval: timedelta = DEFAULT_BATCH_INTERVAL, + *, + codec: PayloadCodec | None = None, + describe: Callable[[], Awaitable[WorkflowExecutionDescription]] | None = None, + ) -> None: + """Prefer :meth:`create` or :meth:`from_within_activity`. + + ``codec`` is applied to every body this client sends and receives, so + a namespace whose payloads are encoded agrees with the Worker, whose + payload visitor applies the same codec to the Workflow's publishes. + ``describe`` is how an unpinned handle learns which run it follows; + see :meth:`WorkflowStreamHandle.pin`. + """ + self._handle = handle + self._converter = converter + self._codec = codec + self._batch_interval = batch_interval + self._describe = describe + self._buffered: list[tuple[str, Any]] = [] + self._flusher: asyncio.Task[None] | None = None + self._wake = asyncio.Event() + self._closed = False + + @classmethod + def create( + cls, + client: Client, + workflow_id: str, + *, + owner_run_id: str = "", + batch_interval: timedelta = DEFAULT_BATCH_INTERVAL, + ) -> WorkflowStreamClient: + """Open the stream owned by ``workflow_id``. + + Without ``owner_run_id`` the current run is looked up on the first + call and the handle pinned to it, so a reader following across a + continue-as-new sees the run end rather than being moved to the + successor's stream at a stale offset. + """ + return cls( + _stream_client(client).workflow_stream( + workflow_id, owner_run_id=owner_run_id + ), + client.data_converter.payload_converter, + batch_interval, + codec=client.data_converter.payload_codec, + describe=client.get_workflow_handle(workflow_id).describe, + ) + + @classmethod + def from_within_activity( + cls, *, batch_interval: timedelta = DEFAULT_BATCH_INTERVAL + ) -> WorkflowStreamClient: + """Open the stream owned by the Workflow that scheduled this Activity.""" + info = activity.info() + if info.workflow_id is None: + raise RuntimeError( + "no Workflow stream to open: this Activity was not started by a " + "Workflow" + ) + # The Activity's output belongs to the run that scheduled it, and the + # Activity already knows which run that is. + return cls.create( + activity.client(), + info.workflow_id, + owner_run_id=info.workflow_run_id or "", + batch_interval=batch_interval, + ) + + async def __aenter__(self) -> WorkflowStreamClient: + """Start the background flusher.""" + self._flusher = asyncio.create_task(self._run_flusher()) + return self + + async def __aexit__(self, *_exc: object) -> None: + """Drain what is buffered before letting the caller go. + + An Activity that returned with a batch still buffered would have + reported work its readers never saw. The flusher is asked to stop + rather than cancelled: a cancel landing inside its append would + unwind with the batch it had already taken off the buffer. + """ + self._closed = True + self._wake.set() + if self._flusher is not None: + await self._flusher + self._flusher = None + await self.flush() + + async def _pin(self) -> None: + if self._handle.owner_run_id or self._describe is None: + return + self._handle.pin((await self._describe()).run_id) + + @overload + def topic(self, name: str) -> TopicHandle[Any]: ... + + @overload + def topic(self, name: str, *, type: type[T]) -> TopicHandle[T]: ... + + def topic(self, name: str, *, type: type = object) -> TopicHandle[Any]: + """Bind a topic on this Workflow's stream.""" + return TopicHandle(self, name, type) + + async def get_offset(self) -> int: + """Where the stream currently ends. + + A reader that wants only what comes next starts here. + """ + await self._pin() + return (await self._handle.describe()).head_offset + + async def subscribe( + self, + *, + topics: Sequence[str] = (), + from_offset: int = 0, + result_type: type | None = None, + poll_cooldown: timedelta | None = None, + ) -> AsyncIterator[WorkflowStreamItem[Any]]: + """Yield items from ``from_offset`` as they arrive. + + ``poll_cooldown`` is accepted and ignored. It paced a client that had + to re-ask; the server parks this read until something arrives. + """ + del poll_cooldown + await self._pin() + async for entry in self._handle.follow(from_offset=from_offset, topics=topics): + record = await self._decoded(entry.record) + yield WorkflowStreamItem( + topic=record.topic, + data=_decode(self._converter, record, result_type), + offset=entry.offset, + ) + + async def poll_raw( + self, + *, + topics: Sequence[str] = (), + from_offset: int = 0, + wait: bool = True, + ) -> RawPage: + """One read, with the records left as they were stored. + + For a caller that forwards items on rather than using them. A gateway + would only have to encode again what this decoded. + """ + await self._pin() + page = await self._handle.poll( + from_offset=from_offset, topics=topics, wait=wait + ) + return RawPage( + items=[ + WorkflowStreamItem( + topic=entry.record.topic, data=entry.record, offset=entry.offset + ) + for entry in page.entries + ], + next_offset=page.next_offset, + head_offset=page.head_offset, + closed=page.closed, + ) + + async def flush(self) -> None: + """Append everything buffered as one batch.""" + pending, self._buffered = self._buffered, [] + if not pending: + return + await self._pin() + records = [ + await self._encoded(_record(self._converter, topic, value)) + for topic, value in pending + ] + await self._handle.append(*records) + + def _buffer(self, topic: str, value: Any) -> None: + self._buffered.append((topic, value)) + + def _flush_soon(self) -> None: + self._wake.set() + + async def _run_flusher(self) -> None: + """Append on a fixed cadence, so a slow producer still gets delivered.""" + while not self._closed: + try: + await asyncio.wait_for( + self._wake.wait(), self._batch_interval.total_seconds() + ) + except asyncio.TimeoutError: + pass + self._wake.clear() + await self.flush() + + async def _encoded(self, record: StreamRecord) -> StreamRecord: + if self._codec is not None and record.HasField("body"): + record.body.CopyFrom((await self._codec.encode([record.body]))[0]) + return record + + async def _decoded(self, record: StreamRecord) -> StreamRecord: + if self._codec is not None and record.HasField("body"): + record.body.CopyFrom((await self._codec.decode([record.body]))[0]) + return record + + +def _stream_client(client: Client) -> StreamClient: + return shared_client(client.service_client.config.target_host, client.namespace) diff --git a/temporalio/streams/providers/native.py b/temporalio/streams/providers/native.py new file mode 100644 index 000000000..c224afb2a --- /dev/null +++ b/temporalio/streams/providers/native.py @@ -0,0 +1,525 @@ +"""The server-side (native) provider. + +Streams live on the Temporal server, beside the workflow that owns them. A +topic is one owned stream named after the topic, created by whoever touches it +first: the workflow publishes to it with a command the server applies in the +transaction that accepts the Workflow Task, subscribes to it by name and reads +the ranges the server delivers on its Workflow Tasks; outside code appends and +reads through the stream service, and the workflow's records and an outside +producer's land in one log in the order the server accepted them. + +A cursor names the run as well as the offset, because an owned stream belongs +to one run and a successor's starts over at zero. A handle without a run id +reads run after run, learning from the poll that a run's stream is closed and +from the run's close event who came next; with a run id it is pinned. A run +that was reset is followed too: its close event does not name the run reset +from it, describe does, and that run's streams continue the base run's offset +space where it inherited a subscription, so the read resumes at the floor the +stream reports rather than at zero. + +Prototype support for AI-198. It needs a server built from that branch and +opens its own gRPC channel to it, because sdk-core does not know the stream +service yet, which is also why it does not support TLS or API keys. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncGenerator +from typing import Any, Generic, TypeVar + +from temporalio import workflow +from temporalio.client import Client, WorkflowHistoryEventFilterType +from temporalio.client_stream import ( + StreamClient, + WorkflowStreamHandle, + close_shared_clients, + shared_client, +) +from temporalio.converter import PayloadCodec, PayloadConverter +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams._errors import StreamCursorError, StreamNotFoundError +from temporalio.streams._provider import ReadSource, WriteSink +from temporalio.streams._record import BEGINNING, Cursor, RecordKind, StreamRecord +from temporalio.streams._topic import StreamTopic, resolve_topic +from temporalio.streams._wire import ( + RecordDecoder, + WireRecord, + cursor_position, + mint_cursor, + producer_identity, + to_wire, +) +from temporalio.streams.providers import ProviderPlugin + +__all__ = ["NativeProducer", "NativeStreamHandle", "NativeStreams"] + +T = TypeVar("T") + +_PROVIDER = "native" + +logger = logging.getLogger(__name__) + + +def _require_topic(topic: str) -> None: + if not topic: + raise ValueError("topic must not be empty") + + +def _cursor(run_id: str, offset: int) -> Cursor: + return mint_cursor(_PROVIDER, f"{run_id}:{offset}") + + +def _position(after: Cursor) -> tuple[str, int] | None: + """The ``(run id, offset)`` a cursor of this provider names, or ``None`` for BEGINNING.""" + token = cursor_position(after, provider=_PROVIDER) + if token is None: + return None + run_id, _, offset = token.rpartition(":") + try: + if not run_id: + raise ValueError + return run_id, int(offset) + except ValueError: + raise StreamCursorError( + f"cursor {after.token!r} does not name a run and an offset on the " + "native provider" + ) from None + + +async def _encode_body(codec: PayloadCodec | None, record: WireRecord) -> WireRecord: + # The worker's payload visitor runs a codec over the bodies a workflow + # publishes and receives; the outside half has no such pass, so it applies + # the client's codec here or the two sides would not agree. + if codec is None or not record.HasField("body"): + return record + encoded = await codec.encode([record.body]) + record.body.CopyFrom(encoded[0]) + return record + + +async def _decode_body(codec: PayloadCodec | None, record: WireRecord) -> WireRecord: + if codec is None or not record.HasField("body"): + return record + decoded = await codec.decode([record.body]) + record.body.CopyFrom(decoded[0]) + return record + + +class _NativeReadSource: + """One subscription of the running workflow, fed by delivered ranges.""" + + def __init__(self, stream_id: str, run_id: str) -> None: + self._stream_id = stream_id + self._run_id = run_id + self._closed = False + + async def next_batch(self) -> list[tuple[Cursor, WireRecord]]: + if self._closed: + raise StopAsyncIteration + delivered = await workflow.read_stream_records(self._stream_id) + return [(_cursor(self._run_id, item.offset), item.record) for item in delivered] + + def close(self) -> None: + self._closed = True + + +class _NativeWriteSink: + def __init__(self, topic: str) -> None: + self._topic = topic + + def publish(self, record: WireRecord) -> None: + # Held by the runtime until the task completes, when the task's + # records on this topic become one command the server applies with + # the task: rule 1 through the server's own commit. + workflow.append_stream_records([record], stream_id=self._topic) + + +class _NativeWorkflowProvider: + """The workflow half: the server's commands and delivered ranges.""" + + def open_reader(self, topic: str, *, after: Cursor) -> ReadSource: + _require_topic(topic) + run_id = workflow.info().run_id + start = 0 + named = _position(after) + if named is not None: + if named[0] != run_id: + raise StreamCursorError( + f"cursor {after.token!r} names another run; a run's stream is its own" + ) + start = named[1] + 1 + workflow.subscribe_stream(topic, start_offset=start) + return _NativeReadSource(topic, run_id) + + def open_writer(self, topic: str) -> WriteSink: + _require_topic(topic) + return _NativeWriteSink(topic) + + def on_workflow_start(self) -> None: + pass + + async def on_workflow_finish(self) -> None: + pass + + +class NativeProducer(Generic[T]): + """Appends to a topic from outside workflow code. + + Every append is visible as soon as the server accepts it. That is the + point for an activity streaming model output, and it is why an activity + carries its own identity: the retry of a failed attempt has no commit + boundary to sort it out afterwards. + """ + + def __init__( + self, + handle: WorkflowStreamHandle, + pin: Any, + codec: PayloadCodec | None, + converter: PayloadConverter, + topic: str, + producer_id: str, + attempt: int, + ) -> None: + """Bind this producer to ``topic`` on the stream ``handle`` names.""" + self._handle = handle + self._pin = pin + self._codec = codec + self._converter = converter + self._topic = topic + self._producer_id = producer_id + self._attempt = attempt + self._sequence = 0 + self._last = BEGINNING + + @property + def producer_id(self) -> str: + """Who this producer writes as.""" + return self._producer_id + + @property + def attempt(self) -> int: + """The generation this producer is writing.""" + return self._attempt + + @property + def _writer(self) -> str: + # The server dedupes on this and the sequence. The attempt is part of + # it so a retried append is dropped while a new generation writing + # different words at the same sequence is not. + return ( + f"{self._producer_id}#{self._attempt}" + if self._attempt + else self._producer_id + ) + + async def append(self, *values: T) -> Cursor: + """Append ``values`` and return the cursor of the last record as stored. + + A repeat returns where the original landed, because the server + answers a deduplicated batch with the original offsets; an empty call + returns the position of this producer's last record. + """ + if not values: + return self._last + return await self._write( + [ + to_wire( + self._converter, + topic=self._topic, + kind=RecordKind.DATA, + value=value, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence + index, + ) + for index, value in enumerate(values) + ] + ) + + async def finish(self) -> None: + """Write ``FINISH`` for this producer on this topic.""" + await self._write( + [ + to_wire( + self._converter, + topic=self._topic, + kind=RecordKind.FINISH, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence, + ) + ] + ) + + async def _write(self, records: list[WireRecord]) -> Cursor: + # Pinned before the first write, so every cursor this producer hands + # out names the run its records landed in. + if not self._handle.owner_run_id: + self._handle.pin(await self._pin()) + for record in records: + await _encode_body(self._codec, record) + appended = await self._handle.append( + *records, producer_id=self._writer, sequence=self._sequence + ) + self._sequence += len(records) + self._last = _cursor(self._handle.owner_run_id, appended.next_offset - 1) + return self._last + + +class NativeStreamHandle: + """One workflow's topics from outside, over the stream service.""" + + def __init__(self, client: Client, workflow_id: str, run_id: str | None) -> None: + """Address ``workflow_id``'s topics, pinned to ``run_id`` when one is given.""" + self._client = client + self._workflow_id = workflow_id + self._run_id = run_id + self._converter = client.data_converter.payload_converter + self._codec = client.data_converter.payload_codec + self._streams: StreamClient | None = None + + def _service(self) -> StreamClient: + # Resolved on first use, because the shared channel belongs to the + # running loop and a handle may be made before there is one. + if self._streams is None: + self._streams = shared_client( + self._client.service_client.config.target_host, + self._client.namespace, + ) + return self._streams + + def _stream(self, topic: str, run_id: str) -> WorkflowStreamHandle: + return self._service().workflow_stream( + self._workflow_id, topic, owner_run_id=run_id + ) + + def read( + self, + *, + topic: str | StreamTopic[Any], + after: Cursor = BEGINNING, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield records on ``topic`` after ``after`` until the chain, or the pinned run, closes. + + The chain is followed across continue-as-new and across a reset: a + run reset from a closed one is read next, from the floor its stream + reports. A handle pinned to a run that was reset ends with that run. + """ + topic, result_type = resolve_topic(topic, result_type) + # Parsed here so a foreign cursor fails this call, not the first + # iteration of the generator. + named = _position(after) + if named is not None and self._run_id is not None and named[0] != self._run_id: + raise StreamCursorError( + f"cursor {after.token!r} names another run than this handle is pinned to" + ) + return self._read(topic, named, after, result_type) + + async def _read( + self, + topic: str, + named: tuple[str, int] | None, + after: Cursor, + result_type: type | None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + decoder = RecordDecoder( + self._converter, result_type, after=after, warn=logger.warning + ) + if named is not None: + run_id, offset = named[0], named[1] + 1 + else: + run_id, offset = self._run_id or await self._first_run(), 0 + while True: + stream = self._stream(topic, run_id) + while True: + page = await stream.poll(from_offset=offset) + for entry in page.entries: + record = await _decode_body(self._codec, entry.record) + for out in decoder.decode(_cursor(run_id, entry.offset), record): + yield out + offset = page.next_offset + # On a pinned stream the server reports the run's end as closed, + # and a closed stream is finished once its head is delivered. + if page.closed and offset >= page.head_offset: + break + if self._run_id is not None: + return + following = await self._successor(topic, run_id) + if following is None: + return + run_id, offset = following + + async def latest(self, *, topic: str | StreamTopic[Any]) -> Cursor: + """The cursor of the newest record on ``topic``, naming the run it was read from. + + An empty topic on the chain's first run is the beginning of the + stream; on a successor it is a position of its own, because + ``BEGINNING`` would send a chain-following read back to the first run. + """ + topic, _ = resolve_topic(topic) + run_id = self._run_id or await self._current_run() + try: + head = (await self._stream(topic, run_id).describe()).head_offset + except StreamNotFoundError: + # A topic nobody has written yet does not exist on the server, + # which is the same answer as an empty one. + head = 0 + if head > 0: + return _cursor(run_id, head - 1) + if self._run_id is None and await self._predecessor(run_id) is not None: + return _cursor(run_id, -1) + return BEGINNING + + def producer( + self, + *, + topic: str | StreamTopic[Any], + producer_id: str = "", + attempt: int = 0, + ) -> NativeProducer[Any]: + """A producer on ``topic``; inside an activity its identity is the activity's.""" + topic, _ = resolve_topic(topic) + producer_id, attempt = producer_identity(producer_id, attempt) + return NativeProducer( + self._stream(topic, self._run_id or ""), + self._current_run, + self._codec, + self._converter, + topic, + producer_id, + attempt, + ) + + async def _current_run(self) -> str: + try: + description = await self._client.get_workflow_handle( + self._workflow_id + ).describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} was not found" + ) from error + raise + assert description.run_id is not None + return description.run_id + + async def _first_run(self) -> str: + """The oldest retained run of the chain, walking back from the latest.""" + run_id = await self._current_run() + while True: + previous = await self._predecessor(run_id) + if previous is None: + return run_id + run_id = previous + + async def _predecessor(self, run_id: str) -> str | None: + handle = self._client.get_workflow_handle(self._workflow_id, run_id=run_id) + try: + async for event in handle.fetch_history_events(page_size=1): + attributes = event.workflow_execution_started_event_attributes + if attributes.continued_execution_run_id: + return attributes.continued_execution_run_id + # A reset run's start event is the base run's, copied, and the + # original run id it carries is kept across resets, so it names + # the run the chain of resets began from. + original = attributes.original_execution_run_id + if original and original != run_id: + return original + return None + except RPCError as error: + if error.status != RPCStatusCode.NOT_FOUND: + raise + # The run's History is gone: the chain's retained part starts here. + return None + + async def _successor(self, topic: str, run_id: str) -> tuple[str, int] | None: + """The run that carries on after ``run_id``, and where its stream starts. + + A continue-as-new names its successor in the close event, and the + successor's streams start at zero. A reset does not: the base run is + closed with no word of the reset in its own History, and only describe + names the run reset from it. That run reads on streams of its own that + continue the base run's offset space where it inherited a subscription, + and such a stream refuses any offset below its floor, so the read + resumes at the floor the stream reports. + """ + handle = self._client.get_workflow_handle(self._workflow_id, run_id=run_id) + events = handle.fetch_history_events( + event_filter_type=WorkflowHistoryEventFilterType.CLOSE_EVENT + ) + closed = False + try: + async for event in events: + closed = True + if event.HasField( + "workflow_execution_continued_as_new_event_attributes" + ): + attributes = ( + event.workflow_execution_continued_as_new_event_attributes + ) + if not attributes.new_execution_run_id: + return None + return attributes.new_execution_run_id, 0 + except RPCError as error: + if error.status != RPCStatusCode.NOT_FOUND: + raise + return None + if not closed: + return None + reset_run = await self._reset_run(run_id) + if reset_run is None: + return None + return reset_run, await self._floor(topic, reset_run) + + async def _reset_run(self, run_id: str) -> str | None: + """The run ``run_id`` was reset into, which only describe reports.""" + handle = self._client.get_workflow_handle(self._workflow_id, run_id=run_id) + try: + description = await handle.describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + return None + raise + extended = description.raw_description.workflow_extended_info + return extended.reset_run_id or None + + async def _floor(self, topic: str, run_id: str) -> int: + """The first offset a run's stream holds. + + A stream the run inherited a subscription to exists from the reset on + and starts at the inherited offset. One the base run only published to + is created on the run's first publish, at zero, and does not exist + before that. + """ + try: + return (await self._stream(topic, run_id).describe()).base_offset + except StreamNotFoundError: + return 0 + + +class NativeStreams(ProviderPlugin): + """The server-side provider. + + Takes no options: the streams are on the server the client is already + connected to. Construct one, pass it to the worker as a plugin and open + handles from it anywhere else; :meth:`close` releases the channels this + process opened to the stream service. + """ + + def workflow_provider(self) -> _NativeWorkflowProvider: + """The workflow half, over the server's commands and delivered ranges.""" + return _NativeWorkflowProvider() + + def get_stream_handle( + self, client: Client, workflow_id: str, *, run_id: str | None = None + ) -> NativeStreamHandle: + """A handle on ``workflow_id``'s topics; without ``run_id`` it follows the chain.""" + return NativeStreamHandle(client, workflow_id, run_id) + + async def close(self) -> None: + """Close the channels this process opened to the stream service.""" + await close_shared_clients() diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index c7ae8d59d..47a54c218 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -8,15 +8,19 @@ from collections.abc import AsyncIterator, Mapping, Sequence from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass +from typing import Any from typing_extensions import TypedDict +import temporalio.api.enums.v1 import temporalio.api.history.v1 +import temporalio.api.stream.v1 import temporalio.bridge.proto.workflow_activation import temporalio.bridge.worker import temporalio.client import temporalio.converter import temporalio.runtime +import temporalio.service import temporalio.streams import temporalio.worker import temporalio.workflow @@ -58,6 +62,7 @@ def __init__( disable_safe_workflow_eviction: bool = False, header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, stream_provider: temporalio.streams.StreamProvider | None = None, + stream_client: temporalio.client.Client | None = None, ) -> None: """Create a replayer to replay workflows from history. @@ -66,6 +71,25 @@ def __init__( the replayer that were passed to the worker when the workflow originally ran, ``stream_provider`` included when the workflow used streams. + A workflow that read a server-side stream cannot be replayed from its + history alone. History records the offsets each Workflow Task consumed + and never the records; on a live task the server re-supplies them from + the stream. Three cases: + + * The history carries the records in + :py:attr:`temporalio.client.WorkflowHistory.stream_slices`, put + there by :py:meth:`fetch_stream_slices` while the stream was + retained and carried by ``to_json()`` and ``from_json()``. They are + handed to the replay and no server is contacted. + * The history carries none and ``stream_client`` is given: a client + connected to the server that still holds the streams. The replayer + fetches every range the completed tasks recorded from the stream + service and hands the records to the replay, so the workflow sees + what it saw the first time. A range the stream no longer holds fails + that replay with :py:class:`temporalio.streams.StreamNotFoundError`. + * The history carries none and there is no client: replaying it fails + and the message names both remedies. + Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that will be shared across all replay calls and never explicitly shut down. @@ -89,6 +113,7 @@ def __init__( disable_safe_workflow_eviction=disable_safe_workflow_eviction, header_codec_behavior=header_codec_behavior, stream_provider=stream_provider, + stream_client=stream_client, ) self._initial_config = self._config.copy() self._default_workflow_logic_flags = set(_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS) @@ -110,6 +135,31 @@ def _set_default_workflow_logic_flag( else: self._default_workflow_logic_flags.discard(flag) + @staticmethod + async def fetch_stream_slices( + client: temporalio.client.Client, history: temporalio.client.WorkflowHistory + ) -> temporalio.client.WorkflowHistory: + """Return ``history`` with the stream records its tasks consumed attached. + + History records only the offsets each Workflow Task consumed from a + server-side stream, so an export cannot be replayed without the + server that still holds the stream. This fetches every recorded range + from the stream service through ``client`` while the stream is + retained and returns a copy of ``history`` with the records in + :py:attr:`temporalio.client.WorkflowHistory.stream_slices`. Its + ``to_json()`` then carries them, and a replayer given the result, or a + ``from_json()`` of it, needs no server. Ranges recorded before a reset + point are fetched from the run the workflow was reset from. + + Raises :py:class:`temporalio.streams.StreamNotFoundError` for a range + the stream no longer holds. + """ + return temporalio.client.WorkflowHistory( + history.workflow_id, + history.events, + await _stream_slices(client, history), + ) + def config(self, *, active_config: bool = False) -> ReplayerConfig: """Config, as a dictionary, used to create this replayer. @@ -365,9 +415,38 @@ def on_eviction_hook( # Start worker workflow_worker_task = asyncio.create_task(workflow_worker.run()) + stream_client = self._config.get("stream_client") + # Yield iterator async def replay_iterator() -> AsyncIterator[WorkflowReplayResult]: async for history in histories: + # The records a consuming workflow read are not in its + # history unless they were captured into it. Otherwise + # fetch them from the stream service, or report here what + # is missing, rather than let Core fail the first task on + # input it was never given. + stream_slices: list[bytes] = [] + if history.stream_slices: + stream_slices = [ + s.SerializeToString() for s in history.stream_slices + ] + elif stream_client is not None: + try: + fetched = await _stream_slices(stream_client, history) + except temporalio.streams.StreamNotFoundError as err: + yield WorkflowReplayResult( + history=history, replay_failure=err + ) + continue + stream_slices = [s.SerializeToString() for s in fetched] + else: + missing = _stream_records_missing(history) + if missing is not None: + yield WorkflowReplayResult( + history=history, replay_failure=RuntimeError(missing) + ) + continue + # Clear last complete and push history last_replay_complete.clear() await pusher.push_history( @@ -375,6 +454,7 @@ async def replay_iterator() -> AsyncIterator[WorkflowReplayResult]: temporalio.api.history.v1.History( events=history.events ).SerializeToString(), + stream_slices, ) # Wait for worker error or last replay to complete. This @@ -420,6 +500,210 @@ async def replay_iterator() -> AsyncIterator[WorkflowReplayResult]: logger.warning("Failed to finalize shutdown", exc_info=True) +def _consumed_ranges( + history: temporalio.client.WorkflowHistory, +) -> list[tuple[int, temporalio.api.stream.v1.StreamRange]]: + """Every range a completed task recorded, with the id of the event that recorded it.""" + out: list[tuple[int, temporalio.api.stream.v1.StreamRange]] = [] + for event in history.events: + if not event.HasField("workflow_task_completed_event_attributes"): + continue + attributes = event.workflow_task_completed_event_attributes + for consumed in attributes.consumed_stream_ranges: + out.append((event.event_id, consumed)) + return out + + +def _eras( + history: temporalio.client.WorkflowHistory, +) -> list[tuple[str, list[tuple[int, temporalio.api.stream.v1.StreamRange]]]]: + """The recorded ranges grouped by the run whose streams hold them. + + A reset copies the base run's history into the new run, so the ranges + before the reset point were consumed from the base run's streams, and the + run they belong to is only named by the ``WorkflowTaskFailed`` event that + marks the reset point, after them in the history. Ranges after the last + reset point belong to the run itself, which that event names as well; a + history with no reset point belongs to the run its start event names. This + is the split the server makes when it re-supplies a cache miss. + """ + reset = temporalio.api.enums.v1.WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_RESET_WORKFLOW + eras: list[tuple[str, list[tuple[int, temporalio.api.stream.v1.StreamRange]]]] = [] + pending: list[tuple[int, temporalio.api.stream.v1.StreamRange]] = [] + own_run_id = history.run_id + for event in history.events: + if event.HasField("workflow_task_failed_event_attributes"): + failed = event.workflow_task_failed_event_attributes + if failed.cause == reset and failed.base_run_id: + eras.append((failed.base_run_id, pending)) + pending = [] + own_run_id = failed.new_run_id or own_run_id + continue + if not event.HasField("workflow_task_completed_event_attributes"): + continue + attributes = event.workflow_task_completed_event_attributes + for consumed in attributes.consumed_stream_ranges: + pending.append((event.event_id, consumed)) + eras.append((own_run_id, pending)) + return eras + + +def _stream_records_missing(history: temporalio.client.WorkflowHistory) -> str | None: + """Why this history cannot be replayed without a stream client, or ``None``.""" + ranges = [r for _, r in _consumed_ranges(history) if r.to_offset > r.from_offset] + if not ranges: + return None + streams = sorted({r.stream_id for r in ranges}) + return ( + f"workflow {history.workflow_id!r} consumed records from stream(s) " + f"{', '.join(repr(s) for s in streams)} in {len(ranges)} task(s), and History " + "records only the offsets. Either create the Replayer with stream_client= (a " + "Client connected to the server that still holds the streams) so the records " + "can be fetched from the stream service, or replay a history exported with " + "them: Replayer.fetch_stream_slices(client, history) attaches the records " + "while the stream is retained and WorkflowHistory.to_json() carries them." + ) + + +async def _stream_slices( + client: temporalio.client.Client, history: temporalio.client.WorkflowHistory +) -> list[temporalio.api.stream.v1.StreamSlice]: + """Fetch the records the history's tasks consumed, as ``StreamSlice`` messages. + + The shape is the one the server puts on a poll response when it re-supplies + a cache miss: one slice per recorded range, tagged with the completion that + recorded it, an empty range included, fetched from the run whose stream + holds it (the run reset from, for a range recorded before a reset point). + Raises :py:class:`temporalio.streams.StreamNotFoundError` for a range the + stream no longer holds. + """ + if not _consumed_ranges(history): + return [] + # Imported here: the stream client needs the grpc extra, which a replayer + # without a stream client never touches. + from temporalio.client_stream import shared_client + + streams = shared_client(client.service_client.config.target_host, client.namespace) + # The handle that served each run's stream, so later ranges of the same + # stream go straight to it. + served_by: dict[tuple[str, str], Any] = {} + slices: list[temporalio.api.stream.v1.StreamSlice] = [] + for run_id, ranges in _eras(history): + for event_id, consumed in ranges: + stream_slice = temporalio.api.stream.v1.StreamSlice( + stream_id=consumed.stream_id, + run_id=run_id, + from_offset=consumed.from_offset, + to_offset=consumed.to_offset, + workflow_task_completed_event_id=event_id, + ) + if consumed.to_offset > consumed.from_offset: + handle, records, owner_run_id = await _fetch_range( + streams, + history.workflow_id, + run_id, + consumed, + served_by.get((run_id, consumed.stream_id)), + ) + served_by[(run_id, consumed.stream_id)] = handle + stream_slice.run_id = owner_run_id or run_id + stream_slice.records.extend(records) + slices.append(stream_slice) + return slices + + +async def _fetch_range( + streams: Any, + workflow_id: str, + run_id: str, + consumed: temporalio.api.stream.v1.StreamRange, + known: Any, +) -> tuple[Any, list[temporalio.api.stream.v1.StreamRecord], str]: + """The records at exactly the recorded range, with the handle that served them. + + A subscribed name is resolved as the server resolves it: a stream the + workflow owns by that name first, else a standalone stream by that id. The + owned stream cannot be asked whether it exists, since a name nobody wrote + reads as empty, so the owned stream is probed for the range's first record + and the standalone one is tried when it has nothing there. + """ + gone = ( + f"stream {consumed.stream_id!r} no longer holds offsets " + f"[{consumed.from_offset}, {consumed.to_offset}) that a completed task of " + f"workflow {workflow_id!r} run {run_id!r} consumed; its records cannot be " + "replayed" + ) + candidates = ( + [known] + if known is not None + else [ + streams.workflow_stream( + workflow_id, consumed.stream_id, owner_run_id=run_id + ), + streams.get(consumed.stream_id), + ] + ) + last: Exception | None = None + for handle in candidates: + try: + records, owner_run_id = await _read_range(handle, consumed, gone) + except temporalio.streams.StreamNotFoundError as error: + last = error + continue + if records is not None: + return handle, records, owner_run_id + raise temporalio.streams.StreamNotFoundError(gone) from last + + +async def _read_range( + handle: Any, consumed: temporalio.api.stream.v1.StreamRange, gone: str +) -> tuple[list[temporalio.api.stream.v1.StreamRecord] | None, str]: + """Read ``[from_offset, to_offset)`` from one stream. + + ``None`` when the stream has nothing at the range's first offset, which is + how a name that is not this stream's reads; a stream that has the start but + not the rest, or refuses the offset as truncated or past its head, raises. + """ + records: list[temporalio.api.stream.v1.StreamRecord] = [] + owner_run_id = "" + offset = consumed.from_offset + while offset < consumed.to_offset: + try: + page = await handle.poll( + from_offset=offset, + max_records=consumed.to_offset - offset, + wait=False, + ) + except temporalio.streams.StreamNotFoundError: + if not records: + return None, "" + raise + except temporalio.service.RPCError as error: + # Below the truncation floor or past the head: the server refuses + # the offset rather than answering short. + if error.status in ( + temporalio.service.RPCStatusCode.FAILED_PRECONDITION, + temporalio.service.RPCStatusCode.INVALID_ARGUMENT, + temporalio.service.RPCStatusCode.OUT_OF_RANGE, + ): + raise temporalio.streams.StreamNotFoundError(gone) from error + raise + owner_run_id = page.run_id or owner_run_id + if not page.entries: + if not records: + return None, "" + raise temporalio.streams.StreamNotFoundError(gone) + for entry in page.entries: + if entry.offset != offset or offset >= consumed.to_offset: + raise temporalio.streams.StreamNotFoundError( + f"{gone}: the stream answered offset {entry.offset} where " + f"{offset} was due" + ) + records.append(entry.record) + offset += 1 + return records, owner_run_id + + class ReplayerConfig(TypedDict, total=False): """TypedDict of config originally passed to :py:class:`Replayer`.""" @@ -438,6 +722,7 @@ class ReplayerConfig(TypedDict, total=False): disable_safe_workflow_eviction: bool header_codec_behavior: HeaderCodecBehavior stream_provider: temporalio.streams.StreamProvider | None + stream_client: temporalio.client.Client | None @dataclass(frozen=True) @@ -455,6 +740,12 @@ class WorkflowReplayResult: :py:class:`temporalio.workflow.NondeterminismError` was encountered during replay - likely indicating your workflow code is incompatible with the history. + + A workflow that read a server-side stream reports here, before any task + runs, when its records could not be fetched: a + :py:class:`temporalio.streams.StreamNotFoundError` when the stream no + longer holds a range a task consumed, or a ``RuntimeError`` when the + replayer was given no ``stream_client`` to fetch them with. """ diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 26f796e57..9b1e1ea0b 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -47,6 +47,7 @@ import temporalio.api.common.v1 import temporalio.api.enums.v1 import temporalio.api.sdk.v1 +import temporalio.api.stream.v1 import temporalio.bridge.proto.activity_result import temporalio.bridge.proto.child_workflow import temporalio.bridge.proto.common @@ -272,6 +273,128 @@ def create_instance(self, det: WorkflowInstanceDetails) -> WorkflowInstance: _ExceptionHandler: TypeAlias = Callable[[asyncio.AbstractEventLoop, _Context], Any] +# Match the server's per-batch limits. A record over its limit is refused where +# it is published, because a rejected command would be reissued on every +# replay; a task's records are split into commands that fit the batch limits. +_MAX_STREAM_RECORDS_PER_BATCH = 1000 +_MAX_STREAM_RECORD_BYTES = 1 << 20 +_MAX_STREAM_BATCH_BYTES = 2 << 20 + + +def _stream_batches( + records: Sequence[temporalio.api.stream.v1.StreamRecord], +) -> Iterator[list[temporalio.api.stream.v1.StreamRecord]]: + """Split one task's records for a stream into batches the server accepts.""" + batch: list[temporalio.api.stream.v1.StreamRecord] = [] + size = 0 + for record in records: + record_size = record.ByteSize() + if batch and ( + len(batch) >= _MAX_STREAM_RECORDS_PER_BATCH + or size + record_size > _MAX_STREAM_BATCH_BYTES + ): + yield batch + batch, size = [], 0 + batch.append(record) + size += record_size + if batch: + yield batch + + +def _is_completion_command( + command: temporalio.bridge.proto.workflow_commands.WorkflowCommand, +) -> bool: + return ( + command.HasField("complete_workflow_execution") + or command.HasField("continue_as_new_workflow_execution") + or command.HasField("fail_workflow_execution") + or command.HasField("cancel_workflow_execution") + ) + + +class _StreamBuffer: + """Holds the stream ranges delivered to a workflow so far. + + Delivery is driven by the server, not by whether workflow code happens to be + reading. A range arrives once, is recorded in History as consumed, and is + never sent again, so anything not yet read has to be kept here rather than + dropped. + """ + + def __init__(self, stream_id: str = "") -> None: + self._stream_id = stream_id + self._records: list[temporalio.workflow.DeliveredStreamRecord] = [] + self._waiters: list[asyncio.Future] = [] + # Where the next range has to start. Unknown until the first one + # arrives, because a subscription may start wherever the stream is and + # the server is the one that resolves that. + self._next_offset: int | None = None + + def extend( + self, + records: Sequence[temporalio.api.stream.v1.StreamRecord], + from_offset: int = 0, + to_offset: int | None = None, + ) -> None: + if to_offset is None: + to_offset = from_offset + len(records) + # A range is recorded as consumed once and never resent, so one that + # repeats, skips or mis-sizes would hand the workflow duplicate or + # shifted records with nothing to say so. Failing the task is what + # makes the fault visible. + if to_offset - from_offset != len(records): + raise RuntimeError( + f"stream {self._stream_id!r} delivered {len(records)} records " + f"for offsets [{from_offset}, {to_offset})" + ) + if self._next_offset is not None and from_offset != self._next_offset: + raise RuntimeError( + f"stream {self._stream_id!r} delivered offsets [{from_offset}, " + f"{to_offset}) but the last range ended at {self._next_offset}" + ) + self._next_offset = to_offset + # An empty range still counts as a delivery, but there is nothing to + # hand a reader, so only a non-empty one wakes anyone. + if not records: + return + # Offsets are dense inside a delivered range and the range arrives in + # order, so counting from its start is the position rather than an + # estimate of it. The per-record field is not on the activation, and a + # reader that has to resume elsewhere needs a position it can name. + for index, record in enumerate(records): + # Copied so the buffer outlives the activation that carried it. + kept = temporalio.api.stream.v1.StreamRecord() + kept.CopyFrom(record) + self._records.append( + temporalio.workflow.DeliveredStreamRecord( + record=kept, offset=from_offset + index + ) + ) + waiters, self._waiters = self._waiters, [] + for waiter in waiters: + if not waiter.done(): + waiter.set_result(None) + + def take(self) -> list[temporalio.workflow.DeliveredStreamRecord]: + taken, self._records = self._records, [] + return taken + + def put_back( + self, records: Sequence[temporalio.workflow.DeliveredStreamRecord] + ) -> None: + """Return an unread tail to the front of the buffer.""" + self._records[:0] = records + + def wait_future(self) -> asyncio.Future: + loop = asyncio.get_event_loop() + fut = loop.create_future() + self._waiters.append(fut) + return fut + + def __len__(self) -> int: + return len(self._records) + + class _WorkflowInstanceImpl( # type: ignore[reportImplicitAbstractClass] WorkflowInstance, temporalio.workflow._Runtime, asyncio.AbstractEventLoop ): @@ -401,6 +524,16 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: str, list[temporalio.bridge.proto.workflow_activation.SignalWorkflow] ] = {} + # Stream ranges delivered to this workflow, keyed by stream id. Ranges + # arrive whether or not anything is reading yet, because the server has + # already recorded them as consumed and will not send them again. + self._stream_buffers: dict[str, _StreamBuffer] = {} + # Records this task's publishes append, by stream, until the task + # completes and they become commands. + self._stream_appends: dict[ + str, list[temporalio.api.stream.v1.StreamRecord] + ] = {} + # When we evict, we have to mark the workflow as deleting so we don't # add any commands and we swallow exceptions on tear down self._deleting = False @@ -461,6 +594,9 @@ def activate( temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() ) self._current_completion.successful.SetInParent() + # A failed task's publishes never became commands, so nothing carries + # over into this one. + self._stream_appends = {} self._current_activation_error: Exception | None = None self._deployment_version_for_current_task = ( @@ -552,6 +688,9 @@ def activate( ) activation_err = None + if activation_err is None and not self._deleting: + self._flush_stream_appends() + # Apply versioning behavior if one was established if self._versioning_behavior: self._current_completion.successful.versioning_behavior = ( @@ -616,6 +755,8 @@ def _apply( ) -> None: if job.HasField("cancel_workflow"): self._apply_cancel_workflow(job.cancel_workflow) + elif job.HasField("deliver_stream_records"): + self._apply_deliver_stream_records(job.deliver_stream_records) elif job.HasField("do_update"): self._apply_do_update(job.do_update) elif job.HasField("fire_timer"): @@ -1141,6 +1282,19 @@ def _apply_resolve_signal_external_workflow( else: fut.set_result(None) + def _apply_deliver_stream_records( + self, + job: temporalio.bridge.proto.workflow_activation.DeliverStreamRecords, + ) -> None: + buffer = self._stream_buffers.get(job.stream_id) + if buffer is None: + # Nothing subscribed. The range is already recorded as consumed and + # will not be sent again, so buffering it is the only way a + # subscription made later in the same task still sees it. + buffer = _StreamBuffer(job.stream_id) + self._stream_buffers[job.stream_id] = buffer + buffer.extend(job.records, job.from_offset, job.to_offset) + def _apply_signal_workflow( self, job: temporalio.bridge.proto.workflow_activation.SignalWorkflow ) -> None: @@ -1305,6 +1459,76 @@ def workflow_get_current_deployment_version( def get_info(self) -> temporalio.workflow.Info: return self._info + def workflow_subscribe_stream(self, stream_id: str, start_offset: int) -> None: + # Reissued on every replay, so the buffer has to exist before the first + # range arrives and the command has to be harmless the second time. A + # repeat subscription leaves the server-side cursor where it is. + self._stream_buffers.setdefault(stream_id, _StreamBuffer(stream_id)) + command = self._add_command() + command.subscribe_stream.stream_id = stream_id + command.subscribe_stream.start_offset = start_offset + + def workflow_append_stream_records( + self, + stream_id: str, + records: Sequence[temporalio.api.stream.v1.StreamRecord], + ) -> None: + self._assert_not_read_only("append stream records") + if not records: + raise ValueError("append_stream_records needs at least one record") + kept: list[temporalio.api.stream.v1.StreamRecord] = [] + for record in records: + if record.ByteSize() > _MAX_STREAM_RECORD_BYTES: + raise ValueError( + f"a stream record is limited to {_MAX_STREAM_RECORD_BYTES} " + f"bytes, got {record.ByteSize()}" + ) + copy = temporalio.api.stream.v1.StreamRecord() + copy.CopyFrom(record) + # The workflow is the producer here, whatever the caller set. + copy.producer_id = "" + kept.append(copy) + # Held until the task completes, so a task's publishes on one stream + # become one command and one History event however many there were. + self._stream_appends.setdefault(stream_id, []).extend(kept) + + def _flush_stream_appends(self) -> None: + appends, self._stream_appends = self._stream_appends, {} + if not appends: + return + commands = self._current_completion.successful.commands + # Ahead of any command that ends the run, because the server accepts + # nothing after one of those. + insert_at = len(commands) + for index, command in enumerate(commands): + if _is_completion_command(command): + insert_at = index + break + for stream_id, records in appends.items(): + for batch in _stream_batches(records): + command = temporalio.bridge.proto.workflow_commands.WorkflowCommand() + command.append_stream_records.stream_id = stream_id + command.append_stream_records.records.extend(batch) + commands.insert(insert_at, command) + insert_at += 1 + + async def workflow_read_stream_records( + self, stream_id: str, max_records: int + ) -> list[temporalio.workflow.DeliveredStreamRecord]: + # Ranges arrive on Workflow Tasks, and a query activation carries none, + # so without this the read waits on a future nothing can resolve and the + # query times out with nothing to say why. + self._assert_not_read_only("read stream") + buffer = self._stream_buffers.setdefault(stream_id, _StreamBuffer(stream_id)) + while not len(buffer): + await buffer.wait_future() + taken = buffer.take() + if max_records and len(taken) > max_records: + # Put the tail back rather than dropping it: nothing will resend it. + buffer.put_back(taken[max_records:]) + taken = taken[:max_records] + return taken + def workflow_get_current_history_length(self) -> int: return self._current_history_length diff --git a/temporalio/workflow/__init__.py b/temporalio/workflow/__init__.py index f63135bf4..18d04cb0c 100644 --- a/temporalio/workflow/__init__.py +++ b/temporalio/workflow/__init__.py @@ -58,6 +58,7 @@ wait, ) from ._context import ( + DeliveredStreamRecord, Info, ParentInfo, RootInfo, @@ -65,6 +66,7 @@ _current_update_info, _Runtime, _set_current_update_info, + append_stream_records, cancellation_reason, current_update_info, deprecate_patch, @@ -86,9 +88,11 @@ payload_converter, random, random_seed, + read_stream_records, register_random_seed_callback, set_current_details, sleep, + subscribe_stream, time, time_ns, upsert_memo, @@ -233,6 +237,10 @@ "upsert_search_attributes", "uuid4", "uuid7", + "DeliveredStreamRecord", + "read_stream_records", + "append_stream_records", + "subscribe_stream", "wait_condition", "DynamicWorkflowConfig", "defn", diff --git a/temporalio/workflow/_context.py b/temporalio/workflow/_context.py index 25523836a..8b6c07782 100644 --- a/temporalio/workflow/_context.py +++ b/temporalio/workflow/_context.py @@ -14,6 +14,7 @@ from nexusrpc import InputT, OutputT import temporalio.api.common.v1 +import temporalio.api.stream.v1 import temporalio.common import temporalio.converter @@ -313,6 +314,21 @@ def workflow_get_current_deployment_version( self, ) -> temporalio.common.WorkerDeploymentVersion | None: ... + @abstractmethod + def workflow_subscribe_stream(self, stream_id: str, start_offset: int) -> None: ... + + @abstractmethod + def workflow_append_stream_records( + self, + stream_id: str, + records: Sequence[temporalio.api.stream.v1.StreamRecord], + ) -> None: ... + + @abstractmethod + async def workflow_read_stream_records( + self, stream_id: str, max_records: int + ) -> list[DeliveredStreamRecord]: ... + @abstractmethod def workflow_get_current_history_length(self) -> int: ... @@ -949,6 +965,87 @@ async def sleep(duration: float | timedelta, *, summary: str | None = None) -> N ) +def subscribe_stream(stream_id: str, *, start_offset: int = 0) -> None: + """Subscribe this workflow to a server-side stream. + + From here on its Workflow Tasks carry the ranges it has not consumed yet, + and :func:`read_stream_records` returns them. Safe to call again: a second + subscription to a stream this run already consumes does not move its + cursor, though it does write one event. Calling it on every replay is + harmless because replay matches the command to the event already recorded. + + Only the stream id and start offset go to the server. The rest of the + stream's addressing is resolved there, because a workflow cannot look it up + without doing I/O and a value it carried would be a reading rather than a + fact. A name this workflow has not written yet names a stream it owns, and + subscribing creates it. + + Args: + stream_id: Stream to consume: the name of one this workflow owns, or + the id of a standalone stream. + start_offset: Where to start. Negative means from wherever the stream is + when the subscription is registered; the server resolves that once + and records it, so replay does not resolve it again. + """ + _Runtime.current().workflow_subscribe_stream(stream_id, start_offset) + + +def append_stream_records( + records: Sequence[temporalio.api.stream.v1.StreamRecord], + *, + stream_id: str = "", +) -> None: + """Publish records to a server-side stream this workflow owns. + + Returns at once. The records become one command when this Workflow Task + completes, so they are visible when the task is accepted and never if it + fails. Their bodies go to the stream's own log rather than into History, + which gets one fixed-size event naming the offset range, so a task that + publishes a thousand records costs History the same as one that publishes + one. Readers do not have to exist yet, and adding one costs the writer + nothing. + + Args: + records: Records to append, in order. The server stores each with an + empty ``producer_id``, because the workflow is the producer. + stream_id: Stream to publish to. Empty means the workflow's default + output stream. + + Raises: + ValueError: ``records`` is empty or one of them is over the server's + per-record size limit. + """ + _Runtime.current().workflow_append_stream_records(stream_id, records) + + +@dataclass(frozen=True) +class DeliveredStreamRecord: + """One record a consuming workflow was given, with where it sat.""" + + record: temporalio.api.stream.v1.StreamRecord + offset: int + """Its position in the whole stream, which is what a reader resumes from.""" + + +async def read_stream_records( + stream_id: str, *, max_records: int = 0 +) -> list[DeliveredStreamRecord]: + """Read the next records of a server-side stream this workflow consumes. + + Waits until at least one record is available. Ranges arrive on Workflow + Tasks, and only the offsets they covered are written to History, so this is + deterministic on replay: the server re-supplies the same ranges by reading + the stream again. Subscribe first with :func:`subscribe_stream`; this only + reads what has already been delivered to this workflow. + + Args: + stream_id: Stream to read from. + max_records: Most records to return at once, or 0 for everything + available. + """ + return await _Runtime.current().workflow_read_stream_records(stream_id, max_records) + + async def wait_condition( fn: Callable[[], bool], *, diff --git a/tests/contrib/test_server_streams.py b/tests/contrib/test_server_streams.py new file mode 100644 index 000000000..540f7f7d7 --- /dev/null +++ b/tests/contrib/test_server_streams.py @@ -0,0 +1,161 @@ +"""The Workflow Streams surface over a server-side stream. + +Both producers and the consumer, since the point of the surface is that an +application does not have to know which of them wrote a given item. Needs a +Temporal server built from the AI-198 branch: + + TEMPORAL_STREAM_TARGET=127.0.0.1:7333 uv run pytest tests/contrib/test_server_streams.py +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from dataclasses import dataclass +from datetime import timedelta + +import pytest + +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.contrib.server_streams import WorkflowStream, WorkflowStreamClient +from temporalio.worker import Worker + +TARGET = os.environ.get("TEMPORAL_STREAM_TARGET") + +pytestmark = pytest.mark.skipif( + not TARGET, + reason="set TEMPORAL_STREAM_TARGET to a server with the stream service", +) + +TOPIC = "turn_events" + + +@dataclass +class Event: + source: str + text: str + + +@activity.defn +async def emit_from_activity(count: int) -> None: + async with WorkflowStreamClient.from_within_activity() as client: + events = client.topic(TOPIC, type=Event) + for i in range(count): + events.publish(Event(source="activity", text=f"token {i}")) + + +@workflow.defn +class Emitting: + def __init__(self) -> None: + self._events = WorkflowStream().topic(TOPIC, type=Event) + + @workflow.run + async def run(self, count: int) -> None: + self._events.publish(Event(source="workflow", text="turn started")) + await workflow.execute_activity( + emit_from_activity, + count, + start_to_close_timeout=timedelta(seconds=30), + ) + self._events.publish(Event(source="workflow", text="turn ended")) + + +async def test_both_producers_reach_one_subscriber() -> None: + client = await Client.connect(TARGET or "") + task_queue = "ss-tq-" + uuid.uuid4().hex[:8] + wf_id = "ss-wf-" + uuid.uuid4().hex[:8] + tokens = 4 + + seen: list[Event] = [] + offsets: list[int] = [] + + async with Worker( + client, + task_queue=task_queue, + workflows=[Emitting], + activities=[emit_from_activity], + max_cached_workflows=0, + ): + handle = await client.start_workflow( + Emitting.run, tokens, id=wf_id, task_queue=task_queue + ) + + # Subscribed before the Workflow has published anything, which is what + # a consumer attaching to a session does. + stream = WorkflowStreamClient.create(client, wf_id) + + async def read() -> None: + async for item in stream.subscribe( + topics=[TOPIC], from_offset=0, result_type=Event + ): + seen.append(item.data) + offsets.append(item.offset) + if len(seen) == tokens + 2: + return + + reading = asyncio.ensure_future(read()) + await asyncio.wait_for(handle.result(), timeout=60) + await asyncio.wait_for(reading, timeout=60) + + # The Workflow's own publishes bracket the Activity's, and both are on one + # log in the order the server took them. + assert [e.source for e in seen] == ["workflow"] + ["activity"] * tokens + [ + "workflow" + ] + assert seen[0].text == "turn started" + assert seen[-1].text == "turn ended" + assert offsets == list(range(tokens + 2)) + + # A consumer that arrives after the fact reads the same thing, and is not + # left tailing: the Workflow has ended, so its stream is finished and the + # subscription ends on its own. + late = WorkflowStreamClient.create(client, wf_id) + assert await late.get_offset() == tokens + 2 + replayed = [ + item.data.text + async for item in late.subscribe( + topics=[TOPIC], from_offset=0, result_type=Event + ) + ] + assert replayed == [e.text for e in seen] + + +async def test_a_reader_resumes_from_an_offset_it_was_given() -> None: + client = await Client.connect(TARGET or "") + task_queue = "ss-resume-tq-" + uuid.uuid4().hex[:8] + wf_id = "ss-resume-wf-" + uuid.uuid4().hex[:8] + + async with Worker( + client, + task_queue=task_queue, + workflows=[Emitting], + activities=[emit_from_activity], + max_cached_workflows=0, + ): + await client.execute_workflow(Emitting.run, 3, id=wf_id, task_queue=task_queue) + + stream = WorkflowStreamClient.create(client, wf_id) + first = [ + item + async for item in stream.subscribe( + topics=[TOPIC], from_offset=0, result_type=Event + ) + ] + assert len(first) == 5 + + # Resuming past the second item skips exactly the two before it, so the + # offset a reader was handed is the position it means. + resumed = [ + item.data.text + async for item in stream.subscribe( + topics=[TOPIC], from_offset=first[1].offset + 1, result_type=Event + ) + ] + assert resumed == [item.data.text for item in first[2:]] + + # The raw page carries the stored records themselves. + raw = await stream.poll_raw(topics=[TOPIC], from_offset=0, wait=False) + assert [item.data.topic for item in raw.items] == [TOPIC] * 5 + assert raw.closed and raw.next_offset == raw.head_offset == 5 diff --git a/tests/streams/test_streams_conformance.py b/tests/streams/test_streams_conformance.py index 9002ea16a..3bdc93e1a 100644 --- a/tests/streams/test_streams_conformance.py +++ b/tests/streams/test_streams_conformance.py @@ -21,6 +21,7 @@ from __future__ import annotations import asyncio +import os import uuid from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import dataclass @@ -28,8 +29,9 @@ import pytest +from temporalio import workflow from temporalio.api.common.v1 import Payload -from temporalio.client import Client +from temporalio.client import Client, WorkflowHandle from temporalio.common import RawValue from temporalio.converter import DataConverter from temporalio.streams import ( @@ -46,6 +48,8 @@ ) from temporalio.streams._policy import AttemptTracker from temporalio.streams.providers.memory import MemoryStreams +from temporalio.streams.providers.native import NativeStreams +from tests.helpers import new_worker # Defined once and shared by every case, the way an application shares them # between its workflow, its activities and its backend. @@ -91,9 +95,56 @@ async def _memory_case(_client: Client) -> AsyncIterator[ProviderCase]: provider.reset() +@workflow.defn +class StreamHost: + """Owns a stream and lingers, so outside code has a running workflow to address.""" + + def __init__(self) -> None: + self._released = False + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: self._released) + + +async def _native_case(client: Client) -> AsyncIterator[ProviderCase]: + # The store is a server built from the stream-carrying branch, which the + # test environment's own server is not; TEMPORAL_ADDRESS names it. + address = os.environ.get("TEMPORAL_ADDRESS") + if address: + client = await Client.connect( + address, namespace=os.environ.get("TEMPORAL_NAMESPACE", "default") + ) + provider = NativeStreams() + # Registered once, on the client: the host's worker inherits it and the + # cases open handles through client.get_stream_handle. + config = client.config() + config["plugins"] = [provider] + client = Client(**config) + hosts: dict[str, WorkflowHandle[Any, Any]] = {} + async with new_worker(client, StreamHost) as worker: + + async def host(workflow_id: str) -> None: + if workflow_id not in hosts: + hosts[workflow_id] = await client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + + yield ProviderCase("native", provider, client, host=host) + for handle in hosts.values(): + await handle.terminate() + await provider.close() + + SETUPS: dict[str, Callable[[Client], AsyncIterator[ProviderCase]]] = { "memory": _memory_case } +if os.environ.get("STREAMS_LIVE") == "native": + SETUPS["native"] = _native_case _CAPABILITIES = { "reports_positions": lambda case: case.reports_positions, diff --git a/tests/test_client_stream.py b/tests/test_client_stream.py new file mode 100644 index 000000000..e3f16f38e --- /dev/null +++ b/tests/test_client_stream.py @@ -0,0 +1,192 @@ +"""End-to-end tests for the server-side stream client. + +These need a Temporal server built from the AI-198 branch, because the stream +service does not exist on a released server. Point them at one: + + TEMPORAL_STREAM_TARGET=127.0.0.1:7333 uv run pytest tests/test_client_stream.py + +Skipped otherwise, rather than silently passing against a server that has no +idea what a stream is. +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from collections.abc import AsyncIterator + +import pytest + +from temporalio.api.common.v1 import Payload +from temporalio.api.stream.v1 import StreamRecord, StreamRecordKind +from temporalio.client_stream import StreamClient, StreamHandle +from temporalio.service import RPCError +from temporalio.streams import StreamNotFoundError + +TARGET = os.environ.get("TEMPORAL_STREAM_TARGET") + +pytestmark = pytest.mark.skipif( + not TARGET, + reason="set TEMPORAL_STREAM_TARGET to a server with the stream service", +) + + +def rec(body: bytes, topic: str = "") -> StreamRecord: + return StreamRecord( + body=Payload(data=body, metadata={"encoding": b"binary/plain"}), topic=topic + ) + + +def data(entries: list) -> list[bytes]: + return [entry.record.body.data for entry in entries] + + +@pytest.fixture +async def streams() -> AsyncIterator[StreamClient]: + client = StreamClient.connect( + TARGET or "", os.environ.get("TEMPORAL_NAMESPACE", "default") + ) + try: + yield client + finally: + await client.close() + + +@pytest.fixture +async def stream(streams: StreamClient) -> StreamHandle: + return await streams.create("py-test-" + uuid.uuid4().hex[:8], max_items=1000) + + +async def test_append_returns_the_offset_it_landed_at(stream: StreamHandle) -> None: + first = await stream.append(rec(b"alpha"), rec(b"beta")) + assert (first.first_offset, first.next_offset, first.count) == (0, 2, 2) + assert not first.deduplicated + assert (await stream.append(rec(b"gamma"))).first_offset == 2 + + +async def test_read_from_an_offset(stream: StreamHandle) -> None: + await stream.append(rec(b"alpha"), rec(b"beta"), rec(b"gamma")) + + entries, next_offset = await stream.read() + assert data(entries) == [b"alpha", b"beta", b"gamma"] + assert [entry.offset for entry in entries] == [0, 1, 2] + assert next_offset == 3 + + entries, next_offset = await stream.read(from_offset=1) + assert data(entries) == [b"beta", b"gamma"] + assert next_offset == 3 + + +async def test_read_past_the_end_returns_nothing(stream: StreamHandle) -> None: + await stream.append(rec(b"alpha")) + + entries, next_offset = await stream.read(from_offset=1) + assert entries == [] + assert next_offset == 1, "a caught-up reader keeps its cursor" + + +# Without an identity the append is at-least-once, so this is the only way a +# producer that retries can avoid writing the same record twice. +async def test_append_is_idempotent_for_a_named_producer(stream: StreamHandle) -> None: + first = await stream.append(rec(b"once"), producer_id="p1", sequence=1) + retry = await stream.append(rec(b"once"), producer_id="p1", sequence=1) + assert first.first_offset == retry.first_offset == 0 + assert not first.deduplicated + # The retry is told it wrote nothing and where the original landed. + assert retry.deduplicated + assert retry.next_offset == first.next_offset + + entries, _ = await stream.read() + assert data(entries) == [b"once"] + + +async def test_topics_filter_a_read(stream: StreamHandle) -> None: + await stream.append(rec(b"tok", topic="tokens")) + await stream.append(rec(b"prog", topic="progress")) + await stream.append(rec(b"tok2", topic="tokens")) + + entries, next_offset = await stream.read(topics=["tokens"]) + assert data(entries) == [b"tok", b"tok2"] + assert [entry.offset for entry in entries] == [0, 2] + assert next_offset == 3, "the cursor advances past filtered-out offsets too" + + +# The record comes back as it went in: kind, producer identity and sequence +# are the server's to store, not to interpret. +async def test_the_record_roundtrips_field_for_field(stream: StreamHandle) -> None: + sent = rec(b"x", topic="t") + sent.producer_id = "model" + sent.attempt = 2 + sent.sequence = 7 + sent.metadata["trace"].CopyFrom(Payload(data=b"abc")) + finish = StreamRecord( + topic="t", kind=StreamRecordKind.STREAM_RECORD_KIND_FINISH, producer_id="model" + ) + await stream.append(sent, finish) + + entries, _ = await stream.read() + got = entries[0].record + assert (got.topic, got.producer_id, got.attempt, got.sequence) == ( + "t", + "model", + 2, + 7, + ) + assert got.kind == StreamRecordKind.STREAM_RECORD_KIND_DATA + assert got.metadata["trace"].data == b"abc" + assert entries[1].record.kind == StreamRecordKind.STREAM_RECORD_KIND_FINISH + assert not entries[1].record.HasField("body") + + +# A closed stream stays readable, which is what removes the shutdown handshake +# between producer and consumer. +async def test_follow_drains_then_ends_when_the_stream_closes( + stream: StreamHandle, +) -> None: + received: list[bytes] = [] + + async def follower() -> None: + async for entry in stream.follow(): + received.append(entry.record.body.data) + + task = asyncio.create_task(follower()) + await stream.append(rec(b"one")) + await asyncio.sleep(0.2) + await stream.append(rec(b"two")) + await asyncio.sleep(0.2) + await stream.close() + + await asyncio.wait_for(task, timeout=30) + assert received == [b"one", b"two"] + + +async def test_follow_started_late_still_sees_everything( + stream: StreamHandle, +) -> None: + await stream.append(rec(b"one"), rec(b"two")) + await stream.close() + + received = [entry.record.body.data async for entry in stream.follow()] + assert received == [b"one", b"two"] + + +async def test_describe_reports_the_frontier(stream: StreamHandle) -> None: + await stream.append(rec(b"one"), rec(b"two")) + + state = await stream.describe() + assert state.head_offset == 2 + assert not state.closed + + await stream.close() + assert (await stream.describe()).closed + + +# The transport's own exception never escapes: a missing stream is the SDK's +# not-found error and anything else is its RPC error. +async def test_failures_surface_as_sdk_errors(streams: StreamClient) -> None: + missing = streams.get("py-test-missing-" + uuid.uuid4().hex[:8]) + with pytest.raises(StreamNotFoundError): + await missing.describe() + with pytest.raises((StreamNotFoundError, RPCError)): + await missing.append(rec(b"x")) diff --git a/tests/worker/test_replayer.py b/tests/worker/test_replayer.py index 52fed7416..ddce04826 100644 --- a/tests/worker/test_replayer.py +++ b/tests/worker/test_replayer.py @@ -8,8 +8,12 @@ from pathlib import Path from typing import Any +import google.protobuf.json_format import pytest +import temporalio.api.common.v1 +import temporalio.api.history.v1 +import temporalio.api.stream.v1 import temporalio.worker._workflow_instance from temporalio import activity, workflow from temporalio.client import Client, WorkflowFailureError, WorkflowHistory @@ -121,6 +125,48 @@ async def test_replayer_workflow_complete(client: Client) -> None: ) +def test_workflow_history_json_carries_stream_slices_only_when_present() -> None: + """The JSON shape is unchanged without slices, and round-trips them when present.""" + with Path(__file__).with_name("test_replayer_complete_history.json").open("r") as f: + history = WorkflowHistory.from_json("fake", f.read()) + assert list(history.stream_slices) == [] + # Byte for byte what the history proto's JSON was before slices existed. + assert history.to_json() == google.protobuf.json_format.MessageToJson( + temporalio.api.history.v1.History(events=history.events) + ) + assert "streamSlices" not in history.to_json_dict() + + record = temporalio.api.stream.v1.StreamRecord( + body=temporalio.api.common.v1.Payload( + data=b'{"n": 1}', metadata={"encoding": b"json/plain"} + ), + topic="inputs", + kind=temporalio.api.stream.v1.StreamRecordKind.STREAM_RECORD_KIND_DATA, + producer_id="model", + attempt=1, + sequence=0, + ) + stream_slice = temporalio.api.stream.v1.StreamSlice( + stream_id="inputs", + run_id="run-1", + from_offset=0, + to_offset=1, + records=[record], + workflow_task_completed_event_id=4, + ) + bundle = WorkflowHistory("fake", history.events, [stream_slice]) + + text = bundle.to_json() + assert "streamSlices" in text + restored = WorkflowHistory.from_json("fake", text) + assert list(restored.stream_slices) == [stream_slice] + assert list(restored.events) == list(history.events) + # The dict form reads back the same, and a plain export stays plain. + from_dict = WorkflowHistory.from_json("fake", bundle.to_json_dict()) + assert list(from_dict.stream_slices) == [stream_slice] + assert WorkflowHistory("fake", restored.events).to_json() == history.to_json() + + @pytest.mark.skipif(sys.version_info < (3, 12), reason="Skipping for < 3.12") async def test_replayer_workflow_complete_json() -> None: # See `test_replayer_workflow_complete` for full skip description. diff --git a/tests/worker/test_workflow_stream.py b/tests/worker/test_workflow_stream.py new file mode 100644 index 000000000..9083e74a9 --- /dev/null +++ b/tests/worker/test_workflow_stream.py @@ -0,0 +1,288 @@ +"""In-workflow consumption and publication of a server-side stream. + +Ranges arrive on Workflow Tasks and only the offsets they covered are written to +History, so replay is served by the server reading the stream again. These tests +drive the buffering, the workflow-facing read and the per-task publish directly, +which is the part this SDK owns; the delivery decision itself lives in the +server and sdk-core. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +import temporalio.api.common.v1 +import temporalio.api.stream.v1 as api_stream +from temporalio.bridge.proto.workflow_commands import WorkflowCommand +from temporalio.worker._workflow_instance import ( + _MAX_STREAM_BATCH_BYTES, + _MAX_STREAM_RECORD_BYTES, + _MAX_STREAM_RECORDS_PER_BATCH, + _StreamBuffer, + _WorkflowInstanceImpl, +) +from temporalio.workflow import ReadOnlyContextError + + +def record(body: bytes, topic: str = "") -> api_stream.StreamRecord: + return api_stream.StreamRecord( + body=temporalio.api.common.v1.Payload(data=body), topic=topic + ) + + +def bodies(delivered: list[Any]) -> list[bytes]: + return [item.record.body.data for item in delivered] + + +async def test_buffer_hands_over_in_order() -> None: + buffer = _StreamBuffer() + buffer.extend([record(b"one"), record(b"two")]) + + assert bodies(buffer.take()) == [b"one", b"two"] + assert len(buffer) == 0 + + +# A reader that arrives before the data must not miss it, and one that arrives +# after must not block: the range is delivered once and never resent. +async def test_buffer_wakes_a_waiting_reader() -> None: + buffer = _StreamBuffer() + waiter = buffer.wait_future() + assert not waiter.done() + + buffer.extend([record(b"late")]) + await asyncio.wait_for(waiter, timeout=1) + assert bodies(buffer.take()) == [b"late"] + + +# An empty range is still a delivery the server recorded, but there is nothing +# to hand a reader, so it must not wake one into returning nothing. +async def test_empty_range_does_not_wake_a_reader() -> None: + buffer = _StreamBuffer() + waiter = buffer.wait_future() + + buffer.extend([]) + + assert not waiter.done() + assert len(buffer) == 0 + + +async def test_buffer_keeps_data_delivered_before_anyone_reads() -> None: + buffer = _StreamBuffer() + buffer.extend([record(b"early")]) + + # No waiting: the data is already there. + assert len(buffer) == 1 + waiter = buffer.wait_future() + assert not waiter.done(), "a fresh waiter is only resolved by new data" + assert bodies(buffer.take()) == [b"early"] + + +class _ReadOnlyStub: + """Enough of the workflow instance to drive the real read. + + The cap lives inside ``workflow_read_stream_records``, so a test that + reimplements it proves nothing about the code that ships. Borrowing the + method off the real class is what keeps the test on the shipped path. + """ + + workflow_read_stream_records = _WorkflowInstanceImpl.workflow_read_stream_records + + def __init__(self, read_only: bool = False) -> None: + self._stream_buffers: dict[str, _StreamBuffer] = {} + self._read_only = read_only + + def _assert_not_read_only(self, action: str) -> None: + if self._read_only: + raise ReadOnlyContextError(f"cannot {action} in a read-only context") + + async def read(self, stream: str, max_records: int) -> list[bytes]: + instance: Any = self + return bodies( + await _WorkflowInstanceImpl.workflow_read_stream_records( + instance, stream, max_records + ) + ) + + +@pytest.mark.parametrize("max_records", [1, 2, 5]) +async def test_read_respects_a_cap_without_losing_the_tail(max_records: int) -> None: + stub = _ReadOnlyStub() + expected = [b"a", b"b", b"c"] + stub._stream_buffers["s"] = _StreamBuffer() + stub._stream_buffers["s"].extend([record(b) for b in expected]) + + got = await stub.read("s", max_records) + + assert got == expected[:max_records] + # Whatever the cap left behind has to still be there: nothing resends it. + assert len(stub._stream_buffers["s"]) == max(0, len(expected) - max_records) + + if len(stub._stream_buffers["s"]): + rest = await stub.read("s", 0) + assert got + rest == expected + else: + assert got == expected + + +# A query activation carries no ranges, so a read there would wait on a future +# nothing can resolve and the query would time out saying nothing. +async def test_read_is_refused_in_a_read_only_context() -> None: + stub = _ReadOnlyStub(read_only=True) + stub._stream_buffers["s"] = _StreamBuffer() + stub._stream_buffers["s"].extend([record(b"a")]) + + with pytest.raises(ReadOnlyContextError, match="read stream"): + await stub.read("s", 0) + # Refused before the buffer was touched, so the range is still there for + # the task that is allowed to read it. + assert len(stub._stream_buffers["s"]) == 1 + + +# A range is recorded as consumed once and never resent, so the buffer is the +# only place a repeated, skipped or mis-sized delivery can still be noticed. +async def test_ranges_have_to_abut_the_last_one() -> None: + buffer = _StreamBuffer("s") + buffer.extend([record(b"a"), record(b"b")], 0, 2) + # An empty range moves the expectation too: the server recorded it. + buffer.extend([], 2, 2) + buffer.extend([record(b"c")], 2, 3) + assert [item.offset for item in buffer.take()] == [0, 1, 2] + + with pytest.raises(RuntimeError, match=r"\[2, 3\).*ended at 3"): + buffer.extend([record(b"c")], 2, 3) + with pytest.raises(RuntimeError, match=r"\[5, 6\).*ended at 3"): + buffer.extend([record(b"f")], 5, 6) + + +async def test_a_range_has_to_carry_as_many_records_as_it_spans() -> None: + buffer = _StreamBuffer("s") + with pytest.raises(RuntimeError, match=r"2 records for offsets \[0, 1\)"): + buffer.extend([record(b"a"), record(b"b")], 0, 1) + assert len(buffer) == 0 + + +class _Completion: + def __init__(self) -> None: + self.commands: list[WorkflowCommand] = [] + + +class _Successful: + def __init__(self) -> None: + self.successful = _Completion() + + +class _CommandStub: + """Drives the real publish path and keeps the commands it issued. + + The completion's command list is a plain list here, which supports the + same ``insert`` the protobuf container does. + """ + + workflow_append_stream_records = ( + _WorkflowInstanceImpl.workflow_append_stream_records + ) + _flush_stream_appends = _WorkflowInstanceImpl._flush_stream_appends + + def __init__(self) -> None: + self._stream_appends: dict[str, list[api_stream.StreamRecord]] = {} + self._current_completion = _Successful() + + def _assert_not_read_only(self, _action: str) -> None: + pass + + @property + def commands(self) -> list[WorkflowCommand]: + return self._current_completion.successful.commands + + def publish(self, *records: api_stream.StreamRecord, stream_id: str = "") -> None: + instance: Any = self + _WorkflowInstanceImpl.workflow_append_stream_records( + instance, stream_id, list(records) + ) + + def flush(self) -> None: + instance: Any = self + _WorkflowInstanceImpl._flush_stream_appends(instance) + + +# A task's publishes on one stream become one command, whatever their number, +# because the event the command produces is what bounds a workflow's History. +def test_a_tasks_publishes_on_one_stream_become_one_command() -> None: + stub = _CommandStub() + stub.publish(record(b"a"), record(b"b")) + stub.publish(record(b"c")) + stub.publish(record(b"d"), stream_id="other") + assert stub.commands == [] + + stub.flush() + + by_stream = { + command.append_stream_records.stream_id: [ + r.body.data for r in command.append_stream_records.records + ] + for command in stub.commands + } + assert by_stream == {"": [b"a", b"b", b"c"], "other": [b"d"]} + # Flushed once: a second flush has nothing left to say. + stub.flush() + assert len(stub.commands) == 2 + + +# The workflow is the producer of what it publishes, whatever the caller set. +def test_the_workflows_records_carry_no_producer() -> None: + stub = _CommandStub() + stub.publish(api_stream.StreamRecord(producer_id="someone", attempt=3)) + stub.flush() + assert stub.commands[0].append_stream_records.records[0].producer_id == "" + + +# The server accepts nothing after a command that ends the run, so the +# publishes have to go ahead of it. +def test_publishes_are_flushed_ahead_of_the_completion_command() -> None: + stub = _CommandStub() + stub.publish(record(b"a")) + done = WorkflowCommand() + done.complete_workflow_execution.SetInParent() + stub.commands.append(done) + + stub.flush() + + assert [c.WhichOneof("variant") for c in stub.commands] == [ + "append_stream_records", + "complete_workflow_execution", + ] + + +# The server refuses an oversized record, and a refused command is reissued on +# every replay, so the limit has to be applied before the record is buffered. +def test_a_record_over_the_server_limit_is_refused_before_the_command() -> None: + stub = _CommandStub() + stub.publish(record(b"x" * (_MAX_STREAM_RECORD_BYTES - 16))) + with pytest.raises(ValueError, match=f"{_MAX_STREAM_RECORD_BYTES} bytes"): + stub.publish(record(b"x" * (_MAX_STREAM_RECORD_BYTES + 1))) + stub.flush() + assert len(stub.commands) == 1 + + +# A task that publishes more than one batch holds is split into commands the +# server accepts, by count and by bytes, rather than refused. +def test_a_task_over_the_batch_limits_is_split_into_commands() -> None: + stub = _CommandStub() + stub.publish(*(record(b"x") for _ in range(_MAX_STREAM_RECORDS_PER_BATCH + 1))) + stub.flush() + assert [len(c.append_stream_records.records) for c in stub.commands] == [ + _MAX_STREAM_RECORDS_PER_BATCH, + 1, + ] + + stub = _CommandStub() + # Two records that fit one batch together, then one whose framing alone + # overflows the room they leave. + big = record(b"x" * (_MAX_STREAM_RECORD_BYTES - 16)) + room = _MAX_STREAM_BATCH_BYTES - 2 * big.ByteSize() + stub.publish(big, big, record(b"y" * room)) + stub.flush() + assert [len(c.append_stream_records.records) for c in stub.commands] == [2, 1] diff --git a/tests/worker/test_workflow_stream_e2e.py b/tests/worker/test_workflow_stream_e2e.py new file mode 100644 index 000000000..652aae699 --- /dev/null +++ b/tests/worker/test_workflow_stream_e2e.py @@ -0,0 +1,962 @@ +"""The native provider inside a real workflow, against a server that has streams. + +The two rules the memory provider cannot keep are the measurement here: a +publish commits with its Workflow Task and never lands if the task fails, and +a read is a recorded observation the server re-supplies on replay. Needs a +Temporal server built from the AI-198 branch, because neither the stream +service nor the commands exist on a released one: + + TEMPORAL_STREAM_TARGET=127.0.0.1:7333 uv run pytest tests/worker/test_workflow_stream_e2e.py + +Skipped otherwise, rather than passing against a server that has no idea what a +stream is. +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from collections.abc import Sequence +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.api.common.v1 import Payload, WorkflowExecution +from temporalio.api.enums.v1 import EventType, WorkflowTaskFailedCause +from temporalio.api.history.v1 import HistoryEvent +from temporalio.api.stream.v1 import StreamRange, StreamRecord +from temporalio.api.workflowservice.v1 import ResetWorkflowExecutionRequest +from temporalio.client import Client, WorkflowExecutionStatus, WorkflowHistory +from temporalio.client_stream import StreamClient +from temporalio.converter import DataConverter, PayloadCodec +from temporalio.streams import RecordKind, StreamNotFoundError +from temporalio.streams.providers.native import NativeStreams +from temporalio.worker import Replayer, Worker +from temporalio.workflow import NondeterminismError +from tests.streams.test_streams_conformance import take + +TARGET = os.environ.get("TEMPORAL_STREAM_TARGET") + +pytestmark = pytest.mark.skipif( + not TARGET, + reason="set TEMPORAL_STREAM_TARGET to a server with the stream commands", +) + +INPUTS = "inputs" +DECISIONS = "decisions" + +EVENT_STREAM_SUBSCRIBED = EventType.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED +EVENT_STREAM_RECORDS_APPENDED = EventType.EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED + + +async def _connect(provider: NativeStreams | None = None) -> Client: + # Registered once, on the client: the worker inherits it and the tests + # open handles through client.get_stream_handle. The contrib tests below + # need no provider. + return await Client.connect(TARGET or "", plugins=[provider] if provider else []) + + +async def _event_counts(client: Client, workflow_id: str) -> dict[Any, int]: + counts = { + EVENT_STREAM_RECORDS_APPENDED: 0, + EVENT_STREAM_SUBSCRIBED: 0, + EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED: 0, + } + async for event in client.get_workflow_handle(workflow_id).fetch_history_events(): + if event.event_type in counts: + counts[event.event_type] += 1 + return counts + + +@workflow.defn +class ContractLoop: + """Reads ``inputs``, publishes a decision per value, reports control records.""" + + def __init__(self) -> None: + self._trace: list[dict[str, Any]] = [] + + @workflow.query + def trace(self) -> list[dict[str, Any]]: + """What the loop has decided so far, for a query against replayed state.""" + return self._trace + + @workflow.run + async def run(self) -> list[dict[str, Any]]: + inputs = workflow.stream_reader(INPUTS, result_type=dict) + decisions = workflow.stream_writer(DECISIONS) + trace = self._trace + async for record in inputs: + if record.kind is RecordKind.SUPERSEDED: + assert record.supersession is not None + trace.append( + { + "kind": "superseded", + "replaced": record.supersession.previous_attempt, + } + ) + decisions.publish( + {"retracting_attempt": record.supersession.previous_attempt} + ) + continue + if record.kind is RecordKind.FINISH: + trace.append({"kind": "finish", "producer": record.producer_id}) + break + assert record.value is not None + decisions.publish({"decided": record.value["n"]}) + trace.append({"kind": "decision", "n": record.value["n"]}) + decisions.finish() + return trace + + +async def test_the_interface_loop_runs_on_the_server_with_a_cold_cache() -> None: + """Rule 2 on the native provider: every task replays from the server. + + With the cache off, each Workflow Task rebuilds the workflow from History + and the server re-supplies the ranges earlier tasks consumed, so the loop + completing at all means the same records came back in the same order. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "loop-tq-" + uuid.uuid4().hex[:8] + workflow_id = "loop-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[ContractLoop], + max_cached_workflows=0, + ): + handle = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + first = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await first.append({"n": 1}, {"n": 2}) + second = stream.producer(topic=INPUTS, producer_id="model", attempt=2) + await second.append({"n": 3}) + await second.finish() + trace = await asyncio.wait_for(handle.result(), 60) + + assert trace == [ + {"kind": "decision", "n": 1}, + {"kind": "decision", "n": 2}, + {"kind": "superseded", "replaced": 1}, + {"kind": "decision", "n": 3}, + {"kind": "finish", "producer": "model"}, + ] + + # The read ends by itself: the workflow is closed and the tail + # delivered, with the workflow's own records carrying no producer. + async def read_everything() -> list[Any]: + return [ + (r.producer_id, r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + + assert await asyncio.wait_for(read_everything(), 60) == [ + ("", RecordKind.DATA, {"decided": 1}), + ("", RecordKind.DATA, {"decided": 2}), + ("", RecordKind.DATA, {"retracting_attempt": 1}), + ("", RecordKind.DATA, {"decided": 3}), + ("", RecordKind.FINISH, None), + ] + counts = await _event_counts(client, workflow_id) + assert counts[EVENT_STREAM_SUBSCRIBED] == 1 + # Several tasks published, one event each; the loop spanned more than + # one task or the cold cache proved nothing. + assert counts[EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED] >= 2 + assert ( + 1 + <= counts[EVENT_STREAM_RECORDS_APPENDED] + <= counts[EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED] + ) + finally: + await provider.close() + + +# Run ids whose first workflow task already failed, shared with the workflow +# thread so the retry can tell it is the retry. Outside the sandbox on +# purpose: the sandbox re-imports this module per run and would hide the set. +_failed_once: set[str] = set() + + +@workflow.defn(sandboxed=False) +class PublishThenFail: + """Publishes, then fails its first workflow task; the retry publishes again.""" + + @workflow.run + async def run(self) -> None: + decisions = workflow.stream_writer(DECISIONS) + run_id = workflow.info().run_id + committed = run_id in _failed_once + decisions.publish({"committed": committed}) + if not committed: + _failed_once.add(run_id) + raise RuntimeError("the first task fails after publishing") + decisions.finish() + + +async def test_a_failed_task_publishes_nothing() -> None: + """Rule 1 on the native provider: the server applies the command with the task.""" + provider = NativeStreams() + client = await _connect(provider) + task_queue = "fail-tq-" + uuid.uuid4().hex[:8] + workflow_id = "fail-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishThenFail], + ): + handle = await client.start_workflow( + PublishThenFail.run, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + records = await take( + stream.read(topic=DECISIONS, result_type=dict), 2, timeout=60 + ) + await handle.result() + assert [(r.kind, r.value) for r in records] == [ + (RecordKind.DATA, {"committed": True}), + (RecordKind.FINISH, None), + ] + finally: + await provider.close() + + +@workflow.defn +class Relay: + """Publishes one record per run and continues as new once.""" + + @workflow.run + async def run(self, run: int) -> None: + decisions = workflow.stream_writer(DECISIONS) + decisions.publish({"run": run}) + if run == 0: + workflow.continue_as_new(run + 1) + decisions.finish() + + +async def test_a_handle_without_a_run_id_reads_across_continue_as_new() -> None: + provider = NativeStreams() + client = await _connect(provider) + task_queue = "relay-tq-" + uuid.uuid4().hex[:8] + workflow_id = "relay-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker(client, task_queue=task_queue, workflows=[Relay]): + handle = await client.start_workflow( + Relay.run, 0, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + + async def read_everything() -> list[Any]: + return [ + (r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + + records = await asyncio.wait_for(read_everything(), 60) + await handle.result() + # The chain is followed: the successor's record arrives on the same + # read, each cursor names its run, and the read ends with the chain. + assert records == [ + (RecordKind.DATA, {"run": 0}), + (RecordKind.DATA, {"run": 1}), + (RecordKind.FINISH, None), + ] + # Pinned to the last run, a handle sees that run's stream alone. + last_run = (await client.get_workflow_handle(workflow_id).describe()).run_id + pinned = client.get_stream_handle(workflow_id, run_id=last_run) + only_last = [ + r.value async for r in pinned.read(topic=DECISIONS, result_type=dict) + ] + assert only_last == [{"run": 1}, None] + finally: + await provider.close() + + +@workflow.defn +class PublishAndRead: + """Publishes through the low-level surface and reads its own records back.""" + + @workflow.run + async def run(self) -> list[str]: + workflow.append_stream_records( + [_record(b"alpha", "progress"), _record(b"beta", "progress")], + stream_id="output", + ) + workflow.append_stream_records([_record(b"gamma")], stream_id="output") + # A name this workflow has not written yet still names a stream it + # owns, so subscribing creates the one the later publish lands in. + workflow.subscribe_stream("output", start_offset=0) + + received: list[str] = [] + while len(received) < 3: + for item in await workflow.read_stream_records("output"): + received.append(item.record.body.data.decode()) + return received + + +def _record(body: bytes, topic: str = "") -> StreamRecord: + return StreamRecord( + body=Payload(data=body, metadata={"encoding": b"binary/plain"}), topic=topic + ) + + +async def test_a_tasks_publishes_become_one_event() -> None: + client = await _connect() + task_queue = "publish-tq-" + uuid.uuid4().hex[:8] + workflow_id = "publish-wf-" + uuid.uuid4().hex[:8] + + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishAndRead], + # Every task after the first is a replay, so completing at all means the + # reissued publish matched the event the first run wrote. + max_cached_workflows=0, + ): + result = await asyncio.wait_for( + client.execute_workflow( + PublishAndRead.run, id=workflow_id, task_queue=task_queue + ), + timeout=60, + ) + assert result == ["alpha", "beta", "gamma"] + + counts = await _event_counts(client, workflow_id) + # Two calls in one task carrying three records, so one event: the event is + # per task and stream, which is what makes publishing often free. + assert counts[EVENT_STREAM_RECORDS_APPENDED] == 1 + assert counts[EVENT_STREAM_SUBSCRIBED] == 1 + + +@workflow.defn +class ConsumeAcrossTasks: + """Reads a standalone stream over several Workflow Tasks, then reports what it saw. + + The turn structure is the point: each read that finds nothing blocks, + which ends a Workflow Task, so the run spans several. With the cache on, + every task after the first is sticky. + """ + + @workflow.run + async def run(self, stream_id: str, expected: int) -> list[str]: + workflow.subscribe_stream(stream_id, start_offset=0) + seen: list[str] = [] + while len(seen) < expected: + for item in await workflow.read_stream_records(stream_id): + seen.append(item.record.body.data.decode()) + return seen + + +async def test_a_cached_workflow_consumes_across_sticky_tasks() -> None: + """The workflow cache stays on, which is what a real worker does. + + The server sends no replay slice for a sticky task, while the sticky + history still carries the previous task's consumed range, and the two + together have to agree on every task after the first consumed range. + """ + client = await _connect() + streams = StreamClient.connect(TARGET or "") + task_queue = "sticky-tq-" + uuid.uuid4().hex[:8] + workflow_id = "sticky-wf-" + uuid.uuid4().hex[:8] + stream_id = "sticky-src-" + uuid.uuid4().hex[:8] + + batches = [["a1", "a2"], ["b1"], ["c1", "c2", "c3"]] + expected = [tok for batch in batches for tok in batch] + + try: + await streams.create(stream_id) + async with Worker( + client, task_queue=task_queue, workflows=[ConsumeAcrossTasks] + ): + handle = await client.start_workflow( + ConsumeAcrossTasks.run, + args=[stream_id, len(expected)], + id=workflow_id, + task_queue=task_queue, + ) + + # Spaced out so the workflow drains, blocks and ends a task between + # them. Without the gap the appends coalesce into one task and the + # sticky path is never taken. + for batch in batches: + await streams.get(stream_id).append( + *[_record(t.encode()) for t in batch] + ) + await asyncio.sleep(0.4) + + assert await asyncio.wait_for(handle.result(), timeout=60) == expected + + counts = await _event_counts(client, workflow_id) + assert counts[EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED] >= 3, ( + "sticky path not exercised" + ) + finally: + await streams.close() + + +class _EncryptingCodec(PayloadCodec): + """A codec whose output never contains its input. + + The SDK's own test codec marks payloads as encrypted but leaves the bytes + as they are, so it cannot show that a stored body is not plaintext. This + one keeps its metadata convention and runs a repeating-key XOR over the + serialized payload, which is enough for the plaintext markers the tests + look for to be absent from what the server stores. + """ + + _KEY = b"ai198" + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [ + Payload( + metadata={"encoding": b"binary/encrypted"}, + data=self._xor(p.SerializeToString()), + ) + for p in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + out: list[Payload] = [] + for p in payloads: + if p.metadata.get("encoding", b"") != b"binary/encrypted": + out.append(p) + continue + out.append(Payload.FromString(self._xor(p.data))) + return out + + def _xor(self, data: bytes) -> bytes: + key = self._KEY + return bytes(b ^ key[i % len(key)] for i, b in enumerate(data)) + + +async def _drive_contract_loop(client: Client) -> WorkflowHistory: + """Run ``ContractLoop`` to completion on ``client`` and return its history.""" + task_queue = "codec-tq-" + uuid.uuid4().hex[:8] + workflow_id = "codec-wf-" + uuid.uuid4().hex[:8] + async with Worker(client, task_queue=task_queue, workflows=[ContractLoop]): + handle = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + await producer.finish() + trace = await asyncio.wait_for(handle.result(), 60) + # The workflow decoded what the outside producer appended. + assert trace == [ + {"kind": "decision", "n": 1}, + {"kind": "decision", "n": 2}, + {"kind": "finish", "producer": "model"}, + ] + # The outside read decodes what the workflow published. + decisions = [ + (r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + assert decisions == [ + (RecordKind.DATA, {"decided": 1}), + (RecordKind.DATA, {"decided": 2}), + (RecordKind.FINISH, None), + ] + return await handle.fetch_history() + + +async def test_a_codec_encodes_records_on_both_halves() -> None: + """A payload codec on the client covers the workflow's records and an outside producer's. + + The worker's payload visitor runs the codec over the bodies a workflow + publishes and receives; the outside half applies the client's codec to each + body it sends and reads. Read raw, without the codec, every stored body is + ciphertext, and each side still reads the other's records in the clear. + """ + plain = await _connect() + config = plain.config() + config["data_converter"] = DataConverter(payload_codec=_EncryptingCodec()) + provider = NativeStreams() + config["plugins"] = [provider] + client = Client(**config) + raw = StreamClient.connect(TARGET or "") + try: + history = await _drive_contract_loop(client) + run_id = history.run_id + + # What the server holds, read through the stream service with no codec. + inputs = await raw.workflow_stream( + history.workflow_id, INPUTS, owner_run_id=run_id + ).poll(from_offset=0, wait=False) + decisions = await raw.workflow_stream( + history.workflow_id, DECISIONS, owner_run_id=run_id + ).poll(from_offset=0, wait=False) + + # The outside producer's two records, stored encoded. + stored_inputs = [e.record for e in inputs.entries if e.record.HasField("body")] + assert len(stored_inputs) == 2 + for record in stored_inputs: + assert record.body.metadata["encoding"] == b"binary/encrypted" + assert b'"n"' not in record.body.data + # The workflow's two decisions, stored encoded by the worker's visitor. + stored_decisions = [ + e.record for e in decisions.entries if e.record.HasField("body") + ] + assert len(stored_decisions) == 2 + for record in stored_decisions: + assert record.body.metadata["encoding"] == b"binary/encrypted" + assert b"decided" not in record.body.data + finally: + await raw.close() + await provider.close() + + +def _consumed_ranges(history: WorkflowHistory) -> list[StreamRange]: + return [ + consumed + for event in history.events + if event.HasField("workflow_task_completed_event_attributes") + for consumed in event.workflow_task_completed_event_attributes.consumed_stream_ranges + ] + + +def _copy_of( + history: WorkflowHistory, workflow_id: str | None = None +) -> WorkflowHistory: + events: list[HistoryEvent] = [] + for event in history.events: + copied = HistoryEvent() + copied.CopyFrom(event) + events.append(copied) + return WorkflowHistory(workflow_id or history.workflow_id, events) + + +async def test_a_replayer_with_a_client_replays_a_consuming_workflow() -> None: + """Rule 2 through the ``Replayer``: the records come back from the stream service. + + History holds the offsets each task consumed and never the records, so the + replayer is given a client to the server that still holds the streams and + fetches every recorded range before pushing the history. The same delivery + path as a live cache miss then hands each range to the task that consumed + it, and the reissued publishes match their events. + """ + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + # Something was consumed, or the replay would prove nothing. + assert any(r.to_offset > r.from_offset for r in _consumed_ranges(history)) + + replayer = Replayer( + workflows=[ContractLoop], plugins=[provider], stream_client=client + ) + result = await replayer.replay_workflow(history) + assert result.replay_failure is None + finally: + await provider.close() + + +async def test_a_replayer_fails_a_tampered_range_as_nondeterministic() -> None: + """A history whose recorded ranges were changed replays on other input and fails. + + Every recorded range is emptied. The task that read two records and + published two decisions is replayed with nothing to read, so it issues no + publish, and Core finds the recorded publish event with no command for it. + """ + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + tampered = _copy_of(history) + for consumed in _consumed_ranges(tampered): + consumed.to_offset = consumed.from_offset + + replayer = Replayer( + workflows=[ContractLoop], plugins=[provider], stream_client=client + ) + with pytest.raises(NondeterminismError): + await replayer.replay_workflow(tampered) + finally: + await provider.close() + + +async def test_a_replayer_fails_loudly_when_the_stream_is_gone() -> None: + """A range the stream service no longer serves is a ``StreamNotFoundError``, not a replay.""" + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + # The same history under a workflow id that owns no stream: the + # records it consumed are nowhere to be fetched from. + gone = _copy_of(history, workflow_id="gone-wf-" + uuid.uuid4().hex[:8]) + + replayer = Replayer( + workflows=[ContractLoop], plugins=[provider], stream_client=client + ) + with pytest.raises(StreamNotFoundError, match="cannot be replayed"): + await replayer.replay_workflow(gone) + finally: + await provider.close() + + +async def test_a_replayer_without_a_client_says_what_it_needs() -> None: + """Without a stream client a consuming workflow's history is refused, with the remedy.""" + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + replayer = Replayer(workflows=[ContractLoop], plugins=[provider]) + + with pytest.raises(RuntimeError, match="stream_client=") as raised: + await replayer.replay_workflow(history) + assert f"'{INPUTS}'" in str(raised.value) + + # The aggregating call reports it per run rather than aborting. + async def histories(): + yield history + + results = await replayer.replay_workflows( + histories(), raise_on_replay_failure=False + ) + assert isinstance(results.replay_failures[history.run_id], RuntimeError) + finally: + await provider.close() + + +async def test_a_replayer_fetches_a_standalone_stream_by_its_id() -> None: + """A subscribed name that is no stream of the workflow's is a standalone stream's id. + + The server resolves a subscription the same way, an owned stream by that + name first, so the replayer has to look in both places to hand the replay + the records the workflow actually read. + """ + client = await _connect() + streams = StreamClient.connect(TARGET or "") + task_queue = "replay-src-tq-" + uuid.uuid4().hex[:8] + workflow_id = "replay-src-wf-" + uuid.uuid4().hex[:8] + stream_id = "replay-src-" + uuid.uuid4().hex[:8] + tokens = ["a1", "a2", "b1"] + try: + await streams.create(stream_id) + async with Worker( + client, task_queue=task_queue, workflows=[ConsumeAcrossTasks] + ): + handle = await client.start_workflow( + ConsumeAcrossTasks.run, + args=[stream_id, len(tokens)], + id=workflow_id, + task_queue=task_queue, + ) + await streams.get(stream_id).append(*[_record(t.encode()) for t in tokens]) + assert await asyncio.wait_for(handle.result(), timeout=60) == tokens + history = await handle.fetch_history() + + result = await Replayer( + workflows=[ConsumeAcrossTasks], stream_client=client + ).replay_workflow(history) + assert result.replay_failure is None + finally: + await streams.close() + + +TWO_DECISIONS = [{"kind": "decision", "n": 1}, {"kind": "decision", "n": 2}] + + +async def _feed_two(client: Client, workflow_id: str) -> Any: + """Append two inputs and wait until both decisions are out, so their tasks have completed.""" + stream = client.get_stream_handle(workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + await take(stream.read(topic=DECISIONS, result_type=dict), 2, timeout=60) + return producer + + +async def test_a_query_against_a_cold_worker_answers_from_replayed_state() -> None: + """A query task built through matching carries the recorded ranges. + + With the cache off every task is a full replay. A query dispatched to a + worker that holds nothing has to rebuild the run from History, and the + records the run read are not in it, so the server attaches them to the + query task the way it does to a task after a cache miss. Without them the + replay would have nothing to read and the query would fail. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "query-cold-tq-" + uuid.uuid4().hex[:8] + workflow_id = "query-cold-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[ContractLoop], + max_cached_workflows=0, + ): + handle = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + producer = await _feed_two(client, workflow_id) + assert await handle.query(ContractLoop.trace) == TWO_DECISIONS + await producer.finish() + await asyncio.wait_for(handle.result(), 60) + finally: + await provider.close() + + +async def test_a_sticky_query_after_an_eviction_still_answers() -> None: + """A query sent to the sticky queue of a run the worker evicted still answers. + + The sticky task carries the history since the last task and no records, + which a worker that lost the run cannot use. Core sees that the history it + fetched itself records a consumed range it was sent no records for, and + lets the query go unanswered rather than answer it from the wrong state. + The server's sticky attempt then times out, stickiness is reset, and the + query is dispatched again on the normal queue, which carries the records. + The sticky timeout is shortened so the test does not wait the default out. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "query-sticky-tq-" + uuid.uuid4().hex[:8] + first_id = "query-sticky-a-" + uuid.uuid4().hex[:8] + second_id = "query-sticky-b-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[ContractLoop], + max_cached_workflows=1, + sticky_queue_schedule_to_start_timeout=timedelta(seconds=2), + ): + first = await client.start_workflow( + ContractLoop.run, id=first_id, task_queue=task_queue + ) + first_producer = await _feed_two(client, first_id) + # A second run on a one-slot cache pushes the first out of it. + second = await client.start_workflow( + ContractLoop.run, id=second_id, task_queue=task_queue + ) + second_producer = await _feed_two(client, second_id) + + answer = await first.query( + ContractLoop.trace, rpc_timeout=timedelta(seconds=60) + ) + assert answer == TWO_DECISIONS + + await first_producer.finish() + await second_producer.finish() + await asyncio.wait_for(first.result(), 60) + await asyncio.wait_for(second.result(), 60) + finally: + await provider.close() + + +async def _collect( + client: Client, workflow_id: str, topic: str, run_id: str | None +) -> list[tuple[Any, str, int]]: + """Every record on ``topic`` as ``(value, run, offset)``, the run and offset from the cursor.""" + handle = client.get_stream_handle(workflow_id, run_id=run_id) + out: list[tuple[Any, str, int]] = [] + async for record in handle.read(topic=topic, result_type=dict): + # native:: + _, run, offset = record.cursor.token.split(":") + out.append((record.value, run, int(offset))) + return out + + +async def test_a_reset_run_is_followed_and_replayed() -> None: + """A run reset from a consuming one carries its subscriptions on, and readers follow. + + The reset re-runs the task named by the reset point, so the base run's + history is copied up to that task and the ranges recorded in the copy are + what the reset run replays, from the base run's streams. The inherited + ``inputs`` stream is the reset run's own from the inherited cursor on: it + starts empty at that offset, and the input the re-run task had consumed is + not delivered again, so the next input takes that offset. The + ``decisions`` stream, which the base run only published to, starts at zero. + A handle without a run id follows the base run into the reset run and starts + each stream at the floor it reports; a handle pinned to the base run ends + with it; the ``Replayer`` fetches each era of the reset run's history from + the run whose stream holds it. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "reset-tq-" + uuid.uuid4().hex[:8] + workflow_id = "reset-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker(client, task_queue=task_queue, workflows=[ContractLoop]): + base = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + base_run = base.result_run_id + assert base_run + producer = await _feed_two(client, workflow_id) + # A third input in a task of its own, which is the task the reset + # re-runs: its consumption is dropped with it. + await producer.append({"n": 3}) + base_stream = client.get_stream_handle(workflow_id, run_id=base_run) + await take( + base_stream.read(topic=DECISIONS, result_type=dict), 3, timeout=60 + ) + + # Reset to the completion of the last consuming task, while the base + # run waits for more input. + completion_id = 0 + async for event in client.get_workflow_handle( + workflow_id, run_id=base_run + ).fetch_history_events(): + if event.HasField("workflow_task_completed_event_attributes"): + completed = event.workflow_task_completed_event_attributes + if any( + r.to_offset > r.from_offset + for r in completed.consumed_stream_ranges + ): + completion_id = event.event_id + assert completion_id + reset = await client.workflow_service.reset_workflow_execution( + ResetWorkflowExecutionRequest( + namespace=client.namespace, + workflow_execution=WorkflowExecution( + workflow_id=workflow_id, run_id=base_run + ), + reason="re-run the last consuming task", + workflow_task_finish_event_id=completion_id, + request_id=uuid.uuid4().hex, + ) + ) + reset_run = reset.run_id + assert reset_run and reset_run != base_run + + # A fresh producer pins to the current run, the reset run, whose + # inherited inputs stream continues at the inherited offset. + continued = client.get_stream_handle(workflow_id).producer( + topic=INPUTS, producer_id="model2", attempt=1 + ) + await continued.append({"n": 4}) + await continued.finish() + trace = await asyncio.wait_for( + client.get_workflow_handle(workflow_id, run_id=reset_run).result(), + 60, + ) + # The first two decisions were replayed from the base run's stream; + # the third input went with the task the reset re-ran. + assert trace == TWO_DECISIONS + [ + {"kind": "decision", "n": 4}, + {"kind": "finish", "producer": "model2"}, + ] + + # The base run was terminated by the reset, and describe is the one + # place that names the run it was reset into. + described = await client.get_workflow_handle( + workflow_id, run_id=base_run + ).describe() + assert described.status == WorkflowExecutionStatus.TERMINATED + extended = described.raw_description.workflow_extended_info + assert extended.reset_run_id == reset_run + + # A reader pinned to the base run ends with the base run. + pinned = await asyncio.wait_for( + _collect(client, workflow_id, DECISIONS, base_run), 30 + ) + assert pinned == [ + ({"decided": 1}, base_run, 0), + ({"decided": 2}, base_run, 1), + ({"decided": 3}, base_run, 2), + ] + + # A chain-following reader crosses from the base run into the reset + # run on both topics, with no gap and no refusal: the published one + # restarts at zero, the inherited one continues at the floor. + decisions = await asyncio.wait_for( + _collect(client, workflow_id, DECISIONS, None), 30 + ) + assert decisions == [ + ({"decided": 1}, base_run, 0), + ({"decided": 2}, base_run, 1), + ({"decided": 3}, base_run, 2), + ({"decided": 4}, reset_run, 0), + (None, reset_run, 1), + ] + inputs = await asyncio.wait_for( + _collect(client, workflow_id, INPUTS, None), 30 + ) + assert inputs == [ + ({"n": 1}, base_run, 0), + ({"n": 2}, base_run, 1), + ({"n": 3}, base_run, 2), + ({"n": 4}, reset_run, 2), + (None, reset_run, 3), + ] + latest = await client.get_stream_handle(workflow_id).latest(topic=INPUTS) + assert latest.token == f"native:{reset_run}:3" + + # The reset run's history: the base run's events, the reset marker + # naming both runs, then its own. The replayer fetches the first era + # from the base run's stream and the rest from the reset run's. + history = await client.get_workflow_handle( + workflow_id, run_id=reset_run + ).fetch_history() + reset_cause = WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_RESET_WORKFLOW + markers = [ + (failed.base_run_id, failed.new_run_id) + for event in history.events + if event.HasField("workflow_task_failed_event_attributes") + for failed in [event.workflow_task_failed_event_attributes] + if failed.cause == reset_cause + ] + assert markers == [(base_run, reset_run)] + result = await Replayer( + workflows=[ContractLoop], plugins=[provider], stream_client=client + ).replay_workflow(history) + assert result.replay_failure is None + + # Exported with its records, the reset run's history replays with no + # server: the slices come from both runs' streams. + bundle = await Replayer.fetch_stream_slices(client, history) + assert {s.run_id for s in bundle.stream_slices if s.records} == { + base_run, + reset_run, + } + restored = WorkflowHistory.from_json(workflow_id, bundle.to_json()) + offline = await Replayer( + workflows=[ContractLoop], plugins=[provider] + ).replay_workflow(restored) + assert offline.replay_failure is None + finally: + await provider.close() + + +async def test_an_exported_history_replays_offline_with_its_records() -> None: + """A history exported with its stream records is the whole replay input. + + ``fetch_stream_slices`` captures the records while the stream is retained, + ``to_json`` writes them beside the events, ``from_json`` reads them back, + and a replayer with no client replays the result. A plain export carries + none and is refused with both remedies named. + """ + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + assert "streamSlices" not in history.to_json() + + bundle = await Replayer.fetch_stream_slices(client, history) + assert list(bundle.events) == list(history.events) + assert any(s.records for s in bundle.stream_slices) + text = bundle.to_json() + assert "streamSlices" in text + restored = WorkflowHistory.from_json(history.workflow_id, text) + assert list(restored.stream_slices) == list(bundle.stream_slices) + assert list(restored.events) == list(history.events) + + offline = Replayer(workflows=[ContractLoop], plugins=[provider]) + result = await offline.replay_workflow(restored) + assert result.replay_failure is None + + # With the records stripped it is a plain export again. + stripped = WorkflowHistory(restored.workflow_id, restored.events) + with pytest.raises(RuntimeError, match="stream_client=") as raised: + await offline.replay_workflow(stripped) + assert "fetch_stream_slices" in str(raised.value) + finally: + await provider.close()