diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5393b0efa..50894a2355 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,56 @@ jobs: npx doctoc README.md [[ -z $(git status --porcelain README.md) ]] || (git diff README.md; echo "README changed"; exit 1) + # The client-side (Redis) stream provider's own evidence. Its tests need a store + # this repo does not otherwise stand up, so without this job nothing that proves + # retention, cursor ownership, the staged commit or the paired producer write ever + # runs anywhere but a developer's machine. + streams-redis: + timeout-minutes: 30 + runs-on: ubuntu-latest + services: + redis: + image: redis:8-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: temporalio/bridge -> target + key: streams-redis-${{ env.pythonLocation }} + - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 + with: + version: "23.x" + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + - run: uv tool install poethepoet + - run: uv sync --all-extras + - run: poe build-develop + # The dev server comes from the test environment, the store from the service + # above. Run serially: the cases measure real timing and share one Redis. + - run: uv run pytest tests/streams -p no:randomly -s + timeout-minutes: 20 + env: + STREAMS_LIVE: redis + TEMPORAL_TEST_REDIS_URL: redis://127.0.0.1:6379 + AI198_REDIS_URL: redis://127.0.0.1:6379 + # Also without the store, so the gate itself keeps working and the memory + # provider's conformance run stays honest. + - run: uv run pytest tests/streams -p no:randomly -s + timeout-minutes: 10 + # Verify the optional FIPS build: the Rust core must link aws-lc-fips-sys # (aws-lc-rs FIPS mode) and must NOT link `ring` (the cargo-tree guard, ported # from sdk-ruby PR #466's `fips_tree` guard); then run the test suite against the diff --git a/.gitignore b/.gitignore index 8cd439e052..42baf649d6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ temporalio/bridge/temporal_sdk_bridge* tags /.claude tmpclaude-* + +# Demo run output, written per provider; the numbers live in the doc. +streams_demo/results-*/ diff --git a/.gitmodules b/.gitmodules index a4c9116312..733443020f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "sdk-core"] path = temporalio/bridge/sdk-core - url = https://github.com/temporalio/sdk-rust.git + url = https://github.com/moedash/sdk-rust.git + branch = moe/AI-198-core-all diff --git a/CHANGELOG.md b/CHANGELOG.md index 03a2243edd..71c9c18b4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,97 @@ to include examples, links to docs, or any other relevant information. worker-side factories registered with `StrandsPlugin(sandboxes=...)`. - Added the `temporalio.contrib.gcp.cloud_run.id` module with the `CloudRunIdPlugin` client plugin to set the worker identity on Cloud Run. +- **Experimental**: `temporalio.streams` defines one stream interface a workflow + can read, decide on, and write. A provider is registered once as a plugin, + `Client.connect(plugins=[provider])`, and workers built from that client + inherit it; each context then asks for its stream the same way: + `workflow.stream_reader()` and `workflow.stream_writer()` in workflow code, + `activity.stream_handle()` in an activity, and `client.get_stream_handle()` + anywhere a client is held. A topic is a typed definition, + `streams.topic("inputs", Token)`, shared by workflow, activity and client + code; a plain string names a topic decided at runtime. The record on the wire + is `temporal.api.stream.v1.StreamRecord` on every provider, and + code; a plain string names a topic decided at runtime, and a call that names + no topic addresses the default topic, `streams.DEFAULT_TOPIC` (`"output"`, + the server's default stream name). The record on the wire + is `temporal.api.stream.v1.StreamRecord` on every provider. A stream is + handed to another process as a `streams.StreamRef`, plain data naming the + owner and the topic, which `client.get_stream_handle(ref)` and + `activity.stream_handle(ref)` open; `client.create_stream(stream_id, ...)` + creates a standalone stream with a retention policy, and its handle's + `close()` seals it. A provider runs record bodies through the client's data + converter, so a payload codec and external storage apply to them. + `temporalio.streams.providers.memory.MemoryStreams` is the in-memory + reference provider the conformance tests run against. +- **Experimental**: `temporalio.streams.providers.redis.RedisStreams` serves the + stream interface over External Workflow Streams, holding one topic as an + input and an output stream. +- **Experimental**: `temporalio.streams.providers.workflow_streams.WorkflowStreamsProvider` + serves the stream interface over the shipped Workflow Streams transport as a + worker plugin, so a workflow reads and publishes through + `temporalio.contrib.workflow_streams` without naming it. Records are the + `StreamRecord` proto inside the shipped item payload, and a handle without a + run id follows continue-as-new run by run. +- **Experimental**: `temporalio.streams.providers.nexus.NexusStreams` puts one + Nexus endpoint in front of a storage provider, so a caller reaches a stream + through the endpoint and never names the store, and + `TemporalStreamsHandler` serves that endpoint by fronting the provider's own + handles. Its contract is defined in `temporal_streams.nexusrpc.yaml` and the + bindings are generated from it; a record crosses as the serialized + `StreamRecord` proto. Configure the front with `data_converter=` to run a + payload codec on the caller side, so records are encoded before they leave + the process. +- **Experimental**: server-side streams. A workflow publishes to a stream it + owns with a command the server applies in its Workflow Task's commit, and + reads the ranges the server delivers on its Workflow Tasks, through + `temporalio.workflow.append_stream_records`, `subscribe_stream` and + `read_stream_records`. `temporalio.client_stream` and + `temporalio.contrib.server_streams` reach the same stream from outside a + workflow, and `temporalio.streams.providers.native.NativeStreams` puts it + behind the shared stream interface with one owned stream per topic. Requires + a server that serves the stream service. `Replayer(stream_client=)` replays a + workflow that read such a stream while the server still holds it: History + records only the offsets each task consumed, so the replayer fetches the + records from the stream service and hands them to the replay with the + history. A range the stream no longer holds fails the replay with + `StreamNotFoundError`. A handle without a run id follows a workflow reset as + it follows a continue-as-new, reading the reset run from the floor its stream + reports, and the replayer fetches the ranges recorded before a reset point + from the run the workflow was reset from. For offline replay, + `Replayer.fetch_stream_slices(client, history)` attaches the records to a + `WorkflowHistory` while the stream is retained, `to_json()` and `from_json()` + carry them as `streamSlices` beside the events, and a history that carries + them replays with no server. +- Added experimental External Workflow Streams in + `temporalio.contrib.external_workflow_streams`. Workflow stream payloads are + stored in a configured external backend instead of Temporal History, with a + Redis Streams provider included. Workflows subscribe with `external_stream`, + external processes publish with `ExternalStreamProducer`, and Workers are + configured with `external_stream_backend`. +- Added the output direction for External Workflow Streams. + Workflows publish with `external_output_stream`, Activities and external + processes use `ExternalOutputStreamProducer`, and external consumers resume + through `ExternalOutputStreamClient`. Workflow output is staged outside + History and becomes readable only after its compact Workflow Task marker is + committed. +- `ExternalStreamSubscription.records()` yields each value with the provider + offset it was read from, for a reader that has to name where it got to. +- Added the `temporalio.contrib.gcp.cloud_run.id` module with the `CloudRunIdPlugin` client plugin to set the worker identity on Cloud Run. + `StreamRecord` proto, and both operations address a stream by a `StreamRef` + naming its owner (a workflow, an activity or a standalone stream) and topic, + which the handler maps onto the store's accessor for that owner. Configure + the front with `data_converter=` to run a payload codec on the caller side, + so records are encoded before they leave the process. +- **Experimental**: `temporalio.streams.providers.workflow_streams.WorkflowStreamsProvider` + serves the stream interface over the shipped Workflow Streams transport as a + worker plugin, so a workflow reads and publishes through + `temporalio.contrib.workflow_streams` without naming it. Records are the + `StreamRecord` proto inside the shipped item payload, and a handle without a + run id follows continue-as-new run by run and a reset into the run reset to. + An outside publish is an Update that answers with the batch's position and + refuses a conflicting repeat, falling back to the shipped Signal on a + workflow whose worker predates it. A workflow's activity keeps its own + streams in the workflow's log under `activity//`. ### Changed @@ -52,6 +143,22 @@ to include examples, links to docs, or any other relevant information. ### Fixed +- Preserve empty activations in the shared External Workflow Streams input and + output replay schedule. Workflows that read input, publish decisions, and + schedule Activities now reproduce that schedule during replay. Inconsistent + prerelease markers are rejected explicitly rather than guessing where omitted + activations belonged. +- Resume external input waits when cold replay encounters a wake in an already + loaded History page, including Workers with workflow caching disabled. +- Avoid an unnecessary output replacement Workflow Task after stream input has + resumed the Workflow and it is waiting on an Activity or timer. +- Keep an incomplete retained external stream task alive when workflow caching + is disabled; evict it after its normal task boundary instead of repeatedly + interrupting input readiness with shutdown markers. +- `WorkflowEnvironment.start_time_skipping()`: concurrent `WorkflowHandle.result()` waiters now + share one time-skipping unlock. The test server holds one lock per in-flight task, so a second + unlock let the clock jump while another workflow still had a task in flight, and that task then + timed out. - `GoogleAdkPlugin` now passes the optional `anthropic`, `litellm`, and `openai` SDKs through the workflow sandbox. - `contrib.deepagents`: prevent duplicate input messages after continue-as-new. @@ -72,6 +179,28 @@ to include examples, links to docs, or any other relevant information. ### Added #### Standalone Activity operator commands +- **Experimental**: `temporalio.streams` defines one stream interface a workflow + can read, decide on, and write. A provider is registered once as a plugin, + `Client.connect(plugins=[provider])`, and workers built from that client + inherit it; each context then asks for its stream the same way: + `workflow.stream_reader()` and `workflow.stream_writer()` in workflow code, + `activity.stream_handle()` in an activity, and `client.get_stream_handle()` + anywhere a client is held. A topic is a typed definition, + `streams.topic("inputs", Token)`, shared by workflow, activity and client + code; a plain string names a topic decided at runtime. The record on the wire + is `temporal.api.stream.v1.StreamRecord` on every provider. A stream is + handed to another process as a `streams.StreamRef`, plain data naming the + owner and, when it has one, the topic, which `client.get_stream_handle(ref)` + and `activity.stream_handle(ref)` open; `client.create_stream(stream_id, ...)` + creates a standalone stream with a retention policy, and its handle's + `close()` seals it. A provider runs record bodies through the client's data + converter, so a payload codec and external storage apply to them. + `temporalio.streams.providers.memory.MemoryStreams` is the in-memory + reference provider the conformance tests run against, and + `temporalio.streams.providers.redis.RedisStreams` serves the same interface + over External Workflow Streams, one topic as an input and an output stream. +- `ExternalStreamSubscription.records()` yields each value with the provider + offset it was read from, for a reader that has to name where it got to. - `ActivityHandle` now supports operator commands for standalone activities: `pause`, `unpause`, `update_options` and `restore_original_options`. diff --git a/pyproject.toml b/pyproject.toml index a83b34cd70..2c08b9c06d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,7 @@ dev = [ "pytest-rerunfailures>=16.1", "pytest-xdist>=3.6,<4", "moto[s3,server]>=5", + "redis>=5,<9", "langgraph>=1.1.0", "langsmith>=0.7.34,<0.9", "deepagents>=0.6.12,<0.7; python_version >= '3.11'", @@ -125,9 +126,11 @@ format = [ ] gen-docs = "uv run scripts/gen_docs.py" gen-nexus-system-api = "uv run scripts/gen_nexus_system_api.py" +gen-streams-nexus-api = "uv run scripts/gen_streams_nexus_api.py" gen-protos = [ { cmd = "uv run scripts/gen_protos.py" }, { ref = "gen-nexus-system-api" }, + { ref = "gen-streams-nexus-api" }, { cmd = "uv run scripts/gen_payload_visitor.py" }, { cmd = "uv run scripts/gen_bridge_client.py" }, { ref = "format" }, @@ -135,6 +138,7 @@ gen-protos = [ gen-protos-docker = [ { cmd = "uv run scripts/gen_protos_docker.py" }, { ref = "gen-nexus-system-api" }, + { ref = "gen-streams-nexus-api" }, { cmd = "uv run scripts/gen_payload_visitor.py" }, { cmd = "uv run scripts/gen_bridge_client.py" }, { ref = "format" }, @@ -198,16 +202,21 @@ exclude = [ 'temporalio/api', 'temporalio/bridge/proto', 'temporalio/nexus/system/workflow_service', + 'temporalio/streams/providers/_nexus_generated', ] [[tool.mypy.overrides]] module = "temporalio.nexus.system.workflow_service.*" ignore_errors = true +[[tool.mypy.overrides]] +module = "temporalio.streams.providers._nexus_generated.*" +ignore_errors = true + [tool.pydocstyle] convention = "google" # https://github.com/PyCQA/pydocstyle/issues/363#issuecomment-625563088 -match_dir = "^(?!(docs|scripts|tests|api|proto|system|\\.)).*" +match_dir = "^(?!(docs|scripts|tests|api|proto|system|_nexus_generated|\\.)).*" add_ignore = [ # We like to wrap at a certain number of chars, even long summary sentences. # https://github.com/PyCQA/pydocstyle/issues/184 @@ -241,6 +250,7 @@ privacy = [ "HIDDEN:temporalio.worker.workflow_sandbox.importer", "HIDDEN:temporalio.worker.workflow_sandbox.in_sandbox", "HIDDEN:**.*_pb2*", + "HIDDEN:temporalio.streams.providers._nexus_generated._definitions", ] project-name = "Temporal Python" sidebar-expand-depth = 2 @@ -267,7 +277,7 @@ reportUnnecessaryIsInstance = "none" reportUnnecessaryTypeIgnoreComment = "none" reportUnusedCallResult = "none" reportUnknownLambdaType = "none" -include = ["temporalio", "tests"] +include = ["temporalio", "tests", "streams_demo"] exclude = [ # Exclude auto generated files "temporalio/api", diff --git a/scripts/gen_payload_visitor.py b/scripts/gen_payload_visitor.py index 7fab85dcda..1df4e044ce 100644 --- a/scripts/gen_payload_visitor.py +++ b/scripts/gen_payload_visitor.py @@ -63,13 +63,10 @@ def name_for(desc: Descriptor) -> str: def field_is_repeated(field: FieldDescriptor) -> bool: - return bool( - getattr( - field, - "is_repeated", - getattr(field, "label") == FieldDescriptor.LABEL_REPEATED, - ) - ) + is_repeated = getattr(field, "is_repeated", None) + if is_repeated is not None: + return bool(is_repeated) + return getattr(field, "label") == FieldDescriptor.LABEL_REPEATED def emit_loop( diff --git a/scripts/gen_stream_api_protos.py b/scripts/gen_stream_api_protos.py new file mode 100644 index 0000000000..eeb7bb3509 --- /dev/null +++ b/scripts/gen_stream_api_protos.py @@ -0,0 +1,101 @@ +"""Regenerate the stream additions to the vendored public API protos. + +The stream protos live on a branch of the api repo that the Core submodule +does not pin yet. Regenerating everything from that branch would also pull in +whatever else moved on the api main line since the pin, so this stages the +API protos the submodule pins, applies the branch's own diff on top, and +regenerates only the files that diff touches. Every other module under +``temporalio/api`` stays byte-identical. + + uv run --python 3.10 --no-project --with "grpcio-tools==1.48.2" \\ + --with "mypy-protobuf==3.3.0" --with "protobuf<4" \\ + scripts/gen_stream_api_protos.py /path/to/api origin/main..origin/ + +The toolchain pins match ``scripts/_proto/Dockerfile``. Generated code refuses +to load on a protobuf runtime older than the one it was built against, and +these modules end up in applications that pin protobuf themselves. Run +``uv run poe format`` afterwards, as the ``gen-protos`` task does. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +BASE = Path(__file__).parent.parent +sys.path.insert(0, str(BASE / "scripts")) + +import gen_protos # noqa: E402 + +API_OUT = BASE / "temporalio" / "api" + + +def main() -> None: + if len(sys.argv) != 3: + sys.exit("usage: gen_stream_api_protos.py /path/to/api-checkout ..") + api = Path(sys.argv[1]).resolve() + revisions = sys.argv[2] + diff = subprocess.run( + ["git", "-C", str(api), "diff", revisions, "--", "temporal/"], + check=True, + capture_output=True, + text=True, + ).stdout + touched = subprocess.run( + ["git", "-C", str(api), "diff", "--name-only", revisions, "--", "temporal/"], + check=True, + capture_output=True, + text=True, + ).stdout.split() + if not touched: + sys.exit(f"{revisions} touches no proto under temporal/") + + with tempfile.TemporaryDirectory() as tmp: + stage = Path(tmp) / "stage" + shutil.copytree(gen_protos.api_proto_dir / "temporal", stage / "temporal") + subprocess.run( + ["patch", "-p1", "--silent"], + check=True, + cwd=stage, + input=diff, + text=True, + ) + out = Path(tmp) / "out" + out.mkdir() + subprocess.check_call( + [ + sys.executable, + "-mgrpc_tools.protoc", + f"--proto_path={stage}", + f"--python_out={out}", + f"--mypy_out={out}", + *touched, + ] + ) + packages: set[Path] = set() + for proto in touched: + relative = Path(proto).relative_to("temporal/api").with_suffix("") + for suffix in ("_pb2.py", "_pb2.pyi"): + generated = ( + out + / "temporal" + / "api" + / relative.with_name(relative.name + suffix) + ) + target = API_OUT / relative.with_name(relative.name + suffix) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(generated, target) + print(f"wrote {target.relative_to(BASE)}") + packages.add(target.parent) + + for package in sorted(packages): + (package.parent / "__init__.py").touch() + gen_protos.fix_generated_output(package) + print(f"rewrote {(package / '__init__.py').relative_to(BASE)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/gen_stream_protos.py b/scripts/gen_stream_protos.py new file mode 100644 index 0000000000..faf5453d50 --- /dev/null +++ b/scripts/gen_stream_protos.py @@ -0,0 +1,193 @@ +"""Regenerate the vendored server-side stream protos. + +The stream service is still defined in the server rather than in the public +API repo, so its protos are copied in and compiled here. Run this whenever the +server-side definitions change: + + uv run python scripts/gen_stream_protos.py /path/to/temporal + +Run it with an interpreter whose ``grpcio-tools`` matches the oldest protobuf +runtime this package has to load on. Generated code refuses to load on a +runtime older than the one it was built against, and these modules end up in +applications that pin protobuf themselves. The stubs come from +``mypy-protobuf`` rather than protoc's own ``--pyi_out``, which that +generation of protoc does not have. + +Two rewrites make the copies compile against this SDK. The server's own +routing and API-category options are dropped, because they mean nothing to a +client and their proto files are not vendored. Paths move under +``temporalio/`` so the generated modules import each other and the public API +messages by the names this package actually publishes. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +BASE = Path(__file__).parent.parent +OUT = BASE / "temporalio" / "api" / "streamservice" / "v1" +# The stream protos reference the public API messages. They normally come from +# the sdk-core submodule; TEMPORAL_API_PROTOS points at a `temporalio/api` +# checkout instead when the submodule is not initialized. +UPSTREAM_API = ( + BASE + / "temporalio" + / "bridge" + / "sdk-core" + / "crates" + / "protos" + / "protos" + / "api_upstream" +) + + +def api_proto_path() -> Path: + override = os.environ.get("TEMPORAL_API_PROTOS") + if override: + return Path(override).resolve() + if ( + UPSTREAM_API / "temporal" / "api" / "common" / "v1" / "message.proto" + ).is_file(): + return UPSTREAM_API + raise SystemExit( + "no api protos found: initialize the sdk-core submodule or set " + "TEMPORAL_API_PROTOS to a temporalio/api checkout" + ) + + +SERVER_PROTO_DIR = Path("chasm/lib/stream/proto/v1") +STAGE_PROTO_DIR = Path("temporalio/api/stream/v1") +FILES = [ + "message.proto", + "stream_state.proto", + "request_response.proto", + "service.proto", +] + +# Server-only options. They carry shard routing and rate-limit category, which +# a client neither reads nor can resolve, since the files defining them are not +# vendored. +DROP_IMPORT = re.compile(r'^import "temporal/server/.*?";\n', re.M) +DROP_OPTION = re.compile(r"^\s*option \(temporal\.server\.api\..*?\n", re.M) + +# protoc names a python module after the proto path. These put the generated +# modules where this package publishes them. +FIX_PY = [ + ( + re.compile(r"from temporalio\.api\.stream\.v1 import"), + "from temporalio.api.streamservice.v1 import", + ), + (re.compile(r"temporalio\.api\.stream\.v1\."), "temporalio.api.streamservice.v1."), + (re.compile(r"from temporal\.api\."), "from temporalio.api."), + (re.compile(r"import temporal\.api\."), "import temporalio.api."), + # mypy-protobuf also writes the public API types fully qualified in the + # stubs, `temporal.api.common.v1.message_pb2.Payload`, which the import + # rewrites above do not reach. The `_pb2` suffix keeps this off the proto + # package names inside the serialized descriptors, which stay `temporal.api`. + # After the streamservice rewrite on purpose, so the public + # `temporal.api.stream.v1` package is not sent to the vendored one. + ( + re.compile(r"\btemporal\.api\.(\w+)\.v1\.(\w+_pb2)\b"), + r"temporalio.api.\1.v1.\2", + ), +] + + +def stage(server: Path, into: Path) -> list[str]: + dest = into / STAGE_PROTO_DIR + dest.mkdir(parents=True) + for name in FILES: + text = (server / SERVER_PROTO_DIR / name).read_text() + text = text.replace(str(SERVER_PROTO_DIR) + "/", str(STAGE_PROTO_DIR) + "/") + text = DROP_IMPORT.sub("", text) + text = DROP_OPTION.sub("", text) + (dest / name).write_text(text) + return [str(STAGE_PROTO_DIR / name) for name in FILES] + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit("usage: gen_stream_protos.py /path/to/temporal-server-checkout") + server = Path(sys.argv[1]).resolve() + if not (server / SERVER_PROTO_DIR).is_dir(): + sys.exit(f"no stream protos under {server / SERVER_PROTO_DIR}") + + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + targets = stage(server, work) + out = work / "out" + out.mkdir() + subprocess.run( + [ + sys.executable, + "-m", + "grpc_tools.protoc", + f"--proto_path={work}", + f"--proto_path={api_proto_path()}", + f"--python_out={out}", + f"--grpc_python_out={out}", + f"--mypy_out={out}", + f"--mypy_grpc_out={out}", + *targets, + ], + check=True, + cwd=work, + ) + + generated = out / STAGE_PROTO_DIR + for path in sorted(generated.iterdir()): + if path.name == "__init__.py": + continue + text = path.read_text() + for pattern, repl in FIX_PY: + text = pattern.sub(repl, text) + (OUT / path.name).write_text(text) + print(f"wrote {OUT / path.name}") + + shutil.rmtree(OUT / "__pycache__", ignore_errors=True) + write_init() + + +MODULES = ["message_pb2", "stream_state_pb2", "request_response_pb2"] + +INIT_HEADER = """\ +# Generated by scripts/gen_stream_protos.py. Do not edit. +# +# Vendored rather than taken from the api submodule, because the stream service +# is still defined in the server. See temporalio/client_stream.py. +""" + + +def write_init() -> None: + """Re-export every generated message and enum, so the package surface + cannot drift from the protos it was built from.""" + import importlib + + sys.path.insert(0, str(BASE)) + blocks, exported = [], [] + for module in MODULES: + mod = importlib.import_module(f"temporalio.api.streamservice.v1.{module}") + names = list(mod.DESCRIPTOR.message_types_by_name) + for enum in mod.DESCRIPTOR.enum_types_by_name.values(): + names.append(enum.name) + names.extend(value.name for value in enum.values) + names.sort() + exported.extend(names) + joined = "".join(f" {name},\n" for name in names) + blocks.append(f"from .{module} import (\n{joined})\n") + + listed = "".join(f' "{name}",\n' for name in sorted(exported)) + (OUT / "__init__.py").write_text( + INIT_HEADER + "\n" + "\n".join(blocks) + f"\n__all__ = [\n{listed}]\n" + ) + print(f"wrote {OUT / '__init__.py'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/gen_streams_nexus_api.py b/scripts/gen_streams_nexus_api.py new file mode 100644 index 0000000000..2d4f16c6d4 --- /dev/null +++ b/scripts/gen_streams_nexus_api.py @@ -0,0 +1,95 @@ +import os +import shutil +import subprocess +import sys +from pathlib import Path + +base_dir = Path(__file__).parent.parent +providers_dir = base_dir / "temporalio" / "streams" / "providers" +contract_path = providers_dir / "temporal_streams.nexusrpc.yaml" +output_dir = providers_dir / "_nexus_generated" +# Pinned to the version CI installs, because nexgen's output changes between +# releases and check-protos compares against what is committed here. +NEX_GEN_VERSION = "0.2.4" + + +def nex_gen_command() -> list[str]: + if bin_path := os.environ.get("NEX_GEN_BIN"): + return [bin_path] + + if shutil.which("nexgen") is None: + subprocess.check_call( + [ + "cargo", + "install", + "--locked", + "nexgen", + "--version", + NEX_GEN_VERSION, + # Same build as the system API script installs, so one binary + # serves both and neither can overwrite the other's output. + "--features", + "advanced", + "--force", + ] + ) + return ["nexgen"] + + +def check_version(command: list[str]) -> None: + # A different release on PATH would regenerate different code locally + # and the drift would only show in CI, so refuse before writing anything. + reported = subprocess.check_output([*command, "--version"], text=True).strip() + found = reported.split()[-1] if reported else "" + if found != NEX_GEN_VERSION: + raise SystemExit( + f"found nexgen {found or '?'} at {command[0]}, but the stream contract is " + f"generated with {NEX_GEN_VERSION}. Install it with `cargo install --locked " + f"nexgen --version {NEX_GEN_VERSION} --features advanced` or point " + "NEX_GEN_BIN at that binary." + ) + + +def generate_streams_nexus_api() -> None: + if not contract_path.exists(): + raise RuntimeError(f"missing stream contract: {contract_path}") + + command = nex_gen_command() + check_version(command) + shutil.rmtree(output_dir, ignore_errors=True) + subprocess.check_call( + [ + *command, + "python", + str(contract_path), + "--output", + str(output_dir), + ] + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "ruff", + "check", + "--select", + "I", + "--fix", + str(output_dir), + ] + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "ruff", + "format", + str(output_dir), + ] + ) + + +if __name__ == "__main__": + print("Generating stream endpoint Nexus API...", file=sys.stderr) + generate_streams_nexus_api() + print("Done", file=sys.stderr) diff --git a/streams_demo/README.md b/streams_demo/README.md new file mode 100644 index 0000000000..a52e56ae8a --- /dev/null +++ b/streams_demo/README.md @@ -0,0 +1,64 @@ +# One workflow file, three stream providers + +`agent_loop.py` and `run_demo.py` never name a provider. The provider is a +configuration choice, and this branch carries all three options plus the +Nexus front, so the same loop runs on each of them by changing one +environment variable. + +The loop: an Activity streams records into the workflow's `inputs` stream, +the workflow reads them, publishes a decision per record onto `decisions`, +runs an ordinary Activity in the same task, and an outside consumer follows +the decisions. + +## Today's Workflow Streams (Option 0) + +Records ride the shipped feature's Signals and Updates and live in the +workflow's History. Nothing to deploy; Option 0's limits apply. + +```sh +temporal server start-dev --headless & +STREAMS_PROVIDER=workflow_streams python streams_demo/run_demo.py +``` + +## Client-side (Redis) + +The store is a Redis the process names; the workflow's own writes go through +the staged commit, so a failed Workflow Task leaks nothing. + +```sh +redis-server --port 6399 & +temporal server start-dev --headless & +AI198_REDIS_URL=redis://127.0.0.1:6399 \ +STREAMS_PROVIDER=redis python streams_demo/run_demo.py +``` + +## Server-side (native) + +Streams live on the Temporal server; publishes are commands applied when the +task is accepted, and consumed ranges arrive on Workflow Tasks. Needs a +server built from `moedash/temporal` `moe/AI-198-server-side-streams`. + +```sh +TEMPORAL_ADDRESS=127.0.0.1:7233 \ +STREAMS_PROVIDER=native python streams_demo/run_demo.py +``` + +## The Nexus front (outside surface) + +Outside producers and consumers can go through one Temporal-authenticated +endpoint instead of naming a store; the handler worker hides whichever +provider it is configured with. Workers still configure a storage provider, +because workflow reads and writes ride the Workflow Task. + +```python +front = NexusStreams(endpoint=endpoint_id) +stream = front.get_stream_handle(client, workflow_id) +producer = stream.producer(topic="inputs", producer_id="model", attempt=1) +``` + +See `tests/streams/test_nexus_provider.py` for the endpoint setup and the +handler worker. + +The conformance suite runs the same expectations on every provider: +`pytest tests/streams/` (memory and Workflow Streams on the test server), plus +`STREAMS_LIVE=native|redis|nexus` for the suites that need a store. diff --git a/streams_demo/agent_loop.py b/streams_demo/agent_loop.py new file mode 100644 index 0000000000..df5eac4d6a --- /dev/null +++ b/streams_demo/agent_loop.py @@ -0,0 +1,110 @@ +"""One agent loop, written once, run on every stream provider. + +Reads input, decides on it, publishes the decision, and runs an ordinary +Activity in the same workflow task; the Activity reports on its own +workflow's stream in turn. Also handles the two control records the contract +defines, so a retried producer and a finished topic are exercised rather than +described. + +This file is byte-identical in the server-side tree and the client-side tree. +Nothing in it names a provider: the workflow asks its runtime, the Activity +asks its context, and the process that runs them registered the provider once. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import Any + +from temporalio import activity, streams, workflow +from temporalio.streams import RecordKind + +# Defined once and shared by the workflow, the Activity and the demo's +# reader, so the type each topic carries is stated in one place. +DECISIONS = streams.topic("decisions", dict[str, Any]) +INPUTS = streams.topic("inputs", dict[str, Any]) +RECEIPTS = streams.topic("receipts", dict[str, Any]) + + +@activity.defn(name="RecordDecision") +async def record_decision(decision: dict[str, Any]) -> str: + """An ordinary command in the same task as the publish. + + Appends its receipt onto its own workflow's stream too, under the + Activity's own identity, so a reader outside sees the decision and the + record of it side by side. + """ + receipt = f"recorded:{decision['source']}:{decision['branch']}" + await activity.stream_handle().producer(topic=RECEIPTS).append({"receipt": receipt}) + return receipt + + +def decide(token: dict[str, Any]) -> dict[str, Any]: + """The decision the workflow is here to make.""" + if token["value"] % 2 == 0: + return { + "source": token["id"], + "branch": "even", + "computed": token["value"] * 10, + } + return {"source": token["id"], "branch": "odd", "computed": token["value"] + 100} + + +@workflow.defn(name="StreamContractDemo", sandboxed=False) +class AgentLoop: + """Read, decide, write, until the producer says it has finished.""" + + @workflow.run + async def run(self, limit: int) -> list[dict[str, Any]]: + """Decide on at most ``limit`` inputs, then return the trace.""" + inputs = workflow.stream_reader(INPUTS) + decisions = workflow.stream_writer(DECISIONS) + trace: list[dict[str, Any]] = [] + accepted = 0 + try: + async for record in inputs: + if record.kind is RecordKind.SUPERSEDED: + # A newer attempt of the same producer started writing. The + # decisions already published stand, so the workflow says so + # rather than pretending they can be withdrawn. + assert record.supersession is not None + trace.append( + { + "kind": "superseded", + "producer": record.producer_id, + "replaced": record.supersession.previous_attempt, + "attempt": record.supersession.attempt, + } + ) + decisions.publish( + {"retracting_attempt": record.supersession.previous_attempt} + ) + continue + if record.kind is RecordKind.FINISH: + # The producer says it is done, which is what ends the loop. + # Counting decisions instead would leave the terminal record + # unread and let the workflow finish while its producer is + # still writing. + trace.append({"kind": "finish", "producer": record.producer_id}) + break + assert record.value is not None + decision = decide(record.value) + decisions.publish(decision) + receipt = await workflow.execute_activity( + record_decision, + decision, + activity_id=f"decision-{decision['source']}", + start_to_close_timeout=timedelta(seconds=10), + ) + trace.append( + {"kind": "decision", "value": decision, "receipt": receipt} + ) + accepted += 1 + if accepted >= limit: + # A bound so a stuck producer cannot run this forever. The + # terminal record above is the ordinary way out. + break + finally: + inputs.close() + decisions.finish() + return trace diff --git a/streams_demo/provider_setup.py b/streams_demo/provider_setup.py new file mode 100644 index 0000000000..9d6e89b652 --- /dev/null +++ b/streams_demo/provider_setup.py @@ -0,0 +1,55 @@ +"""Pick the provider for a demo run from the environment. + +``STREAMS_PROVIDER`` names a provider; this tree carries ``memory``, +``workflow_streams``, ``native`` and ``redis``. The demo needs a Temporal +server to run the workflow either way; ``TEMPORAL_ADDRESS`` points at it, the +native demo needs one built with the stream service, and the Redis demo reads +its store from ``AI198_REDIS_URL`` and ``AI198_REDIS_PREFIX``. +""" + +from __future__ import annotations + +import os + +from temporalio.streams.providers import ProviderPlugin +from temporalio.streams.providers.memory import MemoryStreams + +NAME = os.environ.get("STREAMS_PROVIDER", "memory") + +# The memory provider is not replay-safe, so its demo keeps the cache warm. +# Storage providers run with the smallest cache they support instead. +WORKFLOW_CACHE = int(os.environ.get("STREAMS_WORKFLOW_CACHE", "512")) + + +async def open() -> tuple[str, ProviderPlugin]: + """The server to connect to and the provider the worker and the client share.""" + address = os.environ.get("TEMPORAL_ADDRESS", "localhost:7233") + if NAME == "memory": + return address, MemoryStreams() + if NAME == "workflow_streams": + from temporalio.streams.providers.workflow_streams import ( + WorkflowStreamsProvider, + ) + + return address, WorkflowStreamsProvider() + if NAME == "native": + from temporalio.streams.providers.native import NativeStreams + + return address, NativeStreams() + if NAME == "redis": + from temporalio.streams.providers.redis import RedisStreams + + return address, RedisStreams( + url=os.environ.get("AI198_REDIS_URL", "redis://127.0.0.1:6379"), + key_prefix=os.environ.get("AI198_REDIS_PREFIX", "ai198-contract"), + ) + raise SystemExit(f"this tree carries no stream provider named {NAME!r}") + + +async def close(provider: ProviderPlugin) -> None: + """Let go of whatever :func:`open` acquired. + + Every provider releases what it opened through its own ``close()``, so + the demo's teardown reads the same on all of them. + """ + await provider.close() diff --git a/streams_demo/run_demo.py b/streams_demo/run_demo.py new file mode 100644 index 0000000000..769d7bf189 --- /dev/null +++ b/streams_demo/run_demo.py @@ -0,0 +1,174 @@ +"""Run the shared agent loop against whichever provider is configured. + +Three cases, the same on every provider: + +- read, decide, publish and an ordinary Activity in the same workflow task, + with the smallest workflow cache the provider supports, so that as much of + the run as it allows is rebuilt rather than remembered; +- a producer whose second attempt supersedes its first, which the reader has + to report and the workflow has to act on; +- an outside reader following what the workflow published, and the receipts + the Activity appended on the workflow's stream from inside its own context. + +The provider is registered once, on the client; the worker inherits it and +every context asks for its stream without naming it. Byte-identical in every +tree. ``provider_setup`` is what differs, and it is the only import here that +names a provider. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +import time +import uuid +from pathlib import Path +from typing import Any + +from temporalio.api.enums.v1 import EventType +from temporalio.client import Client +from temporalio.streams import RecordKind, StreamHandle +from temporalio.worker import Worker + +# Run as a script rather than imported as a module, so the two siblings are +# reached by name off this directory rather than through a package path. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import provider_setup # noqa: E402 # pyright: ignore[reportImplicitRelativeImport] +from agent_loop import ( # noqa: E402 # pyright: ignore[reportImplicitRelativeImport] + DECISIONS, + INPUTS, + RECEIPTS, + AgentLoop, + record_decision, +) + +DECISION_LIMIT = 8 +EXPECTED_OUTPUT = 6 + + +async def collect_output(stream: StreamHandle, want: int) -> list[dict]: + """Read ``want`` decisions off the workflow's stream from outside it.""" + seen: list[dict] = [] + async for record in stream.read(topic=DECISIONS): + seen.append({"kind": record.kind.name, "value": record.value}) + if len(seen) >= want: + break + return seen + + +async def main() -> int: + """Run the demo once and write what happened next to this file.""" + out = Path(__file__).resolve().parent / f"results-{provider_setup.NAME}" + out.mkdir(exist_ok=True) + target, provider = await provider_setup.open() + client = await Client.connect(target, namespace="default", plugins=[provider]) + + uid = f"ai198-contract-{provider_setup.NAME}-" + uuid.uuid4().hex + record: dict[str, Any] = { + "provider": provider_setup.NAME, + "workflow_id": uid, + "target": target, + "max_cached_workflows": provider_setup.WORKFLOW_CACHE, + } + + async with Worker( + client, + task_queue=uid, + workflows=[AgentLoop], + activities=[record_decision], + max_cached_workflows=provider_setup.WORKFLOW_CACHE, + ): + handle = await client.start_workflow( + AgentLoop.run, DECISION_LIMIT, id=uid, task_queue=uid + ) + stream = client.get_stream_handle(uid) + output = asyncio.create_task(collect_output(stream, EXPECTED_OUTPUT)) + + # The first attempt writes two records and then stops, as a failed + # activity would. The second writes different inputs under the same + # logical producer, which is what the reader has to report. + first = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await first.append({"id": "r1", "value": 1}, {"id": "r2", "value": 2}) + await asyncio.sleep(0.5) + second = stream.producer(topic=INPUTS, producer_id="model", attempt=2) + await second.append({"id": "r3", "value": 3}, {"id": "r4", "value": 4}) + await second.finish() + + # A failed Workflow Task is not an outcome. The server rejects a + # completion that raced newly buffered events, and the retry usually + # gets through, so reading the first failure as the result reports a + # working run as a broken one. Wait for a terminal event, then report + # the retries separately so they are neither the headline nor hidden. + deadline = time.monotonic() + 120 + # Taken from the enum rather than written out, because guessing these + # numbers is how a run that completed gets reported as terminated. + terminal = { + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED: "completed", + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_FAILED: "workflow_failed", + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT: "execution_timed_out", + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_CANCELED: "canceled", + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_TERMINATED: "terminated", + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW: "continued_as_new", + } + while time.monotonic() < deadline: + history = await handle.fetch_history() + reached = [ + terminal[e.event_type] + for e in history.events + if e.event_type in terminal + ] + if reached: + record["outcome"] = reached[-1] + if record["outcome"] == "completed": + record["trace"] = await handle.result() + break + await asyncio.sleep(0.1) + else: + record["outcome"] = "timed_out_waiting" + history = await handle.fetch_history() + + retried = [ + e + for e in history.events + if e.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_FAILED + ] + record["task_failures"] = [ + { + "event_id": e.event_id, + "cause": int(e.workflow_task_failed_event_attributes.cause), + "message": e.workflow_task_failed_event_attributes.failure.message, + } + for e in retried + ] + + try: + record["observed_output"] = await asyncio.wait_for(output, timeout=20) + except asyncio.TimeoutError: + output.cancel() + record["observed_output"] = "timed_out" + + async def receipts() -> list[Any]: + # Ends by itself once the workflow is closed and the tail served. + return [ + r.value + async for r in stream.read(topic=RECEIPTS) + if r.kind is RecordKind.DATA + ] + + try: + record["receipts"] = await asyncio.wait_for(receipts(), timeout=20) + except asyncio.TimeoutError: + record["receipts"] = "timed_out" + + history = await handle.fetch_history() + record["history_events"] = len(history.events) + (out / "history.json").write_text(history.to_json()) + await provider_setup.close(provider) + (out / "results.json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(record, indent=2)) + return 0 if record["outcome"] == "completed" else 1 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/temporalio/activity.py b/temporalio/activity.py index 768c8d108b..40821db391 100644 --- a/temporalio/activity.py +++ b/temporalio/activity.py @@ -20,6 +20,7 @@ from typing import ( TYPE_CHECKING, Any, + Literal, NoReturn, overload, ) @@ -29,9 +30,11 @@ import temporalio.bridge.proto.activity_task import temporalio.common import temporalio.converter +import temporalio.streams from temporalio.converter._payload_converter import ( _TemporalTransferTypePayloadConverter, ) +from temporalio.streams._ref import open_ref from .types import CallableType @@ -209,6 +212,10 @@ class _Context: runtime_metric_meter: temporalio.common.MetricMeter | None client: Client | None cancellation_details: _ActivityCancellationDetailsHolder + stream_provider: temporalio.streams.StreamProvider | None = None + # A ``def`` activity is handed no client and no stream provider, so what + # it is missing cannot be read off the fields that are absent. + sync: bool = False _logger_details: Mapping[str, Any] | None = None _payload_converter: temporalio.converter.PayloadConverter | None = None _metric_meter: temporalio.common.MetricMeter | None = None @@ -294,6 +301,127 @@ def client() -> Client: return client +def stream_handle( + workflow_id: str | temporalio.streams.StreamRef | None = None, + *, + run_id: str | None = None, + scope: Literal["workflow", "activity"] | None = None, +) -> temporalio.streams.StreamHandle: + """Return a stream handle from the provider the worker was given. + + A :py:class:`temporalio.streams.StreamRef` in place of ``workflow_id``, + such as one this activity received as an argument, opens the stream it + names, whatever owns it, and takes no other argument; the handle's calls + that name no topic then address the ref's topic. + + Which stream a call with no ``workflow_id`` reaches is decided by where + the activity runs, never by what exists: + + - In an activity a workflow scheduled, it is that workflow's stream, + pinned to the run the activity belongs to. + - In a standalone activity, it is the activity's own stream. + - ``scope="activity"`` gives an activity a workflow scheduled its own + streams instead, apart from the workflow's. ``scope="workflow"`` asks + for the workflow explicitly, and a standalone activity has none. + + The rule is static because a stream is created by its first write, so a + rule that looked for one would send attempt 1 to the workflow and a retry + to the stream attempt 1 created. An activity's own streams are one per + activity execution, not per attempt: a retry writes to the same stream, + under a new attempt, and they end when the activity reaches a terminal + status. + + Name a ``workflow_id`` to address another workflow; ``run_id`` then pins + the handle to one run and its absence follows the execution chain. A + ``read``, ``latest`` or ``producer`` that names no topic addresses the + owner's default topic, :py:data:`temporalio.streams.DEFAULT_TOPIC`, the + one :py:func:`temporalio.workflow.stream_reader` and + :py:func:`temporalio.workflow.stream_writer` use without a topic. A + ``read`` starts at :py:data:`temporalio.streams.BEGINNING`, at + :py:data:`temporalio.streams.END` or at the last ``N`` records with + ``last=N``. See :py:mod:`temporalio.streams`. + + Like :py:func:`client`, this is only available in ``async def`` + activities. + + Args: + workflow_id: Another workflow whose stream to address, or a + :py:class:`temporalio.streams.StreamRef` naming the stream. + run_id: The run of ``workflow_id`` to pin to. + scope: ``"activity"`` for this activity's own streams, + ``"workflow"`` for its workflow's. Without it the rule above + decides. + + Returns: + :py:class:`temporalio.streams.StreamHandle` for use in the current + activity. + + Raises: + RuntimeError: When the client is not available, which is what a + ``def`` activity gets, or when ``scope="workflow"`` is asked of + an activity that belongs to no workflow. + temporalio.streams.StreamUnsupportedError: The worker has no stream + provider, or its provider cannot hold a stream an activity owns. + Register one with ``Client.connect(plugins=[provider])`` or + ``Worker(plugins=[provider])``. + ValueError: ``run_id`` was given without ``workflow_id``, + ``scope="activity"`` with one, or a ref with either. + """ + context = _Context.current() + if context.sync: + # A sync activity is handed neither a client nor a provider. Saying + # the worker has none would send the reader to fix a registration + # that is not the problem. + raise RuntimeError( + "No stream handle available. Stream handles are only available in " + "`async def` activities; not in `def` activities, which are handed no " + "client to reach the store with." + ) + provider = context.stream_provider + if provider is None: + raise temporalio.streams.StreamUnsupportedError( + "no stream provider is configured on this worker; register one with " + "Client.connect(plugins=[provider]) or Worker(plugins=[provider])" + ) + if isinstance(workflow_id, temporalio.streams.StreamRef): + if run_id is not None or scope is not None: + raise ValueError( + "a StreamRef names the stream in full, so it takes no run_id or scope" + ) + return open_ref(provider, client(), workflow_id) + if workflow_id is not None: + if scope == "activity": + raise ValueError( + "scope='activity' addresses this activity's own streams, so it takes " + "no workflow_id" + ) + return provider.get_stream_handle(client(), workflow_id, run_id=run_id) + if run_id is not None: + raise ValueError("run_id needs a workflow_id") + info = context.info() + if scope is None: + scope = "workflow" if info.in_workflow else "activity" + if scope == "workflow": + if info.workflow_id is None: + raise RuntimeError( + "this activity belongs to no workflow, so name the workflow_id to " + "address, or leave scope unset for the activity's own streams" + ) + return provider.get_stream_handle( + client(), info.workflow_id, run_id=info.workflow_run_id + ) + if info.workflow_id is not None: + return provider.get_activity_stream_handle( + client(), + info.activity_id, + workflow_id=info.workflow_id, + run_id=info.workflow_run_id, + ) + return provider.get_activity_stream_handle( + client(), info.activity_id, run_id=info.activity_run_id + ) + + def in_activity() -> bool: """Whether the current code is inside an activity. diff --git a/temporalio/api/callback/v1/message_pb2.py b/temporalio/api/callback/v1/message_pb2.py index 5d4ccb9171..b91db112de 100644 --- a/temporalio/api/callback/v1/message_pb2.py +++ b/temporalio/api/callback/v1/message_pb2.py @@ -27,7 +27,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n&temporal/api/callback/v1/message.proto\x12\x18temporal.api.callback.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a"temporal/api/enums/v1/common.proto\x1a%temporal/api/failure/v1/message.proto"\x97\x03\n\x0c\x43\x61llbackInfo\x12\x32\n\x08\x63\x61llback\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Callback\x12\x35\n\x11registration_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x05state\x18\x03 \x01(\x0e\x32$.temporal.api.enums.v1.CallbackState\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x05\x12>\n\x1alast_attempt_complete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x14last_attempt_failure\x18\x06 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12>\n\x1anext_attempt_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x16\n\x0e\x62locked_reason\x18\x08 \x01(\tB\x93\x01\n\x1bio.temporal.api.callback.v1B\x0cMessageProtoP\x01Z\'go.temporal.io/api/callback/v1;callback\xaa\x02\x1aTemporalio.Api.Callback.V1\xea\x02\x1dTemporalio::Api::Callback::V1b\x06proto3' + b'\n&temporal/api/callback/v1/message.proto\x12\x18temporal.api.callback.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a"temporal/api/enums/v1/common.proto\x1a%temporal/api/failure/v1/message.proto"\xab\x03\n\x0c\x43\x61llbackInfo\x12\x32\n\x08\x63\x61llback\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Callback\x12\x35\n\x11registration_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x05state\x18\x03 \x01(\x0e\x32$.temporal.api.enums.v1.CallbackState\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x05\x12>\n\x1alast_attempt_complete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x14last_attempt_failure\x18\x06 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12>\n\x1anext_attempt_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x16\n\x0e\x62locked_reason\x18\x08 \x01(\t\x12\x12\n\nrequest_id\x18\t \x01(\tB\x93\x01\n\x1bio.temporal.api.callback.v1B\x0cMessageProtoP\x01Z\'go.temporal.io/api/callback/v1;callback\xaa\x02\x1aTemporalio.Api.Callback.V1\xea\x02\x1dTemporalio::Api::Callback::V1b\x06proto3' ) @@ -47,5 +47,5 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\033io.temporal.api.callback.v1B\014MessageProtoP\001Z'go.temporal.io/api/callback/v1;callback\252\002\032Temporalio.Api.Callback.V1\352\002\035Temporalio::Api::Callback::V1" _CALLBACKINFO._serialized_start = 215 - _CALLBACKINFO._serialized_end = 622 + _CALLBACKINFO._serialized_end = 642 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/callback/v1/message_pb2.pyi b/temporalio/api/callback/v1/message_pb2.pyi index ea7d6444b3..a9f2ddec4d 100644 --- a/temporalio/api/callback/v1/message_pb2.pyi +++ b/temporalio/api/callback/v1/message_pb2.pyi @@ -34,6 +34,7 @@ class CallbackInfo(google.protobuf.message.Message): LAST_ATTEMPT_FAILURE_FIELD_NUMBER: builtins.int NEXT_ATTEMPT_SCHEDULE_TIME_FIELD_NUMBER: builtins.int BLOCKED_REASON_FIELD_NUMBER: builtins.int + REQUEST_ID_FIELD_NUMBER: builtins.int @property def callback(self) -> temporalio.api.common.v1.message_pb2.Callback: """Information on how this callback should be invoked (e.g. its URL and type).""" @@ -57,6 +58,10 @@ class CallbackInfo(google.protobuf.message.Message): """The time when the next attempt is scheduled.""" blocked_reason: builtins.str """If the state is BLOCKED, blocked reason provides additional information.""" + request_id: builtins.str + """Server-generated request ID used as an idempotency token when invoking callbacks. + It has no relation to caller-side request_id sent in operations like StartNexusOperationExecutionRequest. + """ def __init__( self, *, @@ -71,6 +76,7 @@ class CallbackInfo(google.protobuf.message.Message): next_attempt_schedule_time: google.protobuf.timestamp_pb2.Timestamp | None = ..., blocked_reason: builtins.str = ..., + request_id: builtins.str = ..., ) -> None: ... def HasField( self, @@ -104,6 +110,8 @@ class CallbackInfo(google.protobuf.message.Message): b"next_attempt_schedule_time", "registration_time", b"registration_time", + "request_id", + b"request_id", "state", b"state", ], diff --git a/temporalio/api/command/v1/__init__.py b/temporalio/api/command/v1/__init__.py index e0ce8f712c..4335f783d7 100644 --- a/temporalio/api/command/v1/__init__.py +++ b/temporalio/api/command/v1/__init__.py @@ -1,4 +1,5 @@ from .message_pb2 import ( + AppendStreamRecordsCommandAttributes, CancelTimerCommandAttributes, CancelWorkflowExecutionCommandAttributes, Command, @@ -16,10 +17,14 @@ SignalExternalWorkflowExecutionCommandAttributes, StartChildWorkflowExecutionCommandAttributes, StartTimerCommandAttributes, + SubscribeNotificationChannelCommandAttributes, + SubscribeStreamCommandAttributes, + UnsubscribeNotificationChannelCommandAttributes, UpsertWorkflowSearchAttributesCommandAttributes, ) __all__ = [ + "AppendStreamRecordsCommandAttributes", "CancelTimerCommandAttributes", "CancelWorkflowExecutionCommandAttributes", "Command", @@ -37,5 +42,8 @@ "SignalExternalWorkflowExecutionCommandAttributes", "StartChildWorkflowExecutionCommandAttributes", "StartTimerCommandAttributes", + "SubscribeNotificationChannelCommandAttributes", + "SubscribeStreamCommandAttributes", + "UnsubscribeNotificationChannelCommandAttributes", "UpsertWorkflowSearchAttributesCommandAttributes", ] diff --git a/temporalio/api/command/v1/message_pb2.py b/temporalio/api/command/v1/message_pb2.py index ad004ae03f..52bea94d29 100644 --- a/temporalio/api/command/v1/message_pb2.py +++ b/temporalio/api/command/v1/message_pb2.py @@ -34,6 +34,9 @@ from temporalio.api.sdk.v1 import ( user_metadata_pb2 as temporal_dot_api_dot_sdk_dot_v1_dot_user__metadata__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.taskqueue.v1 import ( message_pb2 as temporal_dot_api_dot_taskqueue_dot_v1_dot_message__pb2, ) @@ -42,7 +45,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n%temporal/api/command/v1/message.proto\x12\x17temporal.api.command.v1\x1a\x1egoogle/protobuf/duration.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a(temporal/api/enums/v1/command_type.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/taskqueue/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto"\xb6\x05\n%ScheduleActivityTaskCommandAttributes\x12\x13\n\x0b\x61\x63tivity_id\x18\x01 \x01(\t\x12;\n\ractivity_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12.\n\x06header\x18\x05 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12/\n\x05input\x18\x06 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12<\n\x19schedule_to_close_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x1f\n\x17request_eager_execution\x18\x0c \x01(\x08\x12\x1d\n\x15use_workflow_build_id\x18\r \x01(\x08\x12\x32\n\x08priority\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.PriorityJ\x04\x08\x03\x10\x04"H\n*RequestCancelActivityTaskCommandAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03"i\n\x1bStartTimerCommandAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"^\n*CompleteWorkflowExecutionCommandAttributes\x12\x30\n\x06result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"[\n&FailWorkflowExecutionCommandAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"0\n\x1c\x43\x61ncelTimerCommandAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t"]\n(CancelWorkflowExecutionCommandAttributes\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"\xb7\x01\n7RequestCancelExternalWorkflowExecutionCommandAttributes\x12\x15\n\tnamespace\x18\x01 \x01(\tB\x02\x18\x01\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x07\x63ontrol\x18\x04 \x01(\tB\x02\x18\x01\x12\x1b\n\x13\x63hild_workflow_only\x18\x05 \x01(\x08\x12\x0e\n\x06reason\x18\x06 \x01(\t"\xaf\x02\n0SignalExternalWorkflowExecutionCommandAttributes\x12\x15\n\tnamespace\x18\x01 \x01(\tB\x02\x18\x01\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x0bsignal_name\x18\x03 \x01(\t\x12/\n\x05input\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x13\n\x07\x63ontrol\x18\x05 \x01(\tB\x02\x18\x01\x12\x1b\n\x13\x63hild_workflow_only\x18\x06 \x01(\x08\x12.\n\x06header\x18\x07 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header"v\n/UpsertWorkflowSearchAttributesCommandAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"`\n)ModifyWorkflowPropertiesCommandAttributes\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xbf\x02\n\x1dRecordMarkerCommandAttributes\x12\x13\n\x0bmarker_name\x18\x01 \x01(\t\x12T\n\x07\x64\x65tails\x18\x02 \x03(\x0b\x32\x43.temporal.api.command.v1.RecordMarkerCommandAttributes.DetailsEntry\x12.\n\x06header\x18\x03 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x1aP\n\x0c\x44\x65tailsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads:\x02\x38\x01"\xac\x07\n/ContinueAsNewWorkflowExecutionCommandAttributes\x12;\n\rworkflow_type\x18\x01 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x03 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x07 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12@\n\tinitiator\x18\x08 \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12\x31\n\x07\x66\x61ilure\x18\t \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\n \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x15\n\rcron_schedule\x18\x0b \x01(\t\x12.\n\x06header\x18\x0c \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12*\n\x04memo\x18\r \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0e \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x1c\n\x10inherit_build_id\x18\x0f \x01(\x08\x42\x02\x18\x01\x12[\n\x1binitial_versioning_behavior\x18\x10 \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior"\xec\x07\n,StartChildWorkflowExecutionCommandAttributes\x12\x15\n\tnamespace\x18\x01 \x01(\tB\x02\x18\x01\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x45\n\x13parent_close_policy\x18\t \x01(\x0e\x32(.temporal.api.enums.v1.ParentClosePolicy\x12\x0f\n\x07\x63ontrol\x18\n \x01(\t\x12N\n\x18workflow_id_reuse_policy\x18\x0b \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\r \x01(\t\x12.\n\x06header\x18\x0e \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12*\n\x04memo\x18\x0f \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x10 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x1c\n\x10inherit_build_id\x18\x11 \x01(\x08\x42\x02\x18\x01\x12\x32\n\x08priority\x18\x12 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x13 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride"6\n ProtocolMessageCommandAttributes\x12\x12\n\nmessage_id\x18\x01 \x01(\t"\xe3\x03\n\'ScheduleNexusOperationCommandAttributes\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x0f\n\x07service\x18\x02 \x01(\t\x12\x11\n\toperation\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12g\n\x0cnexus_header\x18\x06 \x03(\x0b\x32Q.temporal.api.command.v1.ScheduleNexusOperationCommandAttributes.NexusHeaderEntry\x12<\n\x19schedule_to_start_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"J\n,RequestCancelNexusOperationCommandAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03"\x87\x12\n\x07\x43ommand\x12\x38\n\x0c\x63ommand_type\x18\x01 \x01(\x0e\x32".temporal.api.enums.v1.CommandType\x12\x39\n\ruser_metadata\x18\xad\x02 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x43\n\x13\x65vent_group_markers\x18\xae\x02 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12s\n)schedule_activity_task_command_attributes\x18\x02 \x01(\x0b\x32>.temporal.api.command.v1.ScheduleActivityTaskCommandAttributesH\x00\x12^\n\x1estart_timer_command_attributes\x18\x03 \x01(\x0b\x32\x34.temporal.api.command.v1.StartTimerCommandAttributesH\x00\x12}\n.complete_workflow_execution_command_attributes\x18\x04 \x01(\x0b\x32\x43.temporal.api.command.v1.CompleteWorkflowExecutionCommandAttributesH\x00\x12u\n*fail_workflow_execution_command_attributes\x18\x05 \x01(\x0b\x32?.temporal.api.command.v1.FailWorkflowExecutionCommandAttributesH\x00\x12~\n/request_cancel_activity_task_command_attributes\x18\x06 \x01(\x0b\x32\x43.temporal.api.command.v1.RequestCancelActivityTaskCommandAttributesH\x00\x12`\n\x1f\x63\x61ncel_timer_command_attributes\x18\x07 \x01(\x0b\x32\x35.temporal.api.command.v1.CancelTimerCommandAttributesH\x00\x12y\n,cancel_workflow_execution_command_attributes\x18\x08 \x01(\x0b\x32\x41.temporal.api.command.v1.CancelWorkflowExecutionCommandAttributesH\x00\x12\x99\x01\n=request_cancel_external_workflow_execution_command_attributes\x18\t \x01(\x0b\x32P.temporal.api.command.v1.RequestCancelExternalWorkflowExecutionCommandAttributesH\x00\x12\x62\n record_marker_command_attributes\x18\n \x01(\x0b\x32\x36.temporal.api.command.v1.RecordMarkerCommandAttributesH\x00\x12\x89\x01\n5continue_as_new_workflow_execution_command_attributes\x18\x0b \x01(\x0b\x32H.temporal.api.command.v1.ContinueAsNewWorkflowExecutionCommandAttributesH\x00\x12\x82\x01\n1start_child_workflow_execution_command_attributes\x18\x0c \x01(\x0b\x32\x45.temporal.api.command.v1.StartChildWorkflowExecutionCommandAttributesH\x00\x12\x8a\x01\n5signal_external_workflow_execution_command_attributes\x18\r \x01(\x0b\x32I.temporal.api.command.v1.SignalExternalWorkflowExecutionCommandAttributesH\x00\x12\x88\x01\n4upsert_workflow_search_attributes_command_attributes\x18\x0e \x01(\x0b\x32H.temporal.api.command.v1.UpsertWorkflowSearchAttributesCommandAttributesH\x00\x12h\n#protocol_message_command_attributes\x18\x0f \x01(\x0b\x32\x39.temporal.api.command.v1.ProtocolMessageCommandAttributesH\x00\x12{\n-modify_workflow_properties_command_attributes\x18\x11 \x01(\x0b\x32\x42.temporal.api.command.v1.ModifyWorkflowPropertiesCommandAttributesH\x00\x12w\n+schedule_nexus_operation_command_attributes\x18\x12 \x01(\x0b\x32@.temporal.api.command.v1.ScheduleNexusOperationCommandAttributesH\x00\x12\x82\x01\n1request_cancel_nexus_operation_command_attributes\x18\x13 \x01(\x0b\x32\x45.temporal.api.command.v1.RequestCancelNexusOperationCommandAttributesH\x00\x42\x0c\n\nattributesB\x8e\x01\n\x1aio.temporal.api.command.v1B\x0cMessageProtoP\x01Z%go.temporal.io/api/command/v1;command\xaa\x02\x19Temporalio.Api.Command.V1\xea\x02\x1cTemporalio::Api::Command::V1b\x06proto3' + b'\n%temporal/api/command/v1/message.proto\x12\x17temporal.api.command.v1\x1a\x1egoogle/protobuf/duration.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a(temporal/api/enums/v1/command_type.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/taskqueue/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto"\xb6\x05\n%ScheduleActivityTaskCommandAttributes\x12\x13\n\x0b\x61\x63tivity_id\x18\x01 \x01(\t\x12;\n\ractivity_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12.\n\x06header\x18\x05 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12/\n\x05input\x18\x06 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12<\n\x19schedule_to_close_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x1f\n\x17request_eager_execution\x18\x0c \x01(\x08\x12\x1d\n\x15use_workflow_build_id\x18\r \x01(\x08\x12\x32\n\x08priority\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.PriorityJ\x04\x08\x03\x10\x04"H\n*RequestCancelActivityTaskCommandAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03"i\n\x1bStartTimerCommandAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"^\n*CompleteWorkflowExecutionCommandAttributes\x12\x30\n\x06result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"[\n&FailWorkflowExecutionCommandAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"0\n\x1c\x43\x61ncelTimerCommandAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t"]\n(CancelWorkflowExecutionCommandAttributes\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"\xb7\x01\n7RequestCancelExternalWorkflowExecutionCommandAttributes\x12\x15\n\tnamespace\x18\x01 \x01(\tB\x02\x18\x01\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x07\x63ontrol\x18\x04 \x01(\tB\x02\x18\x01\x12\x1b\n\x13\x63hild_workflow_only\x18\x05 \x01(\x08\x12\x0e\n\x06reason\x18\x06 \x01(\t"\xaf\x02\n0SignalExternalWorkflowExecutionCommandAttributes\x12\x15\n\tnamespace\x18\x01 \x01(\tB\x02\x18\x01\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x0bsignal_name\x18\x03 \x01(\t\x12/\n\x05input\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x13\n\x07\x63ontrol\x18\x05 \x01(\tB\x02\x18\x01\x12\x1b\n\x13\x63hild_workflow_only\x18\x06 \x01(\x08\x12.\n\x06header\x18\x07 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header"v\n/UpsertWorkflowSearchAttributesCommandAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"`\n)ModifyWorkflowPropertiesCommandAttributes\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xbf\x02\n\x1dRecordMarkerCommandAttributes\x12\x13\n\x0bmarker_name\x18\x01 \x01(\t\x12T\n\x07\x64\x65tails\x18\x02 \x03(\x0b\x32\x43.temporal.api.command.v1.RecordMarkerCommandAttributes.DetailsEntry\x12.\n\x06header\x18\x03 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x1aP\n\x0c\x44\x65tailsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads:\x02\x38\x01"\xac\x07\n/ContinueAsNewWorkflowExecutionCommandAttributes\x12;\n\rworkflow_type\x18\x01 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x03 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x07 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12@\n\tinitiator\x18\x08 \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12\x31\n\x07\x66\x61ilure\x18\t \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\n \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x15\n\rcron_schedule\x18\x0b \x01(\t\x12.\n\x06header\x18\x0c \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12*\n\x04memo\x18\r \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0e \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x1c\n\x10inherit_build_id\x18\x0f \x01(\x08\x42\x02\x18\x01\x12[\n\x1binitial_versioning_behavior\x18\x10 \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior"\xec\x07\n,StartChildWorkflowExecutionCommandAttributes\x12\x15\n\tnamespace\x18\x01 \x01(\tB\x02\x18\x01\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x45\n\x13parent_close_policy\x18\t \x01(\x0e\x32(.temporal.api.enums.v1.ParentClosePolicy\x12\x0f\n\x07\x63ontrol\x18\n \x01(\t\x12N\n\x18workflow_id_reuse_policy\x18\x0b \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\r \x01(\t\x12.\n\x06header\x18\x0e \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12*\n\x04memo\x18\x0f \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x10 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x1c\n\x10inherit_build_id\x18\x11 \x01(\x08\x42\x02\x18\x01\x12\x32\n\x08priority\x18\x12 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x13 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride"6\n ProtocolMessageCommandAttributes\x12\x12\n\nmessage_id\x18\x01 \x01(\t"\xe3\x03\n\'ScheduleNexusOperationCommandAttributes\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x0f\n\x07service\x18\x02 \x01(\t\x12\x11\n\toperation\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12g\n\x0cnexus_header\x18\x06 \x03(\x0b\x32Q.temporal.api.command.v1.ScheduleNexusOperationCommandAttributes.NexusHeaderEntry\x12<\n\x19schedule_to_start_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"J\n,RequestCancelNexusOperationCommandAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03"\xf4\x15\n\x07\x43ommand\x12\x38\n\x0c\x63ommand_type\x18\x01 \x01(\x0e\x32".temporal.api.enums.v1.CommandType\x12\x39\n\ruser_metadata\x18\xad\x02 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x43\n\x13\x65vent_group_markers\x18\xae\x02 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12s\n)schedule_activity_task_command_attributes\x18\x02 \x01(\x0b\x32>.temporal.api.command.v1.ScheduleActivityTaskCommandAttributesH\x00\x12^\n\x1estart_timer_command_attributes\x18\x03 \x01(\x0b\x32\x34.temporal.api.command.v1.StartTimerCommandAttributesH\x00\x12}\n.complete_workflow_execution_command_attributes\x18\x04 \x01(\x0b\x32\x43.temporal.api.command.v1.CompleteWorkflowExecutionCommandAttributesH\x00\x12u\n*fail_workflow_execution_command_attributes\x18\x05 \x01(\x0b\x32?.temporal.api.command.v1.FailWorkflowExecutionCommandAttributesH\x00\x12~\n/request_cancel_activity_task_command_attributes\x18\x06 \x01(\x0b\x32\x43.temporal.api.command.v1.RequestCancelActivityTaskCommandAttributesH\x00\x12`\n\x1f\x63\x61ncel_timer_command_attributes\x18\x07 \x01(\x0b\x32\x35.temporal.api.command.v1.CancelTimerCommandAttributesH\x00\x12y\n,cancel_workflow_execution_command_attributes\x18\x08 \x01(\x0b\x32\x41.temporal.api.command.v1.CancelWorkflowExecutionCommandAttributesH\x00\x12\x99\x01\n=request_cancel_external_workflow_execution_command_attributes\x18\t \x01(\x0b\x32P.temporal.api.command.v1.RequestCancelExternalWorkflowExecutionCommandAttributesH\x00\x12\x62\n record_marker_command_attributes\x18\n \x01(\x0b\x32\x36.temporal.api.command.v1.RecordMarkerCommandAttributesH\x00\x12\x89\x01\n5continue_as_new_workflow_execution_command_attributes\x18\x0b \x01(\x0b\x32H.temporal.api.command.v1.ContinueAsNewWorkflowExecutionCommandAttributesH\x00\x12\x82\x01\n1start_child_workflow_execution_command_attributes\x18\x0c \x01(\x0b\x32\x45.temporal.api.command.v1.StartChildWorkflowExecutionCommandAttributesH\x00\x12\x8a\x01\n5signal_external_workflow_execution_command_attributes\x18\r \x01(\x0b\x32I.temporal.api.command.v1.SignalExternalWorkflowExecutionCommandAttributesH\x00\x12\x88\x01\n4upsert_workflow_search_attributes_command_attributes\x18\x0e \x01(\x0b\x32H.temporal.api.command.v1.UpsertWorkflowSearchAttributesCommandAttributesH\x00\x12h\n#protocol_message_command_attributes\x18\x0f \x01(\x0b\x32\x39.temporal.api.command.v1.ProtocolMessageCommandAttributesH\x00\x12{\n-modify_workflow_properties_command_attributes\x18\x11 \x01(\x0b\x32\x42.temporal.api.command.v1.ModifyWorkflowPropertiesCommandAttributesH\x00\x12w\n+schedule_nexus_operation_command_attributes\x18\x12 \x01(\x0b\x32@.temporal.api.command.v1.ScheduleNexusOperationCommandAttributesH\x00\x12\x82\x01\n1request_cancel_nexus_operation_command_attributes\x18\x13 \x01(\x0b\x32\x45.temporal.api.command.v1.RequestCancelNexusOperationCommandAttributesH\x00\x12q\n(append_stream_records_command_attributes\x18\x14 \x01(\x0b\x32=.temporal.api.command.v1.AppendStreamRecordsCommandAttributesH\x00\x12h\n#subscribe_stream_command_attributes\x18\x15 \x01(\x0b\x32\x39.temporal.api.command.v1.SubscribeStreamCommandAttributesH\x00\x12\x83\x01\n1subscribe_notification_channel_command_attributes\x18\x16 \x01(\x0b\x32\x46.temporal.api.command.v1.SubscribeNotificationChannelCommandAttributesH\x00\x12\x87\x01\n3unsubscribe_notification_channel_command_attributes\x18\x17 \x01(\x0b\x32H.temporal.api.command.v1.UnsubscribeNotificationChannelCommandAttributesH\x00\x42\x0c\n\nattributes"r\n$AppendStreamRecordsCommandAttributes\x12\x13\n\x0bstream_name\x18\x01 \x01(\t\x12\x35\n\x07records\x18\x02 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord"\x98\x01\n SubscribeStreamCommandAttributes\x12\x19\n\x11stream_name_or_id\x18\x01 \x01(\t\x12\x14\n\x0cstart_offset\x18\x02 \x01(\x03\x12\x43\n\x0estart_position\x18\x03 \x01(\x0b\x32+.temporal.api.stream.v1.StreamStartPosition"@\n-SubscribeNotificationChannelCommandAttributes\x12\x0f\n\x07\x63hannel\x18\x01 \x01(\t"B\n/UnsubscribeNotificationChannelCommandAttributes\x12\x0f\n\x07\x63hannel\x18\x01 \x01(\tB\x8e\x01\n\x1aio.temporal.api.command.v1B\x0cMessageProtoP\x01Z%go.temporal.io/api/command/v1;command\xaa\x02\x19Temporalio.Api.Command.V1\xea\x02\x1cTemporalio::Api::Command::V1b\x06proto3' ) @@ -106,6 +109,18 @@ "RequestCancelNexusOperationCommandAttributes" ] _COMMAND = DESCRIPTOR.message_types_by_name["Command"] +_APPENDSTREAMRECORDSCOMMANDATTRIBUTES = DESCRIPTOR.message_types_by_name[ + "AppendStreamRecordsCommandAttributes" +] +_SUBSCRIBESTREAMCOMMANDATTRIBUTES = DESCRIPTOR.message_types_by_name[ + "SubscribeStreamCommandAttributes" +] +_SUBSCRIBENOTIFICATIONCHANNELCOMMANDATTRIBUTES = DESCRIPTOR.message_types_by_name[ + "SubscribeNotificationChannelCommandAttributes" +] +_UNSUBSCRIBENOTIFICATIONCHANNELCOMMANDATTRIBUTES = DESCRIPTOR.message_types_by_name[ + "UnsubscribeNotificationChannelCommandAttributes" +] ScheduleActivityTaskCommandAttributes = _reflection.GeneratedProtocolMessageType( "ScheduleActivityTaskCommandAttributes", (_message.Message,), @@ -332,6 +347,54 @@ ) _sym_db.RegisterMessage(Command) +AppendStreamRecordsCommandAttributes = _reflection.GeneratedProtocolMessageType( + "AppendStreamRecordsCommandAttributes", + (_message.Message,), + { + "DESCRIPTOR": _APPENDSTREAMRECORDSCOMMANDATTRIBUTES, + "__module__": "temporalio.api.command.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.command.v1.AppendStreamRecordsCommandAttributes) + }, +) +_sym_db.RegisterMessage(AppendStreamRecordsCommandAttributes) + +SubscribeStreamCommandAttributes = _reflection.GeneratedProtocolMessageType( + "SubscribeStreamCommandAttributes", + (_message.Message,), + { + "DESCRIPTOR": _SUBSCRIBESTREAMCOMMANDATTRIBUTES, + "__module__": "temporalio.api.command.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.command.v1.SubscribeStreamCommandAttributes) + }, +) +_sym_db.RegisterMessage(SubscribeStreamCommandAttributes) + +SubscribeNotificationChannelCommandAttributes = ( + _reflection.GeneratedProtocolMessageType( + "SubscribeNotificationChannelCommandAttributes", + (_message.Message,), + { + "DESCRIPTOR": _SUBSCRIBENOTIFICATIONCHANNELCOMMANDATTRIBUTES, + "__module__": "temporalio.api.command.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.command.v1.SubscribeNotificationChannelCommandAttributes) + }, + ) +) +_sym_db.RegisterMessage(SubscribeNotificationChannelCommandAttributes) + +UnsubscribeNotificationChannelCommandAttributes = ( + _reflection.GeneratedProtocolMessageType( + "UnsubscribeNotificationChannelCommandAttributes", + (_message.Message,), + { + "DESCRIPTOR": _UNSUBSCRIBENOTIFICATIONCHANNELCOMMANDATTRIBUTES, + "__module__": "temporalio.api.command.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.command.v1.UnsubscribeNotificationChannelCommandAttributes) + }, + ) +) +_sym_db.RegisterMessage(UnsubscribeNotificationChannelCommandAttributes) + if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\032io.temporal.api.command.v1B\014MessageProtoP\001Z%go.temporal.io/api/command/v1;command\252\002\031Temporalio.Api.Command.V1\352\002\034Temporalio::Api::Command::V1" @@ -383,44 +446,52 @@ _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES_NEXUSHEADERENTRY._serialized_options = ( b"8\001" ) - _SCHEDULEACTIVITYTASKCOMMANDATTRIBUTES._serialized_start = 424 - _SCHEDULEACTIVITYTASKCOMMANDATTRIBUTES._serialized_end = 1118 - _REQUESTCANCELACTIVITYTASKCOMMANDATTRIBUTES._serialized_start = 1120 - _REQUESTCANCELACTIVITYTASKCOMMANDATTRIBUTES._serialized_end = 1192 - _STARTTIMERCOMMANDATTRIBUTES._serialized_start = 1194 - _STARTTIMERCOMMANDATTRIBUTES._serialized_end = 1299 - _COMPLETEWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1301 - _COMPLETEWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 1395 - _FAILWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1397 - _FAILWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 1488 - _CANCELTIMERCOMMANDATTRIBUTES._serialized_start = 1490 - _CANCELTIMERCOMMANDATTRIBUTES._serialized_end = 1538 - _CANCELWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1540 - _CANCELWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 1633 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1636 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 1819 - _SIGNALEXTERNALWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1822 - _SIGNALEXTERNALWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 2125 - _UPSERTWORKFLOWSEARCHATTRIBUTESCOMMANDATTRIBUTES._serialized_start = 2127 - _UPSERTWORKFLOWSEARCHATTRIBUTESCOMMANDATTRIBUTES._serialized_end = 2245 - _MODIFYWORKFLOWPROPERTIESCOMMANDATTRIBUTES._serialized_start = 2247 - _MODIFYWORKFLOWPROPERTIESCOMMANDATTRIBUTES._serialized_end = 2343 - _RECORDMARKERCOMMANDATTRIBUTES._serialized_start = 2346 - _RECORDMARKERCOMMANDATTRIBUTES._serialized_end = 2665 - _RECORDMARKERCOMMANDATTRIBUTES_DETAILSENTRY._serialized_start = 2585 - _RECORDMARKERCOMMANDATTRIBUTES_DETAILSENTRY._serialized_end = 2665 - _CONTINUEASNEWWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 2668 - _CONTINUEASNEWWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 3608 - _STARTCHILDWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 3611 - _STARTCHILDWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 4615 - _PROTOCOLMESSAGECOMMANDATTRIBUTES._serialized_start = 4617 - _PROTOCOLMESSAGECOMMANDATTRIBUTES._serialized_end = 4671 - _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES._serialized_start = 4674 - _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES._serialized_end = 5157 - _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES_NEXUSHEADERENTRY._serialized_start = 5107 - _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES_NEXUSHEADERENTRY._serialized_end = 5157 - _REQUESTCANCELNEXUSOPERATIONCOMMANDATTRIBUTES._serialized_start = 5159 - _REQUESTCANCELNEXUSOPERATIONCOMMANDATTRIBUTES._serialized_end = 5233 - _COMMAND._serialized_start = 5236 - _COMMAND._serialized_end = 7547 + _SCHEDULEACTIVITYTASKCOMMANDATTRIBUTES._serialized_start = 462 + _SCHEDULEACTIVITYTASKCOMMANDATTRIBUTES._serialized_end = 1156 + _REQUESTCANCELACTIVITYTASKCOMMANDATTRIBUTES._serialized_start = 1158 + _REQUESTCANCELACTIVITYTASKCOMMANDATTRIBUTES._serialized_end = 1230 + _STARTTIMERCOMMANDATTRIBUTES._serialized_start = 1232 + _STARTTIMERCOMMANDATTRIBUTES._serialized_end = 1337 + _COMPLETEWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1339 + _COMPLETEWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 1433 + _FAILWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1435 + _FAILWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 1526 + _CANCELTIMERCOMMANDATTRIBUTES._serialized_start = 1528 + _CANCELTIMERCOMMANDATTRIBUTES._serialized_end = 1576 + _CANCELWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1578 + _CANCELWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 1671 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1674 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 1857 + _SIGNALEXTERNALWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 1860 + _SIGNALEXTERNALWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 2163 + _UPSERTWORKFLOWSEARCHATTRIBUTESCOMMANDATTRIBUTES._serialized_start = 2165 + _UPSERTWORKFLOWSEARCHATTRIBUTESCOMMANDATTRIBUTES._serialized_end = 2283 + _MODIFYWORKFLOWPROPERTIESCOMMANDATTRIBUTES._serialized_start = 2285 + _MODIFYWORKFLOWPROPERTIESCOMMANDATTRIBUTES._serialized_end = 2381 + _RECORDMARKERCOMMANDATTRIBUTES._serialized_start = 2384 + _RECORDMARKERCOMMANDATTRIBUTES._serialized_end = 2703 + _RECORDMARKERCOMMANDATTRIBUTES_DETAILSENTRY._serialized_start = 2623 + _RECORDMARKERCOMMANDATTRIBUTES_DETAILSENTRY._serialized_end = 2703 + _CONTINUEASNEWWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 2706 + _CONTINUEASNEWWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 3646 + _STARTCHILDWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_start = 3649 + _STARTCHILDWORKFLOWEXECUTIONCOMMANDATTRIBUTES._serialized_end = 4653 + _PROTOCOLMESSAGECOMMANDATTRIBUTES._serialized_start = 4655 + _PROTOCOLMESSAGECOMMANDATTRIBUTES._serialized_end = 4709 + _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES._serialized_start = 4712 + _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES._serialized_end = 5195 + _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES_NEXUSHEADERENTRY._serialized_start = 5145 + _SCHEDULENEXUSOPERATIONCOMMANDATTRIBUTES_NEXUSHEADERENTRY._serialized_end = 5195 + _REQUESTCANCELNEXUSOPERATIONCOMMANDATTRIBUTES._serialized_start = 5197 + _REQUESTCANCELNEXUSOPERATIONCOMMANDATTRIBUTES._serialized_end = 5271 + _COMMAND._serialized_start = 5274 + _COMMAND._serialized_end = 8078 + _APPENDSTREAMRECORDSCOMMANDATTRIBUTES._serialized_start = 8080 + _APPENDSTREAMRECORDSCOMMANDATTRIBUTES._serialized_end = 8194 + _SUBSCRIBESTREAMCOMMANDATTRIBUTES._serialized_start = 8197 + _SUBSCRIBESTREAMCOMMANDATTRIBUTES._serialized_end = 8349 + _SUBSCRIBENOTIFICATIONCHANNELCOMMANDATTRIBUTES._serialized_start = 8351 + _SUBSCRIBENOTIFICATIONCHANNELCOMMANDATTRIBUTES._serialized_end = 8415 + _UNSUBSCRIBENOTIFICATIONCHANNELCOMMANDATTRIBUTES._serialized_start = 8417 + _UNSUBSCRIBENOTIFICATIONCHANNELCOMMANDATTRIBUTES._serialized_end = 8483 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/command/v1/message_pb2.pyi b/temporalio/api/command/v1/message_pb2.pyi index 18d0abc8be..4a5edb5717 100644 --- a/temporalio/api/command/v1/message_pb2.pyi +++ b/temporalio/api/command/v1/message_pb2.pyi @@ -18,6 +18,7 @@ import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.sdk.v1.event_group_marker_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.taskqueue.v1.message_pb2 import temporalio.api.workflow.v1.message_pb2 @@ -1119,6 +1120,10 @@ class Command(google.protobuf.message.Message): MODIFY_WORKFLOW_PROPERTIES_COMMAND_ATTRIBUTES_FIELD_NUMBER: builtins.int SCHEDULE_NEXUS_OPERATION_COMMAND_ATTRIBUTES_FIELD_NUMBER: builtins.int REQUEST_CANCEL_NEXUS_OPERATION_COMMAND_ATTRIBUTES_FIELD_NUMBER: builtins.int + APPEND_STREAM_RECORDS_COMMAND_ATTRIBUTES_FIELD_NUMBER: builtins.int + SUBSCRIBE_STREAM_COMMAND_ATTRIBUTES_FIELD_NUMBER: builtins.int + SUBSCRIBE_NOTIFICATION_CHANNEL_COMMAND_ATTRIBUTES_FIELD_NUMBER: builtins.int + UNSUBSCRIBE_NOTIFICATION_CHANNEL_COMMAND_ATTRIBUTES_FIELD_NUMBER: builtins.int command_type: temporalio.api.enums.v1.command_type_pb2.CommandType.ValueType @property def user_metadata(self) -> temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata: @@ -1209,6 +1214,22 @@ class Command(google.protobuf.message.Message): def request_cancel_nexus_operation_command_attributes( self, ) -> global___RequestCancelNexusOperationCommandAttributes: ... + @property + def append_stream_records_command_attributes( + self, + ) -> global___AppendStreamRecordsCommandAttributes: ... + @property + def subscribe_stream_command_attributes( + self, + ) -> global___SubscribeStreamCommandAttributes: ... + @property + def subscribe_notification_channel_command_attributes( + self, + ) -> global___SubscribeNotificationChannelCommandAttributes: ... + @property + def unsubscribe_notification_channel_command_attributes( + self, + ) -> global___UnsubscribeNotificationChannelCommandAttributes: ... def __init__( self, *, @@ -1253,10 +1274,20 @@ class Command(google.protobuf.message.Message): | None = ..., request_cancel_nexus_operation_command_attributes: global___RequestCancelNexusOperationCommandAttributes | None = ..., + append_stream_records_command_attributes: global___AppendStreamRecordsCommandAttributes + | None = ..., + subscribe_stream_command_attributes: global___SubscribeStreamCommandAttributes + | None = ..., + subscribe_notification_channel_command_attributes: global___SubscribeNotificationChannelCommandAttributes + | None = ..., + unsubscribe_notification_channel_command_attributes: global___UnsubscribeNotificationChannelCommandAttributes + | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ + "append_stream_records_command_attributes", + b"append_stream_records_command_attributes", "attributes", b"attributes", "cancel_timer_command_attributes", @@ -1291,6 +1322,12 @@ class Command(google.protobuf.message.Message): b"start_child_workflow_execution_command_attributes", "start_timer_command_attributes", b"start_timer_command_attributes", + "subscribe_notification_channel_command_attributes", + b"subscribe_notification_channel_command_attributes", + "subscribe_stream_command_attributes", + b"subscribe_stream_command_attributes", + "unsubscribe_notification_channel_command_attributes", + b"unsubscribe_notification_channel_command_attributes", "upsert_workflow_search_attributes_command_attributes", b"upsert_workflow_search_attributes_command_attributes", "user_metadata", @@ -1300,6 +1337,8 @@ class Command(google.protobuf.message.Message): def ClearField( self, field_name: typing_extensions.Literal[ + "append_stream_records_command_attributes", + b"append_stream_records_command_attributes", "attributes", b"attributes", "cancel_timer_command_attributes", @@ -1338,6 +1377,12 @@ class Command(google.protobuf.message.Message): b"start_child_workflow_execution_command_attributes", "start_timer_command_attributes", b"start_timer_command_attributes", + "subscribe_notification_channel_command_attributes", + b"subscribe_notification_channel_command_attributes", + "subscribe_stream_command_attributes", + b"subscribe_stream_command_attributes", + "unsubscribe_notification_channel_command_attributes", + b"unsubscribe_notification_channel_command_attributes", "upsert_workflow_search_attributes_command_attributes", b"upsert_workflow_search_attributes_command_attributes", "user_metadata", @@ -1365,8 +1410,166 @@ class Command(google.protobuf.message.Message): "modify_workflow_properties_command_attributes", "schedule_nexus_operation_command_attributes", "request_cancel_nexus_operation_command_attributes", + "append_stream_records_command_attributes", + "subscribe_stream_command_attributes", + "subscribe_notification_channel_command_attributes", + "unsubscribe_notification_channel_command_attributes", ] | None ): ... global___Command = Command + +class AppendStreamRecordsCommandAttributes(google.protobuf.message.Message): + """Appends records to a stream the Workflow owns. Applied inside the Workflow + Task's own commit. Produces one `WorkflowStreamRecordsAppended` event + carrying the offset range and none of the payload; it schedules no further + work. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_NAME_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + stream_name: builtins.str + """Name of a stream this Workflow owns, scoped to the Workflow. Created on + first use. Empty means the Workflow's default output stream. A Workflow + cannot append to a stream in another execution, so this is never the id + of a standalone stream. + """ + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: + """Stored in order. The server sets `producer_id` to empty on each record, + because the owning Workflow is the producer here. + """ + def __init__( + self, + *, + stream_name: builtins.str = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "records", b"records", "stream_name", b"stream_name" + ], + ) -> None: ... + +global___AppendStreamRecordsCommandAttributes = AppendStreamRecordsCommandAttributes + +class SubscribeStreamCommandAttributes(google.protobuf.message.Message): + """Subscribe this Workflow to a stream, so later Workflow Tasks carry the ranges + it has not consumed yet. + + The stream's addressing is resolved by the server rather than supplied here. + A Workflow cannot look it up without doing I/O, and a value it carried would + be a reading rather than a fact, so it could differ on replay. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_NAME_OR_ID_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + START_POSITION_FIELD_NUMBER: builtins.int + stream_name_or_id: builtins.str + """Stream to consume, named either way round: a stream this Workflow owns + by the name it appends under, a stream in another execution by its id. + The server tries them in that order, so a Workflow that owns a stream + under this name cannot reach a standalone stream with the same id. When + neither exists the Workflow gets a stream of its own by that name, which + is how a reader subscribes before the first record is written. + """ + start_offset: builtins.int + """Where to start, as an absolute offset. Read only when `start_position` + is unset. A negative value is refused: the head of the stream is asked + for with `start_position.tail`. + """ + @property + def start_position( + self, + ) -> temporalio.api.stream.v1.message_pb2.StreamStartPosition: + """Where to start. The server resolves it once, when it registers the + subscription, and records the resolved absolute offset on the subscribed + event, so replay does not resolve it again. Setting it together with a + non-zero `start_offset` fails the command. + """ + def __init__( + self, + *, + stream_name_or_id: builtins.str = ..., + start_offset: builtins.int = ..., + start_position: temporalio.api.stream.v1.message_pb2.StreamStartPosition + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["start_position", b"start_position"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "start_offset", + b"start_offset", + "start_position", + b"start_position", + "stream_name_or_id", + b"stream_name_or_id", + ], + ) -> None: ... + +global___SubscribeStreamCommandAttributes = SubscribeStreamCommandAttributes + +class SubscribeNotificationChannelCommandAttributes(google.protobuf.message.Message): + """Makes the Workflow a listener of a notification channel for this run. The + next notifications on the channel arrive on the scheduled event of a Workflow + Task. The subscription ends with the run, and a successor subscribes again. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CHANNEL_FIELD_NUMBER: builtins.int + channel: builtins.str + """The channel to listen on, as the writers name it.""" + def __init__( + self, + *, + channel: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["channel", b"channel"] + ) -> None: ... + +global___SubscribeNotificationChannelCommandAttributes = ( + SubscribeNotificationChannelCommandAttributes +) + +class UnsubscribeNotificationChannelCommandAttributes(google.protobuf.message.Message): + """Ends the run's subscription to a notification channel. Notifications already + recorded on a scheduled event still reach that Workflow Task; later ones do + not. A command naming a channel the run is not subscribed to records its + event and changes nothing, so replay matches every command to an event. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CHANNEL_FIELD_NUMBER: builtins.int + channel: builtins.str + """The channel to stop listening on, as the writers name it.""" + def __init__( + self, + *, + channel: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["channel", b"channel"] + ) -> None: ... + +global___UnsubscribeNotificationChannelCommandAttributes = ( + UnsubscribeNotificationChannelCommandAttributes +) diff --git a/temporalio/api/common/v1/message_pb2.py b/temporalio/api/common/v1/message_pb2.py index 538be6a239..4a375947e4 100644 --- a/temporalio/api/common/v1/message_pb2.py +++ b/temporalio/api/common/v1/message_pb2.py @@ -29,7 +29,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n$temporal/api/common/v1/message.proto\x12\x16temporal.api.common.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a"temporal/api/enums/v1/common.proto\x1a&temporal/api/enums/v1/event_type.proto\x1a!temporal/api/enums/v1/reset.proto"T\n\x08\x44\x61taBlob\x12:\n\rencoding_type\x18\x01 \x01(\x0e\x32#.temporal.api.enums.v1.EncodingType\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c"=\n\x08Payloads\x12\x31\n\x08payloads\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload"\x8a\x02\n\x07Payload\x12?\n\x08metadata\x18\x01 \x03(\x0b\x32-.temporal.api.common.v1.Payload.MetadataEntry\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12Q\n\x11\x65xternal_payloads\x18\x03 \x03(\x0b\x32\x36.temporal.api.common.v1.Payload.ExternalPayloadDetails\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a,\n\x16\x45xternalPayloadDetails\x12\x12\n\nsize_bytes\x18\x01 \x01(\x03"\xbe\x01\n\x10SearchAttributes\x12S\n\x0eindexed_fields\x18\x01 \x03(\x0b\x32;.temporal.api.common.v1.SearchAttributes.IndexedFieldsEntry\x1aU\n\x12IndexedFieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x90\x01\n\x04Memo\x12\x38\n\x06\x66ields\x18\x01 \x03(\x0b\x32(.temporal.api.common.v1.Memo.FieldsEntry\x1aN\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x94\x01\n\x06Header\x12:\n\x06\x66ields\x18\x01 \x03(\x0b\x32*.temporal.api.common.v1.Header.FieldsEntry\x1aN\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"8\n\x11WorkflowExecution\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t"d\n\tExecution\x12\x32\n\x04type\x18\x01 \x01(\x0e\x32$.temporal.api.enums.v1.ExecutionType\x12\x13\n\x0b\x62usiness_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"\x1c\n\x0cWorkflowType\x12\x0c\n\x04name\x18\x01 \x01(\t"\x1c\n\x0c\x41\x63tivityType\x12\x0c\n\x04name\x18\x01 \x01(\t"\xd1\x01\n\x0bRetryPolicy\x12\x33\n\x10initial_interval\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x1b\n\x13\x62\x61\x63koff_coefficient\x18\x02 \x01(\x01\x12\x33\n\x10maximum_interval\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x18\n\x10maximum_attempts\x18\x04 \x01(\x05\x12!\n\x19non_retryable_error_types\x18\x05 \x03(\t"F\n\x10MeteringMetadata\x12\x32\n*nonfirst_local_activity_execution_attempts\x18\r \x01(\r">\n\x12WorkerVersionStamp\x12\x10\n\x08\x62uild_id\x18\x01 \x01(\t\x12\x16\n\x0euse_versioning\x18\x03 \x01(\x08"e\n\x19WorkerVersionCapabilities\x12\x10\n\x08\x62uild_id\x18\x01 \x01(\t\x12\x16\n\x0euse_versioning\x18\x02 \x01(\x08\x12\x1e\n\x16\x64\x65ployment_series_name\x18\x04 \x01(\t"\xed\x02\n\x0cResetOptions\x12\x35\n\x13\x66irst_workflow_task\x18\x01 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x12last_workflow_task\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x1a\n\x10workflow_task_id\x18\x03 \x01(\x03H\x00\x12\x12\n\x08\x62uild_id\x18\x04 \x01(\tH\x00\x12G\n\x12reset_reapply_type\x18\n \x01(\x0e\x32\'.temporal.api.enums.v1.ResetReapplyTypeB\x02\x18\x01\x12\x18\n\x10\x63urrent_run_only\x18\x0b \x01(\x08\x12S\n\x1breset_reapply_exclude_types\x18\x0c \x03(\x0e\x32..temporal.api.enums.v1.ResetReapplyExcludeTypeB\x08\n\x06target"\xe4\x02\n\x08\x43\x61llback\x12\x37\n\x05nexus\x18\x02 \x01(\x0b\x32&.temporal.api.common.v1.Callback.NexusH\x00\x12=\n\x08internal\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.Callback.InternalH\x00\x12+\n\x05links\x18\x64 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x1a\x87\x01\n\x05Nexus\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x42\n\x06header\x18\x02 \x03(\x0b\x32\x32.temporal.api.common.v1.Callback.Nexus.HeaderEntry\x1a-\n\x0bHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x18\n\x08Internal\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x42\t\n\x07variantJ\x04\x08\x01\x10\x02"\x8a\x08\n\x04Link\x12\x44\n\x0eworkflow_event\x18\x01 \x01(\x0b\x32*.temporal.api.common.v1.Link.WorkflowEventH\x00\x12:\n\tbatch_job\x18\x02 \x01(\x0b\x32%.temporal.api.common.v1.Link.BatchJobH\x00\x12\x39\n\x08\x61\x63tivity\x18\x03 \x01(\x0b\x32%.temporal.api.common.v1.Link.ActivityH\x00\x12\x46\n\x0fnexus_operation\x18\x04 \x01(\x0b\x32+.temporal.api.common.v1.Link.NexusOperationH\x00\x12\x39\n\x08workflow\x18\x05 \x01(\x0b\x32%.temporal.api.common.v1.Link.WorkflowH\x00\x1a\xb7\x03\n\rWorkflowEvent\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12N\n\tevent_ref\x18\x64 \x01(\x0b\x32\x39.temporal.api.common.v1.Link.WorkflowEvent.EventReferenceH\x00\x12W\n\x0erequest_id_ref\x18\x65 \x01(\x0b\x32=.temporal.api.common.v1.Link.WorkflowEvent.RequestIdReferenceH\x00\x1aX\n\x0e\x45ventReference\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\x03\x12\x34\n\nevent_type\x18\x02 \x01(\x0e\x32 .temporal.api.enums.v1.EventType\x1a^\n\x12RequestIdReference\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x34\n\nevent_type\x18\x02 \x01(\x0e\x32 .temporal.api.enums.v1.EventTypeB\x0b\n\treference\x1a\x1a\n\x08\x42\x61tchJob\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x1a\x42\n\x08\x41\x63tivity\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x1aI\n\x0eNexusOperation\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x1aR\n\x08Workflow\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\tB\t\n\x07variant"\'\n\tPrincipal\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t"O\n\x08Priority\x12\x14\n\x0cpriority_key\x18\x01 \x01(\x05\x12\x14\n\x0c\x66\x61irness_key\x18\x02 \x01(\t\x12\x17\n\x0f\x66\x61irness_weight\x18\x03 \x01(\x02";\n\x0eWorkerSelector\x12\x1d\n\x13worker_instance_key\x18\x01 \x01(\tH\x00\x42\n\n\x08selector"i\n\x11OnConflictOptions\x12\x19\n\x11\x61ttach_request_id\x18\x01 \x01(\x08\x12#\n\x1b\x61ttach_completion_callbacks\x18\x02 \x01(\x08\x12\x14\n\x0c\x61ttach_links\x18\x03 \x01(\x08"\xaa\x01\n\x12TimeSkippingConfig\x12\x0f\n\x07\x65nabled\x18\x01 \x01(\x08\x12\x46\n\x13\x66\x61st_forward_config\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.FastForwardConfig\x12\x1b\n\x13\x64isable_propagation\x18\x03 \x01(\x08\x12\x1e\n\x16max_session_skip_count\x18\x04 \x01(\x05"L\n\x11\x46\x61stForwardConfig\x12\n\n\x02id\x18\x01 \x01(\t\x12+\n\x08\x64uration\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\xb5\x01\n\x1cTimeSkippingStatePropagation\x12;\n\x18initial_skipped_duration\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x18\x66\x61st_forward_target_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1a\n\x12initial_skip_count\x18\x03 \x01(\x05"\xfe\x01\n\x10TimeSkippingInfo\x12\x30\n\x0c\x63urrent_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x44\n\x10\x65\x66\x66\x65\x63tive_config\x18\x02 \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig\x12N\n\x11\x66\x61st_forward_info\x18\x04 \x01(\x0b\x32\x33.temporal.api.common.v1.TimeSkippingFastForwardInfo\x12"\n\x1a\x63urrent_session_skip_count\x18\x06 \x01(\x05"\xb8\x01\n\x1bTimeSkippingFastForwardInfo\x12\x38\n\x15\x66\x61st_forward_duration\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x17\n\x0f\x66\x61st_forward_id\x18\x02 \x01(\t\x12/\n\x0btarget_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rhas_completed\x18\x04 \x01(\x08\x42\x89\x01\n\x19io.temporal.api.common.v1B\x0cMessageProtoP\x01Z#go.temporal.io/api/common/v1;common\xaa\x02\x18Temporalio.Api.Common.V1\xea\x02\x1bTemporalio::Api::Common::V1b\x06proto3' + b'\n$temporal/api/common/v1/message.proto\x12\x16temporal.api.common.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a"temporal/api/enums/v1/common.proto\x1a&temporal/api/enums/v1/event_type.proto\x1a!temporal/api/enums/v1/reset.proto"T\n\x08\x44\x61taBlob\x12:\n\rencoding_type\x18\x01 \x01(\x0e\x32#.temporal.api.enums.v1.EncodingType\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c"=\n\x08Payloads\x12\x31\n\x08payloads\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload"\x8a\x02\n\x07Payload\x12?\n\x08metadata\x18\x01 \x03(\x0b\x32-.temporal.api.common.v1.Payload.MetadataEntry\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12Q\n\x11\x65xternal_payloads\x18\x03 \x03(\x0b\x32\x36.temporal.api.common.v1.Payload.ExternalPayloadDetails\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a,\n\x16\x45xternalPayloadDetails\x12\x12\n\nsize_bytes\x18\x01 \x01(\x03"\xbe\x01\n\x10SearchAttributes\x12S\n\x0eindexed_fields\x18\x01 \x03(\x0b\x32;.temporal.api.common.v1.SearchAttributes.IndexedFieldsEntry\x1aU\n\x12IndexedFieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x90\x01\n\x04Memo\x12\x38\n\x06\x66ields\x18\x01 \x03(\x0b\x32(.temporal.api.common.v1.Memo.FieldsEntry\x1aN\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x94\x01\n\x06Header\x12:\n\x06\x66ields\x18\x01 \x03(\x0b\x32*.temporal.api.common.v1.Header.FieldsEntry\x1aN\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"8\n\x11WorkflowExecution\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t"d\n\tExecution\x12\x32\n\x04type\x18\x01 \x01(\x0e\x32$.temporal.api.enums.v1.ExecutionType\x12\x13\n\x0b\x62usiness_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"\x1c\n\x0cWorkflowType\x12\x0c\n\x04name\x18\x01 \x01(\t"\x1c\n\x0c\x41\x63tivityType\x12\x0c\n\x04name\x18\x01 \x01(\t"\xd1\x01\n\x0bRetryPolicy\x12\x33\n\x10initial_interval\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x1b\n\x13\x62\x61\x63koff_coefficient\x18\x02 \x01(\x01\x12\x33\n\x10maximum_interval\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x18\n\x10maximum_attempts\x18\x04 \x01(\x05\x12!\n\x19non_retryable_error_types\x18\x05 \x03(\t"F\n\x10MeteringMetadata\x12\x32\n*nonfirst_local_activity_execution_attempts\x18\r \x01(\r">\n\x12WorkerVersionStamp\x12\x10\n\x08\x62uild_id\x18\x01 \x01(\t\x12\x16\n\x0euse_versioning\x18\x03 \x01(\x08"e\n\x19WorkerVersionCapabilities\x12\x10\n\x08\x62uild_id\x18\x01 \x01(\t\x12\x16\n\x0euse_versioning\x18\x02 \x01(\x08\x12\x1e\n\x16\x64\x65ployment_series_name\x18\x04 \x01(\t"\xed\x02\n\x0cResetOptions\x12\x35\n\x13\x66irst_workflow_task\x18\x01 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x12last_workflow_task\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x1a\n\x10workflow_task_id\x18\x03 \x01(\x03H\x00\x12\x12\n\x08\x62uild_id\x18\x04 \x01(\tH\x00\x12G\n\x12reset_reapply_type\x18\n \x01(\x0e\x32\'.temporal.api.enums.v1.ResetReapplyTypeB\x02\x18\x01\x12\x18\n\x10\x63urrent_run_only\x18\x0b \x01(\x08\x12S\n\x1breset_reapply_exclude_types\x18\x0c \x03(\x0e\x32..temporal.api.enums.v1.ResetReapplyExcludeTypeB\x08\n\x06target"\xb3\x04\n\x08\x43\x61llback\x12\x37\n\x05nexus\x18\x02 \x01(\x0b\x32&.temporal.api.common.v1.Callback.NexusH\x00\x12=\n\x08internal\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.Callback.InternalH\x00\x12\x46\n\rnexus_handler\x18\x04 \x01(\x0b\x32-.temporal.api.common.v1.Callback.NexusHandlerH\x00\x12+\n\x05links\x18\x64 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x1a\x87\x01\n\x05Nexus\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x42\n\x06header\x18\x02 \x03(\x0b\x32\x32.temporal.api.common.v1.Callback.Nexus.HeaderEntry\x1a-\n\x0bHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x18\n\x08Internal\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x1a\x84\x01\n\x0cNexusHandler\x12\x17\n\x0ftask_queue_name\x18\x01 \x01(\t\x12\x0f\n\x07service\x18\x02 \x01(\t\x12\x11\n\toperation\x18\x03 \x01(\t\x12\x37\n\x0esource_context\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadB\t\n\x07variantJ\x04\x08\x01\x10\x02"\xc6\t\n\x04Link\x12\x44\n\x0eworkflow_event\x18\x01 \x01(\x0b\x32*.temporal.api.common.v1.Link.WorkflowEventH\x00\x12:\n\tbatch_job\x18\x02 \x01(\x0b\x32%.temporal.api.common.v1.Link.BatchJobH\x00\x12\x39\n\x08\x61\x63tivity\x18\x03 \x01(\x0b\x32%.temporal.api.common.v1.Link.ActivityH\x00\x12\x46\n\x0fnexus_operation\x18\x04 \x01(\x0b\x32+.temporal.api.common.v1.Link.NexusOperationH\x00\x12\x39\n\x08workflow\x18\x05 \x01(\x0b\x32%.temporal.api.common.v1.Link.WorkflowH\x00\x12\x39\n\x08\x63\x61llback\x18\x06 \x01(\x0b\x32%.temporal.api.common.v1.Link.CallbackH\x00\x1a\xb7\x03\n\rWorkflowEvent\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12N\n\tevent_ref\x18\x64 \x01(\x0b\x32\x39.temporal.api.common.v1.Link.WorkflowEvent.EventReferenceH\x00\x12W\n\x0erequest_id_ref\x18\x65 \x01(\x0b\x32=.temporal.api.common.v1.Link.WorkflowEvent.RequestIdReferenceH\x00\x1aX\n\x0e\x45ventReference\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\x03\x12\x34\n\nevent_type\x18\x02 \x01(\x0e\x32 .temporal.api.enums.v1.EventType\x1a^\n\x12RequestIdReference\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x34\n\nevent_type\x18\x02 \x01(\x0e\x32 .temporal.api.enums.v1.EventTypeB\x0b\n\treference\x1a\x1a\n\x08\x42\x61tchJob\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x1a\x42\n\x08\x41\x63tivity\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x1aI\n\x0eNexusOperation\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x1aR\n\x08Workflow\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x1a\x7f\n\x08\x43\x61llback\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x34\n\texecution\x18\x02 \x01(\x0b\x32!.temporal.api.common.v1.Execution\x12\x16\n\x0e\x63omponent_path\x18\x03 \x03(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\tB\t\n\x07variant"\'\n\tPrincipal\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t"O\n\x08Priority\x12\x14\n\x0cpriority_key\x18\x01 \x01(\x05\x12\x14\n\x0c\x66\x61irness_key\x18\x02 \x01(\t\x12\x17\n\x0f\x66\x61irness_weight\x18\x03 \x01(\x02";\n\x0eWorkerSelector\x12\x1d\n\x13worker_instance_key\x18\x01 \x01(\tH\x00\x42\n\n\x08selector"i\n\x11OnConflictOptions\x12\x19\n\x11\x61ttach_request_id\x18\x01 \x01(\x08\x12#\n\x1b\x61ttach_completion_callbacks\x18\x02 \x01(\x08\x12\x14\n\x0c\x61ttach_links\x18\x03 \x01(\x08"\xaa\x01\n\x12TimeSkippingConfig\x12\x0f\n\x07\x65nabled\x18\x01 \x01(\x08\x12\x46\n\x13\x66\x61st_forward_config\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.FastForwardConfig\x12\x1b\n\x13\x64isable_propagation\x18\x03 \x01(\x08\x12\x1e\n\x16max_session_skip_count\x18\x04 \x01(\x05"L\n\x11\x46\x61stForwardConfig\x12\n\n\x02id\x18\x01 \x01(\t\x12+\n\x08\x64uration\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\xb5\x01\n\x1cTimeSkippingStatePropagation\x12;\n\x18initial_skipped_duration\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x18\x66\x61st_forward_target_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1a\n\x12initial_skip_count\x18\x03 \x01(\x05"\xfe\x01\n\x10TimeSkippingInfo\x12\x30\n\x0c\x63urrent_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x44\n\x10\x65\x66\x66\x65\x63tive_config\x18\x02 \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig\x12N\n\x11\x66\x61st_forward_info\x18\x04 \x01(\x0b\x32\x33.temporal.api.common.v1.TimeSkippingFastForwardInfo\x12"\n\x1a\x63urrent_session_skip_count\x18\x06 \x01(\x05"\xb8\x01\n\x1bTimeSkippingFastForwardInfo\x12\x38\n\x15\x66\x61st_forward_duration\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x17\n\x0f\x66\x61st_forward_id\x18\x02 \x01(\t\x12/\n\x0btarget_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rhas_completed\x18\x04 \x01(\x08\x42\x89\x01\n\x19io.temporal.api.common.v1B\x0cMessageProtoP\x01Z#go.temporal.io/api/common/v1;common\xaa\x02\x18Temporalio.Api.Common.V1\xea\x02\x1bTemporalio::Api::Common::V1b\x06proto3' ) @@ -63,6 +63,7 @@ _CALLBACK_NEXUS = _CALLBACK.nested_types_by_name["Nexus"] _CALLBACK_NEXUS_HEADERENTRY = _CALLBACK_NEXUS.nested_types_by_name["HeaderEntry"] _CALLBACK_INTERNAL = _CALLBACK.nested_types_by_name["Internal"] +_CALLBACK_NEXUSHANDLER = _CALLBACK.nested_types_by_name["NexusHandler"] _LINK = DESCRIPTOR.message_types_by_name["Link"] _LINK_WORKFLOWEVENT = _LINK.nested_types_by_name["WorkflowEvent"] _LINK_WORKFLOWEVENT_EVENTREFERENCE = _LINK_WORKFLOWEVENT.nested_types_by_name[ @@ -75,6 +76,7 @@ _LINK_ACTIVITY = _LINK.nested_types_by_name["Activity"] _LINK_NEXUSOPERATION = _LINK.nested_types_by_name["NexusOperation"] _LINK_WORKFLOW = _LINK.nested_types_by_name["Workflow"] +_LINK_CALLBACK = _LINK.nested_types_by_name["Callback"] _PRINCIPAL = DESCRIPTOR.message_types_by_name["Principal"] _PRIORITY = DESCRIPTOR.message_types_by_name["Priority"] _WORKERSELECTOR = DESCRIPTOR.message_types_by_name["WorkerSelector"] @@ -334,6 +336,15 @@ # @@protoc_insertion_point(class_scope:temporal.api.common.v1.Callback.Internal) }, ), + "NexusHandler": _reflection.GeneratedProtocolMessageType( + "NexusHandler", + (_message.Message,), + { + "DESCRIPTOR": _CALLBACK_NEXUSHANDLER, + "__module__": "temporalio.api.common.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.common.v1.Callback.NexusHandler) + }, + ), "DESCRIPTOR": _CALLBACK, "__module__": "temporalio.api.common.v1.message_pb2", # @@protoc_insertion_point(class_scope:temporal.api.common.v1.Callback) @@ -343,6 +354,7 @@ _sym_db.RegisterMessage(Callback.Nexus) _sym_db.RegisterMessage(Callback.Nexus.HeaderEntry) _sym_db.RegisterMessage(Callback.Internal) +_sym_db.RegisterMessage(Callback.NexusHandler) Link = _reflection.GeneratedProtocolMessageType( "Link", @@ -411,6 +423,15 @@ # @@protoc_insertion_point(class_scope:temporal.api.common.v1.Link.Workflow) }, ), + "Callback": _reflection.GeneratedProtocolMessageType( + "Callback", + (_message.Message,), + { + "DESCRIPTOR": _LINK_CALLBACK, + "__module__": "temporalio.api.common.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.common.v1.Link.Callback) + }, + ), "DESCRIPTOR": _LINK, "__module__": "temporalio.api.common.v1.message_pb2", # @@protoc_insertion_point(class_scope:temporal.api.common.v1.Link) @@ -424,6 +445,7 @@ _sym_db.RegisterMessage(Link.Activity) _sym_db.RegisterMessage(Link.NexusOperation) _sym_db.RegisterMessage(Link.Workflow) +_sym_db.RegisterMessage(Link.Callback) Principal = _reflection.GeneratedProtocolMessageType( "Principal", @@ -580,45 +602,49 @@ _RESETOPTIONS._serialized_start = 1850 _RESETOPTIONS._serialized_end = 2215 _CALLBACK._serialized_start = 2218 - _CALLBACK._serialized_end = 2574 - _CALLBACK_NEXUS._serialized_start = 2396 - _CALLBACK_NEXUS._serialized_end = 2531 - _CALLBACK_NEXUS_HEADERENTRY._serialized_start = 2486 - _CALLBACK_NEXUS_HEADERENTRY._serialized_end = 2531 - _CALLBACK_INTERNAL._serialized_start = 2533 - _CALLBACK_INTERNAL._serialized_end = 2557 - _LINK._serialized_start = 2577 - _LINK._serialized_end = 3611 - _LINK_WORKFLOWEVENT._serialized_start = 2906 - _LINK_WORKFLOWEVENT._serialized_end = 3345 - _LINK_WORKFLOWEVENT_EVENTREFERENCE._serialized_start = 3148 - _LINK_WORKFLOWEVENT_EVENTREFERENCE._serialized_end = 3236 - _LINK_WORKFLOWEVENT_REQUESTIDREFERENCE._serialized_start = 3238 - _LINK_WORKFLOWEVENT_REQUESTIDREFERENCE._serialized_end = 3332 - _LINK_BATCHJOB._serialized_start = 3347 - _LINK_BATCHJOB._serialized_end = 3373 - _LINK_ACTIVITY._serialized_start = 3375 - _LINK_ACTIVITY._serialized_end = 3441 - _LINK_NEXUSOPERATION._serialized_start = 3443 - _LINK_NEXUSOPERATION._serialized_end = 3516 - _LINK_WORKFLOW._serialized_start = 3518 - _LINK_WORKFLOW._serialized_end = 3600 - _PRINCIPAL._serialized_start = 3613 - _PRINCIPAL._serialized_end = 3652 - _PRIORITY._serialized_start = 3654 - _PRIORITY._serialized_end = 3733 - _WORKERSELECTOR._serialized_start = 3735 - _WORKERSELECTOR._serialized_end = 3794 - _ONCONFLICTOPTIONS._serialized_start = 3796 - _ONCONFLICTOPTIONS._serialized_end = 3901 - _TIMESKIPPINGCONFIG._serialized_start = 3904 - _TIMESKIPPINGCONFIG._serialized_end = 4074 - _FASTFORWARDCONFIG._serialized_start = 4076 - _FASTFORWARDCONFIG._serialized_end = 4152 - _TIMESKIPPINGSTATEPROPAGATION._serialized_start = 4155 - _TIMESKIPPINGSTATEPROPAGATION._serialized_end = 4336 - _TIMESKIPPINGINFO._serialized_start = 4339 - _TIMESKIPPINGINFO._serialized_end = 4593 - _TIMESKIPPINGFASTFORWARDINFO._serialized_start = 4596 - _TIMESKIPPINGFASTFORWARDINFO._serialized_end = 4780 + _CALLBACK._serialized_end = 2781 + _CALLBACK_NEXUS._serialized_start = 2468 + _CALLBACK_NEXUS._serialized_end = 2603 + _CALLBACK_NEXUS_HEADERENTRY._serialized_start = 2558 + _CALLBACK_NEXUS_HEADERENTRY._serialized_end = 2603 + _CALLBACK_INTERNAL._serialized_start = 2605 + _CALLBACK_INTERNAL._serialized_end = 2629 + _CALLBACK_NEXUSHANDLER._serialized_start = 2632 + _CALLBACK_NEXUSHANDLER._serialized_end = 2764 + _LINK._serialized_start = 2784 + _LINK._serialized_end = 4006 + _LINK_WORKFLOWEVENT._serialized_start = 3172 + _LINK_WORKFLOWEVENT._serialized_end = 3611 + _LINK_WORKFLOWEVENT_EVENTREFERENCE._serialized_start = 3414 + _LINK_WORKFLOWEVENT_EVENTREFERENCE._serialized_end = 3502 + _LINK_WORKFLOWEVENT_REQUESTIDREFERENCE._serialized_start = 3504 + _LINK_WORKFLOWEVENT_REQUESTIDREFERENCE._serialized_end = 3598 + _LINK_BATCHJOB._serialized_start = 3613 + _LINK_BATCHJOB._serialized_end = 3639 + _LINK_ACTIVITY._serialized_start = 3641 + _LINK_ACTIVITY._serialized_end = 3707 + _LINK_NEXUSOPERATION._serialized_start = 3709 + _LINK_NEXUSOPERATION._serialized_end = 3782 + _LINK_WORKFLOW._serialized_start = 3784 + _LINK_WORKFLOW._serialized_end = 3866 + _LINK_CALLBACK._serialized_start = 3868 + _LINK_CALLBACK._serialized_end = 3995 + _PRINCIPAL._serialized_start = 4008 + _PRINCIPAL._serialized_end = 4047 + _PRIORITY._serialized_start = 4049 + _PRIORITY._serialized_end = 4128 + _WORKERSELECTOR._serialized_start = 4130 + _WORKERSELECTOR._serialized_end = 4189 + _ONCONFLICTOPTIONS._serialized_start = 4191 + _ONCONFLICTOPTIONS._serialized_end = 4296 + _TIMESKIPPINGCONFIG._serialized_start = 4299 + _TIMESKIPPINGCONFIG._serialized_end = 4469 + _FASTFORWARDCONFIG._serialized_start = 4471 + _FASTFORWARDCONFIG._serialized_end = 4547 + _TIMESKIPPINGSTATEPROPAGATION._serialized_start = 4550 + _TIMESKIPPINGSTATEPROPAGATION._serialized_end = 4731 + _TIMESKIPPINGINFO._serialized_start = 4734 + _TIMESKIPPINGINFO._serialized_end = 4988 + _TIMESKIPPINGFASTFORWARDINFO._serialized_start = 4991 + _TIMESKIPPINGFASTFORWARDINFO._serialized_end = 5175 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/common/v1/message_pb2.pyi b/temporalio/api/common/v1/message_pb2.pyi index f7ed8d0fde..441f1cdf10 100644 --- a/temporalio/api/common/v1/message_pb2.pyi +++ b/temporalio/api/common/v1/message_pb2.pyi @@ -322,9 +322,7 @@ class WorkflowExecution(google.protobuf.message.Message): global___WorkflowExecution = WorkflowExecution class Execution(google.protobuf.message.Message): - """Identifies a specific execution within a namespace. This is used for standalone activities - executions in batch jobs currently. - """ + """Identifies a specific execution within a namespace.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -676,6 +674,10 @@ class Callback(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor class Nexus(google.protobuf.message.Message): + """Nexus callbacks are used to delivery Nexus operation completions, as defined in the Nexus RPC spec: + https://github.com/nexus-rpc/api/blob/main/SPEC.md#callback-urls + """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor class HeaderEntry(google.protobuf.message.Message): @@ -737,14 +739,82 @@ class Callback(google.protobuf.message.Message): self, field_name: typing_extensions.Literal["data", b"data"] ) -> None: ... + class NexusHandler(google.protobuf.message.Message): + """NexusHandler callbacks are requests to invoke a specific shape of Nexus operation on a Temporal worker. + The specified Nexus operation must have the following: + - Input: temporalio.api.notificationservice.v1.OnCompleteRequest + - Output: temporalio.api.notificationservice.v1.OnCompleteResponse + + The targeted Nexus service must be registered within the same namespace as the source operation + the callback is attached to. (While Nexus allows for cross-namespace operations, NexusHandler callbacks + are strictly caller-side.) + + NexusHandler callbacks are only supported for certain types of operations, e.g. standalone Nexus operations. + Attempting to attach a Worker callback for an unsupported operation will result in an INVALID_ARGUMENT + error from the server. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TASK_QUEUE_NAME_FIELD_NUMBER: builtins.int + SERVICE_FIELD_NUMBER: builtins.int + OPERATION_FIELD_NUMBER: builtins.int + SOURCE_CONTEXT_FIELD_NUMBER: builtins.int + task_queue_name: builtins.str + """Nexus task queue the Temporal worker is listening on. + + NOTE: This is not a temporalio.api.taskqueue.v1.TaskQueue to avoid a circular dependency. + """ + service: builtins.str + """Target Nexus service, e.g. "HTTPAdapter".""" + operation: builtins.str + """Target operation, e.g. "DeliverAsWebhook".""" + @property + def source_context(self) -> global___Payload: + """Arbitrary user-supplied data from the source operation's callsite. (As applicable, not all operations + support attaching context data.) + + There are restrictions on the maxium payload size a single callback can carry, as well as the + total sum of all source context payloads attached to an execution. See dynamic configuration: + "callback.nexusHandler.sourceContext.maxSize", "callback.nexusHandler.sourceContext.aggregateMaxSize". + """ + def __init__( + self, + *, + task_queue_name: builtins.str = ..., + service: builtins.str = ..., + operation: builtins.str = ..., + source_context: global___Payload | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["source_context", b"source_context"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "operation", + b"operation", + "service", + b"service", + "source_context", + b"source_context", + "task_queue_name", + b"task_queue_name", + ], + ) -> None: ... + NEXUS_FIELD_NUMBER: builtins.int INTERNAL_FIELD_NUMBER: builtins.int + NEXUS_HANDLER_FIELD_NUMBER: builtins.int LINKS_FIELD_NUMBER: builtins.int @property def nexus(self) -> global___Callback.Nexus: ... @property def internal(self) -> global___Callback.Internal: ... @property + def nexus_handler(self) -> global___Callback.NexusHandler: ... + @property def links( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ @@ -758,12 +828,20 @@ class Callback(google.protobuf.message.Message): *, nexus: global___Callback.Nexus | None = ..., internal: global___Callback.Internal | None = ..., + nexus_handler: global___Callback.NexusHandler | None = ..., links: collections.abc.Iterable[global___Link] | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ - "internal", b"internal", "nexus", b"nexus", "variant", b"variant" + "internal", + b"internal", + "nexus", + b"nexus", + "nexus_handler", + b"nexus_handler", + "variant", + b"variant", ], ) -> builtins.bool: ... def ClearField( @@ -775,13 +853,15 @@ class Callback(google.protobuf.message.Message): b"links", "nexus", b"nexus", + "nexus_handler", + b"nexus_handler", "variant", b"variant", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["variant", b"variant"] - ) -> typing_extensions.Literal["nexus", "internal"] | None: ... + ) -> typing_extensions.Literal["nexus", "internal", "nexus_handler"] | None: ... global___Callback = Callback @@ -1011,11 +1091,63 @@ class Link(google.protobuf.message.Message): ], ) -> None: ... + class Callback(google.protobuf.message.Message): + """A link to a worker callback attached to an execution. An execution (e.g. standalone Nexus operation) can have + multiple callbacks attached, and will be differentiated by the request_id used when the callback is invoked. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + EXECUTION_FIELD_NUMBER: builtins.int + COMPONENT_PATH_FIELD_NUMBER: builtins.int + REQUEST_ID_FIELD_NUMBER: builtins.int + namespace: builtins.str + @property + def execution(self) -> global___Execution: ... + @property + def component_path( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[ + builtins.str + ]: + """In most cases, the Execution is sufficient to identify the callback's source. But the callback could have + been attached some child component of that execution. e.g. a workflow update. The component path describes + the unique component as applicable, typically ending with a unique ID. e.g. ["Update", $workflowUpdateId ] + """ + request_id: builtins.str + """Server-generate request ID sent when the callback was dispatched.""" + def __init__( + self, + *, + namespace: builtins.str = ..., + execution: global___Execution | None = ..., + component_path: collections.abc.Iterable[builtins.str] | None = ..., + request_id: builtins.str = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["execution", b"execution"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "component_path", + b"component_path", + "execution", + b"execution", + "namespace", + b"namespace", + "request_id", + b"request_id", + ], + ) -> None: ... + WORKFLOW_EVENT_FIELD_NUMBER: builtins.int BATCH_JOB_FIELD_NUMBER: builtins.int ACTIVITY_FIELD_NUMBER: builtins.int NEXUS_OPERATION_FIELD_NUMBER: builtins.int WORKFLOW_FIELD_NUMBER: builtins.int + CALLBACK_FIELD_NUMBER: builtins.int @property def workflow_event(self) -> global___Link.WorkflowEvent: ... @property @@ -1026,6 +1158,8 @@ class Link(google.protobuf.message.Message): def nexus_operation(self) -> global___Link.NexusOperation: ... @property def workflow(self) -> global___Link.Workflow: ... + @property + def callback(self) -> global___Link.Callback: ... def __init__( self, *, @@ -1034,6 +1168,7 @@ class Link(google.protobuf.message.Message): activity: global___Link.Activity | None = ..., nexus_operation: global___Link.NexusOperation | None = ..., workflow: global___Link.Workflow | None = ..., + callback: global___Link.Callback | None = ..., ) -> None: ... def HasField( self, @@ -1042,6 +1177,8 @@ class Link(google.protobuf.message.Message): b"activity", "batch_job", b"batch_job", + "callback", + b"callback", "nexus_operation", b"nexus_operation", "variant", @@ -1059,6 +1196,8 @@ class Link(google.protobuf.message.Message): b"activity", "batch_job", b"batch_job", + "callback", + b"callback", "nexus_operation", b"nexus_operation", "variant", @@ -1073,7 +1212,12 @@ class Link(google.protobuf.message.Message): self, oneof_group: typing_extensions.Literal["variant", b"variant"] ) -> ( typing_extensions.Literal[ - "workflow_event", "batch_job", "activity", "nexus_operation", "workflow" + "workflow_event", + "batch_job", + "activity", + "nexus_operation", + "workflow", + "callback", ] | None ): ... diff --git a/temporalio/api/enums/v1/__init__.py b/temporalio/api/enums/v1/__init__.py index 3dc3a179bc..518a2f32bc 100644 --- a/temporalio/api/enums/v1/__init__.py +++ b/temporalio/api/enums/v1/__init__.py @@ -25,6 +25,7 @@ ) from .event_type_pb2 import EventType from .failed_cause_pb2 import ( + ActivityTaskFailedCause, CancelExternalWorkflowExecutionFailedCause, ResourceExhaustedCause, ResourceExhaustedScope, @@ -74,6 +75,7 @@ "ActivityExecutionStatus", "ActivityIdConflictPolicy", "ActivityIdReusePolicy", + "ActivityTaskFailedCause", "ApplicationErrorCategory", "ArchivalState", "BatchOperationState", diff --git a/temporalio/api/enums/v1/command_type_pb2.py b/temporalio/api/enums/v1/command_type_pb2.py index 6d1c3b6e57..dece3c0f44 100644 --- a/temporalio/api/enums/v1/command_type_pb2.py +++ b/temporalio/api/enums/v1/command_type_pb2.py @@ -16,7 +16,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b"\n(temporal/api/enums/v1/command_type.proto\x12\x15temporal.api.enums.v1*\x9c\x06\n\x0b\x43ommandType\x12\x1c\n\x18\x43OMMAND_TYPE_UNSPECIFIED\x10\x00\x12'\n#COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK\x10\x01\x12-\n)COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK\x10\x02\x12\x1c\n\x18\x43OMMAND_TYPE_START_TIMER\x10\x03\x12,\n(COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION\x10\x04\x12(\n$COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION\x10\x05\x12\x1d\n\x19\x43OMMAND_TYPE_CANCEL_TIMER\x10\x06\x12*\n&COMMAND_TYPE_CANCEL_WORKFLOW_EXECUTION\x10\x07\x12;\n7COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION\x10\x08\x12\x1e\n\x1a\x43OMMAND_TYPE_RECORD_MARKER\x10\t\x12\x33\n/COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION\x10\n\x12/\n+COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION\x10\x0b\x12\x33\n/COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION\x10\x0c\x12\x32\n.COMMAND_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES\x10\r\x12!\n\x1d\x43OMMAND_TYPE_PROTOCOL_MESSAGE\x10\x0e\x12+\n'COMMAND_TYPE_MODIFY_WORKFLOW_PROPERTIES\x10\x10\x12)\n%COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION\x10\x11\x12/\n+COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION\x10\x12\x42\x88\x01\n\x18io.temporal.api.enums.v1B\x10\x43ommandTypeProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3" + b"\n(temporal/api/enums/v1/command_type.proto\x12\x15temporal.api.enums.v1*\xcb\x07\n\x0b\x43ommandType\x12\x1c\n\x18\x43OMMAND_TYPE_UNSPECIFIED\x10\x00\x12'\n#COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK\x10\x01\x12-\n)COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK\x10\x02\x12\x1c\n\x18\x43OMMAND_TYPE_START_TIMER\x10\x03\x12,\n(COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION\x10\x04\x12(\n$COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION\x10\x05\x12\x1d\n\x19\x43OMMAND_TYPE_CANCEL_TIMER\x10\x06\x12*\n&COMMAND_TYPE_CANCEL_WORKFLOW_EXECUTION\x10\x07\x12;\n7COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION\x10\x08\x12\x1e\n\x1a\x43OMMAND_TYPE_RECORD_MARKER\x10\t\x12\x33\n/COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION\x10\n\x12/\n+COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION\x10\x0b\x12\x33\n/COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION\x10\x0c\x12\x32\n.COMMAND_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES\x10\r\x12!\n\x1d\x43OMMAND_TYPE_PROTOCOL_MESSAGE\x10\x0e\x12+\n'COMMAND_TYPE_MODIFY_WORKFLOW_PROPERTIES\x10\x10\x12)\n%COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION\x10\x11\x12/\n+COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION\x10\x12\x12&\n\"COMMAND_TYPE_APPEND_STREAM_RECORDS\x10\x13\x12!\n\x1d\x43OMMAND_TYPE_SUBSCRIBE_STREAM\x10\x14\x12/\n+COMMAND_TYPE_SUBSCRIBE_NOTIFICATION_CHANNEL\x10\x15\x12\x31\n-COMMAND_TYPE_UNSUBSCRIBE_NOTIFICATION_CHANNEL\x10\x16\x42\x88\x01\n\x18io.temporal.api.enums.v1B\x10\x43ommandTypeProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3" ) _COMMANDTYPE = DESCRIPTOR.enum_types_by_name["CommandType"] @@ -39,11 +39,15 @@ COMMAND_TYPE_MODIFY_WORKFLOW_PROPERTIES = 16 COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION = 17 COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION = 18 +COMMAND_TYPE_APPEND_STREAM_RECORDS = 19 +COMMAND_TYPE_SUBSCRIBE_STREAM = 20 +COMMAND_TYPE_SUBSCRIBE_NOTIFICATION_CHANNEL = 21 +COMMAND_TYPE_UNSUBSCRIBE_NOTIFICATION_CHANNEL = 22 if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\030io.temporal.api.enums.v1B\020CommandTypeProtoP\001Z!go.temporal.io/api/enums/v1;enums\252\002\027Temporalio.Api.Enums.V1\352\002\032Temporalio::Api::Enums::V1" _COMMANDTYPE._serialized_start = 68 - _COMMANDTYPE._serialized_end = 864 + _COMMANDTYPE._serialized_end = 1039 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/enums/v1/command_type_pb2.pyi b/temporalio/api/enums/v1/command_type_pb2.pyi index 9f1bfd240a..327e663869 100644 --- a/temporalio/api/enums/v1/command_type_pb2.pyi +++ b/temporalio/api/enums/v1/command_type_pb2.pyi @@ -44,6 +44,10 @@ class _CommandTypeEnumTypeWrapper( COMMAND_TYPE_MODIFY_WORKFLOW_PROPERTIES: _CommandType.ValueType # 16 COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION: _CommandType.ValueType # 17 COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION: _CommandType.ValueType # 18 + COMMAND_TYPE_APPEND_STREAM_RECORDS: _CommandType.ValueType # 19 + COMMAND_TYPE_SUBSCRIBE_STREAM: _CommandType.ValueType # 20 + COMMAND_TYPE_SUBSCRIBE_NOTIFICATION_CHANNEL: _CommandType.ValueType # 21 + COMMAND_TYPE_UNSUBSCRIBE_NOTIFICATION_CHANNEL: _CommandType.ValueType # 22 class CommandType(_CommandType, metaclass=_CommandTypeEnumTypeWrapper): """Whenever this list of command types is changed do change the function shouldBufferEvent in mutableStateBuilder.go to make sure to do the correct event ordering.""" @@ -66,4 +70,8 @@ COMMAND_TYPE_PROTOCOL_MESSAGE: CommandType.ValueType # 14 COMMAND_TYPE_MODIFY_WORKFLOW_PROPERTIES: CommandType.ValueType # 16 COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION: CommandType.ValueType # 17 COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION: CommandType.ValueType # 18 +COMMAND_TYPE_APPEND_STREAM_RECORDS: CommandType.ValueType # 19 +COMMAND_TYPE_SUBSCRIBE_STREAM: CommandType.ValueType # 20 +COMMAND_TYPE_SUBSCRIBE_NOTIFICATION_CHANNEL: CommandType.ValueType # 21 +COMMAND_TYPE_UNSUBSCRIBE_NOTIFICATION_CHANNEL: CommandType.ValueType # 22 global___CommandType = CommandType diff --git a/temporalio/api/enums/v1/common_pb2.py b/temporalio/api/enums/v1/common_pb2.py index a646df4ad5..f14182ffc3 100644 --- a/temporalio/api/enums/v1/common_pb2.py +++ b/temporalio/api/enums/v1/common_pb2.py @@ -16,7 +16,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b"\n\"temporal/api/enums/v1/common.proto\x12\x15temporal.api.enums.v1*_\n\x0c\x45ncodingType\x12\x1d\n\x19\x45NCODING_TYPE_UNSPECIFIED\x10\x00\x12\x18\n\x14\x45NCODING_TYPE_PROTO3\x10\x01\x12\x16\n\x12\x45NCODING_TYPE_JSON\x10\x02*\x91\x02\n\x10IndexedValueType\x12\"\n\x1eINDEXED_VALUE_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x17INDEXED_VALUE_TYPE_TEXT\x10\x01\x12\x1e\n\x1aINDEXED_VALUE_TYPE_KEYWORD\x10\x02\x12\x1a\n\x16INDEXED_VALUE_TYPE_INT\x10\x03\x12\x1d\n\x19INDEXED_VALUE_TYPE_DOUBLE\x10\x04\x12\x1b\n\x17INDEXED_VALUE_TYPE_BOOL\x10\x05\x12\x1f\n\x1bINDEXED_VALUE_TYPE_DATETIME\x10\x06\x12#\n\x1fINDEXED_VALUE_TYPE_KEYWORD_LIST\x10\x07*^\n\x08Severity\x12\x18\n\x14SEVERITY_UNSPECIFIED\x10\x00\x12\x11\n\rSEVERITY_HIGH\x10\x01\x12\x13\n\x0fSEVERITY_MEDIUM\x10\x02\x12\x10\n\x0cSEVERITY_LOW\x10\x03*\xde\x01\n\rCallbackState\x12\x1e\n\x1a\x43\x41LLBACK_STATE_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43\x41LLBACK_STATE_STANDBY\x10\x01\x12\x1c\n\x18\x43\x41LLBACK_STATE_SCHEDULED\x10\x02\x12\x1e\n\x1a\x43\x41LLBACK_STATE_BACKING_OFF\x10\x03\x12\x19\n\x15\x43\x41LLBACK_STATE_FAILED\x10\x04\x12\x1c\n\x18\x43\x41LLBACK_STATE_SUCCEEDED\x10\x05\x12\x1a\n\x16\x43\x41LLBACK_STATE_BLOCKED\x10\x06*\xfd\x01\n\x1aPendingNexusOperationState\x12-\n)PENDING_NEXUS_OPERATION_STATE_UNSPECIFIED\x10\x00\x12+\n'PENDING_NEXUS_OPERATION_STATE_SCHEDULED\x10\x01\x12-\n)PENDING_NEXUS_OPERATION_STATE_BACKING_OFF\x10\x02\x12)\n%PENDING_NEXUS_OPERATION_STATE_STARTED\x10\x03\x12)\n%PENDING_NEXUS_OPERATION_STATE_BLOCKED\x10\x04*\xfe\x02\n\x1fNexusOperationCancellationState\x12\x32\n.NEXUS_OPERATION_CANCELLATION_STATE_UNSPECIFIED\x10\x00\x12\x30\n,NEXUS_OPERATION_CANCELLATION_STATE_SCHEDULED\x10\x01\x12\x32\n.NEXUS_OPERATION_CANCELLATION_STATE_BACKING_OFF\x10\x02\x12\x30\n,NEXUS_OPERATION_CANCELLATION_STATE_SUCCEEDED\x10\x03\x12-\n)NEXUS_OPERATION_CANCELLATION_STATE_FAILED\x10\x04\x12\x30\n,NEXUS_OPERATION_CANCELLATION_STATE_TIMED_OUT\x10\x05\x12.\n*NEXUS_OPERATION_CANCELLATION_STATE_BLOCKED\x10\x06*\x97\x01\n\x17WorkflowRuleActionScope\x12*\n&WORKFLOW_RULE_ACTION_SCOPE_UNSPECIFIED\x10\x00\x12'\n#WORKFLOW_RULE_ACTION_SCOPE_WORKFLOW\x10\x01\x12'\n#WORKFLOW_RULE_ACTION_SCOPE_ACTIVITY\x10\x02*m\n\x18\x41pplicationErrorCategory\x12*\n&APPLICATION_ERROR_CATEGORY_UNSPECIFIED\x10\x00\x12%\n!APPLICATION_ERROR_CATEGORY_BENIGN\x10\x01*\x85\x01\n\x0cWorkerStatus\x12\x1d\n\x19WORKER_STATUS_UNSPECIFIED\x10\x00\x12\x19\n\x15WORKER_STATUS_RUNNING\x10\x01\x12\x1f\n\x1bWORKER_STATUS_SHUTTING_DOWN\x10\x02\x12\x1a\n\x16WORKER_STATUS_SHUTDOWN\x10\x03*i\n\rExecutionType\x12\x1e\n\x1a\x45XECUTION_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x17\x45XECUTION_TYPE_WORKFLOW\x10\x01\x12\x1b\n\x17\x45XECUTION_TYPE_ACTIVITY\x10\x02\x42\x83\x01\n\x18io.temporal.api.enums.v1B\x0b\x43ommonProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3" + b"\n\"temporal/api/enums/v1/common.proto\x12\x15temporal.api.enums.v1*_\n\x0c\x45ncodingType\x12\x1d\n\x19\x45NCODING_TYPE_UNSPECIFIED\x10\x00\x12\x18\n\x14\x45NCODING_TYPE_PROTO3\x10\x01\x12\x16\n\x12\x45NCODING_TYPE_JSON\x10\x02*\x91\x02\n\x10IndexedValueType\x12\"\n\x1eINDEXED_VALUE_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x17INDEXED_VALUE_TYPE_TEXT\x10\x01\x12\x1e\n\x1aINDEXED_VALUE_TYPE_KEYWORD\x10\x02\x12\x1a\n\x16INDEXED_VALUE_TYPE_INT\x10\x03\x12\x1d\n\x19INDEXED_VALUE_TYPE_DOUBLE\x10\x04\x12\x1b\n\x17INDEXED_VALUE_TYPE_BOOL\x10\x05\x12\x1f\n\x1bINDEXED_VALUE_TYPE_DATETIME\x10\x06\x12#\n\x1fINDEXED_VALUE_TYPE_KEYWORD_LIST\x10\x07*^\n\x08Severity\x12\x18\n\x14SEVERITY_UNSPECIFIED\x10\x00\x12\x11\n\rSEVERITY_HIGH\x10\x01\x12\x13\n\x0fSEVERITY_MEDIUM\x10\x02\x12\x10\n\x0cSEVERITY_LOW\x10\x03*\xde\x01\n\rCallbackState\x12\x1e\n\x1a\x43\x41LLBACK_STATE_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43\x41LLBACK_STATE_STANDBY\x10\x01\x12\x1c\n\x18\x43\x41LLBACK_STATE_SCHEDULED\x10\x02\x12\x1e\n\x1a\x43\x41LLBACK_STATE_BACKING_OFF\x10\x03\x12\x19\n\x15\x43\x41LLBACK_STATE_FAILED\x10\x04\x12\x1c\n\x18\x43\x41LLBACK_STATE_SUCCEEDED\x10\x05\x12\x1a\n\x16\x43\x41LLBACK_STATE_BLOCKED\x10\x06*\xfd\x01\n\x1aPendingNexusOperationState\x12-\n)PENDING_NEXUS_OPERATION_STATE_UNSPECIFIED\x10\x00\x12+\n'PENDING_NEXUS_OPERATION_STATE_SCHEDULED\x10\x01\x12-\n)PENDING_NEXUS_OPERATION_STATE_BACKING_OFF\x10\x02\x12)\n%PENDING_NEXUS_OPERATION_STATE_STARTED\x10\x03\x12)\n%PENDING_NEXUS_OPERATION_STATE_BLOCKED\x10\x04*\xfe\x02\n\x1fNexusOperationCancellationState\x12\x32\n.NEXUS_OPERATION_CANCELLATION_STATE_UNSPECIFIED\x10\x00\x12\x30\n,NEXUS_OPERATION_CANCELLATION_STATE_SCHEDULED\x10\x01\x12\x32\n.NEXUS_OPERATION_CANCELLATION_STATE_BACKING_OFF\x10\x02\x12\x30\n,NEXUS_OPERATION_CANCELLATION_STATE_SUCCEEDED\x10\x03\x12-\n)NEXUS_OPERATION_CANCELLATION_STATE_FAILED\x10\x04\x12\x30\n,NEXUS_OPERATION_CANCELLATION_STATE_TIMED_OUT\x10\x05\x12.\n*NEXUS_OPERATION_CANCELLATION_STATE_BLOCKED\x10\x06*\x97\x01\n\x17WorkflowRuleActionScope\x12*\n&WORKFLOW_RULE_ACTION_SCOPE_UNSPECIFIED\x10\x00\x12'\n#WORKFLOW_RULE_ACTION_SCOPE_WORKFLOW\x10\x01\x12'\n#WORKFLOW_RULE_ACTION_SCOPE_ACTIVITY\x10\x02*m\n\x18\x41pplicationErrorCategory\x12*\n&APPLICATION_ERROR_CATEGORY_UNSPECIFIED\x10\x00\x12%\n!APPLICATION_ERROR_CATEGORY_BENIGN\x10\x01*\x85\x01\n\x0cWorkerStatus\x12\x1d\n\x19WORKER_STATUS_UNSPECIFIED\x10\x00\x12\x19\n\x15WORKER_STATUS_RUNNING\x10\x01\x12\x1f\n\x1bWORKER_STATUS_SHUTTING_DOWN\x10\x02\x12\x1a\n\x16WORKER_STATUS_SHUTDOWN\x10\x03*\x8d\x01\n\rExecutionType\x12\x1e\n\x1a\x45XECUTION_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x17\x45XECUTION_TYPE_WORKFLOW\x10\x01\x12\x1b\n\x17\x45XECUTION_TYPE_ACTIVITY\x10\x02\x12\"\n\x1e\x45XECUTION_TYPE_NEXUS_OPERATION\x10\x03\x42\x83\x01\n\x18io.temporal.api.enums.v1B\x0b\x43ommonProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3" ) _ENCODINGTYPE = DESCRIPTOR.enum_types_by_name["EncodingType"] @@ -93,6 +93,7 @@ EXECUTION_TYPE_UNSPECIFIED = 0 EXECUTION_TYPE_WORKFLOW = 1 EXECUTION_TYPE_ACTIVITY = 2 +EXECUTION_TYPE_NEXUS_OPERATION = 3 if _descriptor._USE_C_DESCRIPTORS == False: @@ -116,6 +117,6 @@ _APPLICATIONERRORCATEGORY._serialized_end = 1659 _WORKERSTATUS._serialized_start = 1662 _WORKERSTATUS._serialized_end = 1795 - _EXECUTIONTYPE._serialized_start = 1797 - _EXECUTIONTYPE._serialized_end = 1902 + _EXECUTIONTYPE._serialized_start = 1798 + _EXECUTIONTYPE._serialized_end = 1939 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/enums/v1/common_pb2.pyi b/temporalio/api/enums/v1/common_pb2.pyi index 47b6d030b8..c2c73deb27 100644 --- a/temporalio/api/enums/v1/common_pb2.pyi +++ b/temporalio/api/enums/v1/common_pb2.pyi @@ -119,7 +119,7 @@ class _CallbackStateEnumTypeWrapper( CALLBACK_STATE_SUCCEEDED: _CallbackState.ValueType # 5 """Callback has succeeded.""" CALLBACK_STATE_BLOCKED: _CallbackState.ValueType # 6 - """Callback is blocked (eg: by circuit breaker).""" + """Callback is blocked, e.g. by circuit breaker.""" class CallbackState(_CallbackState, metaclass=_CallbackStateEnumTypeWrapper): """State of a callback.""" @@ -137,7 +137,7 @@ CALLBACK_STATE_FAILED: CallbackState.ValueType # 4 CALLBACK_STATE_SUCCEEDED: CallbackState.ValueType # 5 """Callback has succeeded.""" CALLBACK_STATE_BLOCKED: CallbackState.ValueType # 6 -"""Callback is blocked (eg: by circuit breaker).""" +"""Callback is blocked, e.g. by circuit breaker.""" global___CallbackState = CallbackState class _PendingNexusOperationState: @@ -356,6 +356,8 @@ class _ExecutionTypeEnumTypeWrapper( """A workflow execution archetype.""" EXECUTION_TYPE_ACTIVITY: _ExecutionType.ValueType # 2 """An activity execution archetype. This is reserved for standalone activities.""" + EXECUTION_TYPE_NEXUS_OPERATION: _ExecutionType.ValueType # 3 + """A Nexus operation execution archetype. This is reserved for standalone Nexus operations.""" class ExecutionType(_ExecutionType, metaclass=_ExecutionTypeEnumTypeWrapper): ... @@ -364,4 +366,6 @@ EXECUTION_TYPE_WORKFLOW: ExecutionType.ValueType # 1 """A workflow execution archetype.""" EXECUTION_TYPE_ACTIVITY: ExecutionType.ValueType # 2 """An activity execution archetype. This is reserved for standalone activities.""" +EXECUTION_TYPE_NEXUS_OPERATION: ExecutionType.ValueType # 3 +"""A Nexus operation execution archetype. This is reserved for standalone Nexus operations.""" global___ExecutionType = ExecutionType diff --git a/temporalio/api/enums/v1/event_type_pb2.py b/temporalio/api/enums/v1/event_type_pb2.py index 14ff7984d6..c012b3cd23 100644 --- a/temporalio/api/enums/v1/event_type_pb2.py +++ b/temporalio/api/enums/v1/event_type_pb2.py @@ -16,7 +16,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n&temporal/api/enums/v1/event_type.proto\x12\x15temporal.api.enums.v1*\x9e\x16\n\tEventType\x12\x1a\n\x16\x45VENT_TYPE_UNSPECIFIED\x10\x00\x12)\n%EVENT_TYPE_WORKFLOW_EXECUTION_STARTED\x10\x01\x12+\n\'EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED\x10\x02\x12(\n$EVENT_TYPE_WORKFLOW_EXECUTION_FAILED\x10\x03\x12+\n\'EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT\x10\x04\x12&\n"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED\x10\x05\x12$\n EVENT_TYPE_WORKFLOW_TASK_STARTED\x10\x06\x12&\n"EVENT_TYPE_WORKFLOW_TASK_COMPLETED\x10\x07\x12&\n"EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT\x10\x08\x12#\n\x1f\x45VENT_TYPE_WORKFLOW_TASK_FAILED\x10\t\x12&\n"EVENT_TYPE_ACTIVITY_TASK_SCHEDULED\x10\n\x12$\n EVENT_TYPE_ACTIVITY_TASK_STARTED\x10\x0b\x12&\n"EVENT_TYPE_ACTIVITY_TASK_COMPLETED\x10\x0c\x12#\n\x1f\x45VENT_TYPE_ACTIVITY_TASK_FAILED\x10\r\x12&\n"EVENT_TYPE_ACTIVITY_TASK_TIMED_OUT\x10\x0e\x12-\n)EVENT_TYPE_ACTIVITY_TASK_CANCEL_REQUESTED\x10\x0f\x12%\n!EVENT_TYPE_ACTIVITY_TASK_CANCELED\x10\x10\x12\x1c\n\x18\x45VENT_TYPE_TIMER_STARTED\x10\x11\x12\x1a\n\x16\x45VENT_TYPE_TIMER_FIRED\x10\x12\x12\x1d\n\x19\x45VENT_TYPE_TIMER_CANCELED\x10\x13\x12\x32\n.EVENT_TYPE_WORKFLOW_EXECUTION_CANCEL_REQUESTED\x10\x14\x12*\n&EVENT_TYPE_WORKFLOW_EXECUTION_CANCELED\x10\x15\x12\x43\n?EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED\x10\x16\x12@\n\x12\x37\n3EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED\x10?\x12\x39\n5EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED\x10@B\x86\x01\n\x18io.temporal.api.enums.v1B\x0e\x45ventTypeProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3' ) _EVENTTYPE = DESCRIPTOR.enum_types_by_name["EventType"] @@ -82,11 +82,15 @@ EVENT_TYPE_WORKFLOW_EXECUTION_PAUSED = 58 EVENT_TYPE_WORKFLOW_EXECUTION_UNPAUSED = 59 EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED = 60 +EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED = 61 +EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED = 62 +EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED = 63 +EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED = 64 if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\030io.temporal.api.enums.v1B\016EventTypeProtoP\001Z!go.temporal.io/api/enums/v1;enums\252\002\027Temporalio.Api.Enums.V1\352\002\032Temporalio::Api::Enums::V1" _EVENTTYPE._serialized_start = 66 - _EVENTTYPE._serialized_end = 2912 + _EVENTTYPE._serialized_end = 3120 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/enums/v1/event_type_pb2.pyi b/temporalio/api/enums/v1/event_type_pb2.pyi index 16a23b1374..e5ca2bb1f0 100644 --- a/temporalio/api/enums/v1/event_type_pb2.pyi +++ b/temporalio/api/enums/v1/event_type_pb2.pyi @@ -216,6 +216,25 @@ class _EventTypeEnumTypeWrapper( """An event that indicates that the previously paused workflow execution has been unpaused.""" EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED: _EventType.ValueType # 60 """An event that indicates time skipping advanced time or was disabled automatically after a bound was reached.""" + EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED: _EventType.ValueType # 61 + """A Workflow subscribed to a stream. Recorded once per subscription, not + per record: the offsets a task consumed ride WorkflowTaskCompleted and + the payloads never enter History at all. + """ + EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED: _EventType.ValueType # 62 + """A Workflow appended a batch of records to a stream. Recorded per + batch, and carrying only the offset range it landed at: the bodies go to + the stream's own log, never into History. + """ + EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED: _EventType.ValueType # 63 + """A Workflow became a listener of a notification channel for its run. + The notifications themselves ride the WorkflowTaskScheduled event. + """ + EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED: _EventType.ValueType # 64 + """A Workflow stopped listening on a notification channel for its run. + Recorded for every UnsubscribeNotificationChannel command, including one + naming a channel the run was not subscribed to. + """ class EventType(_EventType, metaclass=_EventTypeEnumTypeWrapper): """Whenever this list of events is changed do change the function shouldBufferEvent in mutableStateBuilder.go to make sure to do the correct event ordering""" @@ -408,4 +427,23 @@ EVENT_TYPE_WORKFLOW_EXECUTION_UNPAUSED: EventType.ValueType # 59 """An event that indicates that the previously paused workflow execution has been unpaused.""" EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED: EventType.ValueType # 60 """An event that indicates time skipping advanced time or was disabled automatically after a bound was reached.""" +EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED: EventType.ValueType # 61 +"""A Workflow subscribed to a stream. Recorded once per subscription, not +per record: the offsets a task consumed ride WorkflowTaskCompleted and +the payloads never enter History at all. +""" +EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED: EventType.ValueType # 62 +"""A Workflow appended a batch of records to a stream. Recorded per +batch, and carrying only the offset range it landed at: the bodies go to +the stream's own log, never into History. +""" +EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED: EventType.ValueType # 63 +"""A Workflow became a listener of a notification channel for its run. +The notifications themselves ride the WorkflowTaskScheduled event. +""" +EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED: EventType.ValueType # 64 +"""A Workflow stopped listening on a notification channel for its run. +Recorded for every UnsubscribeNotificationChannel command, including one +naming a channel the run was not subscribed to. +""" global___EventType = EventType diff --git a/temporalio/api/enums/v1/failed_cause_pb2.py b/temporalio/api/enums/v1/failed_cause_pb2.py index 03eb094f97..1a3d4f93e3 100644 --- a/temporalio/api/enums/v1/failed_cause_pb2.py +++ b/temporalio/api/enums/v1/failed_cause_pb2.py @@ -16,11 +16,13 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n(temporal/api/enums/v1/failed_cause.proto\x12\x15temporal.api.enums.v1*\x90\x14\n\x17WorkflowTaskFailedCause\x12*\n&WORKFLOW_TASK_FAILED_CAUSE_UNSPECIFIED\x10\x00\x12\x30\n,WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND\x10\x01\x12?\n;WORKFLOW_TASK_FAILED_CAUSE_BAD_SCHEDULE_ACTIVITY_ATTRIBUTES\x10\x02\x12\x45\nAWORKFLOW_TASK_FAILED_CAUSE_BAD_REQUEST_CANCEL_ACTIVITY_ATTRIBUTES\x10\x03\x12\x39\n5WORKFLOW_TASK_FAILED_CAUSE_BAD_START_TIMER_ATTRIBUTES\x10\x04\x12:\n6WORKFLOW_TASK_FAILED_CAUSE_BAD_CANCEL_TIMER_ATTRIBUTES\x10\x05\x12;\n7WORKFLOW_TASK_FAILED_CAUSE_BAD_RECORD_MARKER_ATTRIBUTES\x10\x06\x12I\nEWORKFLOW_TASK_FAILED_CAUSE_BAD_COMPLETE_WORKFLOW_EXECUTION_ATTRIBUTES\x10\x07\x12\x45\nAWORKFLOW_TASK_FAILED_CAUSE_BAD_FAIL_WORKFLOW_EXECUTION_ATTRIBUTES\x10\x08\x12G\nCWORKFLOW_TASK_FAILED_CAUSE_BAD_CANCEL_WORKFLOW_EXECUTION_ATTRIBUTES\x10\t\x12X\nTWORKFLOW_TASK_FAILED_CAUSE_BAD_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_ATTRIBUTES\x10\n\x12=\n9WORKFLOW_TASK_FAILED_CAUSE_BAD_CONTINUE_AS_NEW_ATTRIBUTES\x10\x0b\x12\x37\n3WORKFLOW_TASK_FAILED_CAUSE_START_TIMER_DUPLICATE_ID\x10\x0c\x12\x36\n2WORKFLOW_TASK_FAILED_CAUSE_RESET_STICKY_TASK_QUEUE\x10\r\x12@\n\n:WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_STREAM_ATTRIBUTES\x10*\x12\x37\n3WORKFLOW_TASK_FAILED_CAUSE_STREAM_RANGE_UNAVAILABLE\x10+\x12L\nHWORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES\x10,\x12N\nJWORKFLOW_TASK_FAILED_CAUSE_BAD_UNSUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES\x10-*\xf3\x01\n\x17\x41\x63tivityTaskFailedCause\x12*\n&ACTIVITY_TASK_FAILED_CAUSE_UNSPECIFIED\x10\x00\x12\x31\n-ACTIVITY_TASK_FAILED_CAUSE_PAYLOADS_TOO_LARGE\x10\x01\x12\x37\n3ACTIVITY_TASK_FAILED_CAUSE_EXTERNAL_STORAGE_FAILURE\x10\x02\x12@\n\n\x1b\x66irst_workflow_task_backoff\x18\x15 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x16 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x17 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x45\n\x16prev_auto_reset_points\x18\x18 \x01(\x0b\x32%.temporal.api.workflow.v1.ResetPoints\x12.\n\x06header\x18\x19 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12&\n\x1eparent_initiated_event_version\x18\x1a \x01(\x03\x12\x13\n\x0bworkflow_id\x18\x1c \x01(\t\x12L\n\x14source_version_stamp\x18\x1d \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\x14\x63ompletion_callbacks\x18\x1e \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12J\n\x17root_workflow_execution\x18\x1f \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1e\n\x12inherited_build_id\x18 \x01(\tB\x02\x18\x01\x12I\n\x13versioning_override\x18! \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12\x33\n\'parent_pinned_worker_deployment_version\x18" \x01(\tB\x02\x18\x01\x12\x32\n\x08priority\x18# \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12U\n\x18inherited_pinned_version\x18% \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12Y\n\x1binherited_auto_upgrade_info\x18\' \x01(\x0b\x32\x34.temporal.api.deployment.v1.InheritedAutoUpgradeInfo\x12 \n\x18\x65\x61ger_execution_accepted\x18& \x01(\x08\x12^\n\x1f\x64\x65\x63lined_target_version_upgrade\x18( \x01(\x0b\x32\x35.temporal.api.history.v1.DeclinedTargetVersionUpgrade\x12H\n\x14time_skipping_config\x18) \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig\x12]\n\x1ftime_skipping_state_propagation\x18+ \x01(\x0b\x32\x34.temporal.api.common.v1.TimeSkippingStatePropagationJ\x04\x08$\x10%J\x04\x08*\x10+R parent_pinned_deployment_versionR\x18initial_skipped_duration"\x88\x01\n\x1c\x44\x65\x63linedTargetVersionUpgrade\x12O\n\x12\x64\x65ployment_version\x18\x01 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x17\n\x0frevision_number\x18\x02 \x01(\x03"\xa5\x01\n)WorkflowExecutionCompletedEventAttributes\x12\x30\n\x06result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x1c\n\x14new_execution_run_id\x18\x03 \x01(\t"\xdb\x01\n&WorkflowExecutionFailedEventAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x36\n\x0bretry_state\x18\x02 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState\x12(\n workflow_task_completed_event_id\x18\x03 \x01(\x03\x12\x1c\n\x14new_execution_run_id\x18\x04 \x01(\t"\x80\x01\n(WorkflowExecutionTimedOutEventAttributes\x12\x36\n\x0bretry_state\x18\x01 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState\x12\x1c\n\x14new_execution_run_id\x18\x02 \x01(\t"\xa5\x07\n.WorkflowExecutionContinuedAsNewEventAttributes\x12\x1c\n\x14new_execution_run_id\x18\x01 \x01(\t\x12;\n\rworkflow_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x03 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x37\n\x14workflow_run_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12(\n workflow_task_completed_event_id\x18\x07 \x01(\x03\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12@\n\tinitiator\x18\t \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12\x35\n\x07\x66\x61ilure\x18\n \x01(\x0b\x32 .temporal.api.failure.v1.FailureB\x02\x18\x01\x12@\n\x16last_completion_result\x18\x0b \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12.\n\x06header\x18\x0c \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12*\n\x04memo\x18\r \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0e \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x1c\n\x10inherit_build_id\x18\x0f \x01(\x08\x42\x02\x18\x01\x12[\n\x1binitial_versioning_behavior\x18\x10 \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior"\xac\x01\n$WorkflowTaskScheduledEventAttributes\x12\x38\n\ntask_queue\x18\x01 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x39\n\x16start_to_close_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x0f\n\x07\x61ttempt\x18\x03 \x01(\x05"\xa0\x03\n"WorkflowTaskStartedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t\x12\x1f\n\x17suggest_continue_as_new\x18\x04 \x01(\x08\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x08 \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\t \x01(\x08\x12\x1a\n\x12history_size_bytes\x18\x05 \x01(\x03\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12%\n\x19\x62uild_id_redirect_counter\x18\x07 \x01(\x03\x42\x02\x18\x01"\x82\x05\n$WorkflowTaskCompletedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x1b\n\x0f\x62inary_checksum\x18\x04 \x01(\tB\x02\x18\x01\x12\x46\n\x0eworker_version\x18\x05 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12H\n\x0csdk_metadata\x18\x06 \x01(\x0b\x32\x32.temporal.api.sdk.v1.WorkflowTaskCompletedMetadata\x12\x43\n\x11metering_metadata\x18\r \x01(\x0b\x32(.temporal.api.common.v1.MeteringMetadata\x12>\n\ndeployment\x18\x07 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12\x46\n\x13versioning_behavior\x18\x08 \x01(\x0e\x32).temporal.api.enums.v1.VersioningBehavior\x12%\n\x19worker_deployment_version\x18\t \x01(\tB\x02\x18\x01\x12\x1e\n\x16worker_deployment_name\x18\n \x01(\t\x12O\n\x12\x64\x65ployment_version\x18\x0b \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion"\x95\x01\n#WorkflowTaskTimedOutEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03\x12\x38\n\x0ctimeout_type\x18\x03 \x01(\x0e\x32".temporal.api.enums.v1.TimeoutType"\x87\x03\n!WorkflowTaskFailedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03\x12=\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowTaskFailedCause\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x13\n\x0b\x62\x61se_run_id\x18\x06 \x01(\t\x12\x12\n\nnew_run_id\x18\x07 \x01(\t\x12\x1a\n\x12\x66ork_event_version\x18\x08 \x01(\x03\x12\x1b\n\x0f\x62inary_checksum\x18\t \x01(\tB\x02\x18\x01\x12\x46\n\x0eworker_version\x18\n \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01"\xc2\x05\n$ActivityTaskScheduledEventAttributes\x12\x13\n\x0b\x61\x63tivity_id\x18\x01 \x01(\t\x12;\n\ractivity_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12.\n\x06header\x18\x05 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12/\n\x05input\x18\x06 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12<\n\x19schedule_to_close_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12(\n workflow_task_completed_event_id\x18\x0b \x01(\x03\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12!\n\x15use_workflow_build_id\x18\r \x01(\x08\x42\x02\x18\x01\x12\x32\n\x08priority\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.PriorityJ\x04\x08\x03\x10\x04"\x9e\x02\n"ActivityTaskStartedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x05\x12\x36\n\x0clast_failure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12%\n\x19\x62uild_id_redirect_counter\x18\x07 \x01(\x03\x42\x02\x18\x01"\xe8\x01\n$ActivityTaskCompletedEventAttributes\x12\x30\n\x06result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1a\n\x12scheduled_event_id\x18\x02 \x01(\x03\x12\x18\n\x10started_event_id\x18\x03 \x01(\x03\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x46\n\x0eworker_version\x18\x05 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01"\x9e\x02\n!ActivityTaskFailedEventAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x1a\n\x12scheduled_event_id\x18\x02 \x01(\x03\x12\x18\n\x10started_event_id\x18\x03 \x01(\x03\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x36\n\x0bretry_state\x18\x05 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01"\xc6\x01\n#ActivityTaskTimedOutEventAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x1a\n\x12scheduled_event_id\x18\x02 \x01(\x03\x12\x18\n\x10started_event_id\x18\x03 \x01(\x03\x12\x36\n\x0bretry_state\x18\x04 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState"r\n*ActivityTaskCancelRequestedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03"\x92\x02\n#ActivityTaskCanceledEventAttributes\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12(\n latest_cancel_requested_event_id\x18\x02 \x01(\x03\x12\x1a\n\x12scheduled_event_id\x18\x03 \x01(\x03\x12\x18\n\x10started_event_id\x18\x04 \x01(\x03\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01"\x93\x01\n\x1bTimerStartedEventAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12(\n workflow_task_completed_event_id\x18\x03 \x01(\x03"G\n\x19TimerFiredEventAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03"\x86\x01\n\x1cTimerCanceledEventAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x03 \x01(\x03\x12\x10\n\x08identity\x18\x04 \x01(\t"\xc7\x01\n/WorkflowExecutionCancelRequestedEventAttributes\x12\r\n\x05\x63\x61use\x18\x01 \x01(\t\x12#\n\x1b\x65xternal_initiated_event_id\x18\x02 \x01(\x03\x12N\n\x1b\x65xternal_workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x04 \x01(\t"\x87\x01\n(WorkflowExecutionCanceledEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x31\n\x07\x64\x65tails\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"\xe9\x02\n\x1dMarkerRecordedEventAttributes\x12\x13\n\x0bmarker_name\x18\x01 \x01(\t\x12T\n\x07\x64\x65tails\x18\x02 \x03(\x0b\x32\x43.temporal.api.history.v1.MarkerRecordedEventAttributes.DetailsEntry\x12(\n workflow_task_completed_event_id\x18\x03 \x01(\x03\x12.\n\x06header\x18\x04 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x1aP\n\x0c\x44\x65tailsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads:\x02\x38\x01"\xbf\x02\n(WorkflowExecutionSignaledEventAttributes\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12/\n\x05input\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t\x12.\n\x06header\x18\x04 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\'\n\x1bskip_generate_workflow_task\x18\x05 \x01(\x08\x42\x02\x18\x01\x12N\n\x1b\x65xternal_workflow_execution\x18\x06 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x12\n\nrequest_id\x18\x07 \x01(\t"\x81\x01\n*WorkflowExecutionTerminatedEventAttributes\x12\x0e\n\x06reason\x18\x01 \x01(\t\x12\x31\n\x07\x64\x65tails\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t"\x9c\x02\n>RequestCancelExternalWorkflowExecutionInitiatedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x07\x63ontrol\x18\x04 \x01(\tB\x02\x18\x01\x12\x1b\n\x13\x63hild_workflow_only\x18\x05 \x01(\x08\x12\x0e\n\x06reason\x18\x06 \x01(\t"\xda\x02\n;RequestCancelExternalWorkflowExecutionFailedEventAttributes\x12P\n\x05\x63\x61use\x18\x01 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x11\n\tnamespace\x18\x03 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x04 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x13\n\x07\x63ontrol\x18\x06 \x01(\tB\x02\x18\x01"\xc5\x01\n7ExternalWorkflowExecutionCancelRequestedEventAttributes\x12\x1a\n\x12initiated_event_id\x18\x01 \x01(\x03\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x04 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution"\xfb\x02\n7SignalExternalWorkflowExecutionInitiatedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\t \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x13\n\x07\x63ontrol\x18\x06 \x01(\tB\x02\x18\x01\x12\x1b\n\x13\x63hild_workflow_only\x18\x07 \x01(\x08\x12.\n\x06header\x18\x08 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header"\xd3\x02\n4SignalExternalWorkflowExecutionFailedEventAttributes\x12P\n\x05\x63\x61use\x18\x01 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x11\n\tnamespace\x18\x03 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x04 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x13\n\x07\x63ontrol\x18\x06 \x01(\tB\x02\x18\x01"\xd3\x01\n0ExternalWorkflowExecutionSignaledEventAttributes\x12\x1a\n\x12initiated_event_id\x18\x01 \x01(\x03\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x05 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x07\x63ontrol\x18\x04 \x01(\tB\x02\x18\x01"\x9e\x01\n-UpsertWorkflowSearchAttributesEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x43\n\x11search_attributes\x18\x02 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"\x8a\x01\n)WorkflowPropertiesModifiedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x33\n\rupserted_memo\x18\x02 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xfc\t\n3StartChildWorkflowExecutionInitiatedEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x12 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x45\n\x13parent_close_policy\x18\t \x01(\x0e\x32(.temporal.api.enums.v1.ParentClosePolicy\x12\x13\n\x07\x63ontrol\x18\n \x01(\tB\x02\x18\x01\x12(\n workflow_task_completed_event_id\x18\x0b \x01(\x03\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12.\n\x06header\x18\x0f \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12*\n\x04memo\x18\x10 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x1c\n\x10inherit_build_id\x18\x13 \x01(\x08\x42\x02\x18\x01\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12H\n\x14time_skipping_config\x18\x15 \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig\x12]\n\x1ftime_skipping_state_propagation\x18\x17 \x01(\x0b\x32\x34.temporal.api.common.v1.TimeSkippingStatePropagation\x12I\n\x13versioning_override\x18\x18 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverrideJ\x04\x08\x16\x10\x17R\x18initial_skipped_duration"\xd6\x02\n0StartChildWorkflowExecutionFailedEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x08 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12L\n\x05\x63\x61use\x18\x04 \x01(\x0e\x32=.temporal.api.enums.v1.StartChildWorkflowExecutionFailedCause\x12\x13\n\x07\x63ontrol\x18\x05 \x01(\tB\x02\x18\x01\x12\x1a\n\x12initiated_event_id\x18\x06 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x07 \x01(\x03"\xa7\x02\n,ChildWorkflowExecutionStartedEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x06 \x01(\t\x12\x1a\n\x12initiated_event_id\x18\x02 \x01(\x03\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x04 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12.\n\x06header\x18\x05 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header"\xc5\x02\n.ChildWorkflowExecutionCompletedEventAttributes\x12\x30\n\x06result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x04 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x18\n\x10started_event_id\x18\x06 \x01(\x03"\xfb\x02\n+ChildWorkflowExecutionFailedEventAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x08 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x04 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x18\n\x10started_event_id\x18\x06 \x01(\x03\x12\x36\n\x0bretry_state\x18\x07 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState"\xc5\x02\n-ChildWorkflowExecutionCanceledEventAttributes\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x04 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x18\n\x10started_event_id\x18\x06 \x01(\x03"\xca\x02\n-ChildWorkflowExecutionTimedOutEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x04 \x01(\x03\x12\x18\n\x10started_event_id\x18\x05 \x01(\x03\x12\x36\n\x0bretry_state\x18\x06 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState"\x94\x02\n/ChildWorkflowExecutionTerminatedEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x06 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x04 \x01(\x03\x12\x18\n\x10started_event_id\x18\x05 \x01(\x03"\xda\x05\n.WorkflowExecutionOptionsUpdatedEventAttributes\x12I\n\x13versioning_override\x18\x01 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12!\n\x19unset_versioning_override\x18\x02 \x01(\x08\x12\x1b\n\x13\x61ttached_request_id\x18\x03 \x01(\t\x12G\n\x1d\x61ttached_completion_callbacks\x18\x04 \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x32\n\x08priority\x18\x06 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12H\n\x14time_skipping_config\x18\x07 \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig\x12$\n\x1ctime_skipping_config_updated\x18\t \x01(\x08\x12\x84\x01\n\x17workflow_update_options\x18\x08 \x03(\x0b\x32\x63.temporal.api.history.v1.WorkflowExecutionOptionsUpdatedEventAttributes.WorkflowUpdateOptionsUpdate\x1a\x96\x01\n\x1bWorkflowUpdateOptionsUpdate\x12\x11\n\tupdate_id\x18\x01 \x01(\t\x12\x1b\n\x13\x61ttached_request_id\x18\x02 \x01(\t\x12G\n\x1d\x61ttached_completion_callbacks\x18\x03 \x03(\x0b\x32 .temporal.api.common.v1.Callback"\xc0\x02\n3WorkflowPropertiesModifiedExternallyEventAttributes\x12\x16\n\x0enew_task_queue\x18\x01 \x01(\t\x12<\n\x19new_workflow_task_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12;\n\x18new_workflow_run_timeout\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x41\n\x1enew_workflow_execution_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x33\n\rupserted_memo\x18\x05 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\x90\x01\n3ActivityPropertiesModifiedExternallyEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12=\n\x10new_retry_policy\x18\x02 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy"\xdc\x01\n.WorkflowExecutionUpdateAcceptedEventAttributes\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12#\n\x1b\x61\x63\x63\x65pted_request_message_id\x18\x02 \x01(\t\x12,\n$accepted_request_sequencing_event_id\x18\x03 \x01(\x03\x12\x39\n\x10\x61\x63\x63\x65pted_request\x18\x04 \x01(\x0b\x32\x1f.temporal.api.update.v1.Request"\xaa\x01\n/WorkflowExecutionUpdateCompletedEventAttributes\x12*\n\x04meta\x18\x01 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x19\n\x11\x61\x63\x63\x65pted_event_id\x18\x03 \x01(\x03\x12\x30\n\x07outcome\x18\x02 \x01(\x0b\x32\x1f.temporal.api.update.v1.Outcome"\x8f\x02\n.WorkflowExecutionUpdateRejectedEventAttributes\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12#\n\x1brejected_request_message_id\x18\x02 \x01(\t\x12,\n$rejected_request_sequencing_event_id\x18\x03 \x01(\x03\x12\x39\n\x10rejected_request\x18\x04 \x01(\x0b\x32\x1f.temporal.api.update.v1.Request\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xa4\x01\n.WorkflowExecutionUpdateAdmittedEventAttributes\x12\x30\n\x07request\x18\x01 \x01(\x0b\x32\x1f.temporal.api.update.v1.Request\x12@\n\x06origin\x18\x02 \x01(\x0e\x32\x30.temporal.api.enums.v1.UpdateAdmittedEventOrigin"^\n&WorkflowExecutionPausedEventAttributes\x12\x10\n\x08identity\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t"`\n(WorkflowExecutionUnpausedEventAttributes\x12\x10\n\x08identity\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t"\xc5\x01\n8WorkflowExecutionTimeSkippingTransitionedEventAttributes\x12/\n\x0btarget_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12#\n\x1b\x64isabled_after_fast_forward\x18\x02 \x01(\x08\x12\x33\n\x0fwall_clock_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\xb4\x04\n&NexusOperationScheduledEventAttributes\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x0f\n\x07service\x18\x02 \x01(\t\x12\x11\n\toperation\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x66\n\x0cnexus_header\x18\x06 \x03(\x0b\x32P.temporal.api.history.v1.NexusOperationScheduledEventAttributes.NexusHeaderEntry\x12(\n workflow_task_completed_event_id\x18\x07 \x01(\x03\x12\x12\n\nrequest_id\x18\x08 \x01(\t\x12\x13\n\x0b\x65ndpoint_id\x18\t \x01(\t\x12<\n\x19schedule_to_start_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"\x89\x01\n$NexusOperationStartedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x18\n\x0coperation_id\x18\x03 \x01(\tB\x02\x18\x01\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x17\n\x0foperation_token\x18\x05 \x01(\t"\x89\x01\n&NexusOperationCompletedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12/\n\x06result\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x12\n\nrequest_id\x18\x03 \x01(\t"\x88\x01\n#NexusOperationFailedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nrequest_id\x18\x03 \x01(\t"\x8a\x01\n%NexusOperationTimedOutEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nrequest_id\x18\x03 \x01(\t"\x8a\x01\n%NexusOperationCanceledEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nrequest_id\x18\x03 \x01(\t"t\n,NexusOperationCancelRequestedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03"\x97\x01\n3NexusOperationCancelRequestCompletedEventAttributes\x12\x1a\n\x12requested_event_id\x18\x01 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x1a\n\x12scheduled_event_id\x18\x03 \x01(\x03"\xc7\x01\n0NexusOperationCancelRequestFailedEventAttributes\x12\x1a\n\x12requested_event_id\x18\x01 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x31\n\x07\x66\x61ilure\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x1a\n\x12scheduled_event_id\x18\x04 \x01(\x03"\xca?\n\x0cHistoryEvent\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\x03\x12.\n\nevent_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x34\n\nevent_type\x18\x03 \x01(\x0e\x32 .temporal.api.enums.v1.EventType\x12\x0f\n\x07version\x18\x04 \x01(\x03\x12\x0f\n\x07task_id\x18\x05 \x01(\x03\x12\x1a\n\x11worker_may_ignore\x18\xac\x02 \x01(\x08\x12\x39\n\ruser_metadata\x18\xad\x02 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12,\n\x05links\x18\xae\x02 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x12\x35\n\tprincipal\x18\xaf\x02 \x01(\x0b\x32!.temporal.api.common.v1.Principal\x12\x43\n\x13\x65vent_group_markers\x18\xb0\x02 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12w\n+workflow_execution_started_event_attributes\x18\x06 \x01(\x0b\x32@.temporal.api.history.v1.WorkflowExecutionStartedEventAttributesH\x00\x12{\n-workflow_execution_completed_event_attributes\x18\x07 \x01(\x0b\x32\x42.temporal.api.history.v1.WorkflowExecutionCompletedEventAttributesH\x00\x12u\n*workflow_execution_failed_event_attributes\x18\x08 \x01(\x0b\x32?.temporal.api.history.v1.WorkflowExecutionFailedEventAttributesH\x00\x12z\n-workflow_execution_timed_out_event_attributes\x18\t \x01(\x0b\x32\x41.temporal.api.history.v1.WorkflowExecutionTimedOutEventAttributesH\x00\x12q\n(workflow_task_scheduled_event_attributes\x18\n \x01(\x0b\x32=.temporal.api.history.v1.WorkflowTaskScheduledEventAttributesH\x00\x12m\n&workflow_task_started_event_attributes\x18\x0b \x01(\x0b\x32;.temporal.api.history.v1.WorkflowTaskStartedEventAttributesH\x00\x12q\n(workflow_task_completed_event_attributes\x18\x0c \x01(\x0b\x32=.temporal.api.history.v1.WorkflowTaskCompletedEventAttributesH\x00\x12p\n(workflow_task_timed_out_event_attributes\x18\r \x01(\x0b\x32<.temporal.api.history.v1.WorkflowTaskTimedOutEventAttributesH\x00\x12k\n%workflow_task_failed_event_attributes\x18\x0e \x01(\x0b\x32:.temporal.api.history.v1.WorkflowTaskFailedEventAttributesH\x00\x12q\n(activity_task_scheduled_event_attributes\x18\x0f \x01(\x0b\x32=.temporal.api.history.v1.ActivityTaskScheduledEventAttributesH\x00\x12m\n&activity_task_started_event_attributes\x18\x10 \x01(\x0b\x32;.temporal.api.history.v1.ActivityTaskStartedEventAttributesH\x00\x12q\n(activity_task_completed_event_attributes\x18\x11 \x01(\x0b\x32=.temporal.api.history.v1.ActivityTaskCompletedEventAttributesH\x00\x12k\n%activity_task_failed_event_attributes\x18\x12 \x01(\x0b\x32:.temporal.api.history.v1.ActivityTaskFailedEventAttributesH\x00\x12p\n(activity_task_timed_out_event_attributes\x18\x13 \x01(\x0b\x32<.temporal.api.history.v1.ActivityTaskTimedOutEventAttributesH\x00\x12^\n\x1etimer_started_event_attributes\x18\x14 \x01(\x0b\x32\x34.temporal.api.history.v1.TimerStartedEventAttributesH\x00\x12Z\n\x1ctimer_fired_event_attributes\x18\x15 \x01(\x0b\x32\x32.temporal.api.history.v1.TimerFiredEventAttributesH\x00\x12~\n/activity_task_cancel_requested_event_attributes\x18\x16 \x01(\x0b\x32\x43.temporal.api.history.v1.ActivityTaskCancelRequestedEventAttributesH\x00\x12o\n\'activity_task_canceled_event_attributes\x18\x17 \x01(\x0b\x32<.temporal.api.history.v1.ActivityTaskCanceledEventAttributesH\x00\x12`\n\x1ftimer_canceled_event_attributes\x18\x18 \x01(\x0b\x32\x35.temporal.api.history.v1.TimerCanceledEventAttributesH\x00\x12\x62\n marker_recorded_event_attributes\x18\x19 \x01(\x0b\x32\x36.temporal.api.history.v1.MarkerRecordedEventAttributesH\x00\x12y\n,workflow_execution_signaled_event_attributes\x18\x1a \x01(\x0b\x32\x41.temporal.api.history.v1.WorkflowExecutionSignaledEventAttributesH\x00\x12}\n.workflow_execution_terminated_event_attributes\x18\x1b \x01(\x0b\x32\x43.temporal.api.history.v1.WorkflowExecutionTerminatedEventAttributesH\x00\x12\x88\x01\n4workflow_execution_cancel_requested_event_attributes\x18\x1c \x01(\x0b\x32H.temporal.api.history.v1.WorkflowExecutionCancelRequestedEventAttributesH\x00\x12y\n,workflow_execution_canceled_event_attributes\x18\x1d \x01(\x0b\x32\x41.temporal.api.history.v1.WorkflowExecutionCanceledEventAttributesH\x00\x12\xa8\x01\nErequest_cancel_external_workflow_execution_initiated_event_attributes\x18\x1e \x01(\x0b\x32W.temporal.api.history.v1.RequestCancelExternalWorkflowExecutionInitiatedEventAttributesH\x00\x12\xa2\x01\nBrequest_cancel_external_workflow_execution_failed_event_attributes\x18\x1f \x01(\x0b\x32T.temporal.api.history.v1.RequestCancelExternalWorkflowExecutionFailedEventAttributesH\x00\x12\x99\x01\n=external_workflow_execution_cancel_requested_event_attributes\x18 \x01(\x0b\x32P.temporal.api.history.v1.ExternalWorkflowExecutionCancelRequestedEventAttributesH\x00\x12\x87\x01\n4workflow_execution_continued_as_new_event_attributes\x18! \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionContinuedAsNewEventAttributesH\x00\x12\x91\x01\n9start_child_workflow_execution_initiated_event_attributes\x18" \x01(\x0b\x32L.temporal.api.history.v1.StartChildWorkflowExecutionInitiatedEventAttributesH\x00\x12\x8b\x01\n6start_child_workflow_execution_failed_event_attributes\x18# \x01(\x0b\x32I.temporal.api.history.v1.StartChildWorkflowExecutionFailedEventAttributesH\x00\x12\x82\x01\n1child_workflow_execution_started_event_attributes\x18$ \x01(\x0b\x32\x45.temporal.api.history.v1.ChildWorkflowExecutionStartedEventAttributesH\x00\x12\x86\x01\n3child_workflow_execution_completed_event_attributes\x18% \x01(\x0b\x32G.temporal.api.history.v1.ChildWorkflowExecutionCompletedEventAttributesH\x00\x12\x80\x01\n0child_workflow_execution_failed_event_attributes\x18& \x01(\x0b\x32\x44.temporal.api.history.v1.ChildWorkflowExecutionFailedEventAttributesH\x00\x12\x84\x01\n2child_workflow_execution_canceled_event_attributes\x18\' \x01(\x0b\x32\x46.temporal.api.history.v1.ChildWorkflowExecutionCanceledEventAttributesH\x00\x12\x85\x01\n3child_workflow_execution_timed_out_event_attributes\x18( \x01(\x0b\x32\x46.temporal.api.history.v1.ChildWorkflowExecutionTimedOutEventAttributesH\x00\x12\x88\x01\n4child_workflow_execution_terminated_event_attributes\x18) \x01(\x0b\x32H.temporal.api.history.v1.ChildWorkflowExecutionTerminatedEventAttributesH\x00\x12\x99\x01\n=signal_external_workflow_execution_initiated_event_attributes\x18* \x01(\x0b\x32P.temporal.api.history.v1.SignalExternalWorkflowExecutionInitiatedEventAttributesH\x00\x12\x93\x01\n:signal_external_workflow_execution_failed_event_attributes\x18+ \x01(\x0b\x32M.temporal.api.history.v1.SignalExternalWorkflowExecutionFailedEventAttributesH\x00\x12\x8a\x01\n5external_workflow_execution_signaled_event_attributes\x18, \x01(\x0b\x32I.temporal.api.history.v1.ExternalWorkflowExecutionSignaledEventAttributesH\x00\x12\x84\x01\n2upsert_workflow_search_attributes_event_attributes\x18- \x01(\x0b\x32\x46.temporal.api.history.v1.UpsertWorkflowSearchAttributesEventAttributesH\x00\x12\x86\x01\n3workflow_execution_update_accepted_event_attributes\x18. \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionUpdateAcceptedEventAttributesH\x00\x12\x86\x01\n3workflow_execution_update_rejected_event_attributes\x18/ \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionUpdateRejectedEventAttributesH\x00\x12\x88\x01\n4workflow_execution_update_completed_event_attributes\x18\x30 \x01(\x0b\x32H.temporal.api.history.v1.WorkflowExecutionUpdateCompletedEventAttributesH\x00\x12\x90\x01\n8workflow_properties_modified_externally_event_attributes\x18\x31 \x01(\x0b\x32L.temporal.api.history.v1.WorkflowPropertiesModifiedExternallyEventAttributesH\x00\x12\x90\x01\n8activity_properties_modified_externally_event_attributes\x18\x32 \x01(\x0b\x32L.temporal.api.history.v1.ActivityPropertiesModifiedExternallyEventAttributesH\x00\x12{\n-workflow_properties_modified_event_attributes\x18\x33 \x01(\x0b\x32\x42.temporal.api.history.v1.WorkflowPropertiesModifiedEventAttributesH\x00\x12\x86\x01\n3workflow_execution_update_admitted_event_attributes\x18\x34 \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionUpdateAdmittedEventAttributesH\x00\x12u\n*nexus_operation_scheduled_event_attributes\x18\x35 \x01(\x0b\x32?.temporal.api.history.v1.NexusOperationScheduledEventAttributesH\x00\x12q\n(nexus_operation_started_event_attributes\x18\x36 \x01(\x0b\x32=.temporal.api.history.v1.NexusOperationStartedEventAttributesH\x00\x12u\n*nexus_operation_completed_event_attributes\x18\x37 \x01(\x0b\x32?.temporal.api.history.v1.NexusOperationCompletedEventAttributesH\x00\x12o\n\'nexus_operation_failed_event_attributes\x18\x38 \x01(\x0b\x32<.temporal.api.history.v1.NexusOperationFailedEventAttributesH\x00\x12s\n)nexus_operation_canceled_event_attributes\x18\x39 \x01(\x0b\x32>.temporal.api.history.v1.NexusOperationCanceledEventAttributesH\x00\x12t\n*nexus_operation_timed_out_event_attributes\x18: \x01(\x0b\x32>.temporal.api.history.v1.NexusOperationTimedOutEventAttributesH\x00\x12\x82\x01\n1nexus_operation_cancel_requested_event_attributes\x18; \x01(\x0b\x32\x45.temporal.api.history.v1.NexusOperationCancelRequestedEventAttributesH\x00\x12\x86\x01\n3workflow_execution_options_updated_event_attributes\x18< \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionOptionsUpdatedEventAttributesH\x00\x12\x91\x01\n9nexus_operation_cancel_request_completed_event_attributes\x18= \x01(\x0b\x32L.temporal.api.history.v1.NexusOperationCancelRequestCompletedEventAttributesH\x00\x12\x8b\x01\n6nexus_operation_cancel_request_failed_event_attributes\x18> \x01(\x0b\x32I.temporal.api.history.v1.NexusOperationCancelRequestFailedEventAttributesH\x00\x12u\n*workflow_execution_paused_event_attributes\x18? \x01(\x0b\x32?.temporal.api.history.v1.WorkflowExecutionPausedEventAttributesH\x00\x12y\n,workflow_execution_unpaused_event_attributes\x18@ \x01(\x0b\x32\x41.temporal.api.history.v1.WorkflowExecutionUnpausedEventAttributesH\x00\x12\x9b\x01\n>workflow_execution_time_skipping_transitioned_event_attributes\x18\x41 \x01(\x0b\x32Q.temporal.api.history.v1.WorkflowExecutionTimeSkippingTransitionedEventAttributesH\x00\x42\x0c\n\nattributes"@\n\x07History\x12\x35\n\x06\x65vents\x18\x01 \x03(\x0b\x32%.temporal.api.history.v1.HistoryEventB\x8e\x01\n\x1aio.temporal.api.history.v1B\x0cMessageProtoP\x01Z%go.temporal.io/api/history/v1;history\xaa\x02\x19Temporalio.Api.History.V1\xea\x02\x1cTemporalio::Api::History::V1b\x06proto3' + b'\n%temporal/api/history/v1/message.proto\x12\x17temporal.api.history.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&temporal/api/enums/v1/event_type.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a"temporal/api/enums/v1/update.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a*temporal/api/notification/v1/message.proto\x1a(temporal/api/deployment/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/taskqueue/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x30temporal/api/sdk/v1/task_complete_metadata.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto"\xda\x12\n\'WorkflowExecutionStartedEventAttributes\x12;\n\rworkflow_type\x18\x01 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12!\n\x19parent_workflow_namespace\x18\x02 \x01(\t\x12$\n\x1cparent_workflow_namespace_id\x18\x1b \x01(\t\x12L\n\x19parent_workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12!\n\x19parent_initiated_event_id\x18\x04 \x01(\x03\x12\x38\n\ntask_queue\x18\x05 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x06 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12"\n\x1a\x63ontinued_execution_run_id\x18\n \x01(\t\x12@\n\tinitiator\x18\x0b \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\x0c \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\r \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12!\n\x19original_execution_run_id\x18\x0e \x01(\t\x12\x10\n\x08identity\x18\x0f \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x10 \x01(\t\x12\x39\n\x0cretry_policy\x18\x11 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x12 \x01(\x05\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rcron_schedule\x18\x14 \x01(\t\x12>\n\x1b\x66irst_workflow_task_backoff\x18\x15 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x16 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x17 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x45\n\x16prev_auto_reset_points\x18\x18 \x01(\x0b\x32%.temporal.api.workflow.v1.ResetPoints\x12.\n\x06header\x18\x19 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12&\n\x1eparent_initiated_event_version\x18\x1a \x01(\x03\x12\x13\n\x0bworkflow_id\x18\x1c \x01(\t\x12L\n\x14source_version_stamp\x18\x1d \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\x14\x63ompletion_callbacks\x18\x1e \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12J\n\x17root_workflow_execution\x18\x1f \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1e\n\x12inherited_build_id\x18 \x01(\tB\x02\x18\x01\x12I\n\x13versioning_override\x18! \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12\x33\n\'parent_pinned_worker_deployment_version\x18" \x01(\tB\x02\x18\x01\x12\x32\n\x08priority\x18# \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12U\n\x18inherited_pinned_version\x18% \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12Y\n\x1binherited_auto_upgrade_info\x18\' \x01(\x0b\x32\x34.temporal.api.deployment.v1.InheritedAutoUpgradeInfo\x12 \n\x18\x65\x61ger_execution_accepted\x18& \x01(\x08\x12^\n\x1f\x64\x65\x63lined_target_version_upgrade\x18( \x01(\x0b\x32\x35.temporal.api.history.v1.DeclinedTargetVersionUpgrade\x12H\n\x14time_skipping_config\x18) \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig\x12]\n\x1ftime_skipping_state_propagation\x18+ \x01(\x0b\x32\x34.temporal.api.common.v1.TimeSkippingStatePropagationJ\x04\x08$\x10%J\x04\x08*\x10+R parent_pinned_deployment_versionR\x18initial_skipped_duration"\x88\x01\n\x1c\x44\x65\x63linedTargetVersionUpgrade\x12O\n\x12\x64\x65ployment_version\x18\x01 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x17\n\x0frevision_number\x18\x02 \x01(\x03"\xa5\x01\n)WorkflowExecutionCompletedEventAttributes\x12\x30\n\x06result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x1c\n\x14new_execution_run_id\x18\x03 \x01(\t"\xdb\x01\n&WorkflowExecutionFailedEventAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x36\n\x0bretry_state\x18\x02 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState\x12(\n workflow_task_completed_event_id\x18\x03 \x01(\x03\x12\x1c\n\x14new_execution_run_id\x18\x04 \x01(\t"\x80\x01\n(WorkflowExecutionTimedOutEventAttributes\x12\x36\n\x0bretry_state\x18\x01 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState\x12\x1c\n\x14new_execution_run_id\x18\x02 \x01(\t"\xa5\x07\n.WorkflowExecutionContinuedAsNewEventAttributes\x12\x1c\n\x14new_execution_run_id\x18\x01 \x01(\t\x12;\n\rworkflow_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x03 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x37\n\x14workflow_run_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12(\n workflow_task_completed_event_id\x18\x07 \x01(\x03\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12@\n\tinitiator\x18\t \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12\x35\n\x07\x66\x61ilure\x18\n \x01(\x0b\x32 .temporal.api.failure.v1.FailureB\x02\x18\x01\x12@\n\x16last_completion_result\x18\x0b \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12.\n\x06header\x18\x0c \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12*\n\x04memo\x18\r \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0e \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x1c\n\x10inherit_build_id\x18\x0f \x01(\x08\x42\x02\x18\x01\x12[\n\x1binitial_versioning_behavior\x18\x10 \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior"\xef\x01\n$WorkflowTaskScheduledEventAttributes\x12\x38\n\ntask_queue\x18\x01 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x39\n\x16start_to_close_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x0f\n\x07\x61ttempt\x18\x03 \x01(\x05\x12\x41\n\rnotifications\x18\x04 \x03(\x0b\x32*.temporal.api.notification.v1.Notification"\xa0\x03\n"WorkflowTaskStartedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t\x12\x1f\n\x17suggest_continue_as_new\x18\x04 \x01(\x08\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x08 \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\t \x01(\x08\x12\x1a\n\x12history_size_bytes\x18\x05 \x01(\x03\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12%\n\x19\x62uild_id_redirect_counter\x18\x07 \x01(\x03\x42\x02\x18\x01"\xcd\x05\n$WorkflowTaskCompletedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x1b\n\x0f\x62inary_checksum\x18\x04 \x01(\tB\x02\x18\x01\x12\x46\n\x0eworker_version\x18\x05 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12H\n\x0csdk_metadata\x18\x06 \x01(\x0b\x32\x32.temporal.api.sdk.v1.WorkflowTaskCompletedMetadata\x12\x43\n\x11metering_metadata\x18\r \x01(\x0b\x32(.temporal.api.common.v1.MeteringMetadata\x12>\n\ndeployment\x18\x07 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12\x46\n\x13versioning_behavior\x18\x08 \x01(\x0e\x32).temporal.api.enums.v1.VersioningBehavior\x12%\n\x19worker_deployment_version\x18\t \x01(\tB\x02\x18\x01\x12\x1e\n\x16worker_deployment_name\x18\n \x01(\t\x12O\n\x12\x64\x65ployment_version\x18\x0b \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x43\n\x16\x63onsumed_stream_ranges\x18\x14 \x03(\x0b\x32#.temporal.api.stream.v1.StreamRangeJ\x04\x08\x0e\x10\x14"\x95\x01\n#WorkflowTaskTimedOutEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03\x12\x38\n\x0ctimeout_type\x18\x03 \x01(\x0e\x32".temporal.api.enums.v1.TimeoutType"\x87\x03\n!WorkflowTaskFailedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03\x12=\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowTaskFailedCause\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x13\n\x0b\x62\x61se_run_id\x18\x06 \x01(\t\x12\x12\n\nnew_run_id\x18\x07 \x01(\t\x12\x1a\n\x12\x66ork_event_version\x18\x08 \x01(\x03\x12\x1b\n\x0f\x62inary_checksum\x18\t \x01(\tB\x02\x18\x01\x12\x46\n\x0eworker_version\x18\n \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01"\xc2\x05\n$ActivityTaskScheduledEventAttributes\x12\x13\n\x0b\x61\x63tivity_id\x18\x01 \x01(\t\x12;\n\ractivity_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12.\n\x06header\x18\x05 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12/\n\x05input\x18\x06 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12<\n\x19schedule_to_close_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12(\n workflow_task_completed_event_id\x18\x0b \x01(\x03\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12!\n\x15use_workflow_build_id\x18\r \x01(\x08\x42\x02\x18\x01\x12\x32\n\x08priority\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.PriorityJ\x04\x08\x03\x10\x04"\x9e\x02\n"ActivityTaskStartedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x05\x12\x36\n\x0clast_failure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12%\n\x19\x62uild_id_redirect_counter\x18\x07 \x01(\x03\x42\x02\x18\x01"\xe8\x01\n$ActivityTaskCompletedEventAttributes\x12\x30\n\x06result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1a\n\x12scheduled_event_id\x18\x02 \x01(\x03\x12\x18\n\x10started_event_id\x18\x03 \x01(\x03\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x46\n\x0eworker_version\x18\x05 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01"\xdd\x02\n!ActivityTaskFailedEventAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x1a\n\x12scheduled_event_id\x18\x02 \x01(\x03\x12\x18\n\x10started_event_id\x18\x03 \x01(\x03\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x36\n\x0bretry_state\x18\x05 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12=\n\x05\x63\x61use\x18\x07 \x01(\x0e\x32..temporal.api.enums.v1.ActivityTaskFailedCause"\xc6\x01\n#ActivityTaskTimedOutEventAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x1a\n\x12scheduled_event_id\x18\x02 \x01(\x03\x12\x18\n\x10started_event_id\x18\x03 \x01(\x03\x12\x36\n\x0bretry_state\x18\x04 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState"r\n*ActivityTaskCancelRequestedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03"\x92\x02\n#ActivityTaskCanceledEventAttributes\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12(\n latest_cancel_requested_event_id\x18\x02 \x01(\x03\x12\x1a\n\x12scheduled_event_id\x18\x03 \x01(\x03\x12\x18\n\x10started_event_id\x18\x04 \x01(\x03\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01"\x93\x01\n\x1bTimerStartedEventAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12(\n workflow_task_completed_event_id\x18\x03 \x01(\x03"G\n\x19TimerFiredEventAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03"\x86\x01\n\x1cTimerCanceledEventAttributes\x12\x10\n\x08timer_id\x18\x01 \x01(\t\x12\x18\n\x10started_event_id\x18\x02 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x03 \x01(\x03\x12\x10\n\x08identity\x18\x04 \x01(\t"\xc7\x01\n/WorkflowExecutionCancelRequestedEventAttributes\x12\r\n\x05\x63\x61use\x18\x01 \x01(\t\x12#\n\x1b\x65xternal_initiated_event_id\x18\x02 \x01(\x03\x12N\n\x1b\x65xternal_workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x04 \x01(\t"\x87\x01\n(WorkflowExecutionCanceledEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x31\n\x07\x64\x65tails\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"\xe9\x02\n\x1dMarkerRecordedEventAttributes\x12\x13\n\x0bmarker_name\x18\x01 \x01(\t\x12T\n\x07\x64\x65tails\x18\x02 \x03(\x0b\x32\x43.temporal.api.history.v1.MarkerRecordedEventAttributes.DetailsEntry\x12(\n workflow_task_completed_event_id\x18\x03 \x01(\x03\x12.\n\x06header\x18\x04 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x1aP\n\x0c\x44\x65tailsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads:\x02\x38\x01"\xbf\x02\n(WorkflowExecutionSignaledEventAttributes\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12/\n\x05input\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t\x12.\n\x06header\x18\x04 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\'\n\x1bskip_generate_workflow_task\x18\x05 \x01(\x08\x42\x02\x18\x01\x12N\n\x1b\x65xternal_workflow_execution\x18\x06 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x12\n\nrequest_id\x18\x07 \x01(\t"\x81\x01\n*WorkflowExecutionTerminatedEventAttributes\x12\x0e\n\x06reason\x18\x01 \x01(\t\x12\x31\n\x07\x64\x65tails\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t"\x9c\x02\n>RequestCancelExternalWorkflowExecutionInitiatedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x07\x63ontrol\x18\x04 \x01(\tB\x02\x18\x01\x12\x1b\n\x13\x63hild_workflow_only\x18\x05 \x01(\x08\x12\x0e\n\x06reason\x18\x06 \x01(\t"\xda\x02\n;RequestCancelExternalWorkflowExecutionFailedEventAttributes\x12P\n\x05\x63\x61use\x18\x01 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x11\n\tnamespace\x18\x03 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x04 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x13\n\x07\x63ontrol\x18\x06 \x01(\tB\x02\x18\x01"\xc5\x01\n7ExternalWorkflowExecutionCancelRequestedEventAttributes\x12\x1a\n\x12initiated_event_id\x18\x01 \x01(\x03\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x04 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution"\xfb\x02\n7SignalExternalWorkflowExecutionInitiatedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\t \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x13\n\x07\x63ontrol\x18\x06 \x01(\tB\x02\x18\x01\x12\x1b\n\x13\x63hild_workflow_only\x18\x07 \x01(\x08\x12.\n\x06header\x18\x08 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header"\xd3\x02\n4SignalExternalWorkflowExecutionFailedEventAttributes\x12P\n\x05\x63\x61use\x18\x01 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x11\n\tnamespace\x18\x03 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x04 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x13\n\x07\x63ontrol\x18\x06 \x01(\tB\x02\x18\x01"\xd3\x01\n0ExternalWorkflowExecutionSignaledEventAttributes\x12\x1a\n\x12initiated_event_id\x18\x01 \x01(\x03\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x05 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x07\x63ontrol\x18\x04 \x01(\tB\x02\x18\x01"\x9e\x01\n-UpsertWorkflowSearchAttributesEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x43\n\x11search_attributes\x18\x02 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"\x8a\x01\n)WorkflowPropertiesModifiedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x33\n\rupserted_memo\x18\x02 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xfc\t\n3StartChildWorkflowExecutionInitiatedEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x12 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x45\n\x13parent_close_policy\x18\t \x01(\x0e\x32(.temporal.api.enums.v1.ParentClosePolicy\x12\x13\n\x07\x63ontrol\x18\n \x01(\tB\x02\x18\x01\x12(\n workflow_task_completed_event_id\x18\x0b \x01(\x03\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12.\n\x06header\x18\x0f \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12*\n\x04memo\x18\x10 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x1c\n\x10inherit_build_id\x18\x13 \x01(\x08\x42\x02\x18\x01\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12H\n\x14time_skipping_config\x18\x15 \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig\x12]\n\x1ftime_skipping_state_propagation\x18\x17 \x01(\x0b\x32\x34.temporal.api.common.v1.TimeSkippingStatePropagation\x12I\n\x13versioning_override\x18\x18 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverrideJ\x04\x08\x16\x10\x17R\x18initial_skipped_duration"\xd6\x02\n0StartChildWorkflowExecutionFailedEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x08 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12L\n\x05\x63\x61use\x18\x04 \x01(\x0e\x32=.temporal.api.enums.v1.StartChildWorkflowExecutionFailedCause\x12\x13\n\x07\x63ontrol\x18\x05 \x01(\tB\x02\x18\x01\x12\x1a\n\x12initiated_event_id\x18\x06 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x07 \x01(\x03"\xa7\x02\n,ChildWorkflowExecutionStartedEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x06 \x01(\t\x12\x1a\n\x12initiated_event_id\x18\x02 \x01(\x03\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x04 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12.\n\x06header\x18\x05 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header"\xc5\x02\n.ChildWorkflowExecutionCompletedEventAttributes\x12\x30\n\x06result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x04 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x18\n\x10started_event_id\x18\x06 \x01(\x03"\xfb\x02\n+ChildWorkflowExecutionFailedEventAttributes\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x08 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x04 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x18\n\x10started_event_id\x18\x06 \x01(\x03\x12\x36\n\x0bretry_state\x18\x07 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState"\xc5\x02\n-ChildWorkflowExecutionCanceledEventAttributes\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x03 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x04 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x05 \x01(\x03\x12\x18\n\x10started_event_id\x18\x06 \x01(\x03"\xca\x02\n-ChildWorkflowExecutionTimedOutEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x07 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x04 \x01(\x03\x12\x18\n\x10started_event_id\x18\x05 \x01(\x03\x12\x36\n\x0bretry_state\x18\x06 \x01(\x0e\x32!.temporal.api.enums.v1.RetryState"\x94\x02\n/ChildWorkflowExecutionTerminatedEventAttributes\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0cnamespace_id\x18\x06 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x1a\n\x12initiated_event_id\x18\x04 \x01(\x03\x12\x18\n\x10started_event_id\x18\x05 \x01(\x03"\xda\x05\n.WorkflowExecutionOptionsUpdatedEventAttributes\x12I\n\x13versioning_override\x18\x01 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12!\n\x19unset_versioning_override\x18\x02 \x01(\x08\x12\x1b\n\x13\x61ttached_request_id\x18\x03 \x01(\t\x12G\n\x1d\x61ttached_completion_callbacks\x18\x04 \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x32\n\x08priority\x18\x06 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12H\n\x14time_skipping_config\x18\x07 \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig\x12$\n\x1ctime_skipping_config_updated\x18\t \x01(\x08\x12\x84\x01\n\x17workflow_update_options\x18\x08 \x03(\x0b\x32\x63.temporal.api.history.v1.WorkflowExecutionOptionsUpdatedEventAttributes.WorkflowUpdateOptionsUpdate\x1a\x96\x01\n\x1bWorkflowUpdateOptionsUpdate\x12\x11\n\tupdate_id\x18\x01 \x01(\t\x12\x1b\n\x13\x61ttached_request_id\x18\x02 \x01(\t\x12G\n\x1d\x61ttached_completion_callbacks\x18\x03 \x03(\x0b\x32 .temporal.api.common.v1.Callback"\xc0\x02\n3WorkflowPropertiesModifiedExternallyEventAttributes\x12\x16\n\x0enew_task_queue\x18\x01 \x01(\t\x12<\n\x19new_workflow_task_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12;\n\x18new_workflow_run_timeout\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x41\n\x1enew_workflow_execution_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x33\n\rupserted_memo\x18\x05 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\x90\x01\n3ActivityPropertiesModifiedExternallyEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12=\n\x10new_retry_policy\x18\x02 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy"|\n\'WorkflowStreamSubscribedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x14\n\x0cstart_offset\x18\x03 \x01(\x03"q\n4WorkflowNotificationChannelSubscribedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x63hannel\x18\x02 \x01(\t"\x90\x01\n6WorkflowNotificationChannelUnsubscribedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x63hannel\x18\x02 \x01(\t\x12\x1b\n\x13subscribed_event_id\x18\x03 \x01(\x03"\x93\x01\n,WorkflowStreamRecordsAppendedEventAttributes\x12(\n workflow_task_completed_event_id\x18\x01 \x01(\x03\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x03\x12\x11\n\tto_offset\x18\x04 \x01(\x03"\xdc\x01\n.WorkflowExecutionUpdateAcceptedEventAttributes\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12#\n\x1b\x61\x63\x63\x65pted_request_message_id\x18\x02 \x01(\t\x12,\n$accepted_request_sequencing_event_id\x18\x03 \x01(\x03\x12\x39\n\x10\x61\x63\x63\x65pted_request\x18\x04 \x01(\x0b\x32\x1f.temporal.api.update.v1.Request"\xaa\x01\n/WorkflowExecutionUpdateCompletedEventAttributes\x12*\n\x04meta\x18\x01 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x19\n\x11\x61\x63\x63\x65pted_event_id\x18\x03 \x01(\x03\x12\x30\n\x07outcome\x18\x02 \x01(\x0b\x32\x1f.temporal.api.update.v1.Outcome"\x8f\x02\n.WorkflowExecutionUpdateRejectedEventAttributes\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12#\n\x1brejected_request_message_id\x18\x02 \x01(\t\x12,\n$rejected_request_sequencing_event_id\x18\x03 \x01(\x03\x12\x39\n\x10rejected_request\x18\x04 \x01(\x0b\x32\x1f.temporal.api.update.v1.Request\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xa4\x01\n.WorkflowExecutionUpdateAdmittedEventAttributes\x12\x30\n\x07request\x18\x01 \x01(\x0b\x32\x1f.temporal.api.update.v1.Request\x12@\n\x06origin\x18\x02 \x01(\x0e\x32\x30.temporal.api.enums.v1.UpdateAdmittedEventOrigin"^\n&WorkflowExecutionPausedEventAttributes\x12\x10\n\x08identity\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t"`\n(WorkflowExecutionUnpausedEventAttributes\x12\x10\n\x08identity\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t"\xc5\x01\n8WorkflowExecutionTimeSkippingTransitionedEventAttributes\x12/\n\x0btarget_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12#\n\x1b\x64isabled_after_fast_forward\x18\x02 \x01(\x08\x12\x33\n\x0fwall_clock_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\xb4\x04\n&NexusOperationScheduledEventAttributes\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x0f\n\x07service\x18\x02 \x01(\t\x12\x11\n\toperation\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x66\n\x0cnexus_header\x18\x06 \x03(\x0b\x32P.temporal.api.history.v1.NexusOperationScheduledEventAttributes.NexusHeaderEntry\x12(\n workflow_task_completed_event_id\x18\x07 \x01(\x03\x12\x12\n\nrequest_id\x18\x08 \x01(\t\x12\x13\n\x0b\x65ndpoint_id\x18\t \x01(\t\x12<\n\x19schedule_to_start_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"\x89\x01\n$NexusOperationStartedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x18\n\x0coperation_id\x18\x03 \x01(\tB\x02\x18\x01\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x17\n\x0foperation_token\x18\x05 \x01(\t"\x89\x01\n&NexusOperationCompletedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12/\n\x06result\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x12\n\nrequest_id\x18\x03 \x01(\t"\x88\x01\n#NexusOperationFailedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nrequest_id\x18\x03 \x01(\t"\x8a\x01\n%NexusOperationTimedOutEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nrequest_id\x18\x03 \x01(\t"\x8a\x01\n%NexusOperationCanceledEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nrequest_id\x18\x03 \x01(\t"t\n,NexusOperationCancelRequestedEventAttributes\x12\x1a\n\x12scheduled_event_id\x18\x01 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03"\x97\x01\n3NexusOperationCancelRequestCompletedEventAttributes\x12\x1a\n\x12requested_event_id\x18\x01 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x1a\n\x12scheduled_event_id\x18\x03 \x01(\x03"\xc7\x01\n0NexusOperationCancelRequestFailedEventAttributes\x12\x1a\n\x12requested_event_id\x18\x01 \x01(\x03\x12(\n workflow_task_completed_event_id\x18\x02 \x01(\x03\x12\x31\n\x07\x66\x61ilure\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x1a\n\x12scheduled_event_id\x18\x04 \x01(\x03"\xf6\x43\n\x0cHistoryEvent\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\x03\x12.\n\nevent_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x34\n\nevent_type\x18\x03 \x01(\x0e\x32 .temporal.api.enums.v1.EventType\x12\x0f\n\x07version\x18\x04 \x01(\x03\x12\x0f\n\x07task_id\x18\x05 \x01(\x03\x12\x1a\n\x11worker_may_ignore\x18\xac\x02 \x01(\x08\x12\x39\n\ruser_metadata\x18\xad\x02 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12,\n\x05links\x18\xae\x02 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x12\x35\n\tprincipal\x18\xaf\x02 \x01(\x0b\x32!.temporal.api.common.v1.Principal\x12\x43\n\x13\x65vent_group_markers\x18\xb0\x02 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12w\n+workflow_execution_started_event_attributes\x18\x06 \x01(\x0b\x32@.temporal.api.history.v1.WorkflowExecutionStartedEventAttributesH\x00\x12{\n-workflow_execution_completed_event_attributes\x18\x07 \x01(\x0b\x32\x42.temporal.api.history.v1.WorkflowExecutionCompletedEventAttributesH\x00\x12u\n*workflow_execution_failed_event_attributes\x18\x08 \x01(\x0b\x32?.temporal.api.history.v1.WorkflowExecutionFailedEventAttributesH\x00\x12z\n-workflow_execution_timed_out_event_attributes\x18\t \x01(\x0b\x32\x41.temporal.api.history.v1.WorkflowExecutionTimedOutEventAttributesH\x00\x12q\n(workflow_task_scheduled_event_attributes\x18\n \x01(\x0b\x32=.temporal.api.history.v1.WorkflowTaskScheduledEventAttributesH\x00\x12m\n&workflow_task_started_event_attributes\x18\x0b \x01(\x0b\x32;.temporal.api.history.v1.WorkflowTaskStartedEventAttributesH\x00\x12q\n(workflow_task_completed_event_attributes\x18\x0c \x01(\x0b\x32=.temporal.api.history.v1.WorkflowTaskCompletedEventAttributesH\x00\x12p\n(workflow_task_timed_out_event_attributes\x18\r \x01(\x0b\x32<.temporal.api.history.v1.WorkflowTaskTimedOutEventAttributesH\x00\x12k\n%workflow_task_failed_event_attributes\x18\x0e \x01(\x0b\x32:.temporal.api.history.v1.WorkflowTaskFailedEventAttributesH\x00\x12q\n(activity_task_scheduled_event_attributes\x18\x0f \x01(\x0b\x32=.temporal.api.history.v1.ActivityTaskScheduledEventAttributesH\x00\x12m\n&activity_task_started_event_attributes\x18\x10 \x01(\x0b\x32;.temporal.api.history.v1.ActivityTaskStartedEventAttributesH\x00\x12q\n(activity_task_completed_event_attributes\x18\x11 \x01(\x0b\x32=.temporal.api.history.v1.ActivityTaskCompletedEventAttributesH\x00\x12k\n%activity_task_failed_event_attributes\x18\x12 \x01(\x0b\x32:.temporal.api.history.v1.ActivityTaskFailedEventAttributesH\x00\x12p\n(activity_task_timed_out_event_attributes\x18\x13 \x01(\x0b\x32<.temporal.api.history.v1.ActivityTaskTimedOutEventAttributesH\x00\x12^\n\x1etimer_started_event_attributes\x18\x14 \x01(\x0b\x32\x34.temporal.api.history.v1.TimerStartedEventAttributesH\x00\x12Z\n\x1ctimer_fired_event_attributes\x18\x15 \x01(\x0b\x32\x32.temporal.api.history.v1.TimerFiredEventAttributesH\x00\x12~\n/activity_task_cancel_requested_event_attributes\x18\x16 \x01(\x0b\x32\x43.temporal.api.history.v1.ActivityTaskCancelRequestedEventAttributesH\x00\x12o\n\'activity_task_canceled_event_attributes\x18\x17 \x01(\x0b\x32<.temporal.api.history.v1.ActivityTaskCanceledEventAttributesH\x00\x12`\n\x1ftimer_canceled_event_attributes\x18\x18 \x01(\x0b\x32\x35.temporal.api.history.v1.TimerCanceledEventAttributesH\x00\x12\x62\n marker_recorded_event_attributes\x18\x19 \x01(\x0b\x32\x36.temporal.api.history.v1.MarkerRecordedEventAttributesH\x00\x12y\n,workflow_execution_signaled_event_attributes\x18\x1a \x01(\x0b\x32\x41.temporal.api.history.v1.WorkflowExecutionSignaledEventAttributesH\x00\x12}\n.workflow_execution_terminated_event_attributes\x18\x1b \x01(\x0b\x32\x43.temporal.api.history.v1.WorkflowExecutionTerminatedEventAttributesH\x00\x12\x88\x01\n4workflow_execution_cancel_requested_event_attributes\x18\x1c \x01(\x0b\x32H.temporal.api.history.v1.WorkflowExecutionCancelRequestedEventAttributesH\x00\x12y\n,workflow_execution_canceled_event_attributes\x18\x1d \x01(\x0b\x32\x41.temporal.api.history.v1.WorkflowExecutionCanceledEventAttributesH\x00\x12\xa8\x01\nErequest_cancel_external_workflow_execution_initiated_event_attributes\x18\x1e \x01(\x0b\x32W.temporal.api.history.v1.RequestCancelExternalWorkflowExecutionInitiatedEventAttributesH\x00\x12\xa2\x01\nBrequest_cancel_external_workflow_execution_failed_event_attributes\x18\x1f \x01(\x0b\x32T.temporal.api.history.v1.RequestCancelExternalWorkflowExecutionFailedEventAttributesH\x00\x12\x99\x01\n=external_workflow_execution_cancel_requested_event_attributes\x18 \x01(\x0b\x32P.temporal.api.history.v1.ExternalWorkflowExecutionCancelRequestedEventAttributesH\x00\x12\x87\x01\n4workflow_execution_continued_as_new_event_attributes\x18! \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionContinuedAsNewEventAttributesH\x00\x12\x91\x01\n9start_child_workflow_execution_initiated_event_attributes\x18" \x01(\x0b\x32L.temporal.api.history.v1.StartChildWorkflowExecutionInitiatedEventAttributesH\x00\x12\x8b\x01\n6start_child_workflow_execution_failed_event_attributes\x18# \x01(\x0b\x32I.temporal.api.history.v1.StartChildWorkflowExecutionFailedEventAttributesH\x00\x12\x82\x01\n1child_workflow_execution_started_event_attributes\x18$ \x01(\x0b\x32\x45.temporal.api.history.v1.ChildWorkflowExecutionStartedEventAttributesH\x00\x12\x86\x01\n3child_workflow_execution_completed_event_attributes\x18% \x01(\x0b\x32G.temporal.api.history.v1.ChildWorkflowExecutionCompletedEventAttributesH\x00\x12\x80\x01\n0child_workflow_execution_failed_event_attributes\x18& \x01(\x0b\x32\x44.temporal.api.history.v1.ChildWorkflowExecutionFailedEventAttributesH\x00\x12\x84\x01\n2child_workflow_execution_canceled_event_attributes\x18\' \x01(\x0b\x32\x46.temporal.api.history.v1.ChildWorkflowExecutionCanceledEventAttributesH\x00\x12\x85\x01\n3child_workflow_execution_timed_out_event_attributes\x18( \x01(\x0b\x32\x46.temporal.api.history.v1.ChildWorkflowExecutionTimedOutEventAttributesH\x00\x12\x88\x01\n4child_workflow_execution_terminated_event_attributes\x18) \x01(\x0b\x32H.temporal.api.history.v1.ChildWorkflowExecutionTerminatedEventAttributesH\x00\x12\x99\x01\n=signal_external_workflow_execution_initiated_event_attributes\x18* \x01(\x0b\x32P.temporal.api.history.v1.SignalExternalWorkflowExecutionInitiatedEventAttributesH\x00\x12\x93\x01\n:signal_external_workflow_execution_failed_event_attributes\x18+ \x01(\x0b\x32M.temporal.api.history.v1.SignalExternalWorkflowExecutionFailedEventAttributesH\x00\x12\x8a\x01\n5external_workflow_execution_signaled_event_attributes\x18, \x01(\x0b\x32I.temporal.api.history.v1.ExternalWorkflowExecutionSignaledEventAttributesH\x00\x12\x84\x01\n2upsert_workflow_search_attributes_event_attributes\x18- \x01(\x0b\x32\x46.temporal.api.history.v1.UpsertWorkflowSearchAttributesEventAttributesH\x00\x12\x86\x01\n3workflow_execution_update_accepted_event_attributes\x18. \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionUpdateAcceptedEventAttributesH\x00\x12\x86\x01\n3workflow_execution_update_rejected_event_attributes\x18/ \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionUpdateRejectedEventAttributesH\x00\x12\x88\x01\n4workflow_execution_update_completed_event_attributes\x18\x30 \x01(\x0b\x32H.temporal.api.history.v1.WorkflowExecutionUpdateCompletedEventAttributesH\x00\x12\x90\x01\n8workflow_properties_modified_externally_event_attributes\x18\x31 \x01(\x0b\x32L.temporal.api.history.v1.WorkflowPropertiesModifiedExternallyEventAttributesH\x00\x12\x90\x01\n8activity_properties_modified_externally_event_attributes\x18\x32 \x01(\x0b\x32L.temporal.api.history.v1.ActivityPropertiesModifiedExternallyEventAttributesH\x00\x12{\n-workflow_properties_modified_event_attributes\x18\x33 \x01(\x0b\x32\x42.temporal.api.history.v1.WorkflowPropertiesModifiedEventAttributesH\x00\x12\x86\x01\n3workflow_execution_update_admitted_event_attributes\x18\x34 \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionUpdateAdmittedEventAttributesH\x00\x12u\n*nexus_operation_scheduled_event_attributes\x18\x35 \x01(\x0b\x32?.temporal.api.history.v1.NexusOperationScheduledEventAttributesH\x00\x12q\n(nexus_operation_started_event_attributes\x18\x36 \x01(\x0b\x32=.temporal.api.history.v1.NexusOperationStartedEventAttributesH\x00\x12u\n*nexus_operation_completed_event_attributes\x18\x37 \x01(\x0b\x32?.temporal.api.history.v1.NexusOperationCompletedEventAttributesH\x00\x12o\n\'nexus_operation_failed_event_attributes\x18\x38 \x01(\x0b\x32<.temporal.api.history.v1.NexusOperationFailedEventAttributesH\x00\x12s\n)nexus_operation_canceled_event_attributes\x18\x39 \x01(\x0b\x32>.temporal.api.history.v1.NexusOperationCanceledEventAttributesH\x00\x12t\n*nexus_operation_timed_out_event_attributes\x18: \x01(\x0b\x32>.temporal.api.history.v1.NexusOperationTimedOutEventAttributesH\x00\x12\x82\x01\n1nexus_operation_cancel_requested_event_attributes\x18; \x01(\x0b\x32\x45.temporal.api.history.v1.NexusOperationCancelRequestedEventAttributesH\x00\x12\x86\x01\n3workflow_execution_options_updated_event_attributes\x18< \x01(\x0b\x32G.temporal.api.history.v1.WorkflowExecutionOptionsUpdatedEventAttributesH\x00\x12\x91\x01\n9nexus_operation_cancel_request_completed_event_attributes\x18= \x01(\x0b\x32L.temporal.api.history.v1.NexusOperationCancelRequestCompletedEventAttributesH\x00\x12\x8b\x01\n6nexus_operation_cancel_request_failed_event_attributes\x18> \x01(\x0b\x32I.temporal.api.history.v1.NexusOperationCancelRequestFailedEventAttributesH\x00\x12u\n*workflow_execution_paused_event_attributes\x18? \x01(\x0b\x32?.temporal.api.history.v1.WorkflowExecutionPausedEventAttributesH\x00\x12y\n,workflow_execution_unpaused_event_attributes\x18@ \x01(\x0b\x32\x41.temporal.api.history.v1.WorkflowExecutionUnpausedEventAttributesH\x00\x12\x9b\x01\n>workflow_execution_time_skipping_transitioned_event_attributes\x18\x41 \x01(\x0b\x32Q.temporal.api.history.v1.WorkflowExecutionTimeSkippingTransitionedEventAttributesH\x00\x12w\n+workflow_stream_subscribed_event_attributes\x18\x42 \x01(\x0b\x32@.temporal.api.history.v1.WorkflowStreamSubscribedEventAttributesH\x00\x12\x82\x01\n1workflow_stream_records_appended_event_attributes\x18\x43 \x01(\x0b\x32\x45.temporal.api.history.v1.WorkflowStreamRecordsAppendedEventAttributesH\x00\x12\x92\x01\n9workflow_notification_channel_subscribed_event_attributes\x18\x44 \x01(\x0b\x32M.temporal.api.history.v1.WorkflowNotificationChannelSubscribedEventAttributesH\x00\x12\x96\x01\n;workflow_notification_channel_unsubscribed_event_attributes\x18\x45 \x01(\x0b\x32O.temporal.api.history.v1.WorkflowNotificationChannelUnsubscribedEventAttributesH\x00\x42\x0c\n\nattributes"@\n\x07History\x12\x35\n\x06\x65vents\x18\x01 \x03(\x0b\x32%.temporal.api.history.v1.HistoryEventB\x8e\x01\n\x1aio.temporal.api.history.v1B\x0cMessageProtoP\x01Z%go.temporal.io/api/history/v1;history\xaa\x02\x19Temporalio.Api.History.V1\xea\x02\x1cTemporalio::Api::History::V1b\x06proto3' ) @@ -215,6 +221,22 @@ _ACTIVITYPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES = DESCRIPTOR.message_types_by_name[ "ActivityPropertiesModifiedExternallyEventAttributes" ] +_WORKFLOWSTREAMSUBSCRIBEDEVENTATTRIBUTES = DESCRIPTOR.message_types_by_name[ + "WorkflowStreamSubscribedEventAttributes" +] +_WORKFLOWNOTIFICATIONCHANNELSUBSCRIBEDEVENTATTRIBUTES = ( + DESCRIPTOR.message_types_by_name[ + "WorkflowNotificationChannelSubscribedEventAttributes" + ] +) +_WORKFLOWNOTIFICATIONCHANNELUNSUBSCRIBEDEVENTATTRIBUTES = ( + DESCRIPTOR.message_types_by_name[ + "WorkflowNotificationChannelUnsubscribedEventAttributes" + ] +) +_WORKFLOWSTREAMRECORDSAPPENDEDEVENTATTRIBUTES = DESCRIPTOR.message_types_by_name[ + "WorkflowStreamRecordsAppendedEventAttributes" +] _WORKFLOWEXECUTIONUPDATEACCEPTEDEVENTATTRIBUTES = DESCRIPTOR.message_types_by_name[ "WorkflowExecutionUpdateAcceptedEventAttributes" ] @@ -819,6 +841,54 @@ ) _sym_db.RegisterMessage(ActivityPropertiesModifiedExternallyEventAttributes) +WorkflowStreamSubscribedEventAttributes = _reflection.GeneratedProtocolMessageType( + "WorkflowStreamSubscribedEventAttributes", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWSTREAMSUBSCRIBEDEVENTATTRIBUTES, + "__module__": "temporalio.api.history.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.history.v1.WorkflowStreamSubscribedEventAttributes) + }, +) +_sym_db.RegisterMessage(WorkflowStreamSubscribedEventAttributes) + +WorkflowNotificationChannelSubscribedEventAttributes = ( + _reflection.GeneratedProtocolMessageType( + "WorkflowNotificationChannelSubscribedEventAttributes", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWNOTIFICATIONCHANNELSUBSCRIBEDEVENTATTRIBUTES, + "__module__": "temporalio.api.history.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.history.v1.WorkflowNotificationChannelSubscribedEventAttributes) + }, + ) +) +_sym_db.RegisterMessage(WorkflowNotificationChannelSubscribedEventAttributes) + +WorkflowNotificationChannelUnsubscribedEventAttributes = ( + _reflection.GeneratedProtocolMessageType( + "WorkflowNotificationChannelUnsubscribedEventAttributes", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWNOTIFICATIONCHANNELUNSUBSCRIBEDEVENTATTRIBUTES, + "__module__": "temporalio.api.history.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.history.v1.WorkflowNotificationChannelUnsubscribedEventAttributes) + }, + ) +) +_sym_db.RegisterMessage(WorkflowNotificationChannelUnsubscribedEventAttributes) + +WorkflowStreamRecordsAppendedEventAttributes = _reflection.GeneratedProtocolMessageType( + "WorkflowStreamRecordsAppendedEventAttributes", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWSTREAMRECORDSAPPENDEDEVENTATTRIBUTES, + "__module__": "temporalio.api.history.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.history.v1.WorkflowStreamRecordsAppendedEventAttributes) + }, +) +_sym_db.RegisterMessage(WorkflowStreamRecordsAppendedEventAttributes) + WorkflowExecutionUpdateAcceptedEventAttributes = ( _reflection.GeneratedProtocolMessageType( "WorkflowExecutionUpdateAcceptedEventAttributes", @@ -1210,140 +1280,148 @@ _NEXUSOPERATIONSTARTEDEVENTATTRIBUTES.fields_by_name[ "operation_id" ]._serialized_options = b"\030\001" - _WORKFLOWEXECUTIONSTARTEDEVENTATTRIBUTES._serialized_start = 663 - _WORKFLOWEXECUTIONSTARTEDEVENTATTRIBUTES._serialized_end = 3057 - _DECLINEDTARGETVERSIONUPGRADE._serialized_start = 3060 - _DECLINEDTARGETVERSIONUPGRADE._serialized_end = 3196 - _WORKFLOWEXECUTIONCOMPLETEDEVENTATTRIBUTES._serialized_start = 3199 - _WORKFLOWEXECUTIONCOMPLETEDEVENTATTRIBUTES._serialized_end = 3364 - _WORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = 3367 - _WORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 3586 - _WORKFLOWEXECUTIONTIMEDOUTEVENTATTRIBUTES._serialized_start = 3589 - _WORKFLOWEXECUTIONTIMEDOUTEVENTATTRIBUTES._serialized_end = 3717 - _WORKFLOWEXECUTIONCONTINUEDASNEWEVENTATTRIBUTES._serialized_start = 3720 - _WORKFLOWEXECUTIONCONTINUEDASNEWEVENTATTRIBUTES._serialized_end = 4653 - _WORKFLOWTASKSCHEDULEDEVENTATTRIBUTES._serialized_start = 4656 - _WORKFLOWTASKSCHEDULEDEVENTATTRIBUTES._serialized_end = 4828 - _WORKFLOWTASKSTARTEDEVENTATTRIBUTES._serialized_start = 4831 - _WORKFLOWTASKSTARTEDEVENTATTRIBUTES._serialized_end = 5247 - _WORKFLOWTASKCOMPLETEDEVENTATTRIBUTES._serialized_start = 5250 - _WORKFLOWTASKCOMPLETEDEVENTATTRIBUTES._serialized_end = 5892 - _WORKFLOWTASKTIMEDOUTEVENTATTRIBUTES._serialized_start = 5895 - _WORKFLOWTASKTIMEDOUTEVENTATTRIBUTES._serialized_end = 6044 - _WORKFLOWTASKFAILEDEVENTATTRIBUTES._serialized_start = 6047 - _WORKFLOWTASKFAILEDEVENTATTRIBUTES._serialized_end = 6438 - _ACTIVITYTASKSCHEDULEDEVENTATTRIBUTES._serialized_start = 6441 - _ACTIVITYTASKSCHEDULEDEVENTATTRIBUTES._serialized_end = 7147 - _ACTIVITYTASKSTARTEDEVENTATTRIBUTES._serialized_start = 7150 - _ACTIVITYTASKSTARTEDEVENTATTRIBUTES._serialized_end = 7436 - _ACTIVITYTASKCOMPLETEDEVENTATTRIBUTES._serialized_start = 7439 - _ACTIVITYTASKCOMPLETEDEVENTATTRIBUTES._serialized_end = 7671 - _ACTIVITYTASKFAILEDEVENTATTRIBUTES._serialized_start = 7674 - _ACTIVITYTASKFAILEDEVENTATTRIBUTES._serialized_end = 7960 - _ACTIVITYTASKTIMEDOUTEVENTATTRIBUTES._serialized_start = 7963 - _ACTIVITYTASKTIMEDOUTEVENTATTRIBUTES._serialized_end = 8161 - _ACTIVITYTASKCANCELREQUESTEDEVENTATTRIBUTES._serialized_start = 8163 - _ACTIVITYTASKCANCELREQUESTEDEVENTATTRIBUTES._serialized_end = 8277 - _ACTIVITYTASKCANCELEDEVENTATTRIBUTES._serialized_start = 8280 - _ACTIVITYTASKCANCELEDEVENTATTRIBUTES._serialized_end = 8554 - _TIMERSTARTEDEVENTATTRIBUTES._serialized_start = 8557 - _TIMERSTARTEDEVENTATTRIBUTES._serialized_end = 8704 - _TIMERFIREDEVENTATTRIBUTES._serialized_start = 8706 - _TIMERFIREDEVENTATTRIBUTES._serialized_end = 8777 - _TIMERCANCELEDEVENTATTRIBUTES._serialized_start = 8780 - _TIMERCANCELEDEVENTATTRIBUTES._serialized_end = 8914 - _WORKFLOWEXECUTIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_start = 8917 - _WORKFLOWEXECUTIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_end = 9116 - _WORKFLOWEXECUTIONCANCELEDEVENTATTRIBUTES._serialized_start = 9119 - _WORKFLOWEXECUTIONCANCELEDEVENTATTRIBUTES._serialized_end = 9254 - _MARKERRECORDEDEVENTATTRIBUTES._serialized_start = 9257 - _MARKERRECORDEDEVENTATTRIBUTES._serialized_end = 9618 - _MARKERRECORDEDEVENTATTRIBUTES_DETAILSENTRY._serialized_start = 9538 - _MARKERRECORDEDEVENTATTRIBUTES_DETAILSENTRY._serialized_end = 9618 - _WORKFLOWEXECUTIONSIGNALEDEVENTATTRIBUTES._serialized_start = 9621 - _WORKFLOWEXECUTIONSIGNALEDEVENTATTRIBUTES._serialized_end = 9940 - _WORKFLOWEXECUTIONTERMINATEDEVENTATTRIBUTES._serialized_start = 9943 - _WORKFLOWEXECUTIONTERMINATEDEVENTATTRIBUTES._serialized_end = 10072 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_start = 10075 + _WORKFLOWEXECUTIONSTARTEDEVENTATTRIBUTES._serialized_start = 745 + _WORKFLOWEXECUTIONSTARTEDEVENTATTRIBUTES._serialized_end = 3139 + _DECLINEDTARGETVERSIONUPGRADE._serialized_start = 3142 + _DECLINEDTARGETVERSIONUPGRADE._serialized_end = 3278 + _WORKFLOWEXECUTIONCOMPLETEDEVENTATTRIBUTES._serialized_start = 3281 + _WORKFLOWEXECUTIONCOMPLETEDEVENTATTRIBUTES._serialized_end = 3446 + _WORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = 3449 + _WORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 3668 + _WORKFLOWEXECUTIONTIMEDOUTEVENTATTRIBUTES._serialized_start = 3671 + _WORKFLOWEXECUTIONTIMEDOUTEVENTATTRIBUTES._serialized_end = 3799 + _WORKFLOWEXECUTIONCONTINUEDASNEWEVENTATTRIBUTES._serialized_start = 3802 + _WORKFLOWEXECUTIONCONTINUEDASNEWEVENTATTRIBUTES._serialized_end = 4735 + _WORKFLOWTASKSCHEDULEDEVENTATTRIBUTES._serialized_start = 4738 + _WORKFLOWTASKSCHEDULEDEVENTATTRIBUTES._serialized_end = 4977 + _WORKFLOWTASKSTARTEDEVENTATTRIBUTES._serialized_start = 4980 + _WORKFLOWTASKSTARTEDEVENTATTRIBUTES._serialized_end = 5396 + _WORKFLOWTASKCOMPLETEDEVENTATTRIBUTES._serialized_start = 5399 + _WORKFLOWTASKCOMPLETEDEVENTATTRIBUTES._serialized_end = 6116 + _WORKFLOWTASKTIMEDOUTEVENTATTRIBUTES._serialized_start = 6119 + _WORKFLOWTASKTIMEDOUTEVENTATTRIBUTES._serialized_end = 6268 + _WORKFLOWTASKFAILEDEVENTATTRIBUTES._serialized_start = 6271 + _WORKFLOWTASKFAILEDEVENTATTRIBUTES._serialized_end = 6662 + _ACTIVITYTASKSCHEDULEDEVENTATTRIBUTES._serialized_start = 6665 + _ACTIVITYTASKSCHEDULEDEVENTATTRIBUTES._serialized_end = 7371 + _ACTIVITYTASKSTARTEDEVENTATTRIBUTES._serialized_start = 7374 + _ACTIVITYTASKSTARTEDEVENTATTRIBUTES._serialized_end = 7660 + _ACTIVITYTASKCOMPLETEDEVENTATTRIBUTES._serialized_start = 7663 + _ACTIVITYTASKCOMPLETEDEVENTATTRIBUTES._serialized_end = 7895 + _ACTIVITYTASKFAILEDEVENTATTRIBUTES._serialized_start = 7898 + _ACTIVITYTASKFAILEDEVENTATTRIBUTES._serialized_end = 8247 + _ACTIVITYTASKTIMEDOUTEVENTATTRIBUTES._serialized_start = 8250 + _ACTIVITYTASKTIMEDOUTEVENTATTRIBUTES._serialized_end = 8448 + _ACTIVITYTASKCANCELREQUESTEDEVENTATTRIBUTES._serialized_start = 8450 + _ACTIVITYTASKCANCELREQUESTEDEVENTATTRIBUTES._serialized_end = 8564 + _ACTIVITYTASKCANCELEDEVENTATTRIBUTES._serialized_start = 8567 + _ACTIVITYTASKCANCELEDEVENTATTRIBUTES._serialized_end = 8841 + _TIMERSTARTEDEVENTATTRIBUTES._serialized_start = 8844 + _TIMERSTARTEDEVENTATTRIBUTES._serialized_end = 8991 + _TIMERFIREDEVENTATTRIBUTES._serialized_start = 8993 + _TIMERFIREDEVENTATTRIBUTES._serialized_end = 9064 + _TIMERCANCELEDEVENTATTRIBUTES._serialized_start = 9067 + _TIMERCANCELEDEVENTATTRIBUTES._serialized_end = 9201 + _WORKFLOWEXECUTIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_start = 9204 + _WORKFLOWEXECUTIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_end = 9403 + _WORKFLOWEXECUTIONCANCELEDEVENTATTRIBUTES._serialized_start = 9406 + _WORKFLOWEXECUTIONCANCELEDEVENTATTRIBUTES._serialized_end = 9541 + _MARKERRECORDEDEVENTATTRIBUTES._serialized_start = 9544 + _MARKERRECORDEDEVENTATTRIBUTES._serialized_end = 9905 + _MARKERRECORDEDEVENTATTRIBUTES_DETAILSENTRY._serialized_start = 9825 + _MARKERRECORDEDEVENTATTRIBUTES_DETAILSENTRY._serialized_end = 9905 + _WORKFLOWEXECUTIONSIGNALEDEVENTATTRIBUTES._serialized_start = 9908 + _WORKFLOWEXECUTIONSIGNALEDEVENTATTRIBUTES._serialized_end = 10227 + _WORKFLOWEXECUTIONTERMINATEDEVENTATTRIBUTES._serialized_start = 10230 + _WORKFLOWEXECUTIONTERMINATEDEVENTATTRIBUTES._serialized_end = 10359 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_start = 10362 _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_end = ( - 10359 + 10646 ) _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = ( - 10362 + 10649 ) - _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 10708 - _EXTERNALWORKFLOWEXECUTIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_start = 10711 - _EXTERNALWORKFLOWEXECUTIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_end = 10908 - _SIGNALEXTERNALWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_start = 10911 - _SIGNALEXTERNALWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_end = 11290 - _SIGNALEXTERNALWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = 11293 - _SIGNALEXTERNALWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 11632 - _EXTERNALWORKFLOWEXECUTIONSIGNALEDEVENTATTRIBUTES._serialized_start = 11635 - _EXTERNALWORKFLOWEXECUTIONSIGNALEDEVENTATTRIBUTES._serialized_end = 11846 - _UPSERTWORKFLOWSEARCHATTRIBUTESEVENTATTRIBUTES._serialized_start = 11849 - _UPSERTWORKFLOWSEARCHATTRIBUTESEVENTATTRIBUTES._serialized_end = 12007 - _WORKFLOWPROPERTIESMODIFIEDEVENTATTRIBUTES._serialized_start = 12010 - _WORKFLOWPROPERTIESMODIFIEDEVENTATTRIBUTES._serialized_end = 12148 - _STARTCHILDWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_start = 12151 - _STARTCHILDWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_end = 13427 - _STARTCHILDWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = 13430 - _STARTCHILDWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 13772 - _CHILDWORKFLOWEXECUTIONSTARTEDEVENTATTRIBUTES._serialized_start = 13775 - _CHILDWORKFLOWEXECUTIONSTARTEDEVENTATTRIBUTES._serialized_end = 14070 - _CHILDWORKFLOWEXECUTIONCOMPLETEDEVENTATTRIBUTES._serialized_start = 14073 - _CHILDWORKFLOWEXECUTIONCOMPLETEDEVENTATTRIBUTES._serialized_end = 14398 - _CHILDWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = 14401 - _CHILDWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 14780 - _CHILDWORKFLOWEXECUTIONCANCELEDEVENTATTRIBUTES._serialized_start = 14783 - _CHILDWORKFLOWEXECUTIONCANCELEDEVENTATTRIBUTES._serialized_end = 15108 - _CHILDWORKFLOWEXECUTIONTIMEDOUTEVENTATTRIBUTES._serialized_start = 15111 - _CHILDWORKFLOWEXECUTIONTIMEDOUTEVENTATTRIBUTES._serialized_end = 15441 - _CHILDWORKFLOWEXECUTIONTERMINATEDEVENTATTRIBUTES._serialized_start = 15444 - _CHILDWORKFLOWEXECUTIONTERMINATEDEVENTATTRIBUTES._serialized_end = 15720 - _WORKFLOWEXECUTIONOPTIONSUPDATEDEVENTATTRIBUTES._serialized_start = 15723 - _WORKFLOWEXECUTIONOPTIONSUPDATEDEVENTATTRIBUTES._serialized_end = 16453 - _WORKFLOWEXECUTIONOPTIONSUPDATEDEVENTATTRIBUTES_WORKFLOWUPDATEOPTIONSUPDATE._serialized_start = 16303 - _WORKFLOWEXECUTIONOPTIONSUPDATEDEVENTATTRIBUTES_WORKFLOWUPDATEOPTIONSUPDATE._serialized_end = 16453 - _WORKFLOWPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES._serialized_start = 16456 - _WORKFLOWPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES._serialized_end = 16776 - _ACTIVITYPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES._serialized_start = 16779 - _ACTIVITYPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES._serialized_end = 16923 - _WORKFLOWEXECUTIONUPDATEACCEPTEDEVENTATTRIBUTES._serialized_start = 16926 - _WORKFLOWEXECUTIONUPDATEACCEPTEDEVENTATTRIBUTES._serialized_end = 17146 - _WORKFLOWEXECUTIONUPDATECOMPLETEDEVENTATTRIBUTES._serialized_start = 17149 - _WORKFLOWEXECUTIONUPDATECOMPLETEDEVENTATTRIBUTES._serialized_end = 17319 - _WORKFLOWEXECUTIONUPDATEREJECTEDEVENTATTRIBUTES._serialized_start = 17322 - _WORKFLOWEXECUTIONUPDATEREJECTEDEVENTATTRIBUTES._serialized_end = 17593 - _WORKFLOWEXECUTIONUPDATEADMITTEDEVENTATTRIBUTES._serialized_start = 17596 - _WORKFLOWEXECUTIONUPDATEADMITTEDEVENTATTRIBUTES._serialized_end = 17760 - _WORKFLOWEXECUTIONPAUSEDEVENTATTRIBUTES._serialized_start = 17762 - _WORKFLOWEXECUTIONPAUSEDEVENTATTRIBUTES._serialized_end = 17856 - _WORKFLOWEXECUTIONUNPAUSEDEVENTATTRIBUTES._serialized_start = 17858 - _WORKFLOWEXECUTIONUNPAUSEDEVENTATTRIBUTES._serialized_end = 17954 - _WORKFLOWEXECUTIONTIMESKIPPINGTRANSITIONEDEVENTATTRIBUTES._serialized_start = 17957 - _WORKFLOWEXECUTIONTIMESKIPPINGTRANSITIONEDEVENTATTRIBUTES._serialized_end = 18154 - _NEXUSOPERATIONSCHEDULEDEVENTATTRIBUTES._serialized_start = 18157 - _NEXUSOPERATIONSCHEDULEDEVENTATTRIBUTES._serialized_end = 18721 - _NEXUSOPERATIONSCHEDULEDEVENTATTRIBUTES_NEXUSHEADERENTRY._serialized_start = 18671 - _NEXUSOPERATIONSCHEDULEDEVENTATTRIBUTES_NEXUSHEADERENTRY._serialized_end = 18721 - _NEXUSOPERATIONSTARTEDEVENTATTRIBUTES._serialized_start = 18724 - _NEXUSOPERATIONSTARTEDEVENTATTRIBUTES._serialized_end = 18861 - _NEXUSOPERATIONCOMPLETEDEVENTATTRIBUTES._serialized_start = 18864 - _NEXUSOPERATIONCOMPLETEDEVENTATTRIBUTES._serialized_end = 19001 - _NEXUSOPERATIONFAILEDEVENTATTRIBUTES._serialized_start = 19004 - _NEXUSOPERATIONFAILEDEVENTATTRIBUTES._serialized_end = 19140 - _NEXUSOPERATIONTIMEDOUTEVENTATTRIBUTES._serialized_start = 19143 - _NEXUSOPERATIONTIMEDOUTEVENTATTRIBUTES._serialized_end = 19281 - _NEXUSOPERATIONCANCELEDEVENTATTRIBUTES._serialized_start = 19284 - _NEXUSOPERATIONCANCELEDEVENTATTRIBUTES._serialized_end = 19422 - _NEXUSOPERATIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_start = 19424 - _NEXUSOPERATIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_end = 19540 - _NEXUSOPERATIONCANCELREQUESTCOMPLETEDEVENTATTRIBUTES._serialized_start = 19543 - _NEXUSOPERATIONCANCELREQUESTCOMPLETEDEVENTATTRIBUTES._serialized_end = 19694 - _NEXUSOPERATIONCANCELREQUESTFAILEDEVENTATTRIBUTES._serialized_start = 19697 - _NEXUSOPERATIONCANCELREQUESTFAILEDEVENTATTRIBUTES._serialized_end = 19896 - _HISTORYEVENT._serialized_start = 19899 - _HISTORYEVENT._serialized_end = 28037 - _HISTORY._serialized_start = 28039 - _HISTORY._serialized_end = 28103 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 10995 + _EXTERNALWORKFLOWEXECUTIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_start = 10998 + _EXTERNALWORKFLOWEXECUTIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_end = 11195 + _SIGNALEXTERNALWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_start = 11198 + _SIGNALEXTERNALWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_end = 11577 + _SIGNALEXTERNALWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = 11580 + _SIGNALEXTERNALWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 11919 + _EXTERNALWORKFLOWEXECUTIONSIGNALEDEVENTATTRIBUTES._serialized_start = 11922 + _EXTERNALWORKFLOWEXECUTIONSIGNALEDEVENTATTRIBUTES._serialized_end = 12133 + _UPSERTWORKFLOWSEARCHATTRIBUTESEVENTATTRIBUTES._serialized_start = 12136 + _UPSERTWORKFLOWSEARCHATTRIBUTESEVENTATTRIBUTES._serialized_end = 12294 + _WORKFLOWPROPERTIESMODIFIEDEVENTATTRIBUTES._serialized_start = 12297 + _WORKFLOWPROPERTIESMODIFIEDEVENTATTRIBUTES._serialized_end = 12435 + _STARTCHILDWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_start = 12438 + _STARTCHILDWORKFLOWEXECUTIONINITIATEDEVENTATTRIBUTES._serialized_end = 13714 + _STARTCHILDWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = 13717 + _STARTCHILDWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 14059 + _CHILDWORKFLOWEXECUTIONSTARTEDEVENTATTRIBUTES._serialized_start = 14062 + _CHILDWORKFLOWEXECUTIONSTARTEDEVENTATTRIBUTES._serialized_end = 14357 + _CHILDWORKFLOWEXECUTIONCOMPLETEDEVENTATTRIBUTES._serialized_start = 14360 + _CHILDWORKFLOWEXECUTIONCOMPLETEDEVENTATTRIBUTES._serialized_end = 14685 + _CHILDWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_start = 14688 + _CHILDWORKFLOWEXECUTIONFAILEDEVENTATTRIBUTES._serialized_end = 15067 + _CHILDWORKFLOWEXECUTIONCANCELEDEVENTATTRIBUTES._serialized_start = 15070 + _CHILDWORKFLOWEXECUTIONCANCELEDEVENTATTRIBUTES._serialized_end = 15395 + _CHILDWORKFLOWEXECUTIONTIMEDOUTEVENTATTRIBUTES._serialized_start = 15398 + _CHILDWORKFLOWEXECUTIONTIMEDOUTEVENTATTRIBUTES._serialized_end = 15728 + _CHILDWORKFLOWEXECUTIONTERMINATEDEVENTATTRIBUTES._serialized_start = 15731 + _CHILDWORKFLOWEXECUTIONTERMINATEDEVENTATTRIBUTES._serialized_end = 16007 + _WORKFLOWEXECUTIONOPTIONSUPDATEDEVENTATTRIBUTES._serialized_start = 16010 + _WORKFLOWEXECUTIONOPTIONSUPDATEDEVENTATTRIBUTES._serialized_end = 16740 + _WORKFLOWEXECUTIONOPTIONSUPDATEDEVENTATTRIBUTES_WORKFLOWUPDATEOPTIONSUPDATE._serialized_start = 16590 + _WORKFLOWEXECUTIONOPTIONSUPDATEDEVENTATTRIBUTES_WORKFLOWUPDATEOPTIONSUPDATE._serialized_end = 16740 + _WORKFLOWPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES._serialized_start = 16743 + _WORKFLOWPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES._serialized_end = 17063 + _ACTIVITYPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES._serialized_start = 17066 + _ACTIVITYPROPERTIESMODIFIEDEXTERNALLYEVENTATTRIBUTES._serialized_end = 17210 + _WORKFLOWSTREAMSUBSCRIBEDEVENTATTRIBUTES._serialized_start = 17212 + _WORKFLOWSTREAMSUBSCRIBEDEVENTATTRIBUTES._serialized_end = 17336 + _WORKFLOWNOTIFICATIONCHANNELSUBSCRIBEDEVENTATTRIBUTES._serialized_start = 17338 + _WORKFLOWNOTIFICATIONCHANNELSUBSCRIBEDEVENTATTRIBUTES._serialized_end = 17451 + _WORKFLOWNOTIFICATIONCHANNELUNSUBSCRIBEDEVENTATTRIBUTES._serialized_start = 17454 + _WORKFLOWNOTIFICATIONCHANNELUNSUBSCRIBEDEVENTATTRIBUTES._serialized_end = 17598 + _WORKFLOWSTREAMRECORDSAPPENDEDEVENTATTRIBUTES._serialized_start = 17601 + _WORKFLOWSTREAMRECORDSAPPENDEDEVENTATTRIBUTES._serialized_end = 17748 + _WORKFLOWEXECUTIONUPDATEACCEPTEDEVENTATTRIBUTES._serialized_start = 17751 + _WORKFLOWEXECUTIONUPDATEACCEPTEDEVENTATTRIBUTES._serialized_end = 17971 + _WORKFLOWEXECUTIONUPDATECOMPLETEDEVENTATTRIBUTES._serialized_start = 17974 + _WORKFLOWEXECUTIONUPDATECOMPLETEDEVENTATTRIBUTES._serialized_end = 18144 + _WORKFLOWEXECUTIONUPDATEREJECTEDEVENTATTRIBUTES._serialized_start = 18147 + _WORKFLOWEXECUTIONUPDATEREJECTEDEVENTATTRIBUTES._serialized_end = 18418 + _WORKFLOWEXECUTIONUPDATEADMITTEDEVENTATTRIBUTES._serialized_start = 18421 + _WORKFLOWEXECUTIONUPDATEADMITTEDEVENTATTRIBUTES._serialized_end = 18585 + _WORKFLOWEXECUTIONPAUSEDEVENTATTRIBUTES._serialized_start = 18587 + _WORKFLOWEXECUTIONPAUSEDEVENTATTRIBUTES._serialized_end = 18681 + _WORKFLOWEXECUTIONUNPAUSEDEVENTATTRIBUTES._serialized_start = 18683 + _WORKFLOWEXECUTIONUNPAUSEDEVENTATTRIBUTES._serialized_end = 18779 + _WORKFLOWEXECUTIONTIMESKIPPINGTRANSITIONEDEVENTATTRIBUTES._serialized_start = 18782 + _WORKFLOWEXECUTIONTIMESKIPPINGTRANSITIONEDEVENTATTRIBUTES._serialized_end = 18979 + _NEXUSOPERATIONSCHEDULEDEVENTATTRIBUTES._serialized_start = 18982 + _NEXUSOPERATIONSCHEDULEDEVENTATTRIBUTES._serialized_end = 19546 + _NEXUSOPERATIONSCHEDULEDEVENTATTRIBUTES_NEXUSHEADERENTRY._serialized_start = 19496 + _NEXUSOPERATIONSCHEDULEDEVENTATTRIBUTES_NEXUSHEADERENTRY._serialized_end = 19546 + _NEXUSOPERATIONSTARTEDEVENTATTRIBUTES._serialized_start = 19549 + _NEXUSOPERATIONSTARTEDEVENTATTRIBUTES._serialized_end = 19686 + _NEXUSOPERATIONCOMPLETEDEVENTATTRIBUTES._serialized_start = 19689 + _NEXUSOPERATIONCOMPLETEDEVENTATTRIBUTES._serialized_end = 19826 + _NEXUSOPERATIONFAILEDEVENTATTRIBUTES._serialized_start = 19829 + _NEXUSOPERATIONFAILEDEVENTATTRIBUTES._serialized_end = 19965 + _NEXUSOPERATIONTIMEDOUTEVENTATTRIBUTES._serialized_start = 19968 + _NEXUSOPERATIONTIMEDOUTEVENTATTRIBUTES._serialized_end = 20106 + _NEXUSOPERATIONCANCELEDEVENTATTRIBUTES._serialized_start = 20109 + _NEXUSOPERATIONCANCELEDEVENTATTRIBUTES._serialized_end = 20247 + _NEXUSOPERATIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_start = 20249 + _NEXUSOPERATIONCANCELREQUESTEDEVENTATTRIBUTES._serialized_end = 20365 + _NEXUSOPERATIONCANCELREQUESTCOMPLETEDEVENTATTRIBUTES._serialized_start = 20368 + _NEXUSOPERATIONCANCELREQUESTCOMPLETEDEVENTATTRIBUTES._serialized_end = 20519 + _NEXUSOPERATIONCANCELREQUESTFAILEDEVENTATTRIBUTES._serialized_start = 20522 + _NEXUSOPERATIONCANCELREQUESTFAILEDEVENTATTRIBUTES._serialized_end = 20721 + _HISTORYEVENT._serialized_start = 20724 + _HISTORYEVENT._serialized_end = 29418 + _HISTORY._serialized_start = 29420 + _HISTORY._serialized_end = 29484 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/history/v1/message_pb2.pyi b/temporalio/api/history/v1/message_pb2.pyi index 783e9d3b63..d8841ae956 100644 --- a/temporalio/api/history/v1/message_pb2.pyi +++ b/temporalio/api/history/v1/message_pb2.pyi @@ -20,9 +20,11 @@ import temporalio.api.enums.v1.failed_cause_pb2 import temporalio.api.enums.v1.update_pb2 import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 +import temporalio.api.notification.v1.message_pb2 import temporalio.api.sdk.v1.event_group_marker_pb2 import temporalio.api.sdk.v1.task_complete_metadata_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.taskqueue.v1.message_pb2 import temporalio.api.update.v1.message_pb2 import temporalio.api.workflow.v1.message_pb2 @@ -852,6 +854,7 @@ class WorkflowTaskScheduledEventAttributes(google.protobuf.message.Message): TASK_QUEUE_FIELD_NUMBER: builtins.int START_TO_CLOSE_TIMEOUT_FIELD_NUMBER: builtins.int ATTEMPT_FIELD_NUMBER: builtins.int + NOTIFICATIONS_FIELD_NUMBER: builtins.int @property def task_queue(self) -> temporalio.api.taskqueue.v1.message_pb2.TaskQueue: """The task queue this workflow task was enqueued in, which could be a normal or sticky queue""" @@ -864,12 +867,26 @@ class WorkflowTaskScheduledEventAttributes(google.protobuf.message.Message): """ attempt: builtins.int """Starting at 1, how many attempts there have been to complete this task""" + @property + def notifications( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.notification.v1.message_pb2.Notification + ]: + """Notifications for channels this Workflow listens to, folded per channel + since the last task was scheduled. In History so a Workflow may act on + them deterministically and replay sees the same. + """ def __init__( self, *, task_queue: temporalio.api.taskqueue.v1.message_pb2.TaskQueue | None = ..., start_to_close_timeout: google.protobuf.duration_pb2.Duration | None = ..., attempt: builtins.int = ..., + notifications: collections.abc.Iterable[ + temporalio.api.notification.v1.message_pb2.Notification + ] + | None = ..., ) -> None: ... def HasField( self, @@ -885,6 +902,8 @@ class WorkflowTaskScheduledEventAttributes(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "attempt", b"attempt", + "notifications", + b"notifications", "start_to_close_timeout", b"start_to_close_timeout", "task_queue", @@ -1010,6 +1029,7 @@ class WorkflowTaskCompletedEventAttributes(google.protobuf.message.Message): WORKER_DEPLOYMENT_VERSION_FIELD_NUMBER: builtins.int WORKER_DEPLOYMENT_NAME_FIELD_NUMBER: builtins.int DEPLOYMENT_VERSION_FIELD_NUMBER: builtins.int + CONSUMED_STREAM_RANGES_FIELD_NUMBER: builtins.int scheduled_event_id: builtins.int """The id of the `WORKFLOW_TASK_SCHEDULED` event this task corresponds to""" started_event_id: builtins.int @@ -1069,6 +1089,17 @@ class WorkflowTaskCompletedEventAttributes(google.protobuf.message.Message): """The Worker Deployment Version that completed this task. Must be set if `versioning_behavior` is set. This value updates workflow execution's `versioning_info.deployment_version`. """ + @property + def consumed_stream_ranges( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRange + ]: + """Offset ranges this Workflow Task consumed from streams it subscribes to. + Recorded on every task where a subscription is active, including when it + observed nothing: an empty range is a fact replay must reproduce, and + omitting it would let replay deliver records the Workflow did not have. + """ def __init__( self, *, @@ -1088,6 +1119,10 @@ class WorkflowTaskCompletedEventAttributes(google.protobuf.message.Message): worker_deployment_name: builtins.str = ..., deployment_version: temporalio.api.deployment.v1.message_pb2.WorkerDeploymentVersion | None = ..., + consumed_stream_ranges: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRange + ] + | None = ..., ) -> None: ... def HasField( self, @@ -1109,6 +1144,8 @@ class WorkflowTaskCompletedEventAttributes(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "binary_checksum", b"binary_checksum", + "consumed_stream_ranges", + b"consumed_stream_ranges", "deployment", b"deployment", "deployment_version", @@ -1555,6 +1592,7 @@ class ActivityTaskFailedEventAttributes(google.protobuf.message.Message): IDENTITY_FIELD_NUMBER: builtins.int RETRY_STATE_FIELD_NUMBER: builtins.int WORKER_VERSION_FIELD_NUMBER: builtins.int + CAUSE_FIELD_NUMBER: builtins.int @property def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: """Failure details""" @@ -1570,6 +1608,8 @@ class ActivityTaskFailedEventAttributes(google.protobuf.message.Message): """Version info of the worker who processed this workflow task. Deprecated. This field should be cleaned up when versioning-2 API is removed. [cleanup-experimental-wv] """ + cause: temporalio.api.enums.v1.failed_cause_pb2.ActivityTaskFailedCause.ValueType + """Why did the task fail? When unset, the failure is treated as an unspecified activity failure.""" def __init__( self, *, @@ -1580,6 +1620,7 @@ class ActivityTaskFailedEventAttributes(google.protobuf.message.Message): retry_state: temporalio.api.enums.v1.workflow_pb2.RetryState.ValueType = ..., worker_version: temporalio.api.common.v1.message_pb2.WorkerVersionStamp | None = ..., + cause: temporalio.api.enums.v1.failed_cause_pb2.ActivityTaskFailedCause.ValueType = ..., ) -> None: ... def HasField( self, @@ -1590,6 +1631,8 @@ class ActivityTaskFailedEventAttributes(google.protobuf.message.Message): def ClearField( self, field_name: typing_extensions.Literal[ + "cause", + b"cause", "failure", b"failure", "identity", @@ -3603,6 +3646,178 @@ global___ActivityPropertiesModifiedExternallyEventAttributes = ( ActivityPropertiesModifiedExternallyEventAttributes ) +class WorkflowStreamSubscribedEventAttributes(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WORKFLOW_TASK_COMPLETED_EVENT_ID_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + workflow_task_completed_event_id: builtins.int + """The WorkflowTaskCompleted event of the task whose command created this + subscription. + """ + stream_id: builtins.str + """The stream the Workflow subscribed to, as the command addressed it: + either the name of a stream this Workflow owns or the id of one in + another execution. + """ + start_offset: builtins.int + """The offset the subscription actually starts from. Resolved by the server + when the subscription is registered and recorded here, so replay reads + the resolved value rather than resolving it again against a stream that + has since moved. + """ + def __init__( + self, + *, + workflow_task_completed_event_id: builtins.int = ..., + stream_id: builtins.str = ..., + start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "start_offset", + b"start_offset", + "stream_id", + b"stream_id", + "workflow_task_completed_event_id", + b"workflow_task_completed_event_id", + ], + ) -> None: ... + +global___WorkflowStreamSubscribedEventAttributes = ( + WorkflowStreamSubscribedEventAttributes +) + +class WorkflowNotificationChannelSubscribedEventAttributes( + google.protobuf.message.Message +): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WORKFLOW_TASK_COMPLETED_EVENT_ID_FIELD_NUMBER: builtins.int + CHANNEL_FIELD_NUMBER: builtins.int + workflow_task_completed_event_id: builtins.int + """The WorkflowTaskCompleted event of the task whose command created this + subscription. + """ + channel: builtins.str + """The channel the Workflow listens on for the rest of this run.""" + def __init__( + self, + *, + workflow_task_completed_event_id: builtins.int = ..., + channel: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "channel", + b"channel", + "workflow_task_completed_event_id", + b"workflow_task_completed_event_id", + ], + ) -> None: ... + +global___WorkflowNotificationChannelSubscribedEventAttributes = ( + WorkflowNotificationChannelSubscribedEventAttributes +) + +class WorkflowNotificationChannelUnsubscribedEventAttributes( + google.protobuf.message.Message +): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WORKFLOW_TASK_COMPLETED_EVENT_ID_FIELD_NUMBER: builtins.int + CHANNEL_FIELD_NUMBER: builtins.int + SUBSCRIBED_EVENT_ID_FIELD_NUMBER: builtins.int + workflow_task_completed_event_id: builtins.int + """The WorkflowTaskCompleted event of the task whose command ended this + subscription. + """ + channel: builtins.str + """The channel the Workflow stopped listening on.""" + subscribed_event_id: builtins.int + """The WorkflowNotificationChannelSubscribed event that recorded the + subscription this command ended. Zero when the run held no subscription + for the channel. + """ + def __init__( + self, + *, + workflow_task_completed_event_id: builtins.int = ..., + channel: builtins.str = ..., + subscribed_event_id: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "channel", + b"channel", + "subscribed_event_id", + b"subscribed_event_id", + "workflow_task_completed_event_id", + b"workflow_task_completed_event_id", + ], + ) -> None: ... + +global___WorkflowNotificationChannelUnsubscribedEventAttributes = ( + WorkflowNotificationChannelUnsubscribedEventAttributes +) + +class WorkflowStreamRecordsAppendedEventAttributes(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WORKFLOW_TASK_COMPLETED_EVENT_ID_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + TO_OFFSET_FIELD_NUMBER: builtins.int + workflow_task_completed_event_id: builtins.int + """The WorkflowTaskCompleted event of the task whose command appended this + batch. + """ + stream_id: builtins.str + """Name of the stream the Workflow appended to.""" + from_offset: builtins.int + """Inclusive. Same range vocabulary as StreamRange and StreamSlice, so a + reader does not have to remember which of the three counts and which + bounds. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + """ + to_offset: builtins.int + """Exclusive. With from_offset this names the range without carrying any of + it, which is what keeps this event a fixed size no matter how large the + batch or its payloads are. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + """ + def __init__( + self, + *, + workflow_task_completed_event_id: builtins.int = ..., + stream_id: builtins.str = ..., + from_offset: builtins.int = ..., + to_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "stream_id", + b"stream_id", + "to_offset", + b"to_offset", + "workflow_task_completed_event_id", + b"workflow_task_completed_event_id", + ], + ) -> None: ... + +global___WorkflowStreamRecordsAppendedEventAttributes = ( + WorkflowStreamRecordsAppendedEventAttributes +) + class WorkflowExecutionUpdateAcceptedEventAttributes(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -4460,6 +4675,12 @@ class HistoryEvent(google.protobuf.message.Message): WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED_EVENT_ATTRIBUTES_FIELD_NUMBER: ( builtins.int ) + WORKFLOW_STREAM_SUBSCRIBED_EVENT_ATTRIBUTES_FIELD_NUMBER: builtins.int + WORKFLOW_STREAM_RECORDS_APPENDED_EVENT_ATTRIBUTES_FIELD_NUMBER: builtins.int + WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED_EVENT_ATTRIBUTES_FIELD_NUMBER: builtins.int + WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED_EVENT_ATTRIBUTES_FIELD_NUMBER: ( + builtins.int + ) event_id: builtins.int """Monotonically increasing event number, starts at 1.""" @property @@ -4745,6 +4966,22 @@ class HistoryEvent(google.protobuf.message.Message): def workflow_execution_time_skipping_transitioned_event_attributes( self, ) -> global___WorkflowExecutionTimeSkippingTransitionedEventAttributes: ... + @property + def workflow_stream_subscribed_event_attributes( + self, + ) -> global___WorkflowStreamSubscribedEventAttributes: ... + @property + def workflow_stream_records_appended_event_attributes( + self, + ) -> global___WorkflowStreamRecordsAppendedEventAttributes: ... + @property + def workflow_notification_channel_subscribed_event_attributes( + self, + ) -> global___WorkflowNotificationChannelSubscribedEventAttributes: ... + @property + def workflow_notification_channel_unsubscribed_event_attributes( + self, + ) -> global___WorkflowNotificationChannelUnsubscribedEventAttributes: ... def __init__( self, *, @@ -4882,6 +5119,14 @@ class HistoryEvent(google.protobuf.message.Message): | None = ..., workflow_execution_time_skipping_transitioned_event_attributes: global___WorkflowExecutionTimeSkippingTransitionedEventAttributes | None = ..., + workflow_stream_subscribed_event_attributes: global___WorkflowStreamSubscribedEventAttributes + | None = ..., + workflow_stream_records_appended_event_attributes: global___WorkflowStreamRecordsAppendedEventAttributes + | None = ..., + workflow_notification_channel_subscribed_event_attributes: global___WorkflowNotificationChannelSubscribedEventAttributes + | None = ..., + workflow_notification_channel_unsubscribed_event_attributes: global___WorkflowNotificationChannelUnsubscribedEventAttributes + | None = ..., ) -> None: ... def HasField( self, @@ -5000,10 +5245,18 @@ class HistoryEvent(google.protobuf.message.Message): b"workflow_execution_update_completed_event_attributes", "workflow_execution_update_rejected_event_attributes", b"workflow_execution_update_rejected_event_attributes", + "workflow_notification_channel_subscribed_event_attributes", + b"workflow_notification_channel_subscribed_event_attributes", + "workflow_notification_channel_unsubscribed_event_attributes", + b"workflow_notification_channel_unsubscribed_event_attributes", "workflow_properties_modified_event_attributes", b"workflow_properties_modified_event_attributes", "workflow_properties_modified_externally_event_attributes", b"workflow_properties_modified_externally_event_attributes", + "workflow_stream_records_appended_event_attributes", + b"workflow_stream_records_appended_event_attributes", + "workflow_stream_subscribed_event_attributes", + b"workflow_stream_subscribed_event_attributes", "workflow_task_completed_event_attributes", b"workflow_task_completed_event_attributes", "workflow_task_failed_event_attributes", @@ -5147,10 +5400,18 @@ class HistoryEvent(google.protobuf.message.Message): b"workflow_execution_update_completed_event_attributes", "workflow_execution_update_rejected_event_attributes", b"workflow_execution_update_rejected_event_attributes", + "workflow_notification_channel_subscribed_event_attributes", + b"workflow_notification_channel_subscribed_event_attributes", + "workflow_notification_channel_unsubscribed_event_attributes", + b"workflow_notification_channel_unsubscribed_event_attributes", "workflow_properties_modified_event_attributes", b"workflow_properties_modified_event_attributes", "workflow_properties_modified_externally_event_attributes", b"workflow_properties_modified_externally_event_attributes", + "workflow_stream_records_appended_event_attributes", + b"workflow_stream_records_appended_event_attributes", + "workflow_stream_subscribed_event_attributes", + b"workflow_stream_subscribed_event_attributes", "workflow_task_completed_event_attributes", b"workflow_task_completed_event_attributes", "workflow_task_failed_event_attributes", @@ -5227,6 +5488,10 @@ class HistoryEvent(google.protobuf.message.Message): "workflow_execution_paused_event_attributes", "workflow_execution_unpaused_event_attributes", "workflow_execution_time_skipping_transitioned_event_attributes", + "workflow_stream_subscribed_event_attributes", + "workflow_stream_records_appended_event_attributes", + "workflow_notification_channel_subscribed_event_attributes", + "workflow_notification_channel_unsubscribed_event_attributes", ] | None ): ... diff --git a/temporalio/api/nexusoperation/__init__.py b/temporalio/api/nexusoperation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/temporalio/api/nexusoperation/v1/__init__.py b/temporalio/api/nexusoperation/v1/__init__.py new file mode 100644 index 0000000000..6357003057 --- /dev/null +++ b/temporalio/api/nexusoperation/v1/__init__.py @@ -0,0 +1,6 @@ +from .message_pb2 import CallbackInfo, OnConflictOptions + +__all__ = [ + "CallbackInfo", + "OnConflictOptions", +] diff --git a/temporalio/api/nexusoperation/v1/message_pb2.py b/temporalio/api/nexusoperation/v1/message_pb2.py new file mode 100644 index 0000000000..a48e8d82ae --- /dev/null +++ b/temporalio/api/nexusoperation/v1/message_pb2.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporal/api/nexusoperation/v1/message.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.callback.v1 import ( + message_pb2 as temporal_dot_api_dot_callback_dot_v1_dot_message__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n,temporal/api/nexusoperation/v1/message.proto\x12\x1etemporal.api.nexusoperation.v1\x1a&temporal/api/callback/v1/message.proto"\x97\x02\n\x0c\x43\x61llbackInfo\x12\x45\n\x07trigger\x18\x01 \x01(\x0b\x32\x34.temporal.api.nexusoperation.v1.CallbackInfo.Trigger\x12\x34\n\x04info\x18\x02 \x01(\x0b\x32&.temporal.api.callback.v1.CallbackInfo\x1a\x14\n\x12OperationCompleted\x1at\n\x07Trigger\x12^\n\x13operation_completed\x18\x01 \x01(\x0b\x32?.temporal.api.nexusoperation.v1.CallbackInfo.OperationCompletedH\x00\x42\t\n\x07variant"i\n\x11OnConflictOptions\x12\x19\n\x11\x61ttach_request_id\x18\x01 \x01(\x08\x12#\n\x1b\x61ttach_completion_callbacks\x18\x02 \x01(\x08\x12\x14\n\x0c\x61ttach_links\x18\x03 \x01(\x08\x42\xb1\x01\n!io.temporal.api.nexusoperation.v1B\x0cMessageProtoP\x01Z3go.temporal.io/api/nexusoperation/v1;nexusoperation\xaa\x02 Temporalio.Api.NexusOperation.V1\xea\x02#Temporalio::Api::NexusOperation::V1b\x06proto3' +) + + +_CALLBACKINFO = DESCRIPTOR.message_types_by_name["CallbackInfo"] +_CALLBACKINFO_OPERATIONCOMPLETED = _CALLBACKINFO.nested_types_by_name[ + "OperationCompleted" +] +_CALLBACKINFO_TRIGGER = _CALLBACKINFO.nested_types_by_name["Trigger"] +_ONCONFLICTOPTIONS = DESCRIPTOR.message_types_by_name["OnConflictOptions"] +CallbackInfo = _reflection.GeneratedProtocolMessageType( + "CallbackInfo", + (_message.Message,), + { + "OperationCompleted": _reflection.GeneratedProtocolMessageType( + "OperationCompleted", + (_message.Message,), + { + "DESCRIPTOR": _CALLBACKINFO_OPERATIONCOMPLETED, + "__module__": "temporalio.api.nexusoperation.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.nexusoperation.v1.CallbackInfo.OperationCompleted) + }, + ), + "Trigger": _reflection.GeneratedProtocolMessageType( + "Trigger", + (_message.Message,), + { + "DESCRIPTOR": _CALLBACKINFO_TRIGGER, + "__module__": "temporalio.api.nexusoperation.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.nexusoperation.v1.CallbackInfo.Trigger) + }, + ), + "DESCRIPTOR": _CALLBACKINFO, + "__module__": "temporalio.api.nexusoperation.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.nexusoperation.v1.CallbackInfo) + }, +) +_sym_db.RegisterMessage(CallbackInfo) +_sym_db.RegisterMessage(CallbackInfo.OperationCompleted) +_sym_db.RegisterMessage(CallbackInfo.Trigger) + +OnConflictOptions = _reflection.GeneratedProtocolMessageType( + "OnConflictOptions", + (_message.Message,), + { + "DESCRIPTOR": _ONCONFLICTOPTIONS, + "__module__": "temporalio.api.nexusoperation.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.nexusoperation.v1.OnConflictOptions) + }, +) +_sym_db.RegisterMessage(OnConflictOptions) + +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b"\n!io.temporal.api.nexusoperation.v1B\014MessageProtoP\001Z3go.temporal.io/api/nexusoperation/v1;nexusoperation\252\002 Temporalio.Api.NexusOperation.V1\352\002#Temporalio::Api::NexusOperation::V1" + _CALLBACKINFO._serialized_start = 121 + _CALLBACKINFO._serialized_end = 400 + _CALLBACKINFO_OPERATIONCOMPLETED._serialized_start = 262 + _CALLBACKINFO_OPERATIONCOMPLETED._serialized_end = 282 + _CALLBACKINFO_TRIGGER._serialized_start = 284 + _CALLBACKINFO_TRIGGER._serialized_end = 400 + _ONCONFLICTOPTIONS._serialized_start = 402 + _ONCONFLICTOPTIONS._serialized_end = 507 +# @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/nexusoperation/v1/message_pb2.pyi b/temporalio/api/nexusoperation/v1/message_pb2.pyi new file mode 100644 index 0000000000..6b81093e7d --- /dev/null +++ b/temporalio/api/nexusoperation/v1/message_pb2.pyi @@ -0,0 +1,125 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import sys + +import google.protobuf.descriptor +import google.protobuf.message + +import temporalio.api.callback.v1.message_pb2 + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class CallbackInfo(google.protobuf.message.Message): + """CallbackInfo contains the state of a callback attached to a standalone Nexus operation.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class OperationCompleted(google.protobuf.message.Message): + """Trigger for when the Nexus operation is completed, covering both success cases as + well as any type of failure. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + + class Trigger(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + OPERATION_COMPLETED_FIELD_NUMBER: builtins.int + @property + def operation_completed(self) -> global___CallbackInfo.OperationCompleted: ... + def __init__( + self, + *, + operation_completed: global___CallbackInfo.OperationCompleted | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "operation_completed", b"operation_completed", "variant", b"variant" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "operation_completed", b"operation_completed", "variant", b"variant" + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["variant", b"variant"] + ) -> typing_extensions.Literal["operation_completed"] | None: ... + + TRIGGER_FIELD_NUMBER: builtins.int + INFO_FIELD_NUMBER: builtins.int + @property + def trigger(self) -> global___CallbackInfo.Trigger: + """Trigger for this callback.""" + @property + def info(self) -> temporalio.api.callback.v1.message_pb2.CallbackInfo: + """Common callback info.""" + def __init__( + self, + *, + trigger: global___CallbackInfo.Trigger | None = ..., + info: temporalio.api.callback.v1.message_pb2.CallbackInfo | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["info", b"info", "trigger", b"trigger"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["info", b"info", "trigger", b"trigger"], + ) -> None: ... + +global___CallbackInfo = CallbackInfo + +class OnConflictOptions(google.protobuf.message.Message): + """When StartNexusOperationExecutionRequest uses the conflict policy NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING + and there is already an existing, running standalone Nexus operation, OnConflictOptions defines actions to be + taken. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ATTACH_REQUEST_ID_FIELD_NUMBER: builtins.int + ATTACH_COMPLETION_CALLBACKS_FIELD_NUMBER: builtins.int + ATTACH_LINKS_FIELD_NUMBER: builtins.int + attach_request_id: builtins.bool + """Attaches the request ID to the running operation.""" + attach_completion_callbacks: builtins.bool + """Attaches the completion callbacks to the running operation.""" + attach_links: builtins.bool + """Attaches any new links to the running operation.""" + def __init__( + self, + *, + attach_request_id: builtins.bool = ..., + attach_completion_callbacks: builtins.bool = ..., + attach_links: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "attach_completion_callbacks", + b"attach_completion_callbacks", + "attach_links", + b"attach_links", + "attach_request_id", + b"attach_request_id", + ], + ) -> None: ... + +global___OnConflictOptions = OnConflictOptions diff --git a/temporalio/api/notification/__init__.py b/temporalio/api/notification/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/temporalio/api/notification/v1/__init__.py b/temporalio/api/notification/v1/__init__.py new file mode 100644 index 0000000000..5814ff02eb --- /dev/null +++ b/temporalio/api/notification/v1/__init__.py @@ -0,0 +1,8 @@ +from .message_pb2 import ChannelKind, ChannelListener, Notification, WorkflowListener + +__all__ = [ + "ChannelKind", + "ChannelListener", + "Notification", + "WorkflowListener", +] diff --git a/temporalio/api/notification/v1/message_pb2.py b/temporalio/api/notification/v1/message_pb2.py new file mode 100644 index 0000000000..cdcdb467d4 --- /dev/null +++ b/temporalio/api/notification/v1/message_pb2.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporal/api/notification/v1/message.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n*temporal/api/notification/v1/message.proto\x12\x1ctemporal.api.notification.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto"\x96\x02\n\x0cNotification\x12\x0f\n\x07\x63hannel\x18\x01 \x01(\t\x12\x10\n\x08position\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63ounter\x18\x03 \x01(\x03\x12J\n\x08metadata\x18\x04 \x03(\x0b\x32\x38.temporal.api.notification.v1.Notification.MetadataEntry\x12\x34\n\tlinked_to\x18\x05 \x01(\x0b\x32!.temporal.api.common.v1.Execution\x1aP\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\xe1\x01\n\x0f\x43hannelListener\x12\x13\n\x0blistener_id\x18\x01 \x01(\t\x12\x42\n\x08workflow\x18\x02 \x01(\x0b\x32..temporal.api.notification.v1.WorkflowListenerH\x00\x12\x34\n\x08\x63\x61llback\x18\x03 \x01(\x0b\x32 .temporal.api.common.v1.CallbackH\x00\x12\x33\n\x0fregistered_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\n\n\x08listener"7\n\x10WorkflowListener\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t*b\n\x0b\x43hannelKind\x12\x1c\n\x18\x43HANNEL_KIND_UNSPECIFIED\x10\x00\x12\x1c\n\x18\x43HANNEL_KIND_INDEPENDENT\x10\x01\x12\x17\n\x13\x43HANNEL_KIND_LINKED\x10\x02\x42\xa7\x01\n\x1fio.temporal.api.notification.v1B\x0cMessageProtoP\x01Z/go.temporal.io/api/notification/v1;notification\xaa\x02\x1eTemporalio.Api.Notification.V1\xea\x02!Temporalio::Api::Notification::V1b\x06proto3' +) + +_CHANNELKIND = DESCRIPTOR.enum_types_by_name["ChannelKind"] +ChannelKind = enum_type_wrapper.EnumTypeWrapper(_CHANNELKIND) +CHANNEL_KIND_UNSPECIFIED = 0 +CHANNEL_KIND_INDEPENDENT = 1 +CHANNEL_KIND_LINKED = 2 + + +_NOTIFICATION = DESCRIPTOR.message_types_by_name["Notification"] +_NOTIFICATION_METADATAENTRY = _NOTIFICATION.nested_types_by_name["MetadataEntry"] +_CHANNELLISTENER = DESCRIPTOR.message_types_by_name["ChannelListener"] +_WORKFLOWLISTENER = DESCRIPTOR.message_types_by_name["WorkflowListener"] +Notification = _reflection.GeneratedProtocolMessageType( + "Notification", + (_message.Message,), + { + "MetadataEntry": _reflection.GeneratedProtocolMessageType( + "MetadataEntry", + (_message.Message,), + { + "DESCRIPTOR": _NOTIFICATION_METADATAENTRY, + "__module__": "temporalio.api.notification.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.notification.v1.Notification.MetadataEntry) + }, + ), + "DESCRIPTOR": _NOTIFICATION, + "__module__": "temporalio.api.notification.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.notification.v1.Notification) + }, +) +_sym_db.RegisterMessage(Notification) +_sym_db.RegisterMessage(Notification.MetadataEntry) + +ChannelListener = _reflection.GeneratedProtocolMessageType( + "ChannelListener", + (_message.Message,), + { + "DESCRIPTOR": _CHANNELLISTENER, + "__module__": "temporalio.api.notification.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.notification.v1.ChannelListener) + }, +) +_sym_db.RegisterMessage(ChannelListener) + +WorkflowListener = _reflection.GeneratedProtocolMessageType( + "WorkflowListener", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWLISTENER, + "__module__": "temporalio.api.notification.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.notification.v1.WorkflowListener) + }, +) +_sym_db.RegisterMessage(WorkflowListener) + +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b"\n\037io.temporal.api.notification.v1B\014MessageProtoP\001Z/go.temporal.io/api/notification/v1;notification\252\002\036Temporalio.Api.Notification.V1\352\002!Temporalio::Api::Notification::V1" + _NOTIFICATION_METADATAENTRY._options = None + _NOTIFICATION_METADATAENTRY._serialized_options = b"8\001" + _CHANNELKIND._serialized_start = 713 + _CHANNELKIND._serialized_end = 811 + _NOTIFICATION._serialized_start = 148 + _NOTIFICATION._serialized_end = 426 + _NOTIFICATION_METADATAENTRY._serialized_start = 346 + _NOTIFICATION_METADATAENTRY._serialized_end = 426 + _CHANNELLISTENER._serialized_start = 429 + _CHANNELLISTENER._serialized_end = 654 + _WORKFLOWLISTENER._serialized_start = 656 + _WORKFLOWLISTENER._serialized_end = 711 +# @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/notification/v1/message_pb2.pyi b/temporalio/api/notification/v1/message_pb2.pyi new file mode 100644 index 0000000000..43c8070b6e --- /dev/null +++ b/temporalio/api/notification/v1/message_pb2.pyi @@ -0,0 +1,247 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import collections.abc +import sys +import typing + +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import google.protobuf.timestamp_pb2 + +import temporalio.api.common.v1.message_pb2 + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _ChannelKind: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _ChannelKindEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ChannelKind.ValueType], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + CHANNEL_KIND_UNSPECIFIED: _ChannelKind.ValueType # 0 + CHANNEL_KIND_INDEPENDENT: _ChannelKind.ValueType # 1 + """Its own execution, keyed by namespace and channel name. Any number of + workflows and callbacks listen to it. + """ + CHANNEL_KIND_LINKED: _ChannelKind.ValueType # 2 + """Kept in one execution's state, keyed by namespace, execution and + channel name. The owning execution is its listener by construction. + """ + +class ChannelKind(_ChannelKind, metaclass=_ChannelKindEnumTypeWrapper): + """Where a channel lives, which decides how a call addresses it.""" + +CHANNEL_KIND_UNSPECIFIED: ChannelKind.ValueType # 0 +CHANNEL_KIND_INDEPENDENT: ChannelKind.ValueType # 1 +"""Its own execution, keyed by namespace and channel name. Any number of +workflows and callbacks listen to it. +""" +CHANNEL_KIND_LINKED: ChannelKind.ValueType # 2 +"""Kept in one execution's state, keyed by namespace, execution and +channel name. The owning execution is its listener by construction. +""" +global___ChannelKind = ChannelKind + +class Notification(google.protobuf.message.Message): + """A notification tells the listeners of a channel that a source they consume + has moved. It is not data: the listener reads the source itself. A channel + is named by the writer and its listeners; for a stream, the provider formats + the stream's identity into the name. Writers never learn who listens. The + server folds notifications per listener while one is pending and no task has + been scheduled for it, keeping the one with the highest counter. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class MetadataEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: temporalio.api.common.v1.message_pb2.Payload | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["key", b"key", "value", b"value"], + ) -> None: ... + + CHANNEL_FIELD_NUMBER: builtins.int + POSITION_FIELD_NUMBER: builtins.int + COUNTER_FIELD_NUMBER: builtins.int + METADATA_FIELD_NUMBER: builtins.int + LINKED_TO_FIELD_NUMBER: builtins.int + channel: builtins.str + """The channel the writer notified. Listeners register on the same name.""" + position: builtins.bytes + """Where the source stands after the write that caused this notification, + in the writer's terms. Opaque to the server. + """ + counter: builtins.int + """Orders notifications from one channel's writers. The writer derives it + from the position, since only the source can order its positions. Among + notifications folded together, the one with the highest counter is kept. + """ + @property + def metadata( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, temporalio.api.common.v1.message_pb2.Payload + ]: + """Details for the listener, such as which topic moved. Bounded in size and + carried as payloads, so a codec applies as to any payload. This is state, + not a log: a fold keeps the latest notification only, so a writer puts + here what is true at `position`, such as which topic moved or a close + flag, never something a consumer must see once per write. + """ + @property + def linked_to(self) -> temporalio.api.common.v1.message_pb2.Execution: + """Set for a channel linked to an execution: the owner and the run that + received the notification. Empty for an independent channel. A listener + that holds both kinds routes the notification by it. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "to" names the owner the channel is linked to. --) + """ + def __init__( + self, + *, + channel: builtins.str = ..., + position: builtins.bytes = ..., + counter: builtins.int = ..., + metadata: collections.abc.Mapping[ + builtins.str, temporalio.api.common.v1.message_pb2.Payload + ] + | None = ..., + linked_to: temporalio.api.common.v1.message_pb2.Execution | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["linked_to", b"linked_to"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "channel", + b"channel", + "counter", + b"counter", + "linked_to", + b"linked_to", + "metadata", + b"metadata", + "position", + b"position", + ], + ) -> None: ... + +global___Notification = Notification + +class ChannelListener(google.protobuf.message.Message): + """A listener of a channel: a Workflow Execution woken with a Workflow Task, or + a callback the server invokes with each notification. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LISTENER_ID_FIELD_NUMBER: builtins.int + WORKFLOW_FIELD_NUMBER: builtins.int + CALLBACK_FIELD_NUMBER: builtins.int + REGISTERED_TIME_FIELD_NUMBER: builtins.int + listener_id: builtins.str + """Assigned by the server when the listener registers.""" + @property + def workflow(self) -> global___WorkflowListener: ... + @property + def callback(self) -> temporalio.api.common.v1.message_pb2.Callback: ... + @property + def registered_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + def __init__( + self, + *, + listener_id: builtins.str = ..., + workflow: global___WorkflowListener | None = ..., + callback: temporalio.api.common.v1.message_pb2.Callback | None = ..., + registered_time: google.protobuf.timestamp_pb2.Timestamp | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "callback", + b"callback", + "listener", + b"listener", + "registered_time", + b"registered_time", + "workflow", + b"workflow", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "callback", + b"callback", + "listener", + b"listener", + "listener_id", + b"listener_id", + "registered_time", + b"registered_time", + "workflow", + b"workflow", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["listener", b"listener"] + ) -> typing_extensions.Literal["workflow", "callback"] | None: ... + +global___ChannelListener = ChannelListener + +class WorkflowListener(google.protobuf.message.Message): + """A Workflow Execution listening on a channel.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WORKFLOW_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + workflow_id: builtins.str + run_id: builtins.str + """The run that subscribed. The server follows a continue-as-new to the + chain's current run when it delivers. + """ + def __init__( + self, + *, + workflow_id: builtins.str = ..., + run_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "run_id", b"run_id", "workflow_id", b"workflow_id" + ], + ) -> None: ... + +global___WorkflowListener = WorkflowListener diff --git a/temporalio/api/notificationservice/__init__.py b/temporalio/api/notificationservice/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/temporalio/api/notificationservice/v1/__init__.py b/temporalio/api/notificationservice/v1/__init__.py new file mode 100644 index 0000000000..7b51b0a83a --- /dev/null +++ b/temporalio/api/notificationservice/v1/__init__.py @@ -0,0 +1,6 @@ +from .request_response_pb2 import OnCompleteRequest, OnCompleteResponse + +__all__ = [ + "OnCompleteRequest", + "OnCompleteResponse", +] diff --git a/temporalio/api/notificationservice/v1/request_response_pb2.py b/temporalio/api/notificationservice/v1/request_response_pb2.py new file mode 100644 index 0000000000..0f7296d1bb --- /dev/null +++ b/temporalio/api/notificationservice/v1/request_response_pb2.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporal/api/notificationservice/v1/request_response.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) +from temporalio.api.failure.v1 import ( + message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n:temporal/api/notificationservice/v1/request_response.proto\x12#temporal.api.notificationservice.v1\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto"\xbf\x01\n\x11OnCompleteRequest\x12\x32\n\x07success\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x12\x33\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x37\n\x0esource_context\x18\x03 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadB\x08\n\x06result"\x14\n\x12OnCompleteResponseB\xd2\x01\n&io.temporal.api.notificationservice.v1B\x14RequestResponseProtoP\x01Z=go.temporal.io/api/notificationservice/v1;notificationservice\xaa\x02%Temporalio.Api.NotificationService.V1\xea\x02(Temporalio::Api::NotificationService::V1b\x06proto3' +) + + +_ONCOMPLETEREQUEST = DESCRIPTOR.message_types_by_name["OnCompleteRequest"] +_ONCOMPLETERESPONSE = DESCRIPTOR.message_types_by_name["OnCompleteResponse"] +OnCompleteRequest = _reflection.GeneratedProtocolMessageType( + "OnCompleteRequest", + (_message.Message,), + { + "DESCRIPTOR": _ONCOMPLETEREQUEST, + "__module__": "temporalio.api.notificationservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.notificationservice.v1.OnCompleteRequest) + }, +) +_sym_db.RegisterMessage(OnCompleteRequest) + +OnCompleteResponse = _reflection.GeneratedProtocolMessageType( + "OnCompleteResponse", + (_message.Message,), + { + "DESCRIPTOR": _ONCOMPLETERESPONSE, + "__module__": "temporalio.api.notificationservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.notificationservice.v1.OnCompleteResponse) + }, +) +_sym_db.RegisterMessage(OnCompleteResponse) + +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b"\n&io.temporal.api.notificationservice.v1B\024RequestResponseProtoP\001Z=go.temporal.io/api/notificationservice/v1;notificationservice\252\002%Temporalio.Api.NotificationService.V1\352\002(Temporalio::Api::NotificationService::V1" + _ONCOMPLETEREQUEST._serialized_start = 177 + _ONCOMPLETEREQUEST._serialized_end = 368 + _ONCOMPLETERESPONSE._serialized_start = 370 + _ONCOMPLETERESPONSE._serialized_end = 390 +# @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/notificationservice/v1/request_response_pb2.pyi b/temporalio/api/notificationservice/v1/request_response_pb2.pyi new file mode 100644 index 0000000000..f83768a3cb --- /dev/null +++ b/temporalio/api/notificationservice/v1/request_response_pb2.pyi @@ -0,0 +1,94 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import sys + +import google.protobuf.descriptor +import google.protobuf.message + +import temporalio.api.common.v1.message_pb2 +import temporalio.api.failure.v1.message_pb2 + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class OnCompleteRequest(google.protobuf.message.Message): + """OnCompleteRequest is the request type to the NotificationService's OnComplete operation, + allowing for defining completion handlers for arbitrary operations. + + Information about the source operation will be available in the form of a commonpb.Link, + which will be available separately from this OnCompleteRequest. e.g. a link to the source + standalone Nexus operation would be found in the nexuspb.StartOperationRequest parameter + sent to the worker callback. (In addition to this OnCompleteRequest.) + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SUCCESS_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + SOURCE_CONTEXT_FIELD_NUMBER: builtins.int + @property + def success(self) -> temporalio.api.common.v1.message_pb2.Payload: + """The operation was successful, and resulted in the given payload.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: + """The operation failed. Includes timeout, cancellation, and application errors.""" + @property + def source_context(self) -> temporalio.api.common.v1.message_pb2.Payload: + """User-supplied data which was added to the source invocation. (As applicable.)""" + def __init__( + self, + *, + success: temporalio.api.common.v1.message_pb2.Payload | None = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + source_context: temporalio.api.common.v1.message_pb2.Payload | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "failure", + b"failure", + "result", + b"result", + "source_context", + b"source_context", + "success", + b"success", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "failure", + b"failure", + "result", + b"result", + "source_context", + b"source_context", + "success", + b"success", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["result", b"result"] + ) -> typing_extensions.Literal["success", "failure"] | None: ... + +global___OnCompleteRequest = OnCompleteRequest + +class OnCompleteResponse(google.protobuf.message.Message): + """OnCompleteResponse is the return type of the OnComplete operation.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___OnCompleteResponse = OnCompleteResponse diff --git a/temporalio/api/notificationservice/v1/request_response_pb2_grpc.py b/temporalio/api/notificationservice/v1/request_response_pb2_grpc.py new file mode 100644 index 0000000000..bf947056a2 --- /dev/null +++ b/temporalio/api/notificationservice/v1/request_response_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc diff --git a/temporalio/api/notificationservice/v1/request_response_pb2_grpc.pyi b/temporalio/api/notificationservice/v1/request_response_pb2_grpc.pyi new file mode 100644 index 0000000000..f3a5a087ef --- /dev/null +++ b/temporalio/api/notificationservice/v1/request_response_pb2_grpc.pyi @@ -0,0 +1,4 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" diff --git a/temporalio/api/replication/v1/message_pb2.py b/temporalio/api/replication/v1/message_pb2.py index 8dbe73e1d1..b77c510320 100644 --- a/temporalio/api/replication/v1/message_pb2.py +++ b/temporalio/api/replication/v1/message_pb2.py @@ -14,6 +14,7 @@ _sym_db = _symbol_database.Default() +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 from temporalio.api.enums.v1 import ( @@ -21,7 +22,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n)temporal/api/replication/v1/message.proto\x12\x1btemporal.api.replication.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/enums/v1/namespace.proto"0\n\x18\x43lusterReplicationConfig\x12\x14\n\x0c\x63luster_name\x18\x01 \x01(\t"\xba\x01\n\x1aNamespaceReplicationConfig\x12\x1b\n\x13\x61\x63tive_cluster_name\x18\x01 \x01(\t\x12G\n\x08\x63lusters\x18\x02 \x03(\x0b\x32\x35.temporal.api.replication.v1.ClusterReplicationConfig\x12\x36\n\x05state\x18\x03 \x01(\x0e\x32\'.temporal.api.enums.v1.ReplicationState"]\n\x0e\x46\x61iloverStatus\x12\x31\n\rfailover_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x18\n\x10\x66\x61ilover_version\x18\x02 \x01(\x03\x42\xa2\x01\n\x1eio.temporal.api.replication.v1B\x0cMessageProtoP\x01Z-go.temporal.io/api/replication/v1;replication\xaa\x02\x1dTemporalio.Api.Replication.V1\xea\x02 Temporalio::Api::Replication::V1b\x06proto3' + b'\n)temporal/api/replication/v1/message.proto\x12\x1btemporal.api.replication.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/enums/v1/namespace.proto"n\n\x18\x43lusterReplicationConfig\x12\x14\n\x0c\x63luster_name\x18\x01 \x01(\t\x12<\n\x19replication_ramp_duration\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\xba\x01\n\x1aNamespaceReplicationConfig\x12\x1b\n\x13\x61\x63tive_cluster_name\x18\x01 \x01(\t\x12G\n\x08\x63lusters\x18\x02 \x03(\x0b\x32\x35.temporal.api.replication.v1.ClusterReplicationConfig\x12\x36\n\x05state\x18\x03 \x01(\x0e\x32\'.temporal.api.enums.v1.ReplicationState"]\n\x0e\x46\x61iloverStatus\x12\x31\n\rfailover_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x18\n\x10\x66\x61ilover_version\x18\x02 \x01(\x03\x42\xa2\x01\n\x1eio.temporal.api.replication.v1B\x0cMessageProtoP\x01Z-go.temporal.io/api/replication/v1;replication\xaa\x02\x1dTemporalio.Api.Replication.V1\xea\x02 Temporalio::Api::Replication::V1b\x06proto3' ) @@ -66,10 +67,10 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\036io.temporal.api.replication.v1B\014MessageProtoP\001Z-go.temporal.io/api/replication/v1;replication\252\002\035Temporalio.Api.Replication.V1\352\002 Temporalio::Api::Replication::V1" - _CLUSTERREPLICATIONCONFIG._serialized_start = 146 - _CLUSTERREPLICATIONCONFIG._serialized_end = 194 - _NAMESPACEREPLICATIONCONFIG._serialized_start = 197 - _NAMESPACEREPLICATIONCONFIG._serialized_end = 383 - _FAILOVERSTATUS._serialized_start = 385 - _FAILOVERSTATUS._serialized_end = 478 + _CLUSTERREPLICATIONCONFIG._serialized_start = 178 + _CLUSTERREPLICATIONCONFIG._serialized_end = 288 + _NAMESPACEREPLICATIONCONFIG._serialized_start = 291 + _NAMESPACEREPLICATIONCONFIG._serialized_end = 477 + _FAILOVERSTATUS._serialized_start = 479 + _FAILOVERSTATUS._serialized_end = 572 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/replication/v1/message_pb2.pyi b/temporalio/api/replication/v1/message_pb2.pyi index e6f04334fa..d3393bb584 100644 --- a/temporalio/api/replication/v1/message_pb2.pyi +++ b/temporalio/api/replication/v1/message_pb2.pyi @@ -8,6 +8,7 @@ import collections.abc import sys import google.protobuf.descriptor +import google.protobuf.duration_pb2 import google.protobuf.internal.containers import google.protobuf.message import google.protobuf.timestamp_pb2 @@ -25,14 +26,33 @@ class ClusterReplicationConfig(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor CLUSTER_NAME_FIELD_NUMBER: builtins.int + REPLICATION_RAMP_DURATION_FIELD_NUMBER: builtins.int cluster_name: builtins.str + @property + def replication_ramp_duration(self) -> google.protobuf.duration_pb2.Duration: + """Ramp duration when this cluster is added as passive by UpdateNamespace; unset or non-positive disables gradual connect. + This field is not persisted and is omitted from namespace responses. + """ def __init__( self, *, cluster_name: builtins.str = ..., + replication_ramp_duration: google.protobuf.duration_pb2.Duration | None = ..., ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "replication_ramp_duration", b"replication_ramp_duration" + ], + ) -> builtins.bool: ... def ClearField( - self, field_name: typing_extensions.Literal["cluster_name", b"cluster_name"] + self, + field_name: typing_extensions.Literal[ + "cluster_name", + b"cluster_name", + "replication_ramp_duration", + b"replication_ramp_duration", + ], ) -> None: ... global___ClusterReplicationConfig = ClusterReplicationConfig diff --git a/temporalio/api/stream/__init__.py b/temporalio/api/stream/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/temporalio/api/stream/v1/__init__.py b/temporalio/api/stream/v1/__init__.py new file mode 100644 index 0000000000..96544007ed --- /dev/null +++ b/temporalio/api/stream/v1/__init__.py @@ -0,0 +1,15 @@ +from .message_pb2 import ( + StreamRange, + StreamRecord, + StreamRecordKind, + StreamSlice, + StreamStartPosition, +) + +__all__ = [ + "StreamRange", + "StreamRecord", + "StreamRecordKind", + "StreamSlice", + "StreamStartPosition", +] diff --git a/temporalio/api/stream/v1/message_pb2.py b/temporalio/api/stream/v1/message_pb2.py new file mode 100644 index 0000000000..57962eb346 --- /dev/null +++ b/temporalio/api/stream/v1/message_pb2.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporal/api/stream/v1/message.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n$temporal/api/stream/v1/message.proto\x12\x16temporal.api.stream.v1\x1a$temporal/api/common/v1/message.proto"\xd4\x02\n\x0cStreamRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x44\n\x08metadata\x18\x02 \x03(\x0b\x32\x32.temporal.api.stream.v1.StreamRecord.MetadataEntry\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x36\n\x04kind\x18\x04 \x01(\x0e\x32(.temporal.api.stream.v1.StreamRecordKind\x12\x13\n\x0bproducer_id\x18\x05 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x06 \x01(\x03\x12\x10\n\x08sequence\x18\x07 \x01(\x03\x1aP\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\xb9\x01\n\x0bStreamSlice\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x03\x12\x11\n\tto_offset\x18\x04 \x01(\x03\x12\x35\n\x07records\x18\x05 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12(\n workflow_task_completed_event_id\x18\x06 \x01(\x03"H\n\x0bStreamRange\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x02 \x01(\x03\x12\x11\n\tto_offset\x18\x03 \x01(\x03"i\n\x13StreamStartPosition\x12\x10\n\x06offset\x18\x01 \x01(\x03H\x00\x12\x10\n\x06last_n\x18\x02 \x01(\x03H\x00\x12\x12\n\x08\x65\x61rliest\x18\x03 \x01(\x08H\x00\x12\x0e\n\x04tail\x18\x04 \x01(\x08H\x00\x42\n\n\x08position*r\n\x10StreamRecordKind\x12"\n\x1eSTREAM_RECORD_KIND_UNSPECIFIED\x10\x00\x12\x1b\n\x17STREAM_RECORD_KIND_DATA\x10\x01\x12\x1d\n\x19STREAM_RECORD_KIND_FINISH\x10\x02\x42\x89\x01\n\x19io.temporal.api.stream.v1B\x0cMessageProtoP\x01Z#go.temporal.io/api/stream/v1;stream\xaa\x02\x18Temporalio.Api.Stream.V1\xea\x02\x1bTemporalio::Api::Stream::V1b\x06proto3' +) + +_STREAMRECORDKIND = DESCRIPTOR.enum_types_by_name["StreamRecordKind"] +StreamRecordKind = enum_type_wrapper.EnumTypeWrapper(_STREAMRECORDKIND) +STREAM_RECORD_KIND_UNSPECIFIED = 0 +STREAM_RECORD_KIND_DATA = 1 +STREAM_RECORD_KIND_FINISH = 2 + + +_STREAMRECORD = DESCRIPTOR.message_types_by_name["StreamRecord"] +_STREAMRECORD_METADATAENTRY = _STREAMRECORD.nested_types_by_name["MetadataEntry"] +_STREAMSLICE = DESCRIPTOR.message_types_by_name["StreamSlice"] +_STREAMRANGE = DESCRIPTOR.message_types_by_name["StreamRange"] +_STREAMSTARTPOSITION = DESCRIPTOR.message_types_by_name["StreamStartPosition"] +StreamRecord = _reflection.GeneratedProtocolMessageType( + "StreamRecord", + (_message.Message,), + { + "MetadataEntry": _reflection.GeneratedProtocolMessageType( + "MetadataEntry", + (_message.Message,), + { + "DESCRIPTOR": _STREAMRECORD_METADATAENTRY, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamRecord.MetadataEntry) + }, + ), + "DESCRIPTOR": _STREAMRECORD, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamRecord) + }, +) +_sym_db.RegisterMessage(StreamRecord) +_sym_db.RegisterMessage(StreamRecord.MetadataEntry) + +StreamSlice = _reflection.GeneratedProtocolMessageType( + "StreamSlice", + (_message.Message,), + { + "DESCRIPTOR": _STREAMSLICE, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamSlice) + }, +) +_sym_db.RegisterMessage(StreamSlice) + +StreamRange = _reflection.GeneratedProtocolMessageType( + "StreamRange", + (_message.Message,), + { + "DESCRIPTOR": _STREAMRANGE, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamRange) + }, +) +_sym_db.RegisterMessage(StreamRange) + +StreamStartPosition = _reflection.GeneratedProtocolMessageType( + "StreamStartPosition", + (_message.Message,), + { + "DESCRIPTOR": _STREAMSTARTPOSITION, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamStartPosition) + }, +) +_sym_db.RegisterMessage(StreamStartPosition) + +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b"\n\031io.temporal.api.stream.v1B\014MessageProtoP\001Z#go.temporal.io/api/stream/v1;stream\252\002\030Temporalio.Api.Stream.V1\352\002\033Temporalio::Api::Stream::V1" + _STREAMRECORD_METADATAENTRY._options = None + _STREAMRECORD_METADATAENTRY._serialized_options = b"8\001" + _STREAMRECORDKIND._serialized_start = 814 + _STREAMRECORDKIND._serialized_end = 928 + _STREAMRECORD._serialized_start = 103 + _STREAMRECORD._serialized_end = 443 + _STREAMRECORD_METADATAENTRY._serialized_start = 363 + _STREAMRECORD_METADATAENTRY._serialized_end = 443 + _STREAMSLICE._serialized_start = 446 + _STREAMSLICE._serialized_end = 631 + _STREAMRANGE._serialized_start = 633 + _STREAMRANGE._serialized_end = 705 + _STREAMSTARTPOSITION._serialized_start = 707 + _STREAMSTARTPOSITION._serialized_end = 812 +# @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/stream/v1/message_pb2.pyi b/temporalio/api/stream/v1/message_pb2.pyi new file mode 100644 index 0000000000..80417d9375 --- /dev/null +++ b/temporalio/api/stream/v1/message_pb2.pyi @@ -0,0 +1,361 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import collections.abc +import sys +import typing + +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message + +import temporalio.api.common.v1.message_pb2 + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _StreamRecordKind: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _StreamRecordKindEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + _StreamRecordKind.ValueType + ], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + STREAM_RECORD_KIND_UNSPECIFIED: _StreamRecordKind.ValueType # 0 + """Read as DATA.""" + STREAM_RECORD_KIND_DATA: _StreamRecordKind.ValueType # 1 + """A value the producer published; `body` carries it.""" + STREAM_RECORD_KIND_FINISH: _StreamRecordKind.ValueType # 2 + """The producer named by `producer_id` writes nothing more on `topic`. + Says nothing about that producer's outcome and does not end the stream. + """ + +class StreamRecordKind(_StreamRecordKind, metaclass=_StreamRecordKindEnumTypeWrapper): + """What a record means to a reader. Kept on the record itself so every store + and every language reads it the same way without a private envelope. + """ + +STREAM_RECORD_KIND_UNSPECIFIED: StreamRecordKind.ValueType # 0 +"""Read as DATA.""" +STREAM_RECORD_KIND_DATA: StreamRecordKind.ValueType # 1 +"""A value the producer published; `body` carries it.""" +STREAM_RECORD_KIND_FINISH: StreamRecordKind.ValueType # 2 +"""The producer named by `producer_id` writes nothing more on `topic`. +Says nothing about that producer's outcome and does not end the stream. +""" +global___StreamRecordKind = StreamRecordKind + +class StreamRecord(google.protobuf.message.Message): + """One entry in a stream. The record is the wire format: stores keep it + serialized as is and readers in every language decode the same bytes. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class MetadataEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: temporalio.api.common.v1.message_pb2.Payload | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["key", b"key", "value", b"value"], + ) -> None: ... + + BODY_FIELD_NUMBER: builtins.int + METADATA_FIELD_NUMBER: builtins.int + TOPIC_FIELD_NUMBER: builtins.int + KIND_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + ATTEMPT_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + @property + def body(self) -> temporalio.api.common.v1.message_pb2.Payload: + """The value the producer published, stored as sent. + + A payload codec applies on the paths this API owns: the append command on + RespondWorkflowTaskCompleted, and the slices on PollWorkflowTaskQueue. + Records a producer writes or reads through the stream service take a + different path, whose messages are not part of this API yet and so are + outside what a codec-applying proxy walks. + """ + @property + def metadata( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, temporalio.api.common.v1.message_pb2.Payload + ]: + """Producer-supplied provenance, stored as sent.""" + topic: builtins.str + """Producer-supplied grouping label, stored as sent.""" + kind: global___StreamRecordKind.ValueType + """How to read this record. Unspecified is read as DATA.""" + producer_id: builtins.str + """Who wrote the record. Empty when the owning Workflow did.""" + attempt: builtins.int + """The producer's attempt. Readers treat a later attempt by the same + producer as superseding what the earlier one wrote. + """ + sequence: builtins.int + """The producer's position within its attempt, zero when it does not number + its records. Stored as sent; the server does not assign, validate or + order by it, and the stream's own offsets are what order a read. + """ + def __init__( + self, + *, + body: temporalio.api.common.v1.message_pb2.Payload | None = ..., + metadata: collections.abc.Mapping[ + builtins.str, temporalio.api.common.v1.message_pb2.Payload + ] + | None = ..., + topic: builtins.str = ..., + kind: global___StreamRecordKind.ValueType = ..., + producer_id: builtins.str = ..., + attempt: builtins.int = ..., + sequence: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["body", b"body"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "attempt", + b"attempt", + "body", + b"body", + "kind", + b"kind", + "metadata", + b"metadata", + "producer_id", + b"producer_id", + "sequence", + b"sequence", + "topic", + b"topic", + ], + ) -> None: ... + +global___StreamRecord = StreamRecord + +class StreamSlice(google.protobuf.message.Message): + """A contiguous range of a stream delivered to a Workflow Task, along with the + offsets it covers. The offsets are what History records; the records + themselves are never written to History. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + TO_OFFSET_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + WORKFLOW_TASK_COMPLETED_EVENT_ID_FIELD_NUMBER: builtins.int + stream_id: builtins.str + """The stream, as the subscribing command addressed it: either the name of + a stream the consuming Workflow owns or the id of one in another + execution. + """ + run_id: builtins.str + """Run id of the execution that owns the stream. Set on both a slice for the + task being started and a re-supplied one. + """ + from_offset: builtins.int + """Inclusive. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + """ + to_offset: builtins.int + """Exclusive. Equal to from_offset when the subscription observed nothing, + which is a fact replay has to reproduce rather than an absence of one. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + """ + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamRecord + ]: ... + workflow_task_completed_event_id: builtins.int + """The WorkflowTaskCompleted event whose consumed_stream_ranges recorded + this range. Set only when the server is re-supplying a range for a task + being replayed; a slice for the task now being started leaves it unset, + because the event closing that task does not exist yet. + + Replay needs this because a Workflow Task response carries one slice set + while a cache miss replays every prior task, so the ranges have to be + matched to the events that recorded them rather than to the response. + """ + def __init__( + self, + *, + stream_id: builtins.str = ..., + run_id: builtins.str = ..., + from_offset: builtins.int = ..., + to_offset: builtins.int = ..., + records: collections.abc.Iterable[global___StreamRecord] | None = ..., + workflow_task_completed_event_id: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "records", + b"records", + "run_id", + b"run_id", + "stream_id", + b"stream_id", + "to_offset", + b"to_offset", + "workflow_task_completed_event_id", + b"workflow_task_completed_event_id", + ], + ) -> None: ... + +global___StreamSlice = StreamSlice + +class StreamRange(google.protobuf.message.Message): + """The offsets a Workflow Task consumed, without the payloads. Recorded on + WorkflowTaskCompleted so History grows with Workflow Tasks rather than with + records. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + TO_OFFSET_FIELD_NUMBER: builtins.int + stream_id: builtins.str + """The stream, as the subscribing command addressed it: either the name of + a stream the consuming Workflow owns or the id of one in another + execution. + """ + from_offset: builtins.int + """Inclusive. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + """ + to_offset: builtins.int + """Exclusive. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + """ + def __init__( + self, + *, + stream_id: builtins.str = ..., + from_offset: builtins.int = ..., + to_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "stream_id", + b"stream_id", + "to_offset", + b"to_offset", + ], + ) -> None: ... + +global___StreamRange = StreamRange + +class StreamStartPosition(google.protobuf.message.Message): + """Where a new subscription or read begins. The server resolves it against the + stream as it stands in the same transaction that registers the reader, so + the result does not race with appends or truncation, and records the + resolved absolute offset. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + OFFSET_FIELD_NUMBER: builtins.int + LAST_N_FIELD_NUMBER: builtins.int + EARLIEST_FIELD_NUMBER: builtins.int + TAIL_FIELD_NUMBER: builtins.int + offset: builtins.int + """Absolute and inclusive. Refused when below the stream's floor.""" + last_n: builtins.int + """The last N records the stream holds, or all of them when it holds + fewer. Counts records of every kind. Must be positive. + """ + earliest: builtins.bool + """The oldest record the stream still holds. Must be true.""" + tail: builtins.bool + """Only records appended after registration: the stream's head offset. + Must be true. + """ + def __init__( + self, + *, + offset: builtins.int = ..., + last_n: builtins.int = ..., + earliest: builtins.bool = ..., + tail: builtins.bool = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "earliest", + b"earliest", + "last_n", + b"last_n", + "offset", + b"offset", + "position", + b"position", + "tail", + b"tail", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "earliest", + b"earliest", + "last_n", + b"last_n", + "offset", + b"offset", + "position", + b"position", + "tail", + b"tail", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["position", b"position"] + ) -> typing_extensions.Literal["offset", "last_n", "earliest", "tail"] | None: ... + +global___StreamStartPosition = StreamStartPosition diff --git a/temporalio/api/streamservice/__init__.py b/temporalio/api/streamservice/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/temporalio/api/streamservice/v1/__init__.py b/temporalio/api/streamservice/v1/__init__.py new file mode 100644 index 0000000000..f635623654 --- /dev/null +++ b/temporalio/api/streamservice/v1/__init__.py @@ -0,0 +1,158 @@ +# Generated by scripts/gen_stream_protos.py. Do not edit. +# +# Vendored rather than taken from the api submodule, because the stream service +# is still defined in the server. See temporalio/client_stream.py. + +from .message_pb2 import ( + StreamRecord, + StreamRecordBatch, +) +from .request_response_pb2 import ( + STREAM_OWNER_KIND_ACTIVITY, + STREAM_OWNER_KIND_UNSPECIFIED, + STREAM_OWNER_KIND_WORKFLOW, + STREAM_OWNER_KIND_WORKFLOW_ACTIVITY, + AddMessagesInput, + AddMessagesOutput, + AddMessagesRequest, + AddMessagesResponse, + AddWorkflowMessagesInput, + AddWorkflowMessagesRequest, + AddWorkflowMessagesResponse, + AdvanceConsumerHeadInput, + AdvanceConsumerHeadOutput, + AdvanceConsumerHeadRequest, + AdvanceConsumerHeadResponse, + CloseStreamInput, + CloseStreamOutput, + CloseStreamRequest, + CloseStreamResponse, + CreateStreamInput, + CreateStreamOutput, + CreateStreamRequest, + CreateStreamResponse, + DeleteStreamInput, + DeleteStreamOutput, + DeleteStreamRequest, + DeleteStreamResponse, + DescribeStreamInput, + DescribeStreamOutput, + DescribeStreamRequest, + DescribeStreamResponse, + DescribeWorkflowStreamInput, + DescribeWorkflowStreamRequest, + DescribeWorkflowStreamResponse, + FinishWritingInput, + FinishWritingOutput, + FinishWritingRequest, + FinishWritingResponse, + ListStreamsInput, + ListStreamsOutput, + ListStreamsRequest, + ListStreamsResponse, + PollMessagesInput, + PollMessagesOutput, + PollMessagesRequest, + PollMessagesResponse, + PollWorkflowMessagesInput, + PollWorkflowMessagesRequest, + PollWorkflowMessagesResponse, + RegisterStreamConsumerInput, + RegisterStreamConsumerOutput, + RegisterStreamConsumerRequest, + RegisterStreamConsumerResponse, + StreamListEntry, + StreamOwner, + StreamOwnerKind, + SubscribeWorkflowInput, + SubscribeWorkflowOutput, + SubscribeWorkflowRequest, + SubscribeWorkflowResponse, + TruncateStreamInput, + TruncateStreamOutput, + TruncateStreamRequest, + TruncateStreamResponse, +) +from .stream_state_pb2 import ( + ConsumerCursor, + ProducerCursor, + StreamBudget, + StreamLifecycle, + StreamState, + WorkflowStreamCursor, +) + +__all__ = [ + "AddMessagesInput", + "AddMessagesOutput", + "AddMessagesRequest", + "AddMessagesResponse", + "AddWorkflowMessagesInput", + "AddWorkflowMessagesRequest", + "AddWorkflowMessagesResponse", + "AdvanceConsumerHeadInput", + "AdvanceConsumerHeadOutput", + "AdvanceConsumerHeadRequest", + "AdvanceConsumerHeadResponse", + "CloseStreamInput", + "CloseStreamOutput", + "CloseStreamRequest", + "CloseStreamResponse", + "ConsumerCursor", + "CreateStreamInput", + "CreateStreamOutput", + "CreateStreamRequest", + "CreateStreamResponse", + "DeleteStreamInput", + "DeleteStreamOutput", + "DeleteStreamRequest", + "DeleteStreamResponse", + "DescribeStreamInput", + "DescribeStreamOutput", + "DescribeStreamRequest", + "DescribeStreamResponse", + "DescribeWorkflowStreamInput", + "DescribeWorkflowStreamRequest", + "DescribeWorkflowStreamResponse", + "FinishWritingInput", + "FinishWritingOutput", + "FinishWritingRequest", + "FinishWritingResponse", + "ListStreamsInput", + "ListStreamsOutput", + "ListStreamsRequest", + "ListStreamsResponse", + "PollMessagesInput", + "PollMessagesOutput", + "PollMessagesRequest", + "PollMessagesResponse", + "PollWorkflowMessagesInput", + "PollWorkflowMessagesRequest", + "PollWorkflowMessagesResponse", + "ProducerCursor", + "RegisterStreamConsumerInput", + "RegisterStreamConsumerOutput", + "RegisterStreamConsumerRequest", + "RegisterStreamConsumerResponse", + "STREAM_OWNER_KIND_ACTIVITY", + "STREAM_OWNER_KIND_UNSPECIFIED", + "STREAM_OWNER_KIND_WORKFLOW", + "STREAM_OWNER_KIND_WORKFLOW_ACTIVITY", + "StreamBudget", + "StreamLifecycle", + "StreamListEntry", + "StreamOwner", + "StreamOwnerKind", + "StreamRecord", + "StreamRecordBatch", + "StreamState", + "SubscribeWorkflowInput", + "SubscribeWorkflowOutput", + "SubscribeWorkflowRequest", + "SubscribeWorkflowResponse", + "TruncateStreamInput", + "TruncateStreamOutput", + "TruncateStreamRequest", + "TruncateStreamResponse", + "WorkflowStreamCursor", +] diff --git a/temporalio/api/streamservice/v1/message_pb2.py b/temporalio/api/streamservice/v1/message_pb2.py new file mode 100644 index 0000000000..cd10df58fa --- /dev/null +++ b/temporalio/api/streamservice/v1/message_pb2.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporalio/api/stream/v1/message.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n&temporalio/api/stream/v1/message.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto"\xf7\x02\n\x0cStreamRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12W\n\x08metadata\x18\x02 \x03(\x0b\x32\x45.temporal.server.chasm.lib.stream.proto.v1.StreamRecord.MetadataEntry\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x10\n\x08sequence\x18\x04 \x01(\x03\x12\x36\n\x04kind\x18\x05 \x01(\x0e\x32(.temporal.api.stream.v1.StreamRecordKind\x12\x0e\n\x06offset\x18\x06 \x01(\x03\x12\x13\n\x0bproducer_id\x18\x07 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x08 \x01(\x03\x1aP\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x8e\x01\n\x11StreamRecordBatch\x12H\n\x07records\x18\x01 \x03(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamRecord\x12/\n\x0b\x61ppended_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampB>Z= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class StreamRecord(google.protobuf.message.Message): + """The stored shape of temporal.api.stream.v1.StreamRecord plus the offset a + read assigns. Field for field the public record, so one crosses the frontend + in either direction without translation. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class MetadataEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: temporalio.api.common.v1.message_pb2.Payload | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["key", b"key", "value", b"value"], + ) -> None: ... + + BODY_FIELD_NUMBER: builtins.int + METADATA_FIELD_NUMBER: builtins.int + TOPIC_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + KIND_FIELD_NUMBER: builtins.int + OFFSET_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + ATTEMPT_FIELD_NUMBER: builtins.int + @property + def body(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + @property + def metadata( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, temporalio.api.common.v1.message_pb2.Payload + ]: + """Producer-supplied provenance, stored as sent.""" + topic: builtins.str + sequence: builtins.int + """The producer's position within its attempt, zero when it does not number + its records. Stored as sent; the global offset is what orders the stream. + """ + kind: temporalio.api.stream.v1.message_pb2.StreamRecordKind.ValueType + """Settled to DATA on append when left unspecified, so a retry hashes the + same bytes and a reader never sees the zero value. + """ + offset: builtins.int + """Position in the whole stream, set on read and never stored. A consumer + that resumes at record granularity needs it, and a topic-filtered read + leaves gaps that make it underivable from the response alone. + """ + producer_id: builtins.str + """Who wrote the record. Empty when the owning Workflow did: its publish + command clears the field, whatever the worker sent. + """ + attempt: builtins.int + """The producer's attempt, stored as sent. Readers treat a later attempt by + the same producer as superseding what the earlier one wrote. + """ + def __init__( + self, + *, + body: temporalio.api.common.v1.message_pb2.Payload | None = ..., + metadata: collections.abc.Mapping[ + builtins.str, temporalio.api.common.v1.message_pb2.Payload + ] + | None = ..., + topic: builtins.str = ..., + sequence: builtins.int = ..., + kind: temporalio.api.stream.v1.message_pb2.StreamRecordKind.ValueType = ..., + offset: builtins.int = ..., + producer_id: builtins.str = ..., + attempt: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["body", b"body"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "attempt", + b"attempt", + "body", + b"body", + "kind", + b"kind", + "metadata", + b"metadata", + "offset", + b"offset", + "producer_id", + b"producer_id", + "sequence", + b"sequence", + "topic", + b"topic", + ], + ) -> None: ... + +global___StreamRecord = StreamRecord + +class StreamRecordBatch(google.protobuf.message.Message): + """One append is one batch, and one batch is one data node. The server stores + this serialized and opaque; it decodes only to trim a partial first page, to + apply a topic filter, or to learn the batch's age. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RECORDS_FIELD_NUMBER: builtins.int + APPENDED_AT_FIELD_NUMBER: builtins.int + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamRecord + ]: ... + @property + def appended_at(self) -> google.protobuf.timestamp_pb2.Timestamp: + """When the batch was appended, for the lifecycle's retention age. Kept here + rather than in the stream's state so the state stays free of a per-batch + index; the age check reads the oldest batches to find it. + """ + def __init__( + self, + *, + records: collections.abc.Iterable[global___StreamRecord] | None = ..., + appended_at: google.protobuf.timestamp_pb2.Timestamp | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["appended_at", b"appended_at"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "appended_at", b"appended_at", "records", b"records" + ], + ) -> None: ... + +global___StreamRecordBatch = StreamRecordBatch diff --git a/temporalio/api/streamservice/v1/message_pb2_grpc.py b/temporalio/api/streamservice/v1/message_pb2_grpc.py new file mode 100644 index 0000000000..bf947056a2 --- /dev/null +++ b/temporalio/api/streamservice/v1/message_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc diff --git a/temporalio/api/streamservice/v1/message_pb2_grpc.pyi b/temporalio/api/streamservice/v1/message_pb2_grpc.pyi new file mode 100644 index 0000000000..f3a5a087ef --- /dev/null +++ b/temporalio/api/streamservice/v1/message_pb2_grpc.pyi @@ -0,0 +1,4 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" diff --git a/temporalio/api/streamservice/v1/request_response_pb2.py b/temporalio/api/streamservice/v1/request_response_pb2.py new file mode 100644 index 0000000000..0ecfa75c6f --- /dev/null +++ b/temporalio/api/streamservice/v1/request_response_pb2.py @@ -0,0 +1,908 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporalio/api/stream/v1/request_response.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) +from temporalio.api.streamservice.v1 import ( + message_pb2 as temporalio_dot_api_dot_stream_dot_v1_dot_message__pb2, +) +from temporalio.api.streamservice.v1 import ( + stream_state_pb2 as temporalio_dot_api_dot_stream_dot_v1_dot_stream__state__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n/temporalio/api/stream/v1/request_response.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a&temporalio/api/stream/v1/message.proto\x1a+temporalio/api/stream/v1/stream_state.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto"\x88\x01\n\x11\x43reateStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12M\n\tlifecycle\x18\x03 \x01(\x0b\x32:.temporal.server.chasm.lib.stream.proto.v1.StreamLifecycle"$\n\x12\x43reateStreamOutput\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xf5\x01\n\x10\x41\x64\x64MessagesInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\t \x01(\t\x12H\n\x07records\x18\x03 \x03(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamRecord\x12\x13\n\x0bproducer_id\x18\x04 \x01(\t\x12\x10\n\x08sequence\x18\x05 \x01(\x03\x12\x17\n\x0f\x65xpected_offset\x18\x06 \x01(\x03\x12\x1b\n\x13use_expected_offset\x18\x08 \x01(\x08J\x04\x08\x07\x10\x08"c\n\x11\x41\x64\x64MessagesOutput\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\x12\x14\n\x0c\x64\x65\x64uplicated\x18\x04 \x01(\x08"O\n\x12\x46inishWritingInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x13\n\x0bproducer_id\x18\x03 \x01(\t"\x15\n\x13\x46inishWritingOutput"\xd9\x01\n\x16SubscribeWorkflowInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x06 \x01(\t\x12\x13\n\x0bstream_name\x18\x03 \x01(\t\x12\x11\n\tstream_id\x18\x05 \x01(\t\x12\x14\n\x0cstart_offset\x18\x04 \x01(\x03\x12\x43\n\x0estart_position\x18\x07 \x01(\x0b\x32+.temporal.api.stream.v1.StreamStartPosition"/\n\x17SubscribeWorkflowOutput\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03"\xe4\x01\n\x11PollMessagesInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x07 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x03\x12\x43\n\x0estart_position\x18\x08 \x01(\x0b\x32+.temporal.api.stream.v1.StreamStartPosition\x12\x14\n\x0cmax_messages\x18\x04 \x01(\x05\x12\x0e\n\x06topics\x18\x05 \x03(\t\x12\x19\n\x11wait_new_messages\x18\x06 \x01(\x08"\xdf\x01\n\x12PollMessagesOutput\x12H\n\x07records\x18\x01 \x03(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamRecord\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x13\n\x0bhead_offset\x18\x03 \x01(\x03\x12\x0e\n\x06\x63losed\x18\x04 \x01(\x08\x12\x35\n\x0c\x63lose_reason\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x0e\n\x06run_id\x18\x06 \x01(\t";\n\x13\x44\x65scribeStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t"\x88\x01\n\x0bStreamOwner\x12H\n\x04kind\x18\x01 \x01(\x0e\x32:.temporal.server.chasm.lib.stream.proto.v1.StreamOwnerKind\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t"\xd0\x02\n\x19PollWorkflowMessagesInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x05owner\x18\t \x01(\x0b\x32\x36.temporal.server.chasm.lib.stream.proto.v1.StreamOwner\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x08 \x01(\t\x12\x13\n\x0bstream_name\x18\x03 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x04 \x01(\x03\x12\x43\n\x0estart_position\x18\n \x01(\x0b\x32+.temporal.api.stream.v1.StreamStartPosition\x12\x14\n\x0cmax_messages\x18\x05 \x01(\x05\x12\x0e\n\x06topics\x18\x06 \x03(\t\x12\x19\n\x11wait_new_messages\x18\x07 \x01(\x08"\xb7\x01\n\x1b\x44\x65scribeWorkflowStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x05owner\x18\x05 \x01(\x0b\x32\x36.temporal.server.chasm.lib.stream.proto.v1.StreamOwner\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x04 \x01(\t\x12\x13\n\x0bstream_name\x18\x03 \x01(\t"\xa5\x02\n\x18\x41\x64\x64WorkflowMessagesInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x05owner\x18\x08 \x01(\x0b\x32\x36.temporal.server.chasm.lib.stream.proto.v1.StreamOwner\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x07 \x01(\t\x12\x13\n\x0bstream_name\x18\x03 \x01(\t\x12H\n\x07records\x18\x04 \x03(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamRecord\x12\x13\n\x0bproducer_id\x18\x05 \x01(\t\x12\x10\n\x08sequence\x18\x06 \x01(\x03"]\n\x14\x44\x65scribeStreamOutput\x12\x45\n\x05state\x18\x01 \x01(\x0b\x32\x36.temporal.server.chasm.lib.stream.proto.v1.StreamState"i\n\x10\x43loseStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12/\n\x06reason\x18\x03 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"\x13\n\x11\x43loseStreamOutput"T\n\x13TruncateStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x17\n\x0fnew_base_offset\x18\x03 \x01(\x03"\x16\n\x14TruncateStreamOutput"H\n\x11\x44\x65leteStreamInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\r\n\x05\x66orce\x18\x03 \x01(\x08"\x14\n\x12\x44\x65leteStreamOutput"\x83\x01\n\x13\x43reateStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12V\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.CreateStreamInput"p\n\x14\x43reateStreamResponse\x12X\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.CreateStreamOutput"\x81\x01\n\x12\x41\x64\x64MessagesRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12U\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32;.temporal.server.chasm.lib.stream.proto.v1.AddMessagesInput"n\n\x13\x41\x64\x64MessagesResponse\x12W\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.AddMessagesOutput"\x85\x01\n\x14\x46inishWritingRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12W\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.FinishWritingInput"r\n\x15\x46inishWritingResponse\x12Y\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32>.temporal.server.chasm.lib.stream.proto.v1.FinishWritingOutput"\x8d\x01\n\x18SubscribeWorkflowRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12[\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x41.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowInput"z\n\x19SubscribeWorkflowResponse\x12]\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32\x42.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowOutput"\x83\x01\n\x13PollMessagesRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12V\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.PollMessagesInput"p\n\x14PollMessagesResponse\x12X\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutput"\x87\x01\n\x15\x44\x65scribeStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12X\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32>.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamInput"t\n\x16\x44\x65scribeStreamResponse\x12Z\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32?.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutput"\x93\x01\n\x1bPollWorkflowMessagesRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12^\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x44.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesInput"x\n\x1cPollWorkflowMessagesResponse\x12X\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutput"\x97\x01\n\x1d\x44\x65scribeWorkflowStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12`\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x46.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamInput"|\n\x1e\x44\x65scribeWorkflowStreamResponse\x12Z\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32?.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutput"\xd5\x01\n\x1bRegisterStreamConsumerInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12\x1c\n\x14\x63onsumer_workflow_id\x18\x03 \x01(\t\x12\x17\n\x0f\x63onsumer_run_id\x18\x05 \x01(\t\x12\x14\n\x0cstart_offset\x18\x04 \x01(\x03\x12\x43\n\x0estart_position\x18\x06 \x01(\x0b\x32+.temporal.api.stream.v1.StreamStartPosition"k\n\x1cRegisterStreamConsumerOutput\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x12\n\nknown_head\x18\x04 \x01(\x03\x12\x15\n\rstream_absent\x18\x05 \x01(\x08J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04"\x80\x01\n\x18\x41\x64vanceConsumerHeadInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0cowner_run_id\x18\x05 \x01(\t\x12\x11\n\tstream_id\x18\x03 \x01(\t\x12\x13\n\x0bhead_offset\x18\x04 \x01(\x03"n\n\x19\x41\x64vanceConsumerHeadOutput\x12\x17\n\x0f\x63onsumer_closed\x18\x01 \x01(\x08\x12\x18\n\x10successor_run_id\x18\x02 \x01(\t\x12\x1e\n\x16successor_start_offset\x18\x03 \x01(\x03"\x91\x01\n\x1a\x41\x64\x64WorkflowMessagesRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12]\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x43.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesInput"v\n\x1b\x41\x64\x64WorkflowMessagesResponse\x12W\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.AddMessagesOutput"\x97\x01\n\x1dRegisterStreamConsumerRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12`\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x46.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerInput"\x84\x01\n\x1eRegisterStreamConsumerResponse\x12\x62\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32G.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerOutput"\x91\x01\n\x1a\x41\x64vanceConsumerHeadRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12]\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32\x43.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadInput"~\n\x1b\x41\x64vanceConsumerHeadResponse\x12_\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32\x44.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadOutput"\x81\x01\n\x12\x43loseStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12U\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32;.temporal.server.chasm.lib.stream.proto.v1.CloseStreamInput"n\n\x13\x43loseStreamResponse\x12W\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.CloseStreamOutput"\x87\x01\n\x15TruncateStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12X\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32>.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamInput"t\n\x16TruncateStreamResponse\x12Z\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32?.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamOutput"`\n\x10ListStreamsInput\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"4\n\x0fStreamListEntry\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t"y\n\x11ListStreamsOutput\x12K\n\x07streams\x18\x01 \x03(\x0b\x32:.temporal.server.chasm.lib.stream.proto.v1.StreamListEntry\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\x81\x01\n\x12ListStreamsRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12U\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32;.temporal.server.chasm.lib.stream.proto.v1.ListStreamsInput"n\n\x13ListStreamsResponse\x12W\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.ListStreamsOutput"\x83\x01\n\x13\x44\x65leteStreamRequest\x12\x14\n\x0cnamespace_id\x18\x01 \x01(\t\x12V\n\x10\x66rontend_request\x18\x02 \x01(\x0b\x32<.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamInput"p\n\x14\x44\x65leteStreamResponse\x12X\n\x11\x66rontend_response\x18\x01 \x01(\x0b\x32=.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamOutput*\x9d\x01\n\x0fStreamOwnerKind\x12!\n\x1dSTREAM_OWNER_KIND_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTREAM_OWNER_KIND_WORKFLOW\x10\x01\x12\x1e\n\x1aSTREAM_OWNER_KIND_ACTIVITY\x10\x02\x12\'\n#STREAM_OWNER_KIND_WORKFLOW_ACTIVITY\x10\x03\x42>Z= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _StreamOwnerKind: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _StreamOwnerKindEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + _StreamOwnerKind.ValueType + ], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + STREAM_OWNER_KIND_UNSPECIFIED: _StreamOwnerKind.ValueType # 0 + STREAM_OWNER_KIND_WORKFLOW: _StreamOwnerKind.ValueType # 1 + STREAM_OWNER_KIND_ACTIVITY: _StreamOwnerKind.ValueType # 2 + """A standalone activity, which is an execution of its own.""" + STREAM_OWNER_KIND_WORKFLOW_ACTIVITY: _StreamOwnerKind.ValueType # 3 + """An activity a workflow scheduled. It is not an execution of its own, so + its streams are reached through the workflow and routed on it. + """ + +class StreamOwnerKind(_StreamOwnerKind, metaclass=_StreamOwnerKindEnumTypeWrapper): + """The kind of execution an owned stream lives in.""" + +STREAM_OWNER_KIND_UNSPECIFIED: StreamOwnerKind.ValueType # 0 +STREAM_OWNER_KIND_WORKFLOW: StreamOwnerKind.ValueType # 1 +STREAM_OWNER_KIND_ACTIVITY: StreamOwnerKind.ValueType # 2 +"""A standalone activity, which is an execution of its own.""" +STREAM_OWNER_KIND_WORKFLOW_ACTIVITY: StreamOwnerKind.ValueType # 3 +"""An activity a workflow scheduled. It is not an execution of its own, so +its streams are reached through the workflow and routed on it. +""" +global___StreamOwnerKind = StreamOwnerKind + +class CreateStreamInput(google.protobuf.message.Message): + """The frontend-facing shapes are defined here rather than in the public API + because streams have no public API yet. Keeping them in a nested + frontend_request mirrors the other CHASM libraries, so promoting them later + is a package move rather than a redesign. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + LIFECYCLE_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + @property + def lifecycle( + self, + ) -> temporalio.api.streamservice.v1.stream_state_pb2.StreamLifecycle: ... + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + lifecycle: temporalio.api.streamservice.v1.stream_state_pb2.StreamLifecycle + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["lifecycle", b"lifecycle"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "lifecycle", + b"lifecycle", + "namespace", + b"namespace", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___CreateStreamInput = CreateStreamInput + +class CreateStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RUN_ID_FIELD_NUMBER: builtins.int + run_id: builtins.str + def __init__( + self, + *, + run_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["run_id", b"run_id"] + ) -> None: ... + +global___CreateStreamOutput = CreateStreamOutput + +class AddMessagesInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + EXPECTED_OFFSET_FIELD_NUMBER: builtins.int + USE_EXPECTED_OFFSET_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + run_id: builtins.str + """Optional. Supplying it skips resolving the stream's current run, which is + a persistence lookup on every call. CreateStream returns it. + """ + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ]: ... + producer_id: builtins.str + """Idempotency, all optional. Supply a producer identity and sequence, or an + expected offset, or neither and accept at-least-once. + """ + sequence: builtins.int + expected_offset: builtins.int + """Guarded by use_expected_offset because proto3 optional is not supported + by this repo's helper generator. + """ + use_expected_offset: builtins.bool + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + run_id: builtins.str = ..., + records: collections.abc.Iterable[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ] + | None = ..., + producer_id: builtins.str = ..., + sequence: builtins.int = ..., + expected_offset: builtins.int = ..., + use_expected_offset: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "expected_offset", + b"expected_offset", + "namespace", + b"namespace", + "producer_id", + b"producer_id", + "records", + b"records", + "run_id", + b"run_id", + "sequence", + b"sequence", + "stream_id", + b"stream_id", + "use_expected_offset", + b"use_expected_offset", + ], + ) -> None: ... + +global___AddMessagesInput = AddMessagesInput + +class AddMessagesOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FIRST_OFFSET_FIELD_NUMBER: builtins.int + NEXT_OFFSET_FIELD_NUMBER: builtins.int + COUNT_FIELD_NUMBER: builtins.int + DEDUPLICATED_FIELD_NUMBER: builtins.int + first_offset: builtins.int + next_offset: builtins.int + count: builtins.int + deduplicated: builtins.bool + """True when a retry matched a recorded sequence and nothing was appended.""" + def __init__( + self, + *, + first_offset: builtins.int = ..., + next_offset: builtins.int = ..., + count: builtins.int = ..., + deduplicated: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "count", + b"count", + "deduplicated", + b"deduplicated", + "first_offset", + b"first_offset", + "next_offset", + b"next_offset", + ], + ) -> None: ... + +global___AddMessagesOutput = AddMessagesOutput + +class FinishWritingInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + producer_id: builtins.str + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + producer_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "producer_id", + b"producer_id", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___FinishWritingInput = FinishWritingInput + +class FinishWritingOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___FinishWritingOutput = FinishWritingOutput + +class SubscribeWorkflowInput(google.protobuf.message.Message): + """Registers a Workflow as a consumer of a stream it owns. The cursor lands in + the Workflow's own state, so from then on each of its Workflow Tasks carries + the next range and records what it consumed. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + START_POSITION_FIELD_NUMBER: builtins.int + namespace: builtins.str + workflow_id: builtins.str + owner_run_id: builtins.str + """Optional. Pins to one run, so a caller that has continued as new is not + silently redirected to the successor's stream, which starts empty and at + offset zero. Empty means whichever run is current. + """ + stream_name: builtins.str + """Name of the stream within the Workflow, for a stream it owns.""" + stream_id: builtins.str + """Id of a standalone stream in another execution. Exactly one of this and + stream_name is set. + """ + start_offset: builtins.int + """Where to start, as an absolute offset. Read only when start_position is + unset. A negative value is refused. + """ + @property + def start_position( + self, + ) -> temporalio.api.stream.v1.message_pb2.StreamStartPosition: + """Where to start. Resolved here rather than at delivery, so the first + recorded range starts from a fact instead of a reading. Refused alongside a + non-zero start_offset. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_name: builtins.str = ..., + stream_id: builtins.str = ..., + start_offset: builtins.int = ..., + start_position: temporalio.api.stream.v1.message_pb2.StreamStartPosition + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["start_position", b"start_position"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "owner_run_id", + b"owner_run_id", + "start_offset", + b"start_offset", + "start_position", + b"start_position", + "stream_id", + b"stream_id", + "stream_name", + b"stream_name", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___SubscribeWorkflowInput = SubscribeWorkflowInput + +class SubscribeWorkflowOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + START_OFFSET_FIELD_NUMBER: builtins.int + start_offset: builtins.int + def __init__( + self, + *, + start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["start_offset", b"start_offset"] + ) -> None: ... + +global___SubscribeWorkflowOutput = SubscribeWorkflowOutput + +class PollMessagesInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + START_POSITION_FIELD_NUMBER: builtins.int + MAX_MESSAGES_FIELD_NUMBER: builtins.int + TOPICS_FIELD_NUMBER: builtins.int + WAIT_NEW_MESSAGES_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + run_id: builtins.str + """Optional, as on AddMessagesInput.""" + from_offset: builtins.int + @property + def start_position( + self, + ) -> temporalio.api.stream.v1.message_pb2.StreamStartPosition: + """Where a first poll starts, when the reader has no offset yet. Resolved in + the same read that serves the records, so it cannot race with truncation, + and next_offset carries the result for the polls after it. Refused + alongside a non-zero from_offset. + """ + max_messages: builtins.int + @property + def topics( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Filters by exact topic. Offsets are assigned over the unfiltered stream, so + next_offset advances past filtered-out records too. + """ + wait_new_messages: builtins.bool + """When set and the reader is caught up, block until something arrives, the + stream closes, or the server's long-poll timeout elapses. A timeout returns + an empty response rather than an error, so the caller simply polls again. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + run_id: builtins.str = ..., + from_offset: builtins.int = ..., + start_position: temporalio.api.stream.v1.message_pb2.StreamStartPosition + | None = ..., + max_messages: builtins.int = ..., + topics: collections.abc.Iterable[builtins.str] | None = ..., + wait_new_messages: builtins.bool = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["start_position", b"start_position"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "max_messages", + b"max_messages", + "namespace", + b"namespace", + "run_id", + b"run_id", + "start_position", + b"start_position", + "stream_id", + b"stream_id", + "topics", + b"topics", + "wait_new_messages", + b"wait_new_messages", + ], + ) -> None: ... + +global___PollMessagesInput = PollMessagesInput + +class PollMessagesOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RECORDS_FIELD_NUMBER: builtins.int + NEXT_OFFSET_FIELD_NUMBER: builtins.int + HEAD_OFFSET_FIELD_NUMBER: builtins.int + CLOSED_FIELD_NUMBER: builtins.int + CLOSE_REASON_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ]: ... + next_offset: builtins.int + head_offset: builtins.int + closed: builtins.bool + @property + def close_reason(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + run_id: builtins.str + """The execution holding the stream. A workflow task slice built from this + read names the run it came from. + """ + def __init__( + self, + *, + records: collections.abc.Iterable[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ] + | None = ..., + next_offset: builtins.int = ..., + head_offset: builtins.int = ..., + closed: builtins.bool = ..., + close_reason: temporalio.api.common.v1.message_pb2.Payload | None = ..., + run_id: builtins.str = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["close_reason", b"close_reason"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "close_reason", + b"close_reason", + "closed", + b"closed", + "head_offset", + b"head_offset", + "next_offset", + b"next_offset", + "records", + b"records", + "run_id", + b"run_id", + ], + ) -> None: ... + +global___PollMessagesOutput = PollMessagesOutput + +class DescribeStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", b"namespace", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___DescribeStreamInput = DescribeStreamInput + +class StreamOwner(google.protobuf.message.Message): + """The execution that owns a stream. An attached stream has no id of its own, + so it is named by this and a stream name, and routed on id. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KIND_FIELD_NUMBER: builtins.int + ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + ACTIVITY_ID_FIELD_NUMBER: builtins.int + kind: global___StreamOwnerKind.ValueType + id: builtins.str + """The workflow id for WORKFLOW and WORKFLOW_ACTIVITY, the activity id for + ACTIVITY. + """ + run_id: builtins.str + """Optional. Pins to one run of the execution named by id, so a caller is not + silently redirected to a later run's stream, which starts empty and at + offset zero. Empty means whichever run is current. + """ + activity_id: builtins.str + """The activity within the workflow, for WORKFLOW_ACTIVITY only.""" + def __init__( + self, + *, + kind: global___StreamOwnerKind.ValueType = ..., + id: builtins.str = ..., + run_id: builtins.str = ..., + activity_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "activity_id", + b"activity_id", + "id", + b"id", + "kind", + b"kind", + "run_id", + b"run_id", + ], + ) -> None: ... + +global___StreamOwner = StreamOwner + +class PollWorkflowMessagesInput(google.protobuf.message.Message): + """A stream an execution owns lives inside it, so it has no standalone id to + address it by. It is named by its owner and its name instead, and routed on + the owner. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + OWNER_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + START_POSITION_FIELD_NUMBER: builtins.int + MAX_MESSAGES_FIELD_NUMBER: builtins.int + TOPICS_FIELD_NUMBER: builtins.int + WAIT_NEW_MESSAGES_FIELD_NUMBER: builtins.int + namespace: builtins.str + @property + def owner(self) -> global___StreamOwner: ... + workflow_id: builtins.str + """Shorthand for an owner of kind WORKFLOW, and refused alongside owner.""" + owner_run_id: builtins.str + stream_name: builtins.str + """Empty means the owner's default output stream.""" + from_offset: builtins.int + @property + def start_position( + self, + ) -> temporalio.api.stream.v1.message_pb2.StreamStartPosition: + """As on PollMessagesInput.""" + max_messages: builtins.int + @property + def topics( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Filters as on PollMessagesInput.""" + wait_new_messages: builtins.bool + def __init__( + self, + *, + namespace: builtins.str = ..., + owner: global___StreamOwner | None = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_name: builtins.str = ..., + from_offset: builtins.int = ..., + start_position: temporalio.api.stream.v1.message_pb2.StreamStartPosition + | None = ..., + max_messages: builtins.int = ..., + topics: collections.abc.Iterable[builtins.str] | None = ..., + wait_new_messages: builtins.bool = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "owner", b"owner", "start_position", b"start_position" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "max_messages", + b"max_messages", + "namespace", + b"namespace", + "owner", + b"owner", + "owner_run_id", + b"owner_run_id", + "start_position", + b"start_position", + "stream_name", + b"stream_name", + "topics", + b"topics", + "wait_new_messages", + b"wait_new_messages", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___PollWorkflowMessagesInput = PollWorkflowMessagesInput + +class DescribeWorkflowStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + OWNER_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + namespace: builtins.str + @property + def owner(self) -> global___StreamOwner: ... + workflow_id: builtins.str + """Shorthand for an owner of kind WORKFLOW, and refused alongside owner.""" + owner_run_id: builtins.str + stream_name: builtins.str + def __init__( + self, + *, + namespace: builtins.str = ..., + owner: global___StreamOwner | None = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_name: builtins.str = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["owner", b"owner"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "owner", + b"owner", + "owner_run_id", + b"owner_run_id", + "stream_name", + b"stream_name", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___DescribeWorkflowStreamInput = DescribeWorkflowStreamInput + +class AddWorkflowMessagesInput(google.protobuf.message.Message): + """Appending to an owned stream from outside its owner. A workflow's own + publishes ride its Workflow Task instead; an activity has no other path. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + OWNER_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + PRODUCER_ID_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + namespace: builtins.str + @property + def owner(self) -> global___StreamOwner: ... + workflow_id: builtins.str + """Shorthand for an owner of kind WORKFLOW, and refused alongside owner.""" + owner_run_id: builtins.str + stream_name: builtins.str + """Empty means the owner's default output stream.""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ]: ... + producer_id: builtins.str + """Optional idempotency, as on AddMessagesInput.""" + sequence: builtins.int + def __init__( + self, + *, + namespace: builtins.str = ..., + owner: global___StreamOwner | None = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_name: builtins.str = ..., + records: collections.abc.Iterable[ + temporalio.api.streamservice.v1.message_pb2.StreamRecord + ] + | None = ..., + producer_id: builtins.str = ..., + sequence: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["owner", b"owner"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "owner", + b"owner", + "owner_run_id", + b"owner_run_id", + "producer_id", + b"producer_id", + "records", + b"records", + "sequence", + b"sequence", + "stream_name", + b"stream_name", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___AddWorkflowMessagesInput = AddWorkflowMessagesInput + +class DescribeStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATE_FIELD_NUMBER: builtins.int + @property + def state(self) -> temporalio.api.streamservice.v1.stream_state_pb2.StreamState: ... + def __init__( + self, + *, + state: temporalio.api.streamservice.v1.stream_state_pb2.StreamState + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["state", b"state"] + ) -> builtins.bool: ... + def ClearField( + self, field_name: typing_extensions.Literal["state", b"state"] + ) -> None: ... + +global___DescribeStreamOutput = DescribeStreamOutput + +class CloseStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + REASON_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + @property + def reason(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + reason: temporalio.api.common.v1.message_pb2.Payload | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["reason", b"reason"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", b"namespace", "reason", b"reason", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___CloseStreamInput = CloseStreamInput + +class CloseStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___CloseStreamOutput = CloseStreamOutput + +class TruncateStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + NEW_BASE_OFFSET_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + new_base_offset: builtins.int + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + new_base_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "new_base_offset", + b"new_base_offset", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___TruncateStreamInput = TruncateStreamInput + +class TruncateStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___TruncateStreamOutput = TruncateStreamOutput + +class DeleteStreamInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + FORCE_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + force: builtins.bool + """Delete even while a workflow consumer is active. Without it the call is + refused, because the consumer's History depends on ranges the deletion + takes with it. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + force: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "force", b"force", "namespace", b"namespace", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___DeleteStreamInput = DeleteStreamInput + +class DeleteStreamOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___DeleteStreamOutput = DeleteStreamOutput + +class CreateStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___CreateStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___CreateStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___CreateStreamRequest = CreateStreamRequest + +class CreateStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___CreateStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___CreateStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___CreateStreamResponse = CreateStreamResponse + +class AddMessagesRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___AddMessagesInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___AddMessagesInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___AddMessagesRequest = AddMessagesRequest + +class AddMessagesResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___AddMessagesOutput: ... + def __init__( + self, + *, + frontend_response: global___AddMessagesOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___AddMessagesResponse = AddMessagesResponse + +class FinishWritingRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___FinishWritingInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___FinishWritingInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___FinishWritingRequest = FinishWritingRequest + +class FinishWritingResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___FinishWritingOutput: ... + def __init__( + self, + *, + frontend_response: global___FinishWritingOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___FinishWritingResponse = FinishWritingResponse + +class SubscribeWorkflowRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___SubscribeWorkflowInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___SubscribeWorkflowInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___SubscribeWorkflowRequest = SubscribeWorkflowRequest + +class SubscribeWorkflowResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___SubscribeWorkflowOutput: ... + def __init__( + self, + *, + frontend_response: global___SubscribeWorkflowOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___SubscribeWorkflowResponse = SubscribeWorkflowResponse + +class PollMessagesRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___PollMessagesInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___PollMessagesInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___PollMessagesRequest = PollMessagesRequest + +class PollMessagesResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___PollMessagesOutput: ... + def __init__( + self, + *, + frontend_response: global___PollMessagesOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___PollMessagesResponse = PollMessagesResponse + +class DescribeStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___DescribeStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___DescribeStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___DescribeStreamRequest = DescribeStreamRequest + +class DescribeStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___DescribeStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___DescribeStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___DescribeStreamResponse = DescribeStreamResponse + +class PollWorkflowMessagesRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___PollWorkflowMessagesInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___PollWorkflowMessagesInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___PollWorkflowMessagesRequest = PollWorkflowMessagesRequest + +class PollWorkflowMessagesResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___PollMessagesOutput: ... + def __init__( + self, + *, + frontend_response: global___PollMessagesOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___PollWorkflowMessagesResponse = PollWorkflowMessagesResponse + +class DescribeWorkflowStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___DescribeWorkflowStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___DescribeWorkflowStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___DescribeWorkflowStreamRequest = DescribeWorkflowStreamRequest + +class DescribeWorkflowStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___DescribeStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___DescribeStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___DescribeWorkflowStreamResponse = DescribeWorkflowStreamResponse + +class RegisterStreamConsumerInput(google.protobuf.message.Message): + """Registering a consumer on a stream in another execution. Split out from + SubscribeWorkflow because the two halves live on different shards: the pin + goes on the stream, the cursor goes on the consuming workflow, and a handler + can only reach the shard it was routed to. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + CONSUMER_WORKFLOW_ID_FIELD_NUMBER: builtins.int + CONSUMER_RUN_ID_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + START_POSITION_FIELD_NUMBER: builtins.int + namespace: builtins.str + stream_id: builtins.str + consumer_workflow_id: builtins.str + """The workflow that will consume. Together with the run it names the pin, + so a later run of the same workflow id registers fresh rather than + inheriting a closed run's floor. + """ + consumer_run_id: builtins.str + start_offset: builtins.int + """An absolute offset, read only when start_position is unset. A negative + value is refused. + """ + @property + def start_position( + self, + ) -> temporalio.api.stream.v1.message_pb2.StreamStartPosition: + """Resolved here, where the frontier is, and returned as start_offset on the + output so the cursor records a fact. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + stream_id: builtins.str = ..., + consumer_workflow_id: builtins.str = ..., + consumer_run_id: builtins.str = ..., + start_offset: builtins.int = ..., + start_position: temporalio.api.stream.v1.message_pb2.StreamStartPosition + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["start_position", b"start_position"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "consumer_run_id", + b"consumer_run_id", + "consumer_workflow_id", + b"consumer_workflow_id", + "namespace", + b"namespace", + "start_offset", + b"start_offset", + "start_position", + b"start_position", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___RegisterStreamConsumerInput = RegisterStreamConsumerInput + +class RegisterStreamConsumerOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + START_OFFSET_FIELD_NUMBER: builtins.int + KNOWN_HEAD_FIELD_NUMBER: builtins.int + STREAM_ABSENT_FIELD_NUMBER: builtins.int + start_offset: builtins.int + known_head: builtins.int + """The frontier at registration, so the cursor starts with a known head + instead of waiting for the first push. + """ + stream_absent: builtins.bool + """No execution in this namespace holds a stream with that id, so nothing + was registered. An answer rather than a NotFound because the caller acts + on it: a subscribe command falls back to a stream of the workflow's own by + that name. Every other NotFound, from a registry miss to a shard that has + moved, stays an error, since binding the workflow to different data on one + of those would be silent and permanent. + """ + def __init__( + self, + *, + start_offset: builtins.int = ..., + known_head: builtins.int = ..., + stream_absent: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "known_head", + b"known_head", + "start_offset", + b"start_offset", + "stream_absent", + b"stream_absent", + ], + ) -> None: ... + +global___RegisterStreamConsumerOutput = RegisterStreamConsumerOutput + +class AdvanceConsumerHeadInput(google.protobuf.message.Message): + """Telling one consumer that the frontier moved. Routed to the consumer, which + is not where the stream lives. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + OWNER_RUN_ID_FIELD_NUMBER: builtins.int + STREAM_ID_FIELD_NUMBER: builtins.int + HEAD_OFFSET_FIELD_NUMBER: builtins.int + namespace: builtins.str + workflow_id: builtins.str + owner_run_id: builtins.str + """The run that subscribed. A successor from continue-as-new holds no cursor + for this stream, so pushing the frontier at it would land nowhere. + """ + stream_id: builtins.str + head_offset: builtins.int + def __init__( + self, + *, + namespace: builtins.str = ..., + workflow_id: builtins.str = ..., + owner_run_id: builtins.str = ..., + stream_id: builtins.str = ..., + head_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "head_offset", + b"head_offset", + "namespace", + b"namespace", + "owner_run_id", + b"owner_run_id", + "stream_id", + b"stream_id", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___AdvanceConsumerHeadInput = AdvanceConsumerHeadInput + +class AdvanceConsumerHeadOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CONSUMER_CLOSED_FIELD_NUMBER: builtins.int + SUCCESSOR_RUN_ID_FIELD_NUMBER: builtins.int + SUCCESSOR_START_OFFSET_FIELD_NUMBER: builtins.int + consumer_closed: builtins.bool + """The run that held the pin is closed and no current run of the workflow + consumes the stream, so the stream can release the pin. + """ + successor_run_id: builtins.str + """The run that held the pin is closed but a successor carries the + subscription, so the stream re-keys the pin to it, with the floor the + successor's cursor started from. + """ + successor_start_offset: builtins.int + def __init__( + self, + *, + consumer_closed: builtins.bool = ..., + successor_run_id: builtins.str = ..., + successor_start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "consumer_closed", + b"consumer_closed", + "successor_run_id", + b"successor_run_id", + "successor_start_offset", + b"successor_start_offset", + ], + ) -> None: ... + +global___AdvanceConsumerHeadOutput = AdvanceConsumerHeadOutput + +class AddWorkflowMessagesRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___AddWorkflowMessagesInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___AddWorkflowMessagesInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___AddWorkflowMessagesRequest = AddWorkflowMessagesRequest + +class AddWorkflowMessagesResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___AddMessagesOutput: ... + def __init__( + self, + *, + frontend_response: global___AddMessagesOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___AddWorkflowMessagesResponse = AddWorkflowMessagesResponse + +class RegisterStreamConsumerRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___RegisterStreamConsumerInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___RegisterStreamConsumerInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___RegisterStreamConsumerRequest = RegisterStreamConsumerRequest + +class RegisterStreamConsumerResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___RegisterStreamConsumerOutput: ... + def __init__( + self, + *, + frontend_response: global___RegisterStreamConsumerOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___RegisterStreamConsumerResponse = RegisterStreamConsumerResponse + +class AdvanceConsumerHeadRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___AdvanceConsumerHeadInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___AdvanceConsumerHeadInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___AdvanceConsumerHeadRequest = AdvanceConsumerHeadRequest + +class AdvanceConsumerHeadResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___AdvanceConsumerHeadOutput: ... + def __init__( + self, + *, + frontend_response: global___AdvanceConsumerHeadOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___AdvanceConsumerHeadResponse = AdvanceConsumerHeadResponse + +class CloseStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___CloseStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___CloseStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___CloseStreamRequest = CloseStreamRequest + +class CloseStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___CloseStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___CloseStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___CloseStreamResponse = CloseStreamResponse + +class TruncateStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___TruncateStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___TruncateStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___TruncateStreamRequest = TruncateStreamRequest + +class TruncateStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___TruncateStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___TruncateStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___TruncateStreamResponse = TruncateStreamResponse + +class ListStreamsInput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + PAGE_SIZE_FIELD_NUMBER: builtins.int + NEXT_PAGE_TOKEN_FIELD_NUMBER: builtins.int + QUERY_FIELD_NUMBER: builtins.int + namespace: builtins.str + page_size: builtins.int + next_page_token: builtins.bytes + query: builtins.str + def __init__( + self, + *, + namespace: builtins.str = ..., + page_size: builtins.int = ..., + next_page_token: builtins.bytes = ..., + query: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "namespace", + b"namespace", + "next_page_token", + b"next_page_token", + "page_size", + b"page_size", + "query", + b"query", + ], + ) -> None: ... + +global___ListStreamsInput = ListStreamsInput + +class StreamListEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + stream_id: builtins.str + run_id: builtins.str + def __init__( + self, + *, + stream_id: builtins.str = ..., + run_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "run_id", b"run_id", "stream_id", b"stream_id" + ], + ) -> None: ... + +global___StreamListEntry = StreamListEntry + +class ListStreamsOutput(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAMS_FIELD_NUMBER: builtins.int + NEXT_PAGE_TOKEN_FIELD_NUMBER: builtins.int + @property + def streams( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamListEntry + ]: ... + next_page_token: builtins.bytes + def __init__( + self, + *, + streams: collections.abc.Iterable[global___StreamListEntry] | None = ..., + next_page_token: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "next_page_token", b"next_page_token", "streams", b"streams" + ], + ) -> None: ... + +global___ListStreamsOutput = ListStreamsOutput + +class ListStreamsRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___ListStreamsInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___ListStreamsInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___ListStreamsRequest = ListStreamsRequest + +class ListStreamsResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___ListStreamsOutput: ... + def __init__( + self, + *, + frontend_response: global___ListStreamsOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___ListStreamsResponse = ListStreamsResponse + +class DeleteStreamRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_ID_FIELD_NUMBER: builtins.int + FRONTEND_REQUEST_FIELD_NUMBER: builtins.int + namespace_id: builtins.str + @property + def frontend_request(self) -> global___DeleteStreamInput: ... + def __init__( + self, + *, + namespace_id: builtins.str = ..., + frontend_request: global___DeleteStreamInput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["frontend_request", b"frontend_request"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_request", b"frontend_request", "namespace_id", b"namespace_id" + ], + ) -> None: ... + +global___DeleteStreamRequest = DeleteStreamRequest + +class DeleteStreamResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FRONTEND_RESPONSE_FIELD_NUMBER: builtins.int + @property + def frontend_response(self) -> global___DeleteStreamOutput: ... + def __init__( + self, + *, + frontend_response: global___DeleteStreamOutput | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "frontend_response", b"frontend_response" + ], + ) -> None: ... + +global___DeleteStreamResponse = DeleteStreamResponse diff --git a/temporalio/api/streamservice/v1/request_response_pb2_grpc.py b/temporalio/api/streamservice/v1/request_response_pb2_grpc.py new file mode 100644 index 0000000000..bf947056a2 --- /dev/null +++ b/temporalio/api/streamservice/v1/request_response_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc diff --git a/temporalio/api/streamservice/v1/request_response_pb2_grpc.pyi b/temporalio/api/streamservice/v1/request_response_pb2_grpc.pyi new file mode 100644 index 0000000000..f3a5a087ef --- /dev/null +++ b/temporalio/api/streamservice/v1/request_response_pb2_grpc.pyi @@ -0,0 +1,4 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" diff --git a/temporalio/api/streamservice/v1/service_pb2.py b/temporalio/api/streamservice/v1/service_pb2.py new file mode 100644 index 0000000000..432abfc5c7 --- /dev/null +++ b/temporalio/api/streamservice/v1/service_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporalio/api/stream/v1/service.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.streamservice.v1 import ( + request_response_pb2 as temporalio_dot_api_dot_stream_dot_v1_dot_request__response__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n&temporalio/api/stream/v1/service.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a/temporalio/api/stream/v1/request_response.proto2\xce\x12\n\rStreamService\x12\x91\x01\n\x0c\x43reateStream\x12>.temporal.server.chasm.lib.stream.proto.v1.CreateStreamRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.CreateStreamResponse"\x00\x12\x8e\x01\n\x0b\x41\x64\x64Messages\x12=.temporal.server.chasm.lib.stream.proto.v1.AddMessagesRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.AddMessagesResponse"\x00\x12\x94\x01\n\rFinishWriting\x12?.temporal.server.chasm.lib.stream.proto.v1.FinishWritingRequest\x1a@.temporal.server.chasm.lib.stream.proto.v1.FinishWritingResponse"\x00\x12\xa0\x01\n\x11SubscribeWorkflow\x12\x43.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowRequest\x1a\x44.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowResponse"\x00\x12\x91\x01\n\x0cPollMessages\x12>.temporal.server.chasm.lib.stream.proto.v1.PollMessagesRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.PollMessagesResponse"\x00\x12\x97\x01\n\x0e\x44\x65scribeStream\x12@.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamRequest\x1a\x41.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamResponse"\x00\x12\xa9\x01\n\x14PollWorkflowMessages\x12\x46.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesRequest\x1aG.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesResponse"\x00\x12\xaf\x01\n\x16\x44\x65scribeWorkflowStream\x12H.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamRequest\x1aI.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamResponse"\x00\x12\xa6\x01\n\x13\x41\x64\x64WorkflowMessages\x12\x45.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesRequest\x1a\x46.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesResponse"\x00\x12\xaf\x01\n\x16RegisterStreamConsumer\x12H.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerRequest\x1aI.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerResponse"\x00\x12\xa6\x01\n\x13\x41\x64vanceConsumerHead\x12\x45.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadRequest\x1a\x46.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadResponse"\x00\x12\x8e\x01\n\x0b\x43loseStream\x12=.temporal.server.chasm.lib.stream.proto.v1.CloseStreamRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.CloseStreamResponse"\x00\x12\x97\x01\n\x0eTruncateStream\x12@.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamRequest\x1a\x41.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamResponse"\x00\x12\x8e\x01\n\x0bListStreams\x12=.temporal.server.chasm.lib.stream.proto.v1.ListStreamsRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.ListStreamsResponse"\x00\x12\x91\x01\n\x0c\x44\x65leteStream\x12>.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamResponse"\x00\x42>Z None: ... + CreateStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.CreateStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.CreateStreamResponse, + ] + AddMessages: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.AddMessagesRequest, + temporalio.api.streamservice.v1.request_response_pb2.AddMessagesResponse, + ] + FinishWriting: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.FinishWritingRequest, + temporalio.api.streamservice.v1.request_response_pb2.FinishWritingResponse, + ] + SubscribeWorkflow: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.SubscribeWorkflowRequest, + temporalio.api.streamservice.v1.request_response_pb2.SubscribeWorkflowResponse, + ] + PollMessages: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.PollMessagesRequest, + temporalio.api.streamservice.v1.request_response_pb2.PollMessagesResponse, + ] + DescribeStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.DescribeStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.DescribeStreamResponse, + ] + PollWorkflowMessages: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.PollWorkflowMessagesRequest, + temporalio.api.streamservice.v1.request_response_pb2.PollWorkflowMessagesResponse, + ] + """The owned-stream calls are routed on the owner, because the stream has no + id of its own. A workflow's activity is routed on the workflow. + """ + DescribeWorkflowStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.DescribeWorkflowStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.DescribeWorkflowStreamResponse, + ] + AddWorkflowMessages: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.AddWorkflowMessagesRequest, + temporalio.api.streamservice.v1.request_response_pb2.AddWorkflowMessagesResponse, + ] + RegisterStreamConsumer: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.RegisterStreamConsumerRequest, + temporalio.api.streamservice.v1.request_response_pb2.RegisterStreamConsumerResponse, + ] + """Internal. History calls these on itself to reach a shard it does not own, + which is the only way a step that spans two executions can work on a + cluster with more than one history host. + """ + AdvanceConsumerHead: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.AdvanceConsumerHeadRequest, + temporalio.api.streamservice.v1.request_response_pb2.AdvanceConsumerHeadResponse, + ] + CloseStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.CloseStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.CloseStreamResponse, + ] + TruncateStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.TruncateStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.TruncateStreamResponse, + ] + ListStreams: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.ListStreamsRequest, + temporalio.api.streamservice.v1.request_response_pb2.ListStreamsResponse, + ] + """Served on the frontend only: it queries visibility rather than a stream, + so there is no business ID to route on and nothing for a shard to answer. + """ + DeleteStream: grpc.UnaryUnaryMultiCallable[ + temporalio.api.streamservice.v1.request_response_pb2.DeleteStreamRequest, + temporalio.api.streamservice.v1.request_response_pb2.DeleteStreamResponse, + ] + +class StreamServiceServicer(metaclass=abc.ABCMeta): + @abc.abstractmethod + def CreateStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.CreateStreamRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.CreateStreamResponse: ... + @abc.abstractmethod + def AddMessages( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.AddMessagesRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.AddMessagesResponse: ... + @abc.abstractmethod + def FinishWriting( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.FinishWritingRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.FinishWritingResponse: ... + @abc.abstractmethod + def SubscribeWorkflow( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.SubscribeWorkflowRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.SubscribeWorkflowResponse + ): ... + @abc.abstractmethod + def PollMessages( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.PollMessagesRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.PollMessagesResponse: ... + @abc.abstractmethod + def DescribeStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.DescribeStreamRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.DescribeStreamResponse + ): ... + @abc.abstractmethod + def PollWorkflowMessages( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.PollWorkflowMessagesRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.PollWorkflowMessagesResponse: + """The owned-stream calls are routed on the owner, because the stream has no + id of its own. A workflow's activity is routed on the workflow. + """ + @abc.abstractmethod + def DescribeWorkflowStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.DescribeWorkflowStreamRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.DescribeWorkflowStreamResponse: ... + @abc.abstractmethod + def AddWorkflowMessages( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.AddWorkflowMessagesRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.AddWorkflowMessagesResponse + ): ... + @abc.abstractmethod + def RegisterStreamConsumer( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.RegisterStreamConsumerRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.RegisterStreamConsumerResponse: + """Internal. History calls these on itself to reach a shard it does not own, + which is the only way a step that spans two executions can work on a + cluster with more than one history host. + """ + @abc.abstractmethod + def AdvanceConsumerHead( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.AdvanceConsumerHeadRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.AdvanceConsumerHeadResponse + ): ... + @abc.abstractmethod + def CloseStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.CloseStreamRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.CloseStreamResponse: ... + @abc.abstractmethod + def TruncateStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.TruncateStreamRequest, + context: grpc.ServicerContext, + ) -> ( + temporalio.api.streamservice.v1.request_response_pb2.TruncateStreamResponse + ): ... + @abc.abstractmethod + def ListStreams( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.ListStreamsRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.ListStreamsResponse: + """Served on the frontend only: it queries visibility rather than a stream, + so there is no business ID to route on and nothing for a shard to answer. + """ + @abc.abstractmethod + def DeleteStream( + self, + request: temporalio.api.streamservice.v1.request_response_pb2.DeleteStreamRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.streamservice.v1.request_response_pb2.DeleteStreamResponse: ... + +def add_StreamServiceServicer_to_server( + servicer: StreamServiceServicer, server: grpc.Server +) -> None: ... diff --git a/temporalio/api/streamservice/v1/stream_state_pb2.py b/temporalio/api/streamservice/v1/stream_state_pb2.py new file mode 100644 index 0000000000..2b4eeafa33 --- /dev/null +++ b/temporalio/api/streamservice/v1/stream_state_pb2.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporalio/api/stream/v1/stream_state.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + +from temporalio.api.common.v1 import ( + message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n+temporalio/api/stream/v1/stream_state.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto"\xe3\x06\n\x0bStreamState\x12\x13\n\x0bhead_offset\x18\x01 \x01(\x03\x12\x13\n\x0b\x62\x61se_offset\x18\x02 \x01(\x03\x12\x0e\n\x06\x63losed\x18\x04 \x01(\x08\x12\x35\n\x0c\x63lose_reason\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\tproducers\x18\t \x03(\x0b\x32\x45.temporal.server.chasm.lib.stream.proto.v1.StreamState.ProducersEntry\x12X\n\tconsumers\x18\n \x03(\x0b\x32\x45.temporal.server.chasm.lib.stream.proto.v1.StreamState.ConsumersEntry\x12M\n\tlifecycle\x18\x0b \x01(\x0b\x32:.temporal.server.chasm.lib.stream.proto.v1.StreamLifecycle\x12\x17\n\x0fredirect_run_id\x18\x0c \x01(\t\x12.\n\nclose_time\x18\r \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12G\n\x06\x62udget\x18\x0e \x01(\x0b\x32\x37.temporal.server.chasm.lib.stream.proto.v1.StreamBudget\x12\x16\n\x0e\x61ppended_bytes\x18\x0f \x01(\x03\x12\x16\n\x0enotify_pending\x18\x10 \x01(\x08\x12\x12\n\nheld_bytes\x18\x11 \x01(\x03\x12\x18\n\x10\x61ge_task_pending\x18\x12 \x01(\x08\x1ak\n\x0eProducersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12H\n\x05value\x18\x02 \x01(\x0b\x32\x39.temporal.server.chasm.lib.stream.proto.v1.ProducerCursor:\x02\x38\x01\x1ak\n\x0e\x43onsumersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12H\n\x05value\x18\x02 \x01(\x0b\x32\x39.temporal.server.chasm.lib.stream.proto.v1.ConsumerCursor:\x02\x38\x01J\x04\x08\x03\x10\x04J\x04\x08\x06\x10\x07J\x04\x08\x07\x10\x08J\x04\x08\x08\x10\t"4\n\x0cStreamBudget\x12\x11\n\tmax_items\x18\x01 \x01(\x03\x12\x11\n\tmax_bytes\x18\x02 \x01(\x03"h\n\x0eProducerCursor\x12\x0b\n\x03seq\x18\x01 \x01(\x03\x12\x14\n\x0c\x66irst_offset\x18\x02 \x01(\x03\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\x12\x14\n\x0c\x63ontent_hash\x18\x04 \x01(\x0c\x12\x0e\n\x06\x66\x65nced\x18\x05 \x01(\x08"}\n\x0e\x43onsumerCursor\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x03\x12\x0e\n\x06\x61\x63tive\x18\x04 \x01(\x08\x12\x10\n\x08\x65xternal\x18\x05 \x01(\x08\x12\x14\n\x0creplay_floor\x18\x06 \x01(\x03"\xc0\x01\n\x14WorkflowStreamCursor\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x04 \x01(\x03\x12\x12\n\nknown_head\x18\x08 \x01(\x03\x12\x10\n\x08\x65xternal\x18\t \x01(\x08\x12\x14\n\x0cpending_from\x18\x05 \x01(\x03\x12\x12\n\npending_to\x18\x06 \x01(\x03\x12\x13\n\x0bhas_pending\x18\x07 \x01(\x08\x12\x14\n\x0cstart_offset\x18\n \x01(\x03J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04"e\n\x0fStreamLifecycle\x12,\n\tretention\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x11\n\tmax_items\x18\x02 \x01(\x03\x12\x11\n\tmax_bytes\x18\x03 \x01(\x03\x42>Z= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class StreamState(google.protobuf.message.Message): + """Size is O(producers + consumers), never O(messages). Payload bytes live in + the component's data nodes, not here, which keeps this off the CHASM + partial-read path. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class ProducersEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___ProducerCursor: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___ProducerCursor | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["key", b"key", "value", b"value"], + ) -> None: ... + + class ConsumersEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___ConsumerCursor: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___ConsumerCursor | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["key", b"key", "value", b"value"], + ) -> None: ... + + HEAD_OFFSET_FIELD_NUMBER: builtins.int + BASE_OFFSET_FIELD_NUMBER: builtins.int + CLOSED_FIELD_NUMBER: builtins.int + CLOSE_REASON_FIELD_NUMBER: builtins.int + PRODUCERS_FIELD_NUMBER: builtins.int + CONSUMERS_FIELD_NUMBER: builtins.int + LIFECYCLE_FIELD_NUMBER: builtins.int + REDIRECT_RUN_ID_FIELD_NUMBER: builtins.int + CLOSE_TIME_FIELD_NUMBER: builtins.int + BUDGET_FIELD_NUMBER: builtins.int + APPENDED_BYTES_FIELD_NUMBER: builtins.int + NOTIFY_PENDING_FIELD_NUMBER: builtins.int + HELD_BYTES_FIELD_NUMBER: builtins.int + AGE_TASK_PENDING_FIELD_NUMBER: builtins.int + head_offset: builtins.int + """Visibility frontier. Readers never observe an offset at or past this.""" + base_offset: builtins.int + """Truncation floor. Offsets below this are gone.""" + closed: builtins.bool + @property + def close_reason(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + @property + def producers( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, global___ProducerCursor + ]: ... + @property + def consumers( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, global___ConsumerCursor + ]: ... + @property + def lifecycle(self) -> global___StreamLifecycle: ... + redirect_run_id: builtins.str + """Set when a successor run takes ownership, so an in-flight poll can follow + the chain instead of stalling on a superseded run. + """ + @property + def close_time(self) -> google.protobuf.timestamp_pb2.Timestamp: + """Wall-clock close time, used to schedule retention deletion.""" + @property + def budget(self) -> global___StreamBudget: + """Set on a stream a workflow owns. Its batches are the owning execution's + mutable state, so appends past the budget are refused rather than left to + the execution size limit, which terminates the workflow. + """ + appended_bytes: builtins.int + """Bytes appended over the stream's life, kept for the budget check. A stream + with a budget carries no lifecycle cap and is never truncated, so nothing + reclaims behind it and this is also what it holds. The item side of the + budget measures head minus base for the same reason: offsets are global + and a stream can begin above zero. + """ + notify_pending: builtins.bool + """A notify task is scheduled and has not run yet. Appends while it is set + schedule none of their own; the task reads the head when it runs, so it + carries every append that landed before it. + """ + held_bytes: builtins.int + """Bytes of the batches the stream still holds, from the floor to the head. + Kept for the lifecycle's byte cap, which the appended total cannot serve + once truncation has reclaimed behind the floor. A batch straddling the + floor counts whole, since it is held whole. + """ + age_task_pending: builtins.bool + """An age check is scheduled and has not run yet. Appends while it is set + schedule none of their own; the check re-arms itself while the stream + holds records and lowers the flag when it holds none. + """ + def __init__( + self, + *, + head_offset: builtins.int = ..., + base_offset: builtins.int = ..., + closed: builtins.bool = ..., + close_reason: temporalio.api.common.v1.message_pb2.Payload | None = ..., + producers: collections.abc.Mapping[builtins.str, global___ProducerCursor] + | None = ..., + consumers: collections.abc.Mapping[builtins.str, global___ConsumerCursor] + | None = ..., + lifecycle: global___StreamLifecycle | None = ..., + redirect_run_id: builtins.str = ..., + close_time: google.protobuf.timestamp_pb2.Timestamp | None = ..., + budget: global___StreamBudget | None = ..., + appended_bytes: builtins.int = ..., + notify_pending: builtins.bool = ..., + held_bytes: builtins.int = ..., + age_task_pending: builtins.bool = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "budget", + b"budget", + "close_reason", + b"close_reason", + "close_time", + b"close_time", + "lifecycle", + b"lifecycle", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "age_task_pending", + b"age_task_pending", + "appended_bytes", + b"appended_bytes", + "base_offset", + b"base_offset", + "budget", + b"budget", + "close_reason", + b"close_reason", + "close_time", + b"close_time", + "closed", + b"closed", + "consumers", + b"consumers", + "head_offset", + b"head_offset", + "held_bytes", + b"held_bytes", + "lifecycle", + b"lifecycle", + "notify_pending", + b"notify_pending", + "producers", + b"producers", + "redirect_run_id", + b"redirect_run_id", + ], + ) -> None: ... + +global___StreamState = StreamState + +class StreamBudget(google.protobuf.message.Message): + """Hard bounds on what a stream may hold. Distinct from StreamLifecycle.max_items, + which reclaims the oldest messages: a budget refuses the newest. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + MAX_ITEMS_FIELD_NUMBER: builtins.int + MAX_BYTES_FIELD_NUMBER: builtins.int + max_items: builtins.int + max_bytes: builtins.int + def __init__( + self, + *, + max_items: builtins.int = ..., + max_bytes: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "max_bytes", b"max_bytes", "max_items", b"max_items" + ], + ) -> None: ... + +global___StreamBudget = StreamBudget + +class ProducerCursor(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + FIRST_OFFSET_FIELD_NUMBER: builtins.int + COUNT_FIELD_NUMBER: builtins.int + CONTENT_HASH_FIELD_NUMBER: builtins.int + FENCED_FIELD_NUMBER: builtins.int + seq: builtins.int + first_offset: builtins.int + count: builtins.int + content_hash: builtins.bytes + """Distinguishes a genuine retry from a client reusing a sequence with + different content, which must be rejected rather than deduplicated. + """ + fenced: builtins.bool + """Set by FinishWriting. Ends this producer's writes without closing the + stream for anyone else. + """ + def __init__( + self, + *, + seq: builtins.int = ..., + first_offset: builtins.int = ..., + count: builtins.int = ..., + content_hash: builtins.bytes = ..., + fenced: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "content_hash", + b"content_hash", + "count", + b"count", + "fenced", + b"fenced", + "first_offset", + b"first_offset", + "seq", + b"seq", + ], + ) -> None: ... + +global___ProducerCursor = ProducerCursor + +class ConsumerCursor(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WORKFLOW_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + OFFSET_FIELD_NUMBER: builtins.int + ACTIVE_FIELD_NUMBER: builtins.int + EXTERNAL_FIELD_NUMBER: builtins.int + REPLAY_FLOOR_FIELD_NUMBER: builtins.int + workflow_id: builtins.str + run_id: builtins.str + offset: builtins.int + """How far this consumer has read. Used to decide whether it needs waking, + not to decide what the stream may drop. + """ + active: builtins.bool + external: builtins.bool + """Set when the consumer is a workflow in another execution, which is the + only case that has to be told the frontier moved. A workflow consuming a + stream it owns sees that while closing its own transaction. + """ + replay_floor: builtins.int + """The oldest offset this consumer's History still depends on. + + It is where the subscription started, not where it has read to. A range + this consumer already consumed is recorded in its History and has to be + re-readable for the workflow to replay, so bytes below the read position + are exactly the ones a replay needs most. It stays put while the consumer + is active and is released when it deregisters. + """ + def __init__( + self, + *, + workflow_id: builtins.str = ..., + run_id: builtins.str = ..., + offset: builtins.int = ..., + active: builtins.bool = ..., + external: builtins.bool = ..., + replay_floor: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "active", + b"active", + "external", + b"external", + "offset", + b"offset", + "replay_floor", + b"replay_floor", + "run_id", + b"run_id", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___ConsumerCursor = ConsumerCursor + +class WorkflowStreamCursor(google.protobuf.message.Message): + """A consuming Workflow's position in a stream. This lives in the consuming + Workflow's own state rather than on the stream, so advancing it commits in + the same transaction as the WorkflowTaskCompleted event that records the + range. Keeping it on the stream would make the advance a cross-execution + write, and a crash between the two would either redeliver or skip. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + OFFSET_FIELD_NUMBER: builtins.int + KNOWN_HEAD_FIELD_NUMBER: builtins.int + EXTERNAL_FIELD_NUMBER: builtins.int + PENDING_FROM_FIELD_NUMBER: builtins.int + PENDING_TO_FIELD_NUMBER: builtins.int + HAS_PENDING_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + stream_id: builtins.str + offset: builtins.int + """Next offset to deliver.""" + known_head: builtins.int + """The stream's frontier as of the last delivery. A workflow consuming a + stream in another execution cannot read the real frontier while closing its + own transaction, so this is what tells it a capped slice left more behind. + """ + external: builtins.bool + """Set when the stream lives in another execution, which decides whether the + frontier is read locally or taken from known_head. + """ + pending_from: builtins.int + """The range attached to the Workflow Task currently in flight. Recorded on + the event that closes that task, then folded into offset. An empty range + is still recorded: a task where the subscription saw nothing is a fact + replay has to reproduce. + """ + pending_to: builtins.int + has_pending: builtins.bool + start_offset: builtins.int + """Where this subscription began reading. Retained separately from offset, + which advances, because replay has to tell "committed nothing yet" apart + from "the events recording what was committed are not in this page". + """ + def __init__( + self, + *, + stream_id: builtins.str = ..., + offset: builtins.int = ..., + known_head: builtins.int = ..., + external: builtins.bool = ..., + pending_from: builtins.int = ..., + pending_to: builtins.int = ..., + has_pending: builtins.bool = ..., + start_offset: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "external", + b"external", + "has_pending", + b"has_pending", + "known_head", + b"known_head", + "offset", + b"offset", + "pending_from", + b"pending_from", + "pending_to", + b"pending_to", + "start_offset", + b"start_offset", + "stream_id", + b"stream_id", + ], + ) -> None: ... + +global___WorkflowStreamCursor = WorkflowStreamCursor + +class StreamLifecycle(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RETENTION_FIELD_NUMBER: builtins.int + MAX_ITEMS_FIELD_NUMBER: builtins.int + MAX_BYTES_FIELD_NUMBER: builtins.int + @property + def retention(self) -> google.protobuf.duration_pb2.Duration: + """How long a record stays readable, and how long a closed stream stays + readable before it is deleted. On an open stream batches older than this + are reclaimed behind the floor, never past an active consumer's floor; on + a closed one it is the time to deletion. + """ + max_items: builtins.int + """Cap on readable messages. Whole batches are reclaimed once the floor + passes them, so a capped stream has bounded storage. + + While a workflow consumer is registered the cap stops being a rolling + window: the consumer's floor sits at the offset it subscribed from, + because replay re-reads every range its History recorded, so the cap + behaves as a lifetime quota measured from that subscription and appends + past it are refused rather than reclaiming behind the consumer. + """ + max_bytes: builtins.int + """Cap on held bytes. Unlike max_items it reclaims nothing: an append that + would take the held bytes past it is refused, and room comes back only as + the record cap, an explicit truncation or the retention age reclaims + batches behind the floor. + """ + def __init__( + self, + *, + retention: google.protobuf.duration_pb2.Duration | None = ..., + max_items: builtins.int = ..., + max_bytes: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["retention", b"retention"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "max_bytes", + b"max_bytes", + "max_items", + b"max_items", + "retention", + b"retention", + ], + ) -> None: ... + +global___StreamLifecycle = StreamLifecycle diff --git a/temporalio/api/streamservice/v1/stream_state_pb2_grpc.py b/temporalio/api/streamservice/v1/stream_state_pb2_grpc.py new file mode 100644 index 0000000000..bf947056a2 --- /dev/null +++ b/temporalio/api/streamservice/v1/stream_state_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc diff --git a/temporalio/api/streamservice/v1/stream_state_pb2_grpc.pyi b/temporalio/api/streamservice/v1/stream_state_pb2_grpc.pyi new file mode 100644 index 0000000000..f3a5a087ef --- /dev/null +++ b/temporalio/api/streamservice/v1/stream_state_pb2_grpc.pyi @@ -0,0 +1,4 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" diff --git a/temporalio/api/workflow/v1/__init__.py b/temporalio/api/workflow/v1/__init__.py index ae647ab673..b708ebed66 100644 --- a/temporalio/api/workflow/v1/__init__.py +++ b/temporalio/api/workflow/v1/__init__.py @@ -1,5 +1,6 @@ from .message_pb2 import ( CallbackInfo, + ChannelSubscriptionInfo, DeploymentTransition, DeploymentVersionTransition, NewWorkflowExecutionInfo, @@ -24,6 +25,7 @@ __all__ = [ "CallbackInfo", + "ChannelSubscriptionInfo", "DeploymentTransition", "DeploymentVersionTransition", "NewWorkflowExecutionInfo", diff --git a/temporalio/api/workflow/v1/message_pb2.py b/temporalio/api/workflow/v1/message_pb2.py index 032f2b2b56..89648e72c6 100644 --- a/temporalio/api/workflow/v1/message_pb2.py +++ b/temporalio/api/workflow/v1/message_pb2.py @@ -40,6 +40,9 @@ from temporalio.api.failure.v1 import ( message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, ) +from temporalio.api.notification.v1 import ( + message_pb2 as temporal_dot_api_dot_notification_dot_v1_dot_message__pb2, +) from temporalio.api.sdk.v1 import ( user_metadata_pb2 as temporal_dot_api_dot_sdk_dot_v1_dot_user__metadata__pb2, ) @@ -48,7 +51,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n&temporal/api/workflow/v1/message.proto\x12\x18temporal.api.workflow.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a google/protobuf/field_mask.proto\x1a&temporal/api/activity/v1/message.proto\x1a"temporal/api/enums/v1/common.proto\x1a&temporal/api/enums/v1/event_type.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/deployment/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/taskqueue/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto"\xf0\t\n\x15WorkflowExecutionInfo\x12<\n\texecution\x18\x01 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x04type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12.\n\nstart_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nclose_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x06status\x18\x05 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowExecutionStatus\x12\x16\n\x0ehistory_length\x18\x06 \x01(\x03\x12\x1b\n\x13parent_namespace_id\x18\x07 \x01(\t\x12\x43\n\x10parent_execution\x18\x08 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x0e\x65xecution_time\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x04memo\x18\n \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0b \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12@\n\x11\x61uto_reset_points\x18\x0c \x01(\x0b\x32%.temporal.api.workflow.v1.ResetPoints\x12\x12\n\ntask_queue\x18\r \x01(\t\x12\x1e\n\x16state_transition_count\x18\x0e \x01(\x03\x12\x1a\n\x12history_size_bytes\x18\x0f \x01(\x03\x12X\n most_recent_worker_version_stamp\x18\x10 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12\x35\n\x12\x65xecution_duration\x18\x11 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x41\n\x0eroot_execution\x18\x12 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1d\n\x11\x61ssigned_build_id\x18\x13 \x01(\tB\x02\x18\x01\x12\x1e\n\x12inherited_build_id\x18\x14 \x01(\tB\x02\x18\x01\x12\x14\n\x0c\x66irst_run_id\x18\x15 \x01(\t\x12R\n\x0fversioning_info\x18\x16 \x01(\x0b\x32\x39.temporal.api.workflow.v1.WorkflowExecutionVersioningInfo\x12\x1e\n\x16worker_deployment_name\x18\x17 \x01(\t\x12\x32\n\x08priority\x18\x18 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12#\n\x1b\x65xternal_payload_size_bytes\x18\x19 \x01(\x03\x12\x1e\n\x16\x65xternal_payload_count\x18\x1a \x01(\x03"\x8c\x05\n\x1dWorkflowExecutionExtendedInfo\x12=\n\x19\x65xecution_expiration_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x37\n\x13run_expiration_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x18\n\x10\x63\x61ncel_requested\x18\x03 \x01(\x08\x12\x33\n\x0flast_reset_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x37\n\x13original_start_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0creset_run_id\x18\x06 \x01(\t\x12\x65\n\x10request_id_infos\x18\x07 \x03(\x0b\x32K.temporal.api.workflow.v1.WorkflowExecutionExtendedInfo.RequestIdInfosEntry\x12H\n\npause_info\x18\x08 \x01(\x0b\x32\x34.temporal.api.workflow.v1.WorkflowExecutionPauseInfo\x12\x44\n\x12time_skipping_info\x18\t \x01(\x0b\x32(.temporal.api.common.v1.TimeSkippingInfo\x1a^\n\x13RequestIdInfosEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x36\n\x05value\x18\x02 \x01(\x0b\x32\'.temporal.api.workflow.v1.RequestIdInfo:\x02\x38\x01"\xfb\x04\n\x1fWorkflowExecutionVersioningInfo\x12;\n\x08\x62\x65havior\x18\x01 \x01(\x0e\x32).temporal.api.enums.v1.VersioningBehavior\x12>\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12\x13\n\x07version\x18\x05 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x07 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12I\n\x13versioning_override\x18\x03 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12Q\n\x15\x64\x65ployment_transition\x18\x04 \x01(\x0b\x32..temporal.api.workflow.v1.DeploymentTransitionB\x02\x18\x01\x12Q\n\x12version_transition\x18\x06 \x01(\x0b\x32\x35.temporal.api.workflow.v1.DeploymentVersionTransition\x12\x17\n\x0frevision_number\x18\x08 \x01(\x03\x12k\n+continue_as_new_initial_versioning_behavior\x18\t \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior"R\n\x14\x44\x65ploymentTransition\x12:\n\ndeployment\x18\x01 \x01(\x0b\x32&.temporal.api.deployment.v1.Deployment"\x83\x01\n\x1b\x44\x65ploymentVersionTransition\x12\x13\n\x07version\x18\x01 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion"\xc7\x02\n\x17WorkflowExecutionConfig\x12\x38\n\ntask_queue\x18\x01 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12=\n\x1aworkflow_execution_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12@\n\x1d\x64\x65\x66\x61ult_workflow_task_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\ruser_metadata\x18\x05 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata"\xbd\r\n\x13PendingActivityInfo\x12\x13\n\x0b\x61\x63tivity_id\x18\x01 \x01(\t\x12;\n\ractivity_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12:\n\x05state\x18\x03 \x01(\x0e\x32+.temporal.api.enums.v1.PendingActivityState\x12;\n\x11heartbeat_details\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x37\n\x13last_heartbeat_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x35\n\x11last_started_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x07 \x01(\x05\x12\x18\n\x10maximum_attempts\x18\x08 \x01(\x05\x12\x32\n\x0escheduled_time\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x0f\x65xpiration_time\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x36\n\x0clast_failure\x18\x0b \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x1c\n\x14last_worker_identity\x18\x0c \x01(\t\x12;\n\x15use_workflow_build_id\x18\r \x01(\x0b\x32\x16.google.protobuf.EmptyB\x02\x18\x01H\x00\x12\x32\n$last_independently_assigned_build_id\x18\x0e \x01(\tB\x02\x18\x01H\x00\x12Q\n\x19last_worker_version_stamp\x18\x0f \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12\x39\n\x16\x63urrent_retry_interval\x18\x10 \x01(\x0b\x32\x19.google.protobuf.Duration\x12>\n\x1alast_attempt_complete_time\x18\x11 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x1anext_attempt_schedule_time\x18\x12 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06paused\x18\x13 \x01(\x08\x12\x43\n\x0flast_deployment\x18\x14 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12*\n\x1elast_worker_deployment_version\x18\x15 \x01(\tB\x02\x18\x01\x12T\n\x17last_deployment_version\x18\x19 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x32\n\x08priority\x18\x16 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12K\n\npause_info\x18\x17 \x01(\x0b\x32\x37.temporal.api.workflow.v1.PendingActivityInfo.PauseInfo\x12\x43\n\x10\x61\x63tivity_options\x18\x18 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions\x1a\xcf\x02\n\tPauseInfo\x12.\n\npause_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12P\n\x06manual\x18\x02 \x01(\x0b\x32>.temporal.api.workflow.v1.PendingActivityInfo.PauseInfo.ManualH\x00\x12L\n\x04rule\x18\x04 \x01(\x0b\x32<.temporal.api.workflow.v1.PendingActivityInfo.PauseInfo.RuleH\x00\x1a*\n\x06Manual\x12\x10\n\x08identity\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x1a\x39\n\x04Rule\x12\x0f\n\x07rule_id\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\tB\x0b\n\tpaused_byB\x13\n\x11\x61ssigned_build_id"\xb9\x01\n\x19PendingChildExecutionInfo\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12\x1a\n\x12workflow_type_name\x18\x03 \x01(\t\x12\x14\n\x0cinitiated_id\x18\x04 \x01(\x03\x12\x45\n\x13parent_close_policy\x18\x05 \x01(\x0e\x32(.temporal.api.enums.v1.ParentClosePolicy"\x8d\x02\n\x17PendingWorkflowTaskInfo\x12>\n\x05state\x18\x01 \x01(\x0e\x32/.temporal.api.enums.v1.PendingWorkflowTaskState\x12\x32\n\x0escheduled_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12;\n\x17original_scheduled_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0cstarted_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x05 \x01(\x05"G\n\x0bResetPoints\x12\x38\n\x06points\x18\x01 \x03(\x0b\x32(.temporal.api.workflow.v1.ResetPointInfo"\xef\x01\n\x0eResetPointInfo\x12\x10\n\x08\x62uild_id\x18\x07 \x01(\t\x12\x1b\n\x0f\x62inary_checksum\x18\x01 \x01(\tB\x02\x18\x01\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12(\n first_workflow_task_completed_id\x18\x03 \x01(\x03\x12/\n\x0b\x63reate_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0b\x65xpire_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nresettable\x18\x06 \x01(\x08"\x85\x07\n\x18NewWorkflowExecutionInfo\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12;\n\rworkflow_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x03 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x18workflow_id_reuse_policy\x18\x08 \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\n \x01(\t\x12*\n\x04memo\x18\x0b \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0c \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\x06header\x18\r \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x38\n\ruser_metadata\x18\x0e \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12I\n\x13versioning_override\x18\x0f \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority"\x82\x06\n\x0c\x43\x61llbackInfo\x12\x32\n\x08\x63\x61llback\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Callback\x12?\n\x07trigger\x18\x02 \x01(\x0b\x32..temporal.api.workflow.v1.CallbackInfo.Trigger\x12\x35\n\x11registration_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x05state\x18\x04 \x01(\x0e\x32$.temporal.api.enums.v1.CallbackState\x12\x0f\n\x07\x61ttempt\x18\x05 \x01(\x05\x12>\n\x1alast_attempt_complete_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x14last_attempt_failure\x18\x07 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12>\n\x1anext_attempt_schedule_time\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x16\n\x0e\x62locked_reason\x18\t \x01(\t\x1a\x10\n\x0eWorkflowClosed\x1a\x35\n UpdateWorkflowExecutionCompleted\x12\x11\n\tupdate_id\x18\x01 \x01(\t\x1a\xde\x01\n\x07Trigger\x12P\n\x0fworkflow_closed\x18\x01 \x01(\x0b\x32\x35.temporal.api.workflow.v1.CallbackInfo.WorkflowClosedH\x00\x12v\n#update_workflow_execution_completed\x18\x02 \x01(\x0b\x32G.temporal.api.workflow.v1.CallbackInfo.UpdateWorkflowExecutionCompletedH\x00\x42\t\n\x07variant"\x8b\x06\n\x19PendingNexusOperationInfo\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x0f\n\x07service\x18\x02 \x01(\t\x12\x11\n\toperation\x18\x03 \x01(\t\x12\x18\n\x0coperation_id\x18\x04 \x01(\tB\x02\x18\x01\x12<\n\x19schedule_to_close_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x32\n\x0escheduled_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\x05state\x18\x07 \x01(\x0e\x32\x31.temporal.api.enums.v1.PendingNexusOperationState\x12\x0f\n\x07\x61ttempt\x18\x08 \x01(\x05\x12>\n\x1alast_attempt_complete_time\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x14last_attempt_failure\x18\n \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12>\n\x1anext_attempt_schedule_time\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12S\n\x11\x63\x61ncellation_info\x18\x0c \x01(\x0b\x32\x38.temporal.api.workflow.v1.NexusOperationCancellationInfo\x12\x1a\n\x12scheduled_event_id\x18\r \x01(\x03\x12\x16\n\x0e\x62locked_reason\x18\x0e \x01(\t\x12\x17\n\x0foperation_token\x18\x0f \x01(\t\x12<\n\x19schedule_to_start_timeout\x18\x10 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\x11 \x01(\x0b\x32\x19.google.protobuf.Duration"\x84\x03\n\x1eNexusOperationCancellationInfo\x12\x32\n\x0erequested_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n\x05state\x18\x02 \x01(\x0e\x32\x36.temporal.api.enums.v1.NexusOperationCancellationState\x12\x0f\n\x07\x61ttempt\x18\x03 \x01(\x05\x12>\n\x1alast_attempt_complete_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x14last_attempt_failure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12>\n\x1anext_attempt_schedule_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x16\n\x0e\x62locked_reason\x18\x07 \x01(\t"\xe3\x01\n\x18WorkflowExecutionOptions\x12I\n\x13versioning_override\x18\x01 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12\x32\n\x08priority\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12H\n\x14time_skipping_config\x18\x03 \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig"\xfa\x05\n\x12VersioningOverride\x12M\n\x06pinned\x18\x03 \x01(\x0b\x32;.temporal.api.workflow.v1.VersioningOverride.PinnedOverrideH\x00\x12\x16\n\x0c\x61uto_upgrade\x18\x04 \x01(\x08H\x00\x12P\n\x08one_time\x18\x05 \x01(\x0b\x32<.temporal.api.workflow.v1.VersioningOverride.OneTimeOverrideH\x00\x12?\n\x08\x62\x65havior\x18\x01 \x01(\x0e\x32).temporal.api.enums.v1.VersioningBehaviorB\x02\x18\x01\x12>\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12\x1a\n\x0epinned_version\x18\t \x01(\tB\x02\x18\x01\x1a\xad\x01\n\x0ePinnedOverride\x12U\n\x08\x62\x65havior\x18\x01 \x01(\x0e\x32\x43.temporal.api.workflow.v1.VersioningOverride.PinnedOverrideBehavior\x12\x44\n\x07version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x1ai\n\x0fOneTimeOverride\x12V\n\x19target_deployment_version\x18\x01 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion"g\n\x16PinnedOverrideBehavior\x12(\n$PINNED_OVERRIDE_BEHAVIOR_UNSPECIFIED\x10\x00\x12#\n\x1fPINNED_OVERRIDE_BEHAVIOR_PINNED\x10\x01\x42\n\n\x08override"i\n\x11OnConflictOptions\x12\x19\n\x11\x61ttach_request_id\x18\x01 \x01(\x08\x12#\n\x1b\x61ttach_completion_callbacks\x18\x02 \x01(\x08\x12\x14\n\x0c\x61ttach_links\x18\x03 \x01(\x08"i\n\rRequestIdInfo\x12\x34\n\nevent_type\x18\x01 \x01(\x0e\x32 .temporal.api.enums.v1.EventType\x12\x10\n\x08\x65vent_id\x18\x02 \x01(\x03\x12\x10\n\x08\x62uffered\x18\x03 \x01(\x08"\xb7\x04\n\x12PostResetOperation\x12V\n\x0fsignal_workflow\x18\x01 \x01(\x0b\x32;.temporal.api.workflow.v1.PostResetOperation.SignalWorkflowH\x00\x12\x65\n\x17update_workflow_options\x18\x02 \x01(\x0b\x32\x42.temporal.api.workflow.v1.PostResetOperation.UpdateWorkflowOptionsH\x00\x1a\xb3\x01\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12/\n\x05input\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12.\n\x06header\x18\x03 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12+\n\x05links\x18\x04 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x1a\xa0\x01\n\x15UpdateWorkflowOptions\x12V\n\x1aworkflow_execution_options\x18\x01 \x01(\x0b\x32\x32.temporal.api.workflow.v1.WorkflowExecutionOptions\x12/\n\x0bupdate_mask\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskB\t\n\x07variant"o\n\x1aWorkflowExecutionPauseInfo\x12\x10\n\x08identity\x18\x01 \x01(\t\x12/\n\x0bpaused_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06reason\x18\x03 \x01(\tB\x93\x01\n\x1bio.temporal.api.workflow.v1B\x0cMessageProtoP\x01Z\'go.temporal.io/api/workflow/v1;workflow\xaa\x02\x1aTemporalio.Api.Workflow.V1\xea\x02\x1dTemporalio::Api::Workflow::V1b\x06proto3' + b'\n&temporal/api/workflow/v1/message.proto\x12\x18temporal.api.workflow.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a google/protobuf/field_mask.proto\x1a&temporal/api/activity/v1/message.proto\x1a"temporal/api/enums/v1/common.proto\x1a&temporal/api/enums/v1/event_type.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/deployment/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a*temporal/api/notification/v1/message.proto\x1a\'temporal/api/taskqueue/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto"\xf0\t\n\x15WorkflowExecutionInfo\x12<\n\texecution\x18\x01 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x04type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12.\n\nstart_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nclose_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x06status\x18\x05 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowExecutionStatus\x12\x16\n\x0ehistory_length\x18\x06 \x01(\x03\x12\x1b\n\x13parent_namespace_id\x18\x07 \x01(\t\x12\x43\n\x10parent_execution\x18\x08 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x0e\x65xecution_time\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x04memo\x18\n \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0b \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12@\n\x11\x61uto_reset_points\x18\x0c \x01(\x0b\x32%.temporal.api.workflow.v1.ResetPoints\x12\x12\n\ntask_queue\x18\r \x01(\t\x12\x1e\n\x16state_transition_count\x18\x0e \x01(\x03\x12\x1a\n\x12history_size_bytes\x18\x0f \x01(\x03\x12X\n most_recent_worker_version_stamp\x18\x10 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12\x35\n\x12\x65xecution_duration\x18\x11 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x41\n\x0eroot_execution\x18\x12 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1d\n\x11\x61ssigned_build_id\x18\x13 \x01(\tB\x02\x18\x01\x12\x1e\n\x12inherited_build_id\x18\x14 \x01(\tB\x02\x18\x01\x12\x14\n\x0c\x66irst_run_id\x18\x15 \x01(\t\x12R\n\x0fversioning_info\x18\x16 \x01(\x0b\x32\x39.temporal.api.workflow.v1.WorkflowExecutionVersioningInfo\x12\x1e\n\x16worker_deployment_name\x18\x17 \x01(\t\x12\x32\n\x08priority\x18\x18 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12#\n\x1b\x65xternal_payload_size_bytes\x18\x19 \x01(\x03\x12\x1e\n\x16\x65xternal_payload_count\x18\x1a \x01(\x03"\x8c\x05\n\x1dWorkflowExecutionExtendedInfo\x12=\n\x19\x65xecution_expiration_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x37\n\x13run_expiration_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x18\n\x10\x63\x61ncel_requested\x18\x03 \x01(\x08\x12\x33\n\x0flast_reset_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x37\n\x13original_start_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0creset_run_id\x18\x06 \x01(\t\x12\x65\n\x10request_id_infos\x18\x07 \x03(\x0b\x32K.temporal.api.workflow.v1.WorkflowExecutionExtendedInfo.RequestIdInfosEntry\x12H\n\npause_info\x18\x08 \x01(\x0b\x32\x34.temporal.api.workflow.v1.WorkflowExecutionPauseInfo\x12\x44\n\x12time_skipping_info\x18\t \x01(\x0b\x32(.temporal.api.common.v1.TimeSkippingInfo\x1a^\n\x13RequestIdInfosEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x36\n\x05value\x18\x02 \x01(\x0b\x32\'.temporal.api.workflow.v1.RequestIdInfo:\x02\x38\x01"\xfb\x04\n\x1fWorkflowExecutionVersioningInfo\x12;\n\x08\x62\x65havior\x18\x01 \x01(\x0e\x32).temporal.api.enums.v1.VersioningBehavior\x12>\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12\x13\n\x07version\x18\x05 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x07 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12I\n\x13versioning_override\x18\x03 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12Q\n\x15\x64\x65ployment_transition\x18\x04 \x01(\x0b\x32..temporal.api.workflow.v1.DeploymentTransitionB\x02\x18\x01\x12Q\n\x12version_transition\x18\x06 \x01(\x0b\x32\x35.temporal.api.workflow.v1.DeploymentVersionTransition\x12\x17\n\x0frevision_number\x18\x08 \x01(\x03\x12k\n+continue_as_new_initial_versioning_behavior\x18\t \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior"R\n\x14\x44\x65ploymentTransition\x12:\n\ndeployment\x18\x01 \x01(\x0b\x32&.temporal.api.deployment.v1.Deployment"\x83\x01\n\x1b\x44\x65ploymentVersionTransition\x12\x13\n\x07version\x18\x01 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion"\xc7\x02\n\x17WorkflowExecutionConfig\x12\x38\n\ntask_queue\x18\x01 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12=\n\x1aworkflow_execution_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12@\n\x1d\x64\x65\x66\x61ult_workflow_task_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\ruser_metadata\x18\x05 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata"\xbd\r\n\x13PendingActivityInfo\x12\x13\n\x0b\x61\x63tivity_id\x18\x01 \x01(\t\x12;\n\ractivity_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12:\n\x05state\x18\x03 \x01(\x0e\x32+.temporal.api.enums.v1.PendingActivityState\x12;\n\x11heartbeat_details\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x37\n\x13last_heartbeat_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x35\n\x11last_started_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x07 \x01(\x05\x12\x18\n\x10maximum_attempts\x18\x08 \x01(\x05\x12\x32\n\x0escheduled_time\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x0f\x65xpiration_time\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x36\n\x0clast_failure\x18\x0b \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x1c\n\x14last_worker_identity\x18\x0c \x01(\t\x12;\n\x15use_workflow_build_id\x18\r \x01(\x0b\x32\x16.google.protobuf.EmptyB\x02\x18\x01H\x00\x12\x32\n$last_independently_assigned_build_id\x18\x0e \x01(\tB\x02\x18\x01H\x00\x12Q\n\x19last_worker_version_stamp\x18\x0f \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12\x39\n\x16\x63urrent_retry_interval\x18\x10 \x01(\x0b\x32\x19.google.protobuf.Duration\x12>\n\x1alast_attempt_complete_time\x18\x11 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x1anext_attempt_schedule_time\x18\x12 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06paused\x18\x13 \x01(\x08\x12\x43\n\x0flast_deployment\x18\x14 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12*\n\x1elast_worker_deployment_version\x18\x15 \x01(\tB\x02\x18\x01\x12T\n\x17last_deployment_version\x18\x19 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x32\n\x08priority\x18\x16 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12K\n\npause_info\x18\x17 \x01(\x0b\x32\x37.temporal.api.workflow.v1.PendingActivityInfo.PauseInfo\x12\x43\n\x10\x61\x63tivity_options\x18\x18 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions\x1a\xcf\x02\n\tPauseInfo\x12.\n\npause_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12P\n\x06manual\x18\x02 \x01(\x0b\x32>.temporal.api.workflow.v1.PendingActivityInfo.PauseInfo.ManualH\x00\x12L\n\x04rule\x18\x04 \x01(\x0b\x32<.temporal.api.workflow.v1.PendingActivityInfo.PauseInfo.RuleH\x00\x1a*\n\x06Manual\x12\x10\n\x08identity\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x1a\x39\n\x04Rule\x12\x0f\n\x07rule_id\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\tB\x0b\n\tpaused_byB\x13\n\x11\x61ssigned_build_id"\xb9\x01\n\x19PendingChildExecutionInfo\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12\x1a\n\x12workflow_type_name\x18\x03 \x01(\t\x12\x14\n\x0cinitiated_id\x18\x04 \x01(\x03\x12\x45\n\x13parent_close_policy\x18\x05 \x01(\x0e\x32(.temporal.api.enums.v1.ParentClosePolicy"\x8d\x02\n\x17PendingWorkflowTaskInfo\x12>\n\x05state\x18\x01 \x01(\x0e\x32/.temporal.api.enums.v1.PendingWorkflowTaskState\x12\x32\n\x0escheduled_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12;\n\x17original_scheduled_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0cstarted_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x05 \x01(\x05"G\n\x0bResetPoints\x12\x38\n\x06points\x18\x01 \x03(\x0b\x32(.temporal.api.workflow.v1.ResetPointInfo"\xef\x01\n\x0eResetPointInfo\x12\x10\n\x08\x62uild_id\x18\x07 \x01(\t\x12\x1b\n\x0f\x62inary_checksum\x18\x01 \x01(\tB\x02\x18\x01\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12(\n first_workflow_task_completed_id\x18\x03 \x01(\x03\x12/\n\x0b\x63reate_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0b\x65xpire_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nresettable\x18\x06 \x01(\x08"\x85\x07\n\x18NewWorkflowExecutionInfo\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12;\n\rworkflow_type\x18\x02 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x03 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x18workflow_id_reuse_policy\x18\x08 \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\n \x01(\t\x12*\n\x04memo\x18\x0b \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0c \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\x06header\x18\r \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x38\n\ruser_metadata\x18\x0e \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12I\n\x13versioning_override\x18\x0f \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority"\x96\x06\n\x0c\x43\x61llbackInfo\x12\x32\n\x08\x63\x61llback\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Callback\x12?\n\x07trigger\x18\x02 \x01(\x0b\x32..temporal.api.workflow.v1.CallbackInfo.Trigger\x12\x35\n\x11registration_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x05state\x18\x04 \x01(\x0e\x32$.temporal.api.enums.v1.CallbackState\x12\x0f\n\x07\x61ttempt\x18\x05 \x01(\x05\x12>\n\x1alast_attempt_complete_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x14last_attempt_failure\x18\x07 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12>\n\x1anext_attempt_schedule_time\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x16\n\x0e\x62locked_reason\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t\x1a\x10\n\x0eWorkflowClosed\x1a\x35\n UpdateWorkflowExecutionCompleted\x12\x11\n\tupdate_id\x18\x01 \x01(\t\x1a\xde\x01\n\x07Trigger\x12P\n\x0fworkflow_closed\x18\x01 \x01(\x0b\x32\x35.temporal.api.workflow.v1.CallbackInfo.WorkflowClosedH\x00\x12v\n#update_workflow_execution_completed\x18\x02 \x01(\x0b\x32G.temporal.api.workflow.v1.CallbackInfo.UpdateWorkflowExecutionCompletedH\x00\x42\t\n\x07variant"\x8b\x06\n\x19PendingNexusOperationInfo\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x0f\n\x07service\x18\x02 \x01(\t\x12\x11\n\toperation\x18\x03 \x01(\t\x12\x18\n\x0coperation_id\x18\x04 \x01(\tB\x02\x18\x01\x12<\n\x19schedule_to_close_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x32\n\x0escheduled_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\x05state\x18\x07 \x01(\x0e\x32\x31.temporal.api.enums.v1.PendingNexusOperationState\x12\x0f\n\x07\x61ttempt\x18\x08 \x01(\x05\x12>\n\x1alast_attempt_complete_time\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x14last_attempt_failure\x18\n \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12>\n\x1anext_attempt_schedule_time\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12S\n\x11\x63\x61ncellation_info\x18\x0c \x01(\x0b\x32\x38.temporal.api.workflow.v1.NexusOperationCancellationInfo\x12\x1a\n\x12scheduled_event_id\x18\r \x01(\x03\x12\x16\n\x0e\x62locked_reason\x18\x0e \x01(\t\x12\x17\n\x0foperation_token\x18\x0f \x01(\t\x12<\n\x19schedule_to_start_timeout\x18\x10 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\x11 \x01(\x0b\x32\x19.google.protobuf.Duration"\x84\x03\n\x1eNexusOperationCancellationInfo\x12\x32\n\x0erequested_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n\x05state\x18\x02 \x01(\x0e\x32\x36.temporal.api.enums.v1.NexusOperationCancellationState\x12\x0f\n\x07\x61ttempt\x18\x03 \x01(\x05\x12>\n\x1alast_attempt_complete_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x14last_attempt_failure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12>\n\x1anext_attempt_schedule_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x16\n\x0e\x62locked_reason\x18\x07 \x01(\t"\xc3\x02\n\x17\x43hannelSubscriptionInfo\x12\x0f\n\x07\x63hannel\x18\x01 \x01(\t\x12\x37\n\x04kind\x18\x02 \x01(\x0e\x32).temporal.api.notification.v1.ChannelKind\x12\x1b\n\x13subscribed_event_id\x18\x03 \x01(\x03\x12\x14\n\x0clast_counter\x18\x04 \x01(\x03\x12H\n\x14pending_notification\x18\x05 \x01(\x0b\x32*.temporal.api.notification.v1.Notification\x12\x19\n\x11scheduled_counter\x18\x06 \x01(\x03\x12\x16\n\x0elistener_count\x18\x07 \x01(\x05\x12\x16\n\x0eretained_count\x18\x08 \x01(\x05\x12\x16\n\x0e\x61\x63\x63\x65pted_count\x18\t \x01(\x03"\xe3\x01\n\x18WorkflowExecutionOptions\x12I\n\x13versioning_override\x18\x01 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12\x32\n\x08priority\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12H\n\x14time_skipping_config\x18\x03 \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig"\xfa\x05\n\x12VersioningOverride\x12M\n\x06pinned\x18\x03 \x01(\x0b\x32;.temporal.api.workflow.v1.VersioningOverride.PinnedOverrideH\x00\x12\x16\n\x0c\x61uto_upgrade\x18\x04 \x01(\x08H\x00\x12P\n\x08one_time\x18\x05 \x01(\x0b\x32<.temporal.api.workflow.v1.VersioningOverride.OneTimeOverrideH\x00\x12?\n\x08\x62\x65havior\x18\x01 \x01(\x0e\x32).temporal.api.enums.v1.VersioningBehaviorB\x02\x18\x01\x12>\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12\x1a\n\x0epinned_version\x18\t \x01(\tB\x02\x18\x01\x1a\xad\x01\n\x0ePinnedOverride\x12U\n\x08\x62\x65havior\x18\x01 \x01(\x0e\x32\x43.temporal.api.workflow.v1.VersioningOverride.PinnedOverrideBehavior\x12\x44\n\x07version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x1ai\n\x0fOneTimeOverride\x12V\n\x19target_deployment_version\x18\x01 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion"g\n\x16PinnedOverrideBehavior\x12(\n$PINNED_OVERRIDE_BEHAVIOR_UNSPECIFIED\x10\x00\x12#\n\x1fPINNED_OVERRIDE_BEHAVIOR_PINNED\x10\x01\x42\n\n\x08override"i\n\x11OnConflictOptions\x12\x19\n\x11\x61ttach_request_id\x18\x01 \x01(\x08\x12#\n\x1b\x61ttach_completion_callbacks\x18\x02 \x01(\x08\x12\x14\n\x0c\x61ttach_links\x18\x03 \x01(\x08"i\n\rRequestIdInfo\x12\x34\n\nevent_type\x18\x01 \x01(\x0e\x32 .temporal.api.enums.v1.EventType\x12\x10\n\x08\x65vent_id\x18\x02 \x01(\x03\x12\x10\n\x08\x62uffered\x18\x03 \x01(\x08"\xb7\x04\n\x12PostResetOperation\x12V\n\x0fsignal_workflow\x18\x01 \x01(\x0b\x32;.temporal.api.workflow.v1.PostResetOperation.SignalWorkflowH\x00\x12\x65\n\x17update_workflow_options\x18\x02 \x01(\x0b\x32\x42.temporal.api.workflow.v1.PostResetOperation.UpdateWorkflowOptionsH\x00\x1a\xb3\x01\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12/\n\x05input\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12.\n\x06header\x18\x03 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12+\n\x05links\x18\x04 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x1a\xa0\x01\n\x15UpdateWorkflowOptions\x12V\n\x1aworkflow_execution_options\x18\x01 \x01(\x0b\x32\x32.temporal.api.workflow.v1.WorkflowExecutionOptions\x12/\n\x0bupdate_mask\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskB\t\n\x07variant"o\n\x1aWorkflowExecutionPauseInfo\x12\x10\n\x08identity\x18\x01 \x01(\t\x12/\n\x0bpaused_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06reason\x18\x03 \x01(\tB\x93\x01\n\x1bio.temporal.api.workflow.v1B\x0cMessageProtoP\x01Z\'go.temporal.io/api/workflow/v1;workflow\xaa\x02\x1aTemporalio.Api.Workflow.V1\xea\x02\x1dTemporalio::Api::Workflow::V1b\x06proto3' ) @@ -94,6 +97,7 @@ _NEXUSOPERATIONCANCELLATIONINFO = DESCRIPTOR.message_types_by_name[ "NexusOperationCancellationInfo" ] +_CHANNELSUBSCRIPTIONINFO = DESCRIPTOR.message_types_by_name["ChannelSubscriptionInfo"] _WORKFLOWEXECUTIONOPTIONS = DESCRIPTOR.message_types_by_name["WorkflowExecutionOptions"] _VERSIONINGOVERRIDE = DESCRIPTOR.message_types_by_name["VersioningOverride"] _VERSIONINGOVERRIDE_PINNEDOVERRIDE = _VERSIONINGOVERRIDE.nested_types_by_name[ @@ -352,6 +356,17 @@ ) _sym_db.RegisterMessage(NexusOperationCancellationInfo) +ChannelSubscriptionInfo = _reflection.GeneratedProtocolMessageType( + "ChannelSubscriptionInfo", + (_message.Message,), + { + "DESCRIPTOR": _CHANNELSUBSCRIPTIONINFO, + "__module__": "temporalio.api.workflow.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflow.v1.ChannelSubscriptionInfo) + }, +) +_sym_db.RegisterMessage(ChannelSubscriptionInfo) + WorkflowExecutionOptions = _reflection.GeneratedProtocolMessageType( "WorkflowExecutionOptions", (_message.Message,), @@ -533,70 +548,72 @@ _VERSIONINGOVERRIDE.fields_by_name[ "pinned_version" ]._serialized_options = b"\030\001" - _WORKFLOWEXECUTIONINFO._serialized_start = 552 - _WORKFLOWEXECUTIONINFO._serialized_end = 1816 - _WORKFLOWEXECUTIONEXTENDEDINFO._serialized_start = 1819 - _WORKFLOWEXECUTIONEXTENDEDINFO._serialized_end = 2471 - _WORKFLOWEXECUTIONEXTENDEDINFO_REQUESTIDINFOSENTRY._serialized_start = 2377 - _WORKFLOWEXECUTIONEXTENDEDINFO_REQUESTIDINFOSENTRY._serialized_end = 2471 - _WORKFLOWEXECUTIONVERSIONINGINFO._serialized_start = 2474 - _WORKFLOWEXECUTIONVERSIONINGINFO._serialized_end = 3109 - _DEPLOYMENTTRANSITION._serialized_start = 3111 - _DEPLOYMENTTRANSITION._serialized_end = 3193 - _DEPLOYMENTVERSIONTRANSITION._serialized_start = 3196 - _DEPLOYMENTVERSIONTRANSITION._serialized_end = 3327 - _WORKFLOWEXECUTIONCONFIG._serialized_start = 3330 - _WORKFLOWEXECUTIONCONFIG._serialized_end = 3657 - _PENDINGACTIVITYINFO._serialized_start = 3660 - _PENDINGACTIVITYINFO._serialized_end = 5385 - _PENDINGACTIVITYINFO_PAUSEINFO._serialized_start = 5029 - _PENDINGACTIVITYINFO_PAUSEINFO._serialized_end = 5364 - _PENDINGACTIVITYINFO_PAUSEINFO_MANUAL._serialized_start = 5250 - _PENDINGACTIVITYINFO_PAUSEINFO_MANUAL._serialized_end = 5292 - _PENDINGACTIVITYINFO_PAUSEINFO_RULE._serialized_start = 5294 - _PENDINGACTIVITYINFO_PAUSEINFO_RULE._serialized_end = 5351 - _PENDINGCHILDEXECUTIONINFO._serialized_start = 5388 - _PENDINGCHILDEXECUTIONINFO._serialized_end = 5573 - _PENDINGWORKFLOWTASKINFO._serialized_start = 5576 - _PENDINGWORKFLOWTASKINFO._serialized_end = 5845 - _RESETPOINTS._serialized_start = 5847 - _RESETPOINTS._serialized_end = 5918 - _RESETPOINTINFO._serialized_start = 5921 - _RESETPOINTINFO._serialized_end = 6160 - _NEWWORKFLOWEXECUTIONINFO._serialized_start = 6163 - _NEWWORKFLOWEXECUTIONINFO._serialized_end = 7064 - _CALLBACKINFO._serialized_start = 7067 - _CALLBACKINFO._serialized_end = 7837 - _CALLBACKINFO_WORKFLOWCLOSED._serialized_start = 7541 - _CALLBACKINFO_WORKFLOWCLOSED._serialized_end = 7557 - _CALLBACKINFO_UPDATEWORKFLOWEXECUTIONCOMPLETED._serialized_start = 7559 - _CALLBACKINFO_UPDATEWORKFLOWEXECUTIONCOMPLETED._serialized_end = 7612 - _CALLBACKINFO_TRIGGER._serialized_start = 7615 - _CALLBACKINFO_TRIGGER._serialized_end = 7837 - _PENDINGNEXUSOPERATIONINFO._serialized_start = 7840 - _PENDINGNEXUSOPERATIONINFO._serialized_end = 8619 - _NEXUSOPERATIONCANCELLATIONINFO._serialized_start = 8622 - _NEXUSOPERATIONCANCELLATIONINFO._serialized_end = 9010 - _WORKFLOWEXECUTIONOPTIONS._serialized_start = 9013 - _WORKFLOWEXECUTIONOPTIONS._serialized_end = 9240 - _VERSIONINGOVERRIDE._serialized_start = 9243 - _VERSIONINGOVERRIDE._serialized_end = 10005 - _VERSIONINGOVERRIDE_PINNEDOVERRIDE._serialized_start = 9608 - _VERSIONINGOVERRIDE_PINNEDOVERRIDE._serialized_end = 9781 - _VERSIONINGOVERRIDE_ONETIMEOVERRIDE._serialized_start = 9783 - _VERSIONINGOVERRIDE_ONETIMEOVERRIDE._serialized_end = 9888 - _VERSIONINGOVERRIDE_PINNEDOVERRIDEBEHAVIOR._serialized_start = 9890 - _VERSIONINGOVERRIDE_PINNEDOVERRIDEBEHAVIOR._serialized_end = 9993 - _ONCONFLICTOPTIONS._serialized_start = 10007 - _ONCONFLICTOPTIONS._serialized_end = 10112 - _REQUESTIDINFO._serialized_start = 10114 - _REQUESTIDINFO._serialized_end = 10219 - _POSTRESETOPERATION._serialized_start = 10222 - _POSTRESETOPERATION._serialized_end = 10789 - _POSTRESETOPERATION_SIGNALWORKFLOW._serialized_start = 10436 - _POSTRESETOPERATION_SIGNALWORKFLOW._serialized_end = 10615 - _POSTRESETOPERATION_UPDATEWORKFLOWOPTIONS._serialized_start = 10618 - _POSTRESETOPERATION_UPDATEWORKFLOWOPTIONS._serialized_end = 10778 - _WORKFLOWEXECUTIONPAUSEINFO._serialized_start = 10791 - _WORKFLOWEXECUTIONPAUSEINFO._serialized_end = 10902 + _WORKFLOWEXECUTIONINFO._serialized_start = 596 + _WORKFLOWEXECUTIONINFO._serialized_end = 1860 + _WORKFLOWEXECUTIONEXTENDEDINFO._serialized_start = 1863 + _WORKFLOWEXECUTIONEXTENDEDINFO._serialized_end = 2515 + _WORKFLOWEXECUTIONEXTENDEDINFO_REQUESTIDINFOSENTRY._serialized_start = 2421 + _WORKFLOWEXECUTIONEXTENDEDINFO_REQUESTIDINFOSENTRY._serialized_end = 2515 + _WORKFLOWEXECUTIONVERSIONINGINFO._serialized_start = 2518 + _WORKFLOWEXECUTIONVERSIONINGINFO._serialized_end = 3153 + _DEPLOYMENTTRANSITION._serialized_start = 3155 + _DEPLOYMENTTRANSITION._serialized_end = 3237 + _DEPLOYMENTVERSIONTRANSITION._serialized_start = 3240 + _DEPLOYMENTVERSIONTRANSITION._serialized_end = 3371 + _WORKFLOWEXECUTIONCONFIG._serialized_start = 3374 + _WORKFLOWEXECUTIONCONFIG._serialized_end = 3701 + _PENDINGACTIVITYINFO._serialized_start = 3704 + _PENDINGACTIVITYINFO._serialized_end = 5429 + _PENDINGACTIVITYINFO_PAUSEINFO._serialized_start = 5073 + _PENDINGACTIVITYINFO_PAUSEINFO._serialized_end = 5408 + _PENDINGACTIVITYINFO_PAUSEINFO_MANUAL._serialized_start = 5294 + _PENDINGACTIVITYINFO_PAUSEINFO_MANUAL._serialized_end = 5336 + _PENDINGACTIVITYINFO_PAUSEINFO_RULE._serialized_start = 5338 + _PENDINGACTIVITYINFO_PAUSEINFO_RULE._serialized_end = 5395 + _PENDINGCHILDEXECUTIONINFO._serialized_start = 5432 + _PENDINGCHILDEXECUTIONINFO._serialized_end = 5617 + _PENDINGWORKFLOWTASKINFO._serialized_start = 5620 + _PENDINGWORKFLOWTASKINFO._serialized_end = 5889 + _RESETPOINTS._serialized_start = 5891 + _RESETPOINTS._serialized_end = 5962 + _RESETPOINTINFO._serialized_start = 5965 + _RESETPOINTINFO._serialized_end = 6204 + _NEWWORKFLOWEXECUTIONINFO._serialized_start = 6207 + _NEWWORKFLOWEXECUTIONINFO._serialized_end = 7108 + _CALLBACKINFO._serialized_start = 7111 + _CALLBACKINFO._serialized_end = 7901 + _CALLBACKINFO_WORKFLOWCLOSED._serialized_start = 7605 + _CALLBACKINFO_WORKFLOWCLOSED._serialized_end = 7621 + _CALLBACKINFO_UPDATEWORKFLOWEXECUTIONCOMPLETED._serialized_start = 7623 + _CALLBACKINFO_UPDATEWORKFLOWEXECUTIONCOMPLETED._serialized_end = 7676 + _CALLBACKINFO_TRIGGER._serialized_start = 7679 + _CALLBACKINFO_TRIGGER._serialized_end = 7901 + _PENDINGNEXUSOPERATIONINFO._serialized_start = 7904 + _PENDINGNEXUSOPERATIONINFO._serialized_end = 8683 + _NEXUSOPERATIONCANCELLATIONINFO._serialized_start = 8686 + _NEXUSOPERATIONCANCELLATIONINFO._serialized_end = 9074 + _CHANNELSUBSCRIPTIONINFO._serialized_start = 9077 + _CHANNELSUBSCRIPTIONINFO._serialized_end = 9400 + _WORKFLOWEXECUTIONOPTIONS._serialized_start = 9403 + _WORKFLOWEXECUTIONOPTIONS._serialized_end = 9630 + _VERSIONINGOVERRIDE._serialized_start = 9633 + _VERSIONINGOVERRIDE._serialized_end = 10395 + _VERSIONINGOVERRIDE_PINNEDOVERRIDE._serialized_start = 9998 + _VERSIONINGOVERRIDE_PINNEDOVERRIDE._serialized_end = 10171 + _VERSIONINGOVERRIDE_ONETIMEOVERRIDE._serialized_start = 10173 + _VERSIONINGOVERRIDE_ONETIMEOVERRIDE._serialized_end = 10278 + _VERSIONINGOVERRIDE_PINNEDOVERRIDEBEHAVIOR._serialized_start = 10280 + _VERSIONINGOVERRIDE_PINNEDOVERRIDEBEHAVIOR._serialized_end = 10383 + _ONCONFLICTOPTIONS._serialized_start = 10397 + _ONCONFLICTOPTIONS._serialized_end = 10502 + _REQUESTIDINFO._serialized_start = 10504 + _REQUESTIDINFO._serialized_end = 10609 + _POSTRESETOPERATION._serialized_start = 10612 + _POSTRESETOPERATION._serialized_end = 11179 + _POSTRESETOPERATION_SIGNALWORKFLOW._serialized_start = 10826 + _POSTRESETOPERATION_SIGNALWORKFLOW._serialized_end = 11005 + _POSTRESETOPERATION_UPDATEWORKFLOWOPTIONS._serialized_start = 11008 + _POSTRESETOPERATION_UPDATEWORKFLOWOPTIONS._serialized_end = 11168 + _WORKFLOWEXECUTIONPAUSEINFO._serialized_start = 11181 + _WORKFLOWEXECUTIONPAUSEINFO._serialized_end = 11292 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/workflow/v1/message_pb2.pyi b/temporalio/api/workflow/v1/message_pb2.pyi index b390d94f46..61d774f636 100644 --- a/temporalio/api/workflow/v1/message_pb2.pyi +++ b/temporalio/api/workflow/v1/message_pb2.pyi @@ -24,6 +24,7 @@ import temporalio.api.enums.v1.common_pb2 import temporalio.api.enums.v1.event_type_pb2 import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 +import temporalio.api.notification.v1.message_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 import temporalio.api.taskqueue.v1.message_pb2 @@ -1553,6 +1554,7 @@ class CallbackInfo(google.protobuf.message.Message): LAST_ATTEMPT_FAILURE_FIELD_NUMBER: builtins.int NEXT_ATTEMPT_SCHEDULE_TIME_FIELD_NUMBER: builtins.int BLOCKED_REASON_FIELD_NUMBER: builtins.int + REQUEST_ID_FIELD_NUMBER: builtins.int @property def callback(self) -> temporalio.api.common.v1.message_pb2.Callback: """Information on how this callback should be invoked (e.g. its URL and type).""" @@ -1578,6 +1580,10 @@ class CallbackInfo(google.protobuf.message.Message): """The time when the next attempt is scheduled.""" blocked_reason: builtins.str """If the state is BLOCKED, blocked reason provides additional information.""" + request_id: builtins.str + """Server-generated request ID used as an idempotency token when invoking callbacks. + It has no relation to caller-side request_id sent in operations like StartWorkflowExecutionRequest. + """ def __init__( self, *, @@ -1593,6 +1599,7 @@ class CallbackInfo(google.protobuf.message.Message): next_attempt_schedule_time: google.protobuf.timestamp_pb2.Timestamp | None = ..., blocked_reason: builtins.str = ..., + request_id: builtins.str = ..., ) -> None: ... def HasField( self, @@ -1628,6 +1635,8 @@ class CallbackInfo(google.protobuf.message.Message): b"next_attempt_schedule_time", "registration_time", b"registration_time", + "request_id", + b"request_id", "state", b"state", "trigger", @@ -1887,6 +1896,94 @@ class NexusOperationCancellationInfo(google.protobuf.message.Message): global___NexusOperationCancellationInfo = NexusOperationCancellationInfo +class ChannelSubscriptionInfo(google.protobuf.message.Message): + """A workflow's standing on a notification channel, as reported by DescribeWorkflowExecution.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CHANNEL_FIELD_NUMBER: builtins.int + KIND_FIELD_NUMBER: builtins.int + SUBSCRIBED_EVENT_ID_FIELD_NUMBER: builtins.int + LAST_COUNTER_FIELD_NUMBER: builtins.int + PENDING_NOTIFICATION_FIELD_NUMBER: builtins.int + SCHEDULED_COUNTER_FIELD_NUMBER: builtins.int + LISTENER_COUNT_FIELD_NUMBER: builtins.int + RETAINED_COUNT_FIELD_NUMBER: builtins.int + ACCEPTED_COUNT_FIELD_NUMBER: builtins.int + channel: builtins.str + """Channel name.""" + kind: temporalio.api.notification.v1.message_pb2.ChannelKind.ValueType + """CHANNEL_KIND_INDEPENDENT for a channel the workflow subscribed to with a + SubscribeNotificationChannel command. CHANNEL_KIND_LINKED for a channel linked to this + workflow, which lists it once the channel holds any state. + """ + subscribed_event_id: builtins.int + """Independent kind: id of the WorkflowNotificationChannelSubscribed event that recorded the + subscription. Zero for the linked kind. + """ + last_counter: builtins.int + """Highest counter the workflow has accepted from the channel. Zero when none has arrived.""" + @property + def pending_notification( + self, + ) -> temporalio.api.notification.v1.message_pb2.Notification: + """The notification held for the workflow's next Workflow Task, when one is pending.""" + scheduled_counter: builtins.int + """Counter carried by the scheduled event of a Workflow Task that has not started yet. Zero + otherwise. + """ + listener_count: builtins.int + """Linked kind: callback listeners registered on the channel.""" + retained_count: builtins.int + """Linked kind: notifications retained for pollers.""" + accepted_count: builtins.int + """Linked kind: notifications the channel has accepted over its life.""" + def __init__( + self, + *, + channel: builtins.str = ..., + kind: temporalio.api.notification.v1.message_pb2.ChannelKind.ValueType = ..., + subscribed_event_id: builtins.int = ..., + last_counter: builtins.int = ..., + pending_notification: temporalio.api.notification.v1.message_pb2.Notification + | None = ..., + scheduled_counter: builtins.int = ..., + listener_count: builtins.int = ..., + retained_count: builtins.int = ..., + accepted_count: builtins.int = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "pending_notification", b"pending_notification" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "accepted_count", + b"accepted_count", + "channel", + b"channel", + "kind", + b"kind", + "last_counter", + b"last_counter", + "listener_count", + b"listener_count", + "pending_notification", + b"pending_notification", + "retained_count", + b"retained_count", + "scheduled_counter", + b"scheduled_counter", + "subscribed_event_id", + b"subscribed_event_id", + ], + ) -> None: ... + +global___ChannelSubscriptionInfo = ChannelSubscriptionInfo + class WorkflowExecutionOptions(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor diff --git a/temporalio/api/workflowservice/v1/__init__.py b/temporalio/api/workflowservice/v1/__init__.py index ab72e6b09e..2ab6533c89 100644 --- a/temporalio/api/workflowservice/v1/__init__.py +++ b/temporalio/api/workflowservice/v1/__init__.py @@ -37,6 +37,8 @@ DescribeActivityExecutionResponse, DescribeBatchOperationRequest, DescribeBatchOperationResponse, + DescribeChannelRequest, + DescribeChannelResponse, DescribeDeploymentRequest, DescribeDeploymentResponse, DescribeNamespaceRequest, @@ -111,6 +113,8 @@ ListWorkflowExecutionsResponse, ListWorkflowRulesRequest, ListWorkflowRulesResponse, + NotifyChannelRequest, + NotifyChannelResponse, PatchScheduleRequest, PatchScheduleResponse, PauseActivityExecutionRequest, @@ -123,6 +127,8 @@ PollActivityExecutionResponse, PollActivityTaskQueueRequest, PollActivityTaskQueueResponse, + PollChannelRequest, + PollChannelResponse, PollNexusOperationExecutionRequest, PollNexusOperationExecutionResponse, PollNexusTaskQueueRequest, @@ -141,6 +147,8 @@ RecordActivityTaskHeartbeatResponse, RecordWorkerHeartbeatRequest, RecordWorkerHeartbeatResponse, + RegisterChannelListenerRequest, + RegisterChannelListenerResponse, RegisterNamespaceRequest, RegisterNamespaceResponse, RequestCancelActivityExecutionRequest, @@ -219,6 +227,8 @@ UnpauseActivityResponse, UnpauseWorkflowExecutionRequest, UnpauseWorkflowExecutionResponse, + UnregisterChannelListenerRequest, + UnregisterChannelListenerResponse, UpdateActivityExecutionOptionsRequest, UpdateActivityExecutionOptionsResponse, UpdateActivityOptionsRequest, @@ -286,6 +296,8 @@ "DescribeActivityExecutionResponse", "DescribeBatchOperationRequest", "DescribeBatchOperationResponse", + "DescribeChannelRequest", + "DescribeChannelResponse", "DescribeDeploymentRequest", "DescribeDeploymentResponse", "DescribeNamespaceRequest", @@ -360,6 +372,8 @@ "ListWorkflowExecutionsResponse", "ListWorkflowRulesRequest", "ListWorkflowRulesResponse", + "NotifyChannelRequest", + "NotifyChannelResponse", "PatchScheduleRequest", "PatchScheduleResponse", "PauseActivityExecutionRequest", @@ -372,6 +386,8 @@ "PollActivityExecutionResponse", "PollActivityTaskQueueRequest", "PollActivityTaskQueueResponse", + "PollChannelRequest", + "PollChannelResponse", "PollNexusOperationExecutionRequest", "PollNexusOperationExecutionResponse", "PollNexusTaskQueueRequest", @@ -390,6 +406,8 @@ "RecordActivityTaskHeartbeatResponse", "RecordWorkerHeartbeatRequest", "RecordWorkerHeartbeatResponse", + "RegisterChannelListenerRequest", + "RegisterChannelListenerResponse", "RegisterNamespaceRequest", "RegisterNamespaceResponse", "RequestCancelActivityExecutionRequest", @@ -468,6 +486,8 @@ "UnpauseActivityResponse", "UnpauseWorkflowExecutionRequest", "UnpauseWorkflowExecutionResponse", + "UnregisterChannelListenerRequest", + "UnregisterChannelListenerResponse", "UpdateActivityExecutionOptionsRequest", "UpdateActivityExecutionOptionsResponse", "UpdateActivityOptionsRequest", diff --git a/temporalio/api/workflowservice/v1/request_response_pb2.py b/temporalio/api/workflowservice/v1/request_response_pb2.py index e1b2e01c4f..82132d6f71 100644 --- a/temporalio/api/workflowservice/v1/request_response_pb2.py +++ b/temporalio/api/workflowservice/v1/request_response_pb2.py @@ -90,6 +90,12 @@ from temporalio.api.nexus.v1 import ( message_pb2 as temporal_dot_api_dot_nexus_dot_v1_dot_message__pb2, ) +from temporalio.api.nexusoperation.v1 import ( + message_pb2 as temporal_dot_api_dot_nexusoperation_dot_v1_dot_message__pb2, +) +from temporalio.api.notification.v1 import ( + message_pb2 as temporal_dot_api_dot_notification_dot_v1_dot_message__pb2, +) from temporalio.api.protocol.v1 import ( message_pb2 as temporal_dot_api_dot_protocol_dot_v1_dot_message__pb2, ) @@ -114,6 +120,9 @@ from temporalio.api.sdk.v1 import ( worker_config_pb2 as temporal_dot_api_dot_sdk_dot_v1_dot_worker__config__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.taskqueue.v1 import ( message_pb2 as temporal_dot_api_dot_taskqueue_dot_v1_dot_message__pb2, ) @@ -131,7 +140,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n6temporal/api/workflowservice/v1/request_response.proto\x12\x1ftemporal.api.workflowservice.v1\x1a+temporal/api/enums/v1/batch_operation.proto\x1a"temporal/api/enums/v1/common.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/enums/v1/namespace.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a!temporal/api/enums/v1/query.proto\x1a!temporal/api/enums/v1/reset.proto\x1a&temporal/api/enums/v1/task_queue.proto\x1a&temporal/api/enums/v1/deployment.proto\x1a"temporal/api/enums/v1/update.proto\x1a)temporal/api/enums/v1/time_skipping.proto\x1a$temporal/api/enums/v1/activity.proto\x1a!temporal/api/enums/v1/nexus.proto\x1a&temporal/api/activity/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/history/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a%temporal/api/command/v1/message.proto\x1a$temporal/api/compute/v1/config.proto\x1a(temporal/api/deployment/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/filter/v1/message.proto\x1a&temporal/api/protocol/v1/message.proto\x1a\'temporal/api/namespace/v1/message.proto\x1a#temporal/api/query/v1/message.proto\x1a)temporal/api/replication/v1/message.proto\x1a#temporal/api/rules/v1/message.proto\x1a\'temporal/api/sdk/v1/worker_config.proto\x1a&temporal/api/schedule/v1/message.proto\x1a\'temporal/api/taskqueue/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a%temporal/api/version/v1/message.proto\x1a#temporal/api/batch/v1/message.proto\x1a\x30temporal/api/sdk/v1/task_complete_metadata.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/worker/v1/message.proto\x1a\x1egoogle/protobuf/duration.proto\x1a google/protobuf/field_mask.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\x88\x05\n\x18RegisterNamespaceRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x13\n\x0bowner_email\x18\x03 \x01(\t\x12\x46\n#workflow_execution_retention_period\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12G\n\x08\x63lusters\x18\x05 \x03(\x0b\x32\x35.temporal.api.replication.v1.ClusterReplicationConfig\x12\x1b\n\x13\x61\x63tive_cluster_name\x18\x06 \x01(\t\x12Q\n\x04\x64\x61ta\x18\x07 \x03(\x0b\x32\x43.temporal.api.workflowservice.v1.RegisterNamespaceRequest.DataEntry\x12\x16\n\x0esecurity_token\x18\x08 \x01(\t\x12\x1b\n\x13is_global_namespace\x18\t \x01(\x08\x12\x44\n\x16history_archival_state\x18\n \x01(\x0e\x32$.temporal.api.enums.v1.ArchivalState\x12\x1c\n\x14history_archival_uri\x18\x0b \x01(\t\x12G\n\x19visibility_archival_state\x18\x0c \x01(\x0e\x32$.temporal.api.enums.v1.ArchivalState\x12\x1f\n\x17visibility_archival_uri\x18\r \x01(\t\x1a+\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"\x1b\n\x19RegisterNamespaceResponse"\x89\x01\n\x15ListNamespacesRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c\x12\x44\n\x10namespace_filter\x18\x03 \x01(\x0b\x32*.temporal.api.namespace.v1.NamespaceFilter"\x81\x01\n\x16ListNamespacesResponse\x12N\n\nnamespaces\x18\x01 \x03(\x0b\x32:.temporal.api.workflowservice.v1.DescribeNamespaceResponse\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"S\n\x18\x44\x65scribeNamespaceRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x18\n\x10weak_consistency\x18\x03 \x01(\x08"\x81\x04\n\x19\x44\x65scribeNamespaceResponse\x12@\n\x0enamespace_info\x18\x01 \x01(\x0b\x32(.temporal.api.namespace.v1.NamespaceInfo\x12:\n\x06\x63onfig\x18\x02 \x01(\x0b\x32*.temporal.api.namespace.v1.NamespaceConfig\x12S\n\x12replication_config\x18\x03 \x01(\x0b\x32\x37.temporal.api.replication.v1.NamespaceReplicationConfig\x12\x18\n\x10\x66\x61ilover_version\x18\x04 \x01(\x03\x12\x1b\n\x13is_global_namespace\x18\x05 \x01(\x08\x12\x45\n\x10\x66\x61ilover_history\x18\x06 \x03(\x0b\x32+.temporal.api.replication.v1.FailoverStatus\x12J\n\x12poller_group_infos\x18\x07 \x03(\x0b\x32*.temporal.api.taskqueue.v1.PollerGroupInfoB\x02\x18\x01\x12G\n\x12poller_groups_info\x18\x08 \x01(\x0b\x32+.temporal.api.taskqueue.v1.PollerGroupsInfo"\xcf\x02\n\x16UpdateNamespaceRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x43\n\x0bupdate_info\x18\x02 \x01(\x0b\x32..temporal.api.namespace.v1.UpdateNamespaceInfo\x12:\n\x06\x63onfig\x18\x03 \x01(\x0b\x32*.temporal.api.namespace.v1.NamespaceConfig\x12S\n\x12replication_config\x18\x04 \x01(\x0b\x32\x37.temporal.api.replication.v1.NamespaceReplicationConfig\x12\x16\n\x0esecurity_token\x18\x05 \x01(\t\x12\x19\n\x11\x64\x65lete_bad_binary\x18\x06 \x01(\t\x12\x19\n\x11promote_namespace\x18\x07 \x01(\x08"\xa3\x02\n\x17UpdateNamespaceResponse\x12@\n\x0enamespace_info\x18\x01 \x01(\x0b\x32(.temporal.api.namespace.v1.NamespaceInfo\x12:\n\x06\x63onfig\x18\x02 \x01(\x0b\x32*.temporal.api.namespace.v1.NamespaceConfig\x12S\n\x12replication_config\x18\x03 \x01(\x0b\x32\x37.temporal.api.replication.v1.NamespaceReplicationConfig\x12\x18\n\x10\x66\x61ilover_version\x18\x04 \x01(\x03\x12\x1b\n\x13is_global_namespace\x18\x05 \x01(\x08"F\n\x19\x44\x65precateNamespaceRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x16\n\x0esecurity_token\x18\x02 \x01(\t"\x1c\n\x1a\x44\x65precateNamespaceResponse"\xd1\x0c\n\x1dStartWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x10\n\x08identity\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t\x12N\n\x18workflow_id_reuse_policy\x18\x0b \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12T\n\x1bworkflow_id_conflict_policy\x18\x16 \x01(\x0e\x32/.temporal.api.enums.v1.WorkflowIdConflictPolicy\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\r \x01(\t\x12*\n\x04memo\x18\x0e \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0f \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\x06header\x18\x10 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x1f\n\x17request_eager_execution\x18\x11 \x01(\x08\x12;\n\x11\x63ontinued_failure\x18\x12 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x13 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x37\n\x14workflow_start_delay\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12>\n\x14\x63ompletion_callbacks\x18\x15 \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12\x38\n\ruser_metadata\x18\x17 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12+\n\x05links\x18\x18 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x12I\n\x13versioning_override\x18\x19 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12H\n\x13on_conflict_options\x18\x1a \x01(\x0b\x32+.temporal.api.workflow.v1.OnConflictOptions\x12\x32\n\x08priority\x18\x1b \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12\\\n\x1f\x65\x61ger_worker_deployment_options\x18\x1c \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12H\n\x14time_skipping_config\x18\x1d \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig"\xaa\x02\n\x1eStartWorkflowExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x06 \x01(\t\x12\x0f\n\x07started\x18\x03 \x01(\x08\x12>\n\x06status\x18\x05 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowExecutionStatus\x12[\n\x13\x65\x61ger_workflow_task\x18\x02 \x01(\x0b\x32>.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse\x12*\n\x04link\x18\x04 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xaa\x02\n"GetWorkflowExecutionHistoryRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x19\n\x11maximum_page_size\x18\x03 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x04 \x01(\x0c\x12\x16\n\x0ewait_new_event\x18\x05 \x01(\x08\x12P\n\x19history_event_filter_type\x18\x06 \x01(\x0e\x32-.temporal.api.enums.v1.HistoryEventFilterType\x12\x15\n\rskip_archival\x18\x07 \x01(\x08"\xba\x01\n#GetWorkflowExecutionHistoryResponse\x12\x31\n\x07history\x18\x01 \x01(\x0b\x32 .temporal.api.history.v1.History\x12\x35\n\x0braw_history\x18\x02 \x03(\x0b\x32 .temporal.api.common.v1.DataBlob\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\x10\n\x08\x61rchived\x18\x04 \x01(\x08"\xb0\x01\n)GetWorkflowExecutionHistoryReverseRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x19\n\x11maximum_page_size\x18\x03 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x04 \x01(\x0c"x\n*GetWorkflowExecutionHistoryReverseResponse\x12\x31\n\x07history\x18\x01 \x01(\x0b\x32 .temporal.api.history.v1.History\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c"\xb8\x03\n\x1cPollWorkflowTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x17\n\x0fpoller_group_id\x18\n \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x1b\n\x13worker_instance_key\x18\x08 \x01(\t\x12!\n\x19worker_control_task_queue\x18\t \x01(\t\x12\x1b\n\x0f\x62inary_checksum\x18\x04 \x01(\tB\x02\x18\x01\x12Z\n\x1bworker_version_capabilities\x18\x05 \x01(\x0b\x32\x31.temporal.api.common.v1.WorkerVersionCapabilitiesB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x06 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptionsJ\x04\x08\x07\x10\x08R\x10worker_heartbeat"\xbf\x08\n\x1dPollWorkflowTaskQueueResponse\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12!\n\x19previous_started_event_id\x18\x04 \x01(\x03\x12\x18\n\x10started_event_id\x18\x05 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x06 \x01(\x05\x12\x1a\n\x12\x62\x61\x63klog_count_hint\x18\x07 \x01(\x03\x12\x31\n\x07history\x18\x08 \x01(\x0b\x32 .temporal.api.history.v1.History\x12\x17\n\x0fnext_page_token\x18\t \x01(\x0c\x12\x33\n\x05query\x18\n \x01(\x0b\x32$.temporal.api.query.v1.WorkflowQuery\x12K\n\x1dworkflow_execution_task_queue\x18\x0b \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x32\n\x0escheduled_time\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0cstarted_time\x18\r \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\\\n\x07queries\x18\x0e \x03(\x0b\x32K.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse.QueriesEntry\x12\x33\n\x08messages\x18\x0f \x03(\x0b\x32!.temporal.api.protocol.v1.Message\x12Q\n\x17poller_scaling_decision\x18\x10 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.PollerScalingDecision\x12\x17\n\x0fpoller_group_id\x18\x11 \x01(\t\x12J\n\x12poller_group_infos\x18\x12 \x03(\x0b\x32*.temporal.api.taskqueue.v1.PollerGroupInfoB\x02\x18\x01\x12G\n\x12poller_groups_info\x18\x13 \x01(\x0b\x32+.temporal.api.taskqueue.v1.PollerGroupsInfo\x1aT\n\x0cQueriesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.temporal.api.query.v1.WorkflowQuery:\x02\x38\x01"\xba\n\n#RespondWorkflowTaskCompletedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x32\n\x08\x63ommands\x18\x02 \x03(\x0b\x32 .temporal.api.command.v1.Command\x12\x10\n\x08identity\x18\x03 \x01(\t\x12O\n\x11sticky_attributes\x18\x04 \x01(\x0b\x32\x34.temporal.api.taskqueue.v1.StickyExecutionAttributes\x12 \n\x18return_new_workflow_task\x18\x05 \x01(\x08\x12&\n\x1e\x66orce_create_new_workflow_task\x18\x06 \x01(\x08\x12\x1b\n\x0f\x62inary_checksum\x18\x07 \x01(\tB\x02\x18\x01\x12m\n\rquery_results\x18\x08 \x03(\x0b\x32V.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest.QueryResultsEntry\x12\x11\n\tnamespace\x18\t \x01(\t\x12\x13\n\x0bresource_id\x18\x12 \x01(\t\x12L\n\x14worker_version_stamp\x18\n \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12\x33\n\x08messages\x18\x0b \x03(\x0b\x32!.temporal.api.protocol.v1.Message\x12H\n\x0csdk_metadata\x18\x0c \x01(\x0b\x32\x32.temporal.api.sdk.v1.WorkflowTaskCompletedMetadata\x12\x43\n\x11metering_metadata\x18\r \x01(\x0b\x32(.temporal.api.common.v1.MeteringMetadata\x12g\n\x0c\x63\x61pabilities\x18\x0e \x01(\x0b\x32Q.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest.Capabilities\x12>\n\ndeployment\x18\x0f \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12\x46\n\x13versioning_behavior\x18\x10 \x01(\x0e\x32).temporal.api.enums.v1.VersioningBehavior\x12O\n\x12\x64\x65ployment_options\x18\x11 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12\x1b\n\x13worker_instance_key\x18\x13 \x01(\t\x12!\n\x19worker_control_task_queue\x18\x14 \x01(\t\x12\x13\n\x0bpage_number\x18\x15 \x01(\x05\x12\x19\n\x11intermediate_page\x18\x16 \x01(\x08\x1a_\n\x11QueryResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32*.temporal.api.query.v1.WorkflowQueryResult:\x02\x38\x01\x1a\x45\n\x0c\x43\x61pabilities\x12\x35\n-discard_speculative_workflow_task_with_events\x18\x01 \x01(\x08"\xf5\x01\n$RespondWorkflowTaskCompletedResponse\x12U\n\rworkflow_task\x18\x01 \x01(\x0b\x32>.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse\x12V\n\x0e\x61\x63tivity_tasks\x18\x02 \x03(\x0b\x32>.temporal.api.workflowservice.v1.PollActivityTaskQueueResponse\x12\x1e\n\x16reset_history_event_id\x18\x03 \x01(\x03"\x8d\x04\n RespondWorkflowTaskFailedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12=\n\x05\x63\x61use\x18\x02 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowTaskFailedCause\x12\x31\n\x07\x66\x61ilure\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x1b\n\x0f\x62inary_checksum\x18\x05 \x01(\tB\x02\x18\x01\x12\x11\n\tnamespace\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x0b \x01(\t\x12\x33\n\x08messages\x18\x07 \x03(\x0b\x32!.temporal.api.protocol.v1.Message\x12\x46\n\x0eworker_version\x18\x08 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\ndeployment\x18\t \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\n \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions"#\n!RespondWorkflowTaskFailedResponse"\xe6\x03\n\x1cPollActivityTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x17\n\x0fpoller_group_id\x18\n \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x1b\n\x13worker_instance_key\x18\x08 \x01(\t\x12!\n\x19worker_control_task_queue\x18\t \x01(\t\x12I\n\x13task_queue_metadata\x18\x04 \x01(\x0b\x32,.temporal.api.taskqueue.v1.TaskQueueMetadata\x12Z\n\x1bworker_version_capabilities\x18\x05 \x01(\x0b\x32\x31.temporal.api.common.v1.WorkerVersionCapabilitiesB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x06 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptionsJ\x04\x08\x07\x10\x08R\x10worker_heartbeat"\x9d\t\n\x1dPollActivityTaskQueueResponse\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x1a\n\x12workflow_namespace\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x45\n\x12workflow_execution\x18\x04 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\ractivity_type\x18\x05 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12\x13\n\x0b\x61\x63tivity_id\x18\x06 \x01(\t\x12.\n\x06header\x18\x07 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12/\n\x05input\x18\x08 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12;\n\x11heartbeat_details\x18\t \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x32\n\x0escheduled_time\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x42\n\x1e\x63urrent_attempt_scheduled_time\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0cstarted_time\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\r \x01(\x05\x12<\n\x19schedule_to_close_timeout\x18\x0e \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\x0f \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x10 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x11 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12Q\n\x17poller_scaling_decision\x18\x12 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.PollerScalingDecision\x12\x32\n\x08priority\x18\x13 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12\x17\n\x0f\x61\x63tivity_run_id\x18\x14 \x01(\t\x12J\n\x12poller_group_infos\x18\x15 \x03(\x0b\x32*.temporal.api.taskqueue.v1.PollerGroupInfoB\x02\x18\x01\x12G\n\x12poller_groups_info\x18\x16 \x01(\x0b\x32+.temporal.api.taskqueue.v1.PollerGroupsInfo"\xa5\x01\n"RecordActivityTaskHeartbeatRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x31\n\x07\x64\x65tails\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tnamespace\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\x05 \x01(\t"p\n#RecordActivityTaskHeartbeatResponse\x12\x18\n\x10\x63\x61ncel_requested\x18\x01 \x01(\x08\x12\x17\n\x0f\x61\x63tivity_paused\x18\x02 \x01(\x08\x12\x16\n\x0e\x61\x63tivity_reset\x18\x03 \x01(\x08"\xcf\x01\n&RecordActivityTaskHeartbeatByIdRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12\x31\n\x07\x64\x65tails\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x07 \x01(\t"t\n\'RecordActivityTaskHeartbeatByIdResponse\x12\x18\n\x10\x63\x61ncel_requested\x18\x01 \x01(\x08\x12\x17\n\x0f\x61\x63tivity_paused\x18\x02 \x01(\x08\x12\x16\n\x0e\x61\x63tivity_reset\x18\x03 \x01(\x08"\xfe\x02\n#RespondActivityTaskCompletedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x30\n\x06result\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tnamespace\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\x08 \x01(\t\x12\x46\n\x0eworker_version\x18\x05 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\ndeployment\x18\x06 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x07 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions"&\n$RespondActivityTaskCompletedResponse"\xcf\x01\n\'RespondActivityTaskCompletedByIdRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12\x30\n\x06result\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x07 \x01(\t"*\n(RespondActivityTaskCompletedByIdResponse"\xbe\x03\n RespondActivityTaskFailedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tnamespace\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\t \x01(\t\x12@\n\x16last_heartbeat_details\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\ndeployment\x18\x07 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x08 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions"W\n!RespondActivityTaskFailedResponse\x12\x32\n\x08\x66\x61ilures\x18\x01 \x03(\x0b\x32 .temporal.api.failure.v1.Failure"\x8f\x02\n$RespondActivityTaskFailedByIdRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x10\n\x08identity\x18\x06 \x01(\t\x12@\n\x16last_heartbeat_details\x18\x07 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x13\n\x0bresource_id\x18\x08 \x01(\t"[\n%RespondActivityTaskFailedByIdResponse\x12\x32\n\x08\x66\x61ilures\x18\x01 \x03(\x0b\x32 .temporal.api.failure.v1.Failure"\xfe\x02\n"RespondActivityTaskCanceledRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x31\n\x07\x64\x65tails\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tnamespace\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\x08 \x01(\t\x12\x46\n\x0eworker_version\x18\x05 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\ndeployment\x18\x06 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x07 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions"%\n#RespondActivityTaskCanceledResponse"\xa0\x02\n&RespondActivityTaskCanceledByIdRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12\x31\n\x07\x64\x65tails\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x06 \x01(\t\x12O\n\x12\x64\x65ployment_options\x18\x07 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12\x13\n\x0bresource_id\x18\x08 \x01(\t")\n\'RespondActivityTaskCanceledByIdResponse"\x84\x02\n%RequestCancelWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t\x12+\n\x05links\x18\x07 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link"(\n&RequestCancelWorkflowExecutionResponse"\xde\x02\n\x1eSignalWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x0bsignal_name\x18\x03 \x01(\t\x12/\n\x05input\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x13\n\x07\x63ontrol\x18\x07 \x01(\tB\x02\x18\x01\x12.\n\x06header\x18\x08 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12+\n\x05links\x18\n \x03(\x0b\x32\x1c.temporal.api.common.v1.LinkJ\x04\x08\t\x10\n"M\n\x1fSignalWorkflowExecutionResponse\x12*\n\x04link\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xbb\n\n\'SignalWithStartWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x10\n\x08identity\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t\x12N\n\x18workflow_id_reuse_policy\x18\x0b \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12T\n\x1bworkflow_id_conflict_policy\x18\x16 \x01(\x0e\x32/.temporal.api.enums.v1.WorkflowIdConflictPolicy\x12\x13\n\x0bsignal_name\x18\x0c \x01(\t\x12\x36\n\x0csignal_input\x18\r \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x13\n\x07\x63ontrol\x18\x0e \x01(\tB\x02\x18\x01\x12\x39\n\x0cretry_policy\x18\x0f \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x10 \x01(\t\x12*\n\x04memo\x18\x11 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x12 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\x06header\x18\x13 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x37\n\x14workflow_start_delay\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\ruser_metadata\x18\x17 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12+\n\x05links\x18\x18 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x12I\n\x13versioning_override\x18\x19 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12\x32\n\x08priority\x18\x1a \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12H\n\x14time_skipping_config\x18\x1b \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfigJ\x04\x08\x15\x10\x16"\x9e\x01\n(SignalWithStartWorkflowExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x04 \x01(\t\x12\x0f\n\x07started\x18\x02 \x01(\x08\x12\x31\n\x0bsignal_link\x18\x03 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xc1\x03\n\x1dResetWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12%\n\x1dworkflow_task_finish_event_id\x18\x04 \x01(\x03\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12G\n\x12reset_reapply_type\x18\x06 \x01(\x0e\x32\'.temporal.api.enums.v1.ResetReapplyTypeB\x02\x18\x01\x12S\n\x1breset_reapply_exclude_types\x18\x07 \x03(\x0e\x32..temporal.api.enums.v1.ResetReapplyExcludeType\x12K\n\x15post_reset_operations\x18\x08 \x03(\x0b\x32,.temporal.api.workflow.v1.PostResetOperation\x12\x10\n\x08identity\x18\t \x01(\t"0\n\x1eResetWorkflowExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\x9f\x02\n!TerminateWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x31\n\x07\x64\x65tails\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x06 \x01(\t\x12+\n\x05links\x18\x07 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link"$\n"TerminateWorkflowExecutionResponse"z\n\x1e\x44\x65leteWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution"!\n\x1f\x44\x65leteWorkflowExecutionResponse"\xc9\x02\n!ListOpenWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x19\n\x11maximum_page_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\x42\n\x11start_time_filter\x18\x04 \x01(\x0b\x32\'.temporal.api.filter.v1.StartTimeFilter\x12K\n\x10\x65xecution_filter\x18\x05 \x01(\x0b\x32/.temporal.api.filter.v1.WorkflowExecutionFilterH\x00\x12\x41\n\x0btype_filter\x18\x06 \x01(\x0b\x32*.temporal.api.filter.v1.WorkflowTypeFilterH\x00\x42\t\n\x07\x66ilters"\x82\x01\n"ListOpenWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\x8a\x03\n#ListClosedWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x19\n\x11maximum_page_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\x42\n\x11start_time_filter\x18\x04 \x01(\x0b\x32\'.temporal.api.filter.v1.StartTimeFilter\x12K\n\x10\x65xecution_filter\x18\x05 \x01(\x0b\x32/.temporal.api.filter.v1.WorkflowExecutionFilterH\x00\x12\x41\n\x0btype_filter\x18\x06 \x01(\x0b\x32*.temporal.api.filter.v1.WorkflowTypeFilterH\x00\x12=\n\rstatus_filter\x18\x07 \x01(\x0b\x32$.temporal.api.filter.v1.StatusFilterH\x00\x42\t\n\x07\x66ilters"\x84\x01\n$ListClosedWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"m\n\x1dListWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"~\n\x1eListWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"u\n%ListArchivedWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"\x86\x01\n&ListArchivedWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"m\n\x1dScanWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"~\n\x1eScanWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"B\n\x1e\x43ountWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t"\xed\x01\n\x1f\x43ountWorkflowExecutionsResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12\x61\n\x06groups\x18\x02 \x03(\x0b\x32Q.temporal.api.workflowservice.v1.CountWorkflowExecutionsResponse.AggregationGroup\x1aX\n\x10\x41ggregationGroup\x12\x35\n\x0cgroup_values\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\r\n\x05\x63ount\x18\x02 \x01(\x03"\x1c\n\x1aGetSearchAttributesRequest"\xc9\x01\n\x1bGetSearchAttributesResponse\x12T\n\x04keys\x18\x01 \x03(\x0b\x32\x46.temporal.api.workflowservice.v1.GetSearchAttributesResponse.KeysEntry\x1aT\n\tKeysEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x36\n\x05value\x18\x02 \x01(\x0e\x32\'.temporal.api.enums.v1.IndexedValueType:\x02\x38\x01"\xe9\x02\n RespondQueryTaskCompletedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12>\n\x0e\x63ompleted_type\x18\x02 \x01(\x0e\x32&.temporal.api.enums.v1.QueryResultType\x12\x36\n\x0cquery_result\x18\x03 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x15\n\rerror_message\x18\x04 \x01(\t\x12\x11\n\tnamespace\x18\x06 \x01(\t\x12\x31\n\x07\x66\x61ilure\x18\x07 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12=\n\x05\x63\x61use\x18\x08 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowTaskFailedCause\x12\x17\n\x0fpoller_group_id\x18\t \x01(\tJ\x04\x08\x05\x10\x06"#\n!RespondQueryTaskCompletedResponse"n\n\x1bResetStickyTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution"\x1e\n\x1cResetStickyTaskQueueResponse"\x9b\x02\n\x15ShutdownWorkerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x19\n\x11sticky_task_queue\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x41\n\x10worker_heartbeat\x18\x05 \x01(\x0b\x32\'.temporal.api.worker.v1.WorkerHeartbeat\x12\x1b\n\x13worker_instance_key\x18\x06 \x01(\t\x12\x12\n\ntask_queue\x18\x07 \x01(\t\x12>\n\x10task_queue_types\x18\x08 \x03(\x0e\x32$.temporal.api.enums.v1.TaskQueueType"\x18\n\x16ShutdownWorkerResponse"\xe9\x01\n\x14QueryWorkflowRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x33\n\x05query\x18\x03 \x01(\x0b\x32$.temporal.api.query.v1.WorkflowQuery\x12K\n\x16query_reject_condition\x18\x04 \x01(\x0e\x32+.temporal.api.enums.v1.QueryRejectCondition"\xb9\x01\n\x15QueryWorkflowResponse\x12\x36\n\x0cquery_result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12<\n\x0equery_rejected\x18\x02 \x01(\x0b\x32$.temporal.api.query.v1.QueryRejected\x12*\n\x04link\x18\x03 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"s\n DescribeWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution"\x99\x05\n!DescribeWorkflowExecutionResponse\x12K\n\x10\x65xecution_config\x18\x01 \x01(\x0b\x32\x31.temporal.api.workflow.v1.WorkflowExecutionConfig\x12P\n\x17workflow_execution_info\x18\x02 \x01(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12I\n\x12pending_activities\x18\x03 \x03(\x0b\x32-.temporal.api.workflow.v1.PendingActivityInfo\x12M\n\x10pending_children\x18\x04 \x03(\x0b\x32\x33.temporal.api.workflow.v1.PendingChildExecutionInfo\x12P\n\x15pending_workflow_task\x18\x05 \x01(\x0b\x32\x31.temporal.api.workflow.v1.PendingWorkflowTaskInfo\x12\x39\n\tcallbacks\x18\x06 \x03(\x0b\x32&.temporal.api.workflow.v1.CallbackInfo\x12U\n\x18pending_nexus_operations\x18\x07 \x03(\x0b\x32\x33.temporal.api.workflow.v1.PendingNexusOperationInfo\x12W\n\x16workflow_extended_info\x18\x08 \x01(\x0b\x32\x37.temporal.api.workflow.v1.WorkflowExecutionExtendedInfo"\x90\x04\n\x18\x44\x65scribeTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12=\n\x0ftask_queue_type\x18\x03 \x01(\x0e\x32$.temporal.api.enums.v1.TaskQueueType\x12\x14\n\x0creport_stats\x18\x08 \x01(\x08\x12\x15\n\rreport_config\x18\x0b \x01(\x08\x12%\n\x19include_task_queue_status\x18\x04 \x01(\x08\x42\x02\x18\x01\x12\x42\n\x08\x61pi_mode\x18\x05 \x01(\x0e\x32,.temporal.api.enums.v1.DescribeTaskQueueModeB\x02\x18\x01\x12J\n\x08versions\x18\x06 \x01(\x0b\x32\x34.temporal.api.taskqueue.v1.TaskQueueVersionSelectionB\x02\x18\x01\x12\x42\n\x10task_queue_types\x18\x07 \x03(\x0e\x32$.temporal.api.enums.v1.TaskQueueTypeB\x02\x18\x01\x12\x1a\n\x0ereport_pollers\x18\t \x01(\x08\x42\x02\x18\x01\x12$\n\x18report_task_reachability\x18\n \x01(\x08\x42\x02\x18\x01"\xec\x07\n\x19\x44\x65scribeTaskQueueResponse\x12\x36\n\x07pollers\x18\x01 \x03(\x0b\x32%.temporal.api.taskqueue.v1.PollerInfo\x12\x38\n\x05stats\x18\x05 \x01(\x0b\x32).temporal.api.taskqueue.v1.TaskQueueStats\x12q\n\x15stats_by_priority_key\x18\x08 \x03(\x0b\x32R.temporal.api.workflowservice.v1.DescribeTaskQueueResponse.StatsByPriorityKeyEntry\x12K\n\x0fversioning_info\x18\x04 \x01(\x0b\x32\x32.temporal.api.taskqueue.v1.TaskQueueVersioningInfo\x12:\n\x06\x63onfig\x18\x06 \x01(\x0b\x32*.temporal.api.taskqueue.v1.TaskQueueConfig\x12k\n\x14\x65\x66\x66\x65\x63tive_rate_limit\x18\x07 \x01(\x0b\x32M.temporal.api.workflowservice.v1.DescribeTaskQueueResponse.EffectiveRateLimit\x12I\n\x11task_queue_status\x18\x02 \x01(\x0b\x32*.temporal.api.taskqueue.v1.TaskQueueStatusB\x02\x18\x01\x12g\n\rversions_info\x18\x03 \x03(\x0b\x32L.temporal.api.workflowservice.v1.DescribeTaskQueueResponse.VersionsInfoEntryB\x02\x18\x01\x1a\x64\n\x17StatsByPriorityKeyEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32).temporal.api.taskqueue.v1.TaskQueueStats:\x02\x38\x01\x1at\n\x12\x45\x66\x66\x65\x63tiveRateLimit\x12\x1b\n\x13requests_per_second\x18\x01 \x01(\x02\x12\x41\n\x11rate_limit_source\x18\x02 \x01(\x0e\x32&.temporal.api.enums.v1.RateLimitSource\x1a\x64\n\x11VersionsInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12>\n\x05value\x18\x02 \x01(\x0b\x32/.temporal.api.taskqueue.v1.TaskQueueVersionInfo:\x02\x38\x01"\x17\n\x15GetClusterInfoRequest"\xd1\x03\n\x16GetClusterInfoResponse\x12h\n\x11supported_clients\x18\x01 \x03(\x0b\x32M.temporal.api.workflowservice.v1.GetClusterInfoResponse.SupportedClientsEntry\x12\x16\n\x0eserver_version\x18\x02 \x01(\t\x12\x12\n\ncluster_id\x18\x03 \x01(\t\x12:\n\x0cversion_info\x18\x04 \x01(\x0b\x32$.temporal.api.version.v1.VersionInfo\x12\x14\n\x0c\x63luster_name\x18\x05 \x01(\t\x12\x1b\n\x13history_shard_count\x18\x06 \x01(\x05\x12\x19\n\x11persistence_store\x18\x07 \x01(\t\x12\x18\n\x10visibility_store\x18\x08 \x01(\t\x12 \n\x18initial_failover_version\x18\t \x01(\x03\x12"\n\x1a\x66\x61ilover_version_increment\x18\n \x01(\x03\x1a\x37\n\x15SupportedClientsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"\x16\n\x14GetSystemInfoRequest"\xc1\x04\n\x15GetSystemInfoResponse\x12\x16\n\x0eserver_version\x18\x01 \x01(\t\x12Y\n\x0c\x63\x61pabilities\x18\x02 \x01(\x0b\x32\x43.temporal.api.workflowservice.v1.GetSystemInfoResponse.Capabilities\x1a\xb4\x03\n\x0c\x43\x61pabilities\x12\x1f\n\x17signal_and_query_header\x18\x01 \x01(\x08\x12&\n\x1einternal_error_differentiation\x18\x02 \x01(\x08\x12*\n"activity_failure_include_heartbeat\x18\x03 \x01(\x08\x12\x1a\n\x12supports_schedules\x18\x04 \x01(\x08\x12"\n\x1a\x65ncoded_failure_attributes\x18\x05 \x01(\x08\x12!\n\x19\x62uild_id_based_versioning\x18\x06 \x01(\x08\x12\x13\n\x0bupsert_memo\x18\x07 \x01(\x08\x12\x1c\n\x14\x65\x61ger_workflow_start\x18\x08 \x01(\x08\x12\x14\n\x0csdk_metadata\x18\t \x01(\x08\x12\'\n\x1f\x63ount_group_by_execution_status\x18\n \x01(\x08\x12\r\n\x05nexus\x18\x0b \x01(\x08\x12!\n\x19server_scaled_deployments\x18\x0c \x01(\x08\x12(\n server_scaled_provider_cloud_run\x18\r \x01(\x08"m\n\x1eListTaskQueuePartitionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue"\xdf\x01\n\x1fListTaskQueuePartitionsResponse\x12]\n\x1e\x61\x63tivity_task_queue_partitions\x18\x01 \x03(\x0b\x32\x35.temporal.api.taskqueue.v1.TaskQueuePartitionMetadata\x12]\n\x1eworkflow_task_queue_partitions\x18\x02 \x03(\x0b\x32\x35.temporal.api.taskqueue.v1.TaskQueuePartitionMetadata"\xcc\x02\n\x15\x43reateScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12\x34\n\x08schedule\x18\x03 \x01(\x0b\x32".temporal.api.schedule.v1.Schedule\x12>\n\rinitial_patch\x18\x04 \x01(\x0b\x32\'.temporal.api.schedule.v1.SchedulePatch\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12*\n\x04memo\x18\x07 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"0\n\x16\x43reateScheduleResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c"A\n\x17\x44\x65scribeScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t"\x8f\x02\n\x18\x44\x65scribeScheduleResponse\x12\x34\n\x08schedule\x18\x01 \x01(\x0b\x32".temporal.api.schedule.v1.Schedule\x12\x34\n\x04info\x18\x02 \x01(\x0b\x32&.temporal.api.schedule.v1.ScheduleInfo\x12*\n\x04memo\x18\x03 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x04 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x16\n\x0e\x63onflict_token\x18\x05 \x01(\x0c"\xa4\x02\n\x15UpdateScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12\x34\n\x08schedule\x18\x03 \x01(\x0b\x32".temporal.api.schedule.v1.Schedule\x12\x16\n\x0e\x63onflict_token\x18\x04 \x01(\x0c\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x43\n\x11search_attributes\x18\x07 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12*\n\x04memo\x18\x08 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\x18\n\x16UpdateScheduleResponse"\x9c\x01\n\x14PatchScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12\x36\n\x05patch\x18\x03 \x01(\x0b\x32\'.temporal.api.schedule.v1.SchedulePatch\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t"\x17\n\x15PatchScheduleResponse"\xa8\x01\n ListScheduleMatchingTimesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12.\n\nstart_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"S\n!ListScheduleMatchingTimesResponse\x12.\n\nstart_time\x18\x01 \x03(\x0b\x32\x1a.google.protobuf.Timestamp"Q\n\x15\x44\x65leteScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t"\x18\n\x16\x44\x65leteScheduleResponse"l\n\x14ListSchedulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x19\n\x11maximum_page_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"p\n\x15ListSchedulesResponse\x12>\n\tschedules\x18\x01 \x03(\x0b\x32+.temporal.api.schedule.v1.ScheduleListEntry\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"9\n\x15\x43ountSchedulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t"\xdb\x01\n\x16\x43ountSchedulesResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12X\n\x06groups\x18\x02 \x03(\x0b\x32H.temporal.api.workflowservice.v1.CountSchedulesResponse.AggregationGroup\x1aX\n\x10\x41ggregationGroup\x12\x35\n\x0cgroup_values\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\r\n\x05\x63ount\x18\x02 \x01(\x03"\x86\x05\n\'UpdateWorkerBuildIdCompatibilityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12-\n#add_new_build_id_in_new_default_set\x18\x03 \x01(\tH\x00\x12\x87\x01\n\x1b\x61\x64\x64_new_compatible_build_id\x18\x04 \x01(\x0b\x32`.temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest.AddNewCompatibleVersionH\x00\x12!\n\x17promote_set_by_build_id\x18\x05 \x01(\tH\x00\x12%\n\x1bpromote_build_id_within_set\x18\x06 \x01(\tH\x00\x12h\n\nmerge_sets\x18\x07 \x01(\x0b\x32R.temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest.MergeSetsH\x00\x1ao\n\x17\x41\x64\x64NewCompatibleVersion\x12\x14\n\x0cnew_build_id\x18\x01 \x01(\t\x12$\n\x1c\x65xisting_compatible_build_id\x18\x02 \x01(\t\x12\x18\n\x10make_set_default\x18\x03 \x01(\x08\x1aI\n\tMergeSets\x12\x1c\n\x14primary_set_build_id\x18\x01 \x01(\t\x12\x1e\n\x16secondary_set_build_id\x18\x02 \x01(\tB\x0b\n\toperation"@\n(UpdateWorkerBuildIdCompatibilityResponseJ\x04\x08\x01\x10\x02R\x0eversion_set_id"_\n$GetWorkerBuildIdCompatibilityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x10\n\x08max_sets\x18\x03 \x01(\x05"t\n%GetWorkerBuildIdCompatibilityResponse\x12K\n\x12major_version_sets\x18\x01 \x03(\x0b\x32/.temporal.api.taskqueue.v1.CompatibleVersionSet"\xb5\r\n"UpdateWorkerVersioningRulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x16\n\x0e\x63onflict_token\x18\x03 \x01(\x0c\x12\x81\x01\n\x16insert_assignment_rule\x18\x04 \x01(\x0b\x32_.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.InsertBuildIdAssignmentRuleH\x00\x12\x83\x01\n\x17replace_assignment_rule\x18\x05 \x01(\x0b\x32`.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.ReplaceBuildIdAssignmentRuleH\x00\x12\x81\x01\n\x16\x64\x65lete_assignment_rule\x18\x06 \x01(\x0b\x32_.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.DeleteBuildIdAssignmentRuleH\x00\x12\x8c\x01\n\x1c\x61\x64\x64_compatible_redirect_rule\x18\x07 \x01(\x0b\x32\x64.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.AddCompatibleBuildIdRedirectRuleH\x00\x12\x94\x01\n replace_compatible_redirect_rule\x18\x08 \x01(\x0b\x32h.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.ReplaceCompatibleBuildIdRedirectRuleH\x00\x12\x92\x01\n\x1f\x64\x65lete_compatible_redirect_rule\x18\t \x01(\x0b\x32g.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.DeleteCompatibleBuildIdRedirectRuleH\x00\x12l\n\x0f\x63ommit_build_id\x18\n \x01(\x0b\x32Q.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.CommitBuildIdH\x00\x1aq\n\x1bInsertBuildIdAssignmentRule\x12\x12\n\nrule_index\x18\x01 \x01(\x05\x12>\n\x04rule\x18\x02 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.BuildIdAssignmentRule\x1a\x81\x01\n\x1cReplaceBuildIdAssignmentRule\x12\x12\n\nrule_index\x18\x01 \x01(\x05\x12>\n\x04rule\x18\x02 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.BuildIdAssignmentRule\x12\r\n\x05\x66orce\x18\x03 \x01(\x08\x1a@\n\x1b\x44\x65leteBuildIdAssignmentRule\x12\x12\n\nrule_index\x18\x01 \x01(\x05\x12\r\n\x05\x66orce\x18\x02 \x01(\x08\x1aj\n AddCompatibleBuildIdRedirectRule\x12\x46\n\x04rule\x18\x01 \x01(\x0b\x32\x38.temporal.api.taskqueue.v1.CompatibleBuildIdRedirectRule\x1an\n$ReplaceCompatibleBuildIdRedirectRule\x12\x46\n\x04rule\x18\x01 \x01(\x0b\x32\x38.temporal.api.taskqueue.v1.CompatibleBuildIdRedirectRule\x1a>\n#DeleteCompatibleBuildIdRedirectRule\x12\x17\n\x0fsource_build_id\x18\x01 \x01(\t\x1a\x37\n\rCommitBuildId\x12\x17\n\x0ftarget_build_id\x18\x01 \x01(\t\x12\r\n\x05\x66orce\x18\x02 \x01(\x08\x42\x0b\n\toperation"\xfc\x01\n#UpdateWorkerVersioningRulesResponse\x12U\n\x10\x61ssignment_rules\x18\x01 \x03(\x0b\x32;.temporal.api.taskqueue.v1.TimestampedBuildIdAssignmentRule\x12\x66\n\x19\x63ompatible_redirect_rules\x18\x02 \x03(\x0b\x32\x43.temporal.api.taskqueue.v1.TimestampedCompatibleBuildIdRedirectRule\x12\x16\n\x0e\x63onflict_token\x18\x03 \x01(\x0c"H\n\x1fGetWorkerVersioningRulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t"\xf9\x01\n GetWorkerVersioningRulesResponse\x12U\n\x10\x61ssignment_rules\x18\x01 \x03(\x0b\x32;.temporal.api.taskqueue.v1.TimestampedBuildIdAssignmentRule\x12\x66\n\x19\x63ompatible_redirect_rules\x18\x02 \x03(\x0b\x32\x43.temporal.api.taskqueue.v1.TimestampedCompatibleBuildIdRedirectRule\x12\x16\n\x0e\x63onflict_token\x18\x03 \x01(\x0c"\x9c\x01\n GetWorkerTaskReachabilityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tbuild_ids\x18\x02 \x03(\t\x12\x13\n\x0btask_queues\x18\x03 \x03(\t\x12=\n\x0creachability\x18\x04 \x01(\x0e\x32\'.temporal.api.enums.v1.TaskReachability"r\n!GetWorkerTaskReachabilityResponse\x12M\n\x15\x62uild_id_reachability\x18\x01 \x03(\x0b\x32..temporal.api.taskqueue.v1.BuildIdReachability"\x85\x02\n\x1eUpdateWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1e\n\x16\x66irst_execution_run_id\x18\x03 \x01(\t\x12\x37\n\x0bwait_policy\x18\x04 \x01(\x0b\x32".temporal.api.update.v1.WaitPolicy\x12\x30\n\x07request\x18\x05 \x01(\x0b\x32\x1f.temporal.api.update.v1.Request"\x83\x02\n\x1fUpdateWorkflowExecutionResponse\x12\x35\n\nupdate_ref\x18\x01 \x01(\x0b\x32!.temporal.api.update.v1.UpdateRef\x12\x30\n\x07outcome\x18\x02 \x01(\x0b\x32\x1f.temporal.api.update.v1.Outcome\x12K\n\x05stage\x18\x03 \x01(\x0e\x32<.temporal.api.enums.v1.UpdateWorkflowExecutionLifecycleStage\x12*\n\x04link\x18\x04 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xd6\n\n\x1aStartBatchOperationRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x18\n\x10visibility_query\x18\x02 \x01(\t\x12\x0e\n\x06job_id\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x41\n\nexecutions\x18\x05 \x03(\x0b\x32).temporal.api.common.v1.WorkflowExecutionB\x02\x18\x01\x12<\n\x11target_executions\x18\x16 \x03(\x0b\x32!.temporal.api.common.v1.Execution\x12!\n\x19max_operations_per_second\x18\x06 \x01(\x02\x12Q\n\x15termination_operation\x18\n \x01(\x0b\x32\x30.temporal.api.batch.v1.BatchOperationTerminationH\x00\x12G\n\x10signal_operation\x18\x0b \x01(\x0b\x32+.temporal.api.batch.v1.BatchOperationSignalH\x00\x12S\n\x16\x63\x61ncellation_operation\x18\x0c \x01(\x0b\x32\x31.temporal.api.batch.v1.BatchOperationCancellationH\x00\x12K\n\x12\x64\x65letion_operation\x18\r \x01(\x0b\x32-.temporal.api.batch.v1.BatchOperationDeletionH\x00\x12\x45\n\x0freset_operation\x18\x0e \x01(\x0b\x32*.temporal.api.batch.v1.BatchOperationResetH\x00\x12p\n!update_workflow_options_operation\x18\x0f \x01(\x0b\x32\x43.temporal.api.batch.v1.BatchOperationUpdateWorkflowExecutionOptionsH\x00\x12^\n\x1cunpause_activities_operation\x18\x10 \x01(\x0b\x32\x36.temporal.api.batch.v1.BatchOperationUnpauseActivitiesH\x00\x12Z\n\x1areset_activities_operation\x18\x11 \x01(\x0b\x32\x34.temporal.api.batch.v1.BatchOperationResetActivitiesH\x00\x12g\n!update_activity_options_operation\x18\x12 \x01(\x0b\x32:.temporal.api.batch.v1.BatchOperationUpdateActivityOptionsH\x00\x12\\\n\x1b\x63\x61ncel_activities_operation\x18\x13 \x01(\x0b\x32\x35.temporal.api.batch.v1.BatchOperationCancelActivitiesH\x00\x12\x62\n\x1eterminate_activities_operation\x18\x14 \x01(\x0b\x32\x38.temporal.api.batch.v1.BatchOperationTerminateActivitiesH\x00\x12\\\n\x1b\x64\x65lete_activities_operation\x18\x15 \x01(\x0b\x32\x35.temporal.api.batch.v1.BatchOperationDeleteActivitiesH\x00\x42\x0b\n\toperation"\x1d\n\x1bStartBatchOperationResponse"`\n\x19StopBatchOperationRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t"\x1c\n\x1aStopBatchOperationResponse"B\n\x1d\x44\x65scribeBatchOperationRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t"\xd8\x03\n\x1e\x44\x65scribeBatchOperationResponse\x12\x41\n\x0eoperation_type\x18\x01 \x01(\x0e\x32).temporal.api.enums.v1.BatchOperationType\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12\x39\n\x05state\x18\x03 \x01(\x0e\x32*.temporal.api.enums.v1.BatchOperationState\x12.\n\nstart_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nclose_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x15total_operation_count\x18\x06 \x01(\x03\x12 \n\x18\x63omplete_operation_count\x18\x07 \x01(\x03\x12\x1f\n\x17\x66\x61ilure_operation_count\x18\x08 \x01(\x03\x12\x10\n\x08identity\x18\t \x01(\t\x12\x0e\n\x06reason\x18\n \x01(\t\x12\r\n\x05query\x18\x0b \x01(\t\x12\x35\n\nexecutions\x18\x0c \x03(\x0b\x32!.temporal.api.common.v1.Execution"[\n\x1aListBatchOperationsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c"y\n\x1bListBatchOperationsResponse\x12\x41\n\x0eoperation_info\x18\x01 \x03(\x0b\x32).temporal.api.batch.v1.BatchOperationInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\xb9\x01\n"PollWorkflowExecutionUpdateRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x35\n\nupdate_ref\x18\x02 \x01(\x0b\x32!.temporal.api.update.v1.UpdateRef\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x37\n\x0bwait_policy\x18\x04 \x01(\x0b\x32".temporal.api.update.v1.WaitPolicy"\xdb\x01\n#PollWorkflowExecutionUpdateResponse\x12\x30\n\x07outcome\x18\x01 \x01(\x0b\x32\x1f.temporal.api.update.v1.Outcome\x12K\n\x05stage\x18\x02 \x01(\x0e\x32<.temporal.api.enums.v1.UpdateWorkflowExecutionLifecycleStage\x12\x35\n\nupdate_ref\x18\x03 \x01(\x0b\x32!.temporal.api.update.v1.UpdateRef"\xa0\x03\n\x19PollNexusTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x03 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x17\n\x0fpoller_group_id\x18\t \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x1b\n\x13worker_instance_key\x18\x08 \x01(\t\x12Z\n\x1bworker_version_capabilities\x18\x04 \x01(\x0b\x32\x31.temporal.api.common.v1.WorkerVersionCapabilitiesB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x06 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12\x41\n\x10worker_heartbeat\x18\x07 \x03(\x0b\x32\'.temporal.api.worker.v1.WorkerHeartbeat"\xe2\x02\n\x1aPollNexusTaskQueueResponse\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12/\n\x07request\x18\x02 \x01(\x0b\x32\x1e.temporal.api.nexus.v1.Request\x12Q\n\x17poller_scaling_decision\x18\x03 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.PollerScalingDecision\x12\x17\n\x0fpoller_group_id\x18\x04 \x01(\t\x12J\n\x12poller_group_infos\x18\x05 \x03(\x0b\x32*.temporal.api.taskqueue.v1.PollerGroupInfoB\x02\x18\x01\x12G\n\x12poller_groups_info\x18\x06 \x01(\x0b\x32+.temporal.api.taskqueue.v1.PollerGroupsInfo"\xa7\x01\n RespondNexusTaskCompletedRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\ntask_token\x18\x03 \x01(\x0c\x12\x31\n\x08response\x18\x04 \x01(\x0b\x32\x1f.temporal.api.nexus.v1.Response\x12\x17\n\x0fpoller_group_id\x18\x05 \x01(\t"#\n!RespondNexusTaskCompletedResponse"\xdc\x01\n\x1dRespondNexusTaskFailedRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\ntask_token\x18\x03 \x01(\x0c\x12\x36\n\x05\x65rror\x18\x04 \x01(\x0b\x32#.temporal.api.nexus.v1.HandlerErrorB\x02\x18\x01\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x17\n\x0fpoller_group_id\x18\x06 \x01(\t" \n\x1eRespondNexusTaskFailedResponse"\xf4\x02\n\x1c\x45xecuteMultiOperationRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12[\n\noperations\x18\x02 \x03(\x0b\x32G.temporal.api.workflowservice.v1.ExecuteMultiOperationRequest.Operation\x12\x13\n\x0bresource_id\x18\x03 \x01(\t\x1a\xce\x01\n\tOperation\x12X\n\x0estart_workflow\x18\x01 \x01(\x0b\x32>.temporal.api.workflowservice.v1.StartWorkflowExecutionRequestH\x00\x12Z\n\x0fupdate_workflow\x18\x02 \x01(\x0b\x32?.temporal.api.workflowservice.v1.UpdateWorkflowExecutionRequestH\x00\x42\x0b\n\toperation"\xcc\x02\n\x1d\x45xecuteMultiOperationResponse\x12Z\n\tresponses\x18\x01 \x03(\x0b\x32G.temporal.api.workflowservice.v1.ExecuteMultiOperationResponse.Response\x1a\xce\x01\n\x08Response\x12Y\n\x0estart_workflow\x18\x01 \x01(\x0b\x32?.temporal.api.workflowservice.v1.StartWorkflowExecutionResponseH\x00\x12[\n\x0fupdate_workflow\x18\x02 \x01(\x0b\x32@.temporal.api.workflowservice.v1.UpdateWorkflowExecutionResponseH\x00\x42\n\n\x08response"\xd0\x02\n\x1cUpdateActivityOptionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x43\n\x10\x61\x63tivity_options\x18\x04 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions\x12/\n\x0bupdate_mask\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.FieldMask\x12\x0c\n\x02id\x18\x06 \x01(\tH\x00\x12\x0e\n\x04type\x18\x07 \x01(\tH\x00\x12\x13\n\tmatch_all\x18\t \x01(\x08H\x00\x12\x18\n\x10restore_original\x18\x08 \x01(\x08\x42\n\n\x08\x61\x63tivity"\xbf\x02\n%UpdateActivityExecutionOptionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x43\n\x10\x61\x63tivity_options\x18\x06 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions\x12/\n\x0bupdate_mask\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.FieldMask\x12\x18\n\x10restore_original\x18\x08 \x01(\x08\x12\x13\n\x0bresource_id\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t"d\n\x1dUpdateActivityOptionsResponse\x12\x43\n\x10\x61\x63tivity_options\x18\x01 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions"m\n&UpdateActivityExecutionOptionsResponse\x12\x43\n\x10\x61\x63tivity_options\x18\x01 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions"\xc7\x01\n\x14PauseActivityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0c\n\x02id\x18\x04 \x01(\tH\x00\x12\x0e\n\x04type\x18\x05 \x01(\tH\x00\x12\x0e\n\x06reason\x18\x06 \x01(\t\x12\x12\n\nrequest_id\x18\x07 \x01(\tB\n\n\x08\x61\x63tivity"\xb7\x01\n\x1dPauseActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x07 \x01(\t\x12\x12\n\nrequest_id\x18\x08 \x01(\t"\x17\n\x15PauseActivityResponse" \n\x1ePauseActivityExecutionResponse"\x98\x02\n\x16UnpauseActivityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0c\n\x02id\x18\x04 \x01(\tH\x00\x12\x0e\n\x04type\x18\x05 \x01(\tH\x00\x12\x15\n\x0bunpause_all\x18\x06 \x01(\x08H\x00\x12\x16\n\x0ereset_attempts\x18\x07 \x01(\x08\x12\x17\n\x0freset_heartbeat\x18\x08 \x01(\x08\x12)\n\x06jitter\x18\t \x01(\x0b\x32\x19.google.protobuf.DurationB\n\n\x08\x61\x63tivity"\x91\x02\n\x1fUnpauseActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x08 \x01(\t\x12)\n\x06jitter\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bresource_id\x18\n \x01(\t\x12\x12\n\nrequest_id\x18\x0b \x01(\tJ\x04\x08\x06\x10\x07J\x04\x08\x07\x10\x08R\x0ereset_attemptsR\x0freset_heartbeat"\x19\n\x17UnpauseActivityResponse""\n UnpauseActivityExecutionResponse"\xb3\x02\n\x14ResetActivityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0c\n\x02id\x18\x04 \x01(\tH\x00\x12\x0e\n\x04type\x18\x05 \x01(\tH\x00\x12\x13\n\tmatch_all\x18\n \x01(\x08H\x00\x12\x17\n\x0freset_heartbeat\x18\x06 \x01(\x08\x12\x13\n\x0bkeep_paused\x18\x07 \x01(\x08\x12)\n\x06jitter\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12 \n\x18restore_original_options\x18\t \x01(\x08\x42\n\n\x08\x61\x63tivity"\xa2\x02\n\x1dResetActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x13\n\x0bkeep_paused\x18\x06 \x01(\x08\x12)\n\x06jitter\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12 \n\x18restore_original_options\x18\x08 \x01(\x08\x12\x13\n\x0bresource_id\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t\x12\x17\n\x0freset_heartbeat\x18\x0b \x01(\x08"\x17\n\x15ResetActivityResponse" \n\x1eResetActivityExecutionResponse"\x9c\x02\n%UpdateWorkflowExecutionOptionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12V\n\x1aworkflow_execution_options\x18\x03 \x01(\x0b\x32\x32.temporal.api.workflow.v1.WorkflowExecutionOptions\x12/\n\x0bupdate_mask\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.FieldMask\x12\x10\n\x08identity\x18\x05 \x01(\t"\xb1\x01\n&UpdateWorkflowExecutionOptionsResponse\x12V\n\x1aworkflow_execution_options\x18\x01 \x01(\x0b\x32\x32.temporal.api.workflow.v1.WorkflowExecutionOptions\x12/\n\x0bupdate_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"j\n\x19\x44\x65scribeDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12:\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.Deployment"a\n\x1a\x44\x65scribeDeploymentResponse\x12\x43\n\x0f\x64\x65ployment_info\x18\x01 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo"\xc2\x01\n&DescribeWorkerDeploymentVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x07version\x18\x02 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x03 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x1f\n\x17report_task_queue_stats\x18\x04 \x01(\x08"\x8c\x05\n\'DescribeWorkerDeploymentVersionResponse\x12_\n\x1eworker_deployment_version_info\x18\x01 \x01(\x0b\x32\x37.temporal.api.deployment.v1.WorkerDeploymentVersionInfo\x12v\n\x13version_task_queues\x18\x02 \x03(\x0b\x32Y.temporal.api.workflowservice.v1.DescribeWorkerDeploymentVersionResponse.VersionTaskQueue\x1a\x87\x03\n\x10VersionTaskQueue\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32$.temporal.api.enums.v1.TaskQueueType\x12\x38\n\x05stats\x18\x03 \x01(\x0b\x32).temporal.api.taskqueue.v1.TaskQueueStats\x12\x90\x01\n\x15stats_by_priority_key\x18\x04 \x03(\x0b\x32q.temporal.api.workflowservice.v1.DescribeWorkerDeploymentVersionResponse.VersionTaskQueue.StatsByPriorityKeyEntry\x1a\x64\n\x17StatsByPriorityKeyEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32).temporal.api.taskqueue.v1.TaskQueueStats:\x02\x38\x01"M\n\x1f\x44\x65scribeWorkerDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t"\x8c\x01\n DescribeWorkerDeploymentResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c\x12P\n\x16worker_deployment_info\x18\x02 \x01(\x0b\x32\x30.temporal.api.deployment.v1.WorkerDeploymentInfo"l\n\x16ListDeploymentsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\x13\n\x0bseries_name\x18\x04 \x01(\t"w\n\x17ListDeploymentsResponse\x12\x17\n\x0fnext_page_token\x18\x01 \x01(\x0c\x12\x43\n\x0b\x64\x65ployments\x18\x02 \x03(\x0b\x32..temporal.api.deployment.v1.DeploymentListInfo"\xcd\x01\n\x1bSetCurrentDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12:\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.Deployment\x12\x10\n\x08identity\x18\x03 \x01(\t\x12M\n\x0fupdate_metadata\x18\x04 \x01(\x0b\x32\x34.temporal.api.deployment.v1.UpdateDeploymentMetadata"\xb9\x01\n\x1cSetCurrentDeploymentResponse\x12K\n\x17\x63urrent_deployment_info\x18\x01 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo\x12L\n\x18previous_deployment_info\x18\x02 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo"\xe5\x01\n(SetWorkerDeploymentCurrentVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x13\n\x07version\x18\x03 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x07 \x01(\t\x12\x16\n\x0e\x63onflict_token\x18\x04 \x01(\x0c\x12\x10\n\x08identity\x18\x05 \x01(\t\x12"\n\x1aignore_missing_task_queues\x18\x06 \x01(\x08\x12\x18\n\x10\x61llow_no_pollers\x18\t \x01(\x08"\xbf\x01\n)SetWorkerDeploymentCurrentVersionResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c\x12\x1c\n\x10previous_version\x18\x02 \x01(\tB\x02\x18\x01\x12\\\n\x1bprevious_deployment_version\x18\x03 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersionB\x02\x18\x01"\xf9\x01\n(SetWorkerDeploymentRampingVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x13\n\x07version\x18\x03 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x08 \x01(\t\x12\x12\n\npercentage\x18\x04 \x01(\x02\x12\x16\n\x0e\x63onflict_token\x18\x05 \x01(\x0c\x12\x10\n\x08identity\x18\x06 \x01(\t\x12"\n\x1aignore_missing_task_queues\x18\x07 \x01(\x08\x12\x18\n\x10\x61llow_no_pollers\x18\n \x01(\x08"\xe0\x01\n)SetWorkerDeploymentRampingVersionResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c\x12\x1c\n\x10previous_version\x18\x02 \x01(\tB\x02\x18\x01\x12\\\n\x1bprevious_deployment_version\x18\x04 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersionB\x02\x18\x01\x12\x1f\n\x13previous_percentage\x18\x03 \x01(\x02\x42\x02\x18\x01"q\n\x1d\x43reateWorkerDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t"8\n\x1e\x43reateWorkerDeploymentResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c"]\n\x1cListWorkerDeploymentsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c"\x9f\x05\n\x1dListWorkerDeploymentsResponse\x12\x17\n\x0fnext_page_token\x18\x01 \x01(\x0c\x12r\n\x12worker_deployments\x18\x02 \x03(\x0b\x32V.temporal.api.workflowservice.v1.ListWorkerDeploymentsResponse.WorkerDeploymentSummary\x1a\xf0\x03\n\x17WorkerDeploymentSummary\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\x0b\x63reate_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x41\n\x0erouting_config\x18\x03 \x01(\x0b\x32).temporal.api.deployment.v1.RoutingConfig\x12o\n\x16latest_version_summary\x18\x04 \x01(\x0b\x32O.temporal.api.deployment.v1.WorkerDeploymentInfo.WorkerDeploymentVersionSummary\x12p\n\x17\x63urrent_version_summary\x18\x05 \x01(\x0b\x32O.temporal.api.deployment.v1.WorkerDeploymentInfo.WorkerDeploymentVersionSummary\x12p\n\x17ramping_version_summary\x18\x06 \x01(\x0b\x32O.temporal.api.deployment.v1.WorkerDeploymentInfo.WorkerDeploymentVersionSummary"\xf0\x01\n$CreateWorkerDeploymentVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12O\n\x12\x64\x65ployment_version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12>\n\x0e\x63ompute_config\x18\x04 \x01(\x0b\x32&.temporal.api.compute.v1.ComputeConfig\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t"\'\n%CreateWorkerDeploymentVersionResponse"\xc8\x01\n$DeleteWorkerDeploymentVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x07version\x18\x02 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x05 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x15\n\rskip_drainage\x18\x03 \x01(\x08\x12\x10\n\x08identity\x18\x04 \x01(\t"\'\n%DeleteWorkerDeploymentVersionResponse"]\n\x1d\x44\x65leteWorkerDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t" \n\x1e\x44\x65leteWorkerDeploymentResponse"\x84\x04\n1UpdateWorkerDeploymentVersionComputeConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12O\n\x12\x64\x65ployment_version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x99\x01\n\x1d\x63ompute_config_scaling_groups\x18\x06 \x03(\x0b\x32r.temporal.api.workflowservice.v1.UpdateWorkerDeploymentVersionComputeConfigRequest.ComputeConfigScalingGroupsEntry\x12,\n$remove_compute_config_scaling_groups\x18\x07 \x03(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x1a{\n\x1f\x43omputeConfigScalingGroupsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12G\n\x05value\x18\x02 \x01(\x0b\x32\x38.temporal.api.compute.v1.ComputeConfigScalingGroupUpdate:\x02\x38\x01"4\n2UpdateWorkerDeploymentVersionComputeConfigResponse"\xf4\x03\n3ValidateWorkerDeploymentVersionComputeConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12O\n\x12\x64\x65ployment_version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x9b\x01\n\x1d\x63ompute_config_scaling_groups\x18\x06 \x03(\x0b\x32t.temporal.api.workflowservice.v1.ValidateWorkerDeploymentVersionComputeConfigRequest.ComputeConfigScalingGroupsEntry\x12,\n$remove_compute_config_scaling_groups\x18\x07 \x03(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x1a{\n\x1f\x43omputeConfigScalingGroupsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12G\n\x05value\x18\x02 \x01(\x0b\x32\x38.temporal.api.compute.v1.ComputeConfigScalingGroupUpdate:\x02\x38\x01"6\n4ValidateWorkerDeploymentVersionComputeConfigResponse"\xa2\x03\n,UpdateWorkerDeploymentVersionMetadataRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x07version\x18\x02 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x05 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12x\n\x0eupsert_entries\x18\x03 \x03(\x0b\x32`.temporal.api.workflowservice.v1.UpdateWorkerDeploymentVersionMetadataRequest.UpsertEntriesEntry\x12\x16\n\x0eremove_entries\x18\x04 \x03(\t\x12\x10\n\x08identity\x18\x06 \x01(\t\x1aU\n\x12UpsertEntriesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"n\n-UpdateWorkerDeploymentVersionMetadataResponse\x12=\n\x08metadata\x18\x01 \x01(\x0b\x32+.temporal.api.deployment.v1.VersionMetadata"\xbd\x01\n!SetWorkerDeploymentManagerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x1a\n\x10manager_identity\x18\x03 \x01(\tH\x00\x12\x0e\n\x04self\x18\x04 \x01(\x08H\x00\x12\x16\n\x0e\x63onflict_token\x18\x05 \x01(\x0c\x12\x10\n\x08identity\x18\x06 \x01(\tB\x16\n\x14new_manager_identity"c\n"SetWorkerDeploymentManagerResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c\x12%\n\x19previous_manager_identity\x18\x02 \x01(\tB\x02\x18\x01"E\n\x1bGetCurrentDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bseries_name\x18\x02 \x01(\t"k\n\x1cGetCurrentDeploymentResponse\x12K\n\x17\x63urrent_deployment_info\x18\x01 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo"q\n GetDeploymentReachabilityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12:\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.Deployment"\xe3\x01\n!GetDeploymentReachabilityResponse\x12\x43\n\x0f\x64\x65ployment_info\x18\x01 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo\x12\x43\n\x0creachability\x18\x02 \x01(\x0e\x32-.temporal.api.enums.v1.DeploymentReachability\x12\x34\n\x10last_update_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\xb4\x01\n\x19\x43reateWorkflowRuleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x35\n\x04spec\x18\x02 \x01(\x0b\x32\'.temporal.api.rules.v1.WorkflowRuleSpec\x12\x12\n\nforce_scan\x18\x03 \x01(\x08\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t"_\n\x1a\x43reateWorkflowRuleResponse\x12\x31\n\x04rule\x18\x01 \x01(\x0b\x32#.temporal.api.rules.v1.WorkflowRule\x12\x0e\n\x06job_id\x18\x02 \x01(\t"A\n\x1b\x44\x65scribeWorkflowRuleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0f\n\x07rule_id\x18\x02 \x01(\t"Q\n\x1c\x44\x65scribeWorkflowRuleResponse\x12\x31\n\x04rule\x18\x01 \x01(\x0b\x32#.temporal.api.rules.v1.WorkflowRule"?\n\x19\x44\x65leteWorkflowRuleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0f\n\x07rule_id\x18\x02 \x01(\t"\x1c\n\x1a\x44\x65leteWorkflowRuleResponse"F\n\x18ListWorkflowRulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"h\n\x19ListWorkflowRulesResponse\x12\x32\n\x05rules\x18\x01 \x03(\x0b\x32#.temporal.api.rules.v1.WorkflowRule\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\xce\x01\n\x1aTriggerWorkflowRuleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x0c\n\x02id\x18\x04 \x01(\tH\x00\x12\x37\n\x04spec\x18\x05 \x01(\x0b\x32\'.temporal.api.rules.v1.WorkflowRuleSpecH\x00\x12\x10\n\x08identity\x18\x03 \x01(\tB\x06\n\x04rule".\n\x1bTriggerWorkflowRuleResponse\x12\x0f\n\x07\x61pplied\x18\x01 \x01(\x08"\x9b\x01\n\x1cRecordWorkerHeartbeatRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x41\n\x10worker_heartbeat\x18\x03 \x03(\x0b\x32\'.temporal.api.worker.v1.WorkerHeartbeat\x12\x13\n\x0bresource_id\x18\x04 \x01(\t"\x1f\n\x1dRecordWorkerHeartbeatResponse"\x82\x01\n\x12ListWorkersRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t\x12\x1e\n\x16include_system_workers\x18\x05 \x01(\x08"\xa5\x01\n\x13ListWorkersResponse\x12<\n\x0cworkers_info\x18\x01 \x03(\x0b\x32".temporal.api.worker.v1.WorkerInfoB\x02\x18\x01\x12\x37\n\x07workers\x18\x03 \x03(\x0b\x32&.temporal.api.worker.v1.WorkerListInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\xd5\x05\n\x1cUpdateTaskQueueConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\ntask_queue\x18\x03 \x01(\t\x12=\n\x0ftask_queue_type\x18\x04 \x01(\x0e\x32$.temporal.api.enums.v1.TaskQueueType\x12n\n\x17update_queue_rate_limit\x18\x05 \x01(\x0b\x32M.temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest.RateLimitUpdate\x12}\n&update_fairness_key_rate_limit_default\x18\x06 \x01(\x0b\x32M.temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest.RateLimitUpdate\x12\x84\x01\n\x1dset_fairness_weight_overrides\x18\x07 \x03(\x0b\x32].temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest.SetFairnessWeightOverridesEntry\x12\'\n\x1funset_fairness_weight_overrides\x18\x08 \x03(\t\x1a[\n\x0fRateLimitUpdate\x12\x38\n\nrate_limit\x18\x01 \x01(\x0b\x32$.temporal.api.taskqueue.v1.RateLimit\x12\x0e\n\x06reason\x18\x02 \x01(\t\x1a\x41\n\x1fSetFairnessWeightOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01"[\n\x1dUpdateTaskQueueConfigResponse\x12:\n\x06\x63onfig\x18\x01 \x01(\x0b\x32*.temporal.api.taskqueue.v1.TaskQueueConfig"\x9e\x01\n\x18\x46\x65tchWorkerConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x38\n\x08selector\x18\x06 \x01(\x0b\x32&.temporal.api.common.v1.WorkerSelector\x12\x13\n\x0bresource_id\x18\x07 \x01(\t"U\n\x19\x46\x65tchWorkerConfigResponse\x12\x38\n\rworker_config\x18\x01 \x01(\x0b\x32!.temporal.api.sdk.v1.WorkerConfig"\x8a\x02\n\x19UpdateWorkerConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x38\n\rworker_config\x18\x04 \x01(\x0b\x32!.temporal.api.sdk.v1.WorkerConfig\x12/\n\x0bupdate_mask\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.FieldMask\x12\x38\n\x08selector\x18\x06 \x01(\x0b\x32&.temporal.api.common.v1.WorkerSelector\x12\x13\n\x0bresource_id\x18\x07 \x01(\t"d\n\x1aUpdateWorkerConfigResponse\x12:\n\rworker_config\x18\x01 \x01(\x0b\x32!.temporal.api.sdk.v1.WorkerConfigH\x00\x42\n\n\x08response"G\n\x15\x44\x65scribeWorkerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x1b\n\x13worker_instance_key\x18\x02 \x01(\t"Q\n\x16\x44\x65scribeWorkerResponse\x12\x37\n\x0bworker_info\x18\x01 \x01(\x0b\x32".temporal.api.worker.v1.WorkerInfo"W\n\x13\x43ountWorkersRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x1e\n\x16include_system_workers\x18\x03 \x01(\x08"%\n\x14\x43ountWorkersResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03"\x8d\x01\n\x1dPauseWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x0e\n\x06reason\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t" \n\x1ePauseWorkflowExecutionResponse"\x8f\x01\n\x1fUnpauseWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x0e\n\x06reason\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t""\n UnpauseWorkflowExecutionResponse"\x99\t\n\x1dStartActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12;\n\ractivity_type\x18\x05 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12\x38\n\ntask_queue\x18\x06 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12<\n\x19schedule_to_close_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12/\n\x05input\x18\x0c \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x45\n\x0fid_reuse_policy\x18\r \x01(\x0e\x32,.temporal.api.enums.v1.ActivityIdReusePolicy\x12K\n\x12id_conflict_policy\x18\x0e \x01(\x0e\x32/.temporal.api.enums.v1.ActivityIdConflictPolicy\x12\x43\n\x11search_attributes\x18\x0f \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\x06header\x18\x10 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x38\n\ruser_metadata\x18\x11 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x32\n\x08priority\x18\x12 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12>\n\x14\x63ompletion_callbacks\x18\x13 \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12+\n\x05links\x18\x14 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x12\x46\n\x13on_conflict_options\x18\x15 \x01(\x0b\x32).temporal.api.common.v1.OnConflictOptions\x12.\n\x0bstart_delay\x18\x16 \x01(\x0b\x32\x19.google.protobuf.Duration"m\n\x1eStartActivityExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0f\n\x07started\x18\x02 \x01(\x08\x12*\n\x04link\x18\x03 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xe4\x01\n DescribeActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x15\n\rinclude_input\x18\x04 \x01(\x08\x12\x17\n\x0finclude_outcome\x18\x05 \x01(\x08\x12\x17\n\x0flong_poll_token\x18\x06 \x01(\x0c\x12!\n\x19include_heartbeat_details\x18\x07 \x01(\x08\x12\x1c\n\x14include_last_failure\x18\x08 \x01(\x08"\xbc\x02\n!DescribeActivityExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12=\n\x04info\x18\x02 \x01(\x0b\x32/.temporal.api.activity.v1.ActivityExecutionInfo\x12/\n\x05input\x18\x03 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x43\n\x07outcome\x18\x04 \x01(\x0b\x32\x32.temporal.api.activity.v1.ActivityExecutionOutcome\x12\x17\n\x0flong_poll_token\x18\x05 \x01(\x0c\x12\x39\n\tcallbacks\x18\x06 \x03(\x0b\x32&.temporal.api.activity.v1.CallbackInfo"V\n\x1cPollActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"t\n\x1dPollActivityExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x43\n\x07outcome\x18\x02 \x01(\x0b\x32\x32.temporal.api.activity.v1.ActivityExecutionOutcome"m\n\x1dListActivityExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"\x82\x01\n\x1eListActivityExecutionsResponse\x12G\n\nexecutions\x18\x01 \x03(\x0b\x32\x33.temporal.api.activity.v1.ActivityExecutionListInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\xd1\x06\n#StartNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t\x12\x14\n\x0coperation_id\x18\x04 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x05 \x01(\t\x12\x0f\n\x07service\x18\x06 \x01(\t\x12\x11\n\toperation\x18\x07 \x01(\t\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12.\n\x05input\x18\x0b \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12K\n\x0fid_reuse_policy\x18\x0c \x01(\x0e\x32\x32.temporal.api.enums.v1.NexusOperationIdReusePolicy\x12Q\n\x12id_conflict_policy\x18\r \x01(\x0e\x32\x35.temporal.api.enums.v1.NexusOperationIdConflictPolicy\x12\x43\n\x11search_attributes\x18\x0e \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12k\n\x0cnexus_header\x18\x0f \x03(\x0b\x32U.temporal.api.workflowservice.v1.StartNexusOperationExecutionRequest.NexusHeaderEntry\x12\x38\n\ruser_metadata\x18\x10 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"G\n$StartNexusOperationExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0f\n\x07started\x18\x02 \x01(\x08"\xaa\x01\n&DescribeNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x15\n\rinclude_input\x18\x04 \x01(\x08\x12\x17\n\x0finclude_outcome\x18\x05 \x01(\x08\x12\x17\n\x0flong_poll_token\x18\x06 \x01(\x0c"\xb7\x02\n\'DescribeNexusOperationExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12@\n\x04info\x18\x02 \x01(\x0b\x32\x32.temporal.api.nexus.v1.NexusOperationExecutionInfo\x12.\n\x05input\x18\x03 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x31\n\x06result\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x12\x33\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x17\n\x0flong_poll_token\x18\x06 \x01(\x0c\x42\t\n\x07outcome"\xa1\x01\n"PollNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x42\n\nwait_stage\x18\x04 \x01(\x0e\x32..temporal.api.enums.v1.NexusOperationWaitStage"\x85\x02\n#PollNexusOperationExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x42\n\nwait_stage\x18\x02 \x01(\x0e\x32..temporal.api.enums.v1.NexusOperationWaitStage\x12\x17\n\x0foperation_token\x18\x03 \x01(\t\x12\x31\n\x06result\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x12\x33\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07outcome"s\n#ListNexusOperationExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"\x8b\x01\n$ListNexusOperationExecutionsResponse\x12J\n\noperations\x18\x01 \x03(\x0b\x32\x36.temporal.api.nexus.v1.NexusOperationExecutionListInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"B\n\x1e\x43ountActivityExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t"\xed\x01\n\x1f\x43ountActivityExecutionsResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12\x61\n\x06groups\x18\x02 \x03(\x0b\x32Q.temporal.api.workflowservice.v1.CountActivityExecutionsResponse.AggregationGroup\x1aX\n\x10\x41ggregationGroup\x12\x35\n\x0cgroup_values\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\r\n\x05\x63ount\x18\x02 \x01(\x03"H\n$CountNexusOperationExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t"\xf9\x01\n%CountNexusOperationExecutionsResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12g\n\x06groups\x18\x02 \x03(\x0b\x32W.temporal.api.workflowservice.v1.CountNexusOperationExecutionsResponse.AggregationGroup\x1aX\n\x10\x41ggregationGroup\x12\x35\n\x0cgroup_values\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\r\n\x05\x63ount\x18\x02 \x01(\x03"\x95\x01\n%RequestCancelActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t"(\n&RequestCancelActivityExecutionResponse"\x91\x01\n!TerminateActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t"$\n"TerminateActivityExecutionResponse"X\n\x1e\x44\x65leteActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"!\n\x1f\x44\x65leteActivityExecutionResponse"\x9c\x01\n+RequestCancelNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t".\n,RequestCancelNexusOperationExecutionResponse"\x98\x01\n\'TerminateNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t"*\n(TerminateNexusOperationExecutionResponse"_\n$DeleteNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"\'\n%DeleteNexusOperationExecutionResponse"\x9d\x01\n(PollWorkflowExecutionTimeSkippingRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x17\n\x0f\x66\x61st_forward_id\x18\x03 \x01(\t"\xe8\x01\n)PollWorkflowExecutionTimeSkippingResponse\x12T\n\x1b\x66\x61st_forward_polling_result\x18\x01 \x01(\x0e\x32/.temporal.api.enums.v1.FastForwardPollingResult\x12\x15\n\rfailed_reason\x18\x02 \x01(\t\x12N\n\x11\x66\x61st_forward_info\x18\x03 \x01(\x0b\x32\x33.temporal.api.common.v1.TimeSkippingFastForwardInfoB\xbe\x01\n"io.temporal.api.workflowservice.v1B\x14RequestResponseProtoP\x01Z5go.temporal.io/api/workflowservice/v1;workflowservice\xaa\x02!Temporalio.Api.WorkflowService.V1\xea\x02$Temporalio::Api::WorkflowService::V1b\x06proto3' + b'\n6temporal/api/workflowservice/v1/request_response.proto\x12\x1ftemporal.api.workflowservice.v1\x1a+temporal/api/enums/v1/batch_operation.proto\x1a"temporal/api/enums/v1/common.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/enums/v1/namespace.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a!temporal/api/enums/v1/query.proto\x1a!temporal/api/enums/v1/reset.proto\x1a&temporal/api/enums/v1/task_queue.proto\x1a&temporal/api/enums/v1/deployment.proto\x1a"temporal/api/enums/v1/update.proto\x1a)temporal/api/enums/v1/time_skipping.proto\x1a$temporal/api/enums/v1/activity.proto\x1a!temporal/api/enums/v1/nexus.proto\x1a&temporal/api/activity/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a*temporal/api/notification/v1/message.proto\x1a%temporal/api/history/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a%temporal/api/command/v1/message.proto\x1a$temporal/api/compute/v1/config.proto\x1a(temporal/api/deployment/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/filter/v1/message.proto\x1a&temporal/api/protocol/v1/message.proto\x1a\'temporal/api/namespace/v1/message.proto\x1a#temporal/api/query/v1/message.proto\x1a)temporal/api/replication/v1/message.proto\x1a#temporal/api/rules/v1/message.proto\x1a\'temporal/api/sdk/v1/worker_config.proto\x1a&temporal/api/schedule/v1/message.proto\x1a\'temporal/api/taskqueue/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a%temporal/api/version/v1/message.proto\x1a#temporal/api/batch/v1/message.proto\x1a\x30temporal/api/sdk/v1/task_complete_metadata.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a#temporal/api/nexus/v1/message.proto\x1a,temporal/api/nexusoperation/v1/message.proto\x1a$temporal/api/worker/v1/message.proto\x1a\x1egoogle/protobuf/duration.proto\x1a google/protobuf/field_mask.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\x88\x05\n\x18RegisterNamespaceRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x13\n\x0bowner_email\x18\x03 \x01(\t\x12\x46\n#workflow_execution_retention_period\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12G\n\x08\x63lusters\x18\x05 \x03(\x0b\x32\x35.temporal.api.replication.v1.ClusterReplicationConfig\x12\x1b\n\x13\x61\x63tive_cluster_name\x18\x06 \x01(\t\x12Q\n\x04\x64\x61ta\x18\x07 \x03(\x0b\x32\x43.temporal.api.workflowservice.v1.RegisterNamespaceRequest.DataEntry\x12\x16\n\x0esecurity_token\x18\x08 \x01(\t\x12\x1b\n\x13is_global_namespace\x18\t \x01(\x08\x12\x44\n\x16history_archival_state\x18\n \x01(\x0e\x32$.temporal.api.enums.v1.ArchivalState\x12\x1c\n\x14history_archival_uri\x18\x0b \x01(\t\x12G\n\x19visibility_archival_state\x18\x0c \x01(\x0e\x32$.temporal.api.enums.v1.ArchivalState\x12\x1f\n\x17visibility_archival_uri\x18\r \x01(\t\x1a+\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"\x1b\n\x19RegisterNamespaceResponse"\x89\x01\n\x15ListNamespacesRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c\x12\x44\n\x10namespace_filter\x18\x03 \x01(\x0b\x32*.temporal.api.namespace.v1.NamespaceFilter"\x81\x01\n\x16ListNamespacesResponse\x12N\n\nnamespaces\x18\x01 \x03(\x0b\x32:.temporal.api.workflowservice.v1.DescribeNamespaceResponse\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"S\n\x18\x44\x65scribeNamespaceRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x18\n\x10weak_consistency\x18\x03 \x01(\x08"\x81\x04\n\x19\x44\x65scribeNamespaceResponse\x12@\n\x0enamespace_info\x18\x01 \x01(\x0b\x32(.temporal.api.namespace.v1.NamespaceInfo\x12:\n\x06\x63onfig\x18\x02 \x01(\x0b\x32*.temporal.api.namespace.v1.NamespaceConfig\x12S\n\x12replication_config\x18\x03 \x01(\x0b\x32\x37.temporal.api.replication.v1.NamespaceReplicationConfig\x12\x18\n\x10\x66\x61ilover_version\x18\x04 \x01(\x03\x12\x1b\n\x13is_global_namespace\x18\x05 \x01(\x08\x12\x45\n\x10\x66\x61ilover_history\x18\x06 \x03(\x0b\x32+.temporal.api.replication.v1.FailoverStatus\x12J\n\x12poller_group_infos\x18\x07 \x03(\x0b\x32*.temporal.api.taskqueue.v1.PollerGroupInfoB\x02\x18\x01\x12G\n\x12poller_groups_info\x18\x08 \x01(\x0b\x32+.temporal.api.taskqueue.v1.PollerGroupsInfo"\xcf\x02\n\x16UpdateNamespaceRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x43\n\x0bupdate_info\x18\x02 \x01(\x0b\x32..temporal.api.namespace.v1.UpdateNamespaceInfo\x12:\n\x06\x63onfig\x18\x03 \x01(\x0b\x32*.temporal.api.namespace.v1.NamespaceConfig\x12S\n\x12replication_config\x18\x04 \x01(\x0b\x32\x37.temporal.api.replication.v1.NamespaceReplicationConfig\x12\x16\n\x0esecurity_token\x18\x05 \x01(\t\x12\x19\n\x11\x64\x65lete_bad_binary\x18\x06 \x01(\t\x12\x19\n\x11promote_namespace\x18\x07 \x01(\x08"\xa3\x02\n\x17UpdateNamespaceResponse\x12@\n\x0enamespace_info\x18\x01 \x01(\x0b\x32(.temporal.api.namespace.v1.NamespaceInfo\x12:\n\x06\x63onfig\x18\x02 \x01(\x0b\x32*.temporal.api.namespace.v1.NamespaceConfig\x12S\n\x12replication_config\x18\x03 \x01(\x0b\x32\x37.temporal.api.replication.v1.NamespaceReplicationConfig\x12\x18\n\x10\x66\x61ilover_version\x18\x04 \x01(\x03\x12\x1b\n\x13is_global_namespace\x18\x05 \x01(\x08"F\n\x19\x44\x65precateNamespaceRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x16\n\x0esecurity_token\x18\x02 \x01(\t"\x1c\n\x1a\x44\x65precateNamespaceResponse"\xd1\x0c\n\x1dStartWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x10\n\x08identity\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t\x12N\n\x18workflow_id_reuse_policy\x18\x0b \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12T\n\x1bworkflow_id_conflict_policy\x18\x16 \x01(\x0e\x32/.temporal.api.enums.v1.WorkflowIdConflictPolicy\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\r \x01(\t\x12*\n\x04memo\x18\x0e \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x0f \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\x06header\x18\x10 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x1f\n\x17request_eager_execution\x18\x11 \x01(\x08\x12;\n\x11\x63ontinued_failure\x18\x12 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x13 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x37\n\x14workflow_start_delay\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12>\n\x14\x63ompletion_callbacks\x18\x15 \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12\x38\n\ruser_metadata\x18\x17 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12+\n\x05links\x18\x18 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x12I\n\x13versioning_override\x18\x19 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12H\n\x13on_conflict_options\x18\x1a \x01(\x0b\x32+.temporal.api.workflow.v1.OnConflictOptions\x12\x32\n\x08priority\x18\x1b \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12\\\n\x1f\x65\x61ger_worker_deployment_options\x18\x1c \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12H\n\x14time_skipping_config\x18\x1d \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfig"\xaa\x02\n\x1eStartWorkflowExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x06 \x01(\t\x12\x0f\n\x07started\x18\x03 \x01(\x08\x12>\n\x06status\x18\x05 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowExecutionStatus\x12[\n\x13\x65\x61ger_workflow_task\x18\x02 \x01(\x0b\x32>.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse\x12*\n\x04link\x18\x04 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xaa\x02\n"GetWorkflowExecutionHistoryRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x19\n\x11maximum_page_size\x18\x03 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x04 \x01(\x0c\x12\x16\n\x0ewait_new_event\x18\x05 \x01(\x08\x12P\n\x19history_event_filter_type\x18\x06 \x01(\x0e\x32-.temporal.api.enums.v1.HistoryEventFilterType\x12\x15\n\rskip_archival\x18\x07 \x01(\x08"\xba\x01\n#GetWorkflowExecutionHistoryResponse\x12\x31\n\x07history\x18\x01 \x01(\x0b\x32 .temporal.api.history.v1.History\x12\x35\n\x0braw_history\x18\x02 \x03(\x0b\x32 .temporal.api.common.v1.DataBlob\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\x10\n\x08\x61rchived\x18\x04 \x01(\x08"\xb0\x01\n)GetWorkflowExecutionHistoryReverseRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x19\n\x11maximum_page_size\x18\x03 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x04 \x01(\x0c"x\n*GetWorkflowExecutionHistoryReverseResponse\x12\x31\n\x07history\x18\x01 \x01(\x0b\x32 .temporal.api.history.v1.History\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c"\xb8\x03\n\x1cPollWorkflowTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x17\n\x0fpoller_group_id\x18\n \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x1b\n\x13worker_instance_key\x18\x08 \x01(\t\x12!\n\x19worker_control_task_queue\x18\t \x01(\t\x12\x1b\n\x0f\x62inary_checksum\x18\x04 \x01(\tB\x02\x18\x01\x12Z\n\x1bworker_version_capabilities\x18\x05 \x01(\x0b\x32\x31.temporal.api.common.v1.WorkerVersionCapabilitiesB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x06 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptionsJ\x04\x08\x07\x10\x08R\x10worker_heartbeat"\x81\t\n\x1dPollWorkflowTaskQueueResponse\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12!\n\x19previous_started_event_id\x18\x04 \x01(\x03\x12\x18\n\x10started_event_id\x18\x05 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x06 \x01(\x05\x12\x1a\n\x12\x62\x61\x63klog_count_hint\x18\x07 \x01(\x03\x12\x31\n\x07history\x18\x08 \x01(\x0b\x32 .temporal.api.history.v1.History\x12\x17\n\x0fnext_page_token\x18\t \x01(\x0c\x12\x33\n\x05query\x18\n \x01(\x0b\x32$.temporal.api.query.v1.WorkflowQuery\x12K\n\x1dworkflow_execution_task_queue\x18\x0b \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x32\n\x0escheduled_time\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0cstarted_time\x18\r \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\\\n\x07queries\x18\x0e \x03(\x0b\x32K.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse.QueriesEntry\x12\x33\n\x08messages\x18\x0f \x03(\x0b\x32!.temporal.api.protocol.v1.Message\x12Q\n\x17poller_scaling_decision\x18\x10 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.PollerScalingDecision\x12\x17\n\x0fpoller_group_id\x18\x11 \x01(\t\x12J\n\x12poller_group_infos\x18\x12 \x03(\x0b\x32*.temporal.api.taskqueue.v1.PollerGroupInfoB\x02\x18\x01\x12G\n\x12poller_groups_info\x18\x13 \x01(\x0b\x32+.temporal.api.taskqueue.v1.PollerGroupsInfo\x12:\n\rstream_slices\x18\x14 \x03(\x0b\x32#.temporal.api.stream.v1.StreamSlice\x1aT\n\x0cQueriesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.temporal.api.query.v1.WorkflowQuery:\x02\x38\x01J\x04\x08\x15\x10\x16"\xba\n\n#RespondWorkflowTaskCompletedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x32\n\x08\x63ommands\x18\x02 \x03(\x0b\x32 .temporal.api.command.v1.Command\x12\x10\n\x08identity\x18\x03 \x01(\t\x12O\n\x11sticky_attributes\x18\x04 \x01(\x0b\x32\x34.temporal.api.taskqueue.v1.StickyExecutionAttributes\x12 \n\x18return_new_workflow_task\x18\x05 \x01(\x08\x12&\n\x1e\x66orce_create_new_workflow_task\x18\x06 \x01(\x08\x12\x1b\n\x0f\x62inary_checksum\x18\x07 \x01(\tB\x02\x18\x01\x12m\n\rquery_results\x18\x08 \x03(\x0b\x32V.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest.QueryResultsEntry\x12\x11\n\tnamespace\x18\t \x01(\t\x12\x13\n\x0bresource_id\x18\x12 \x01(\t\x12L\n\x14worker_version_stamp\x18\n \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12\x33\n\x08messages\x18\x0b \x03(\x0b\x32!.temporal.api.protocol.v1.Message\x12H\n\x0csdk_metadata\x18\x0c \x01(\x0b\x32\x32.temporal.api.sdk.v1.WorkflowTaskCompletedMetadata\x12\x43\n\x11metering_metadata\x18\r \x01(\x0b\x32(.temporal.api.common.v1.MeteringMetadata\x12g\n\x0c\x63\x61pabilities\x18\x0e \x01(\x0b\x32Q.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest.Capabilities\x12>\n\ndeployment\x18\x0f \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12\x46\n\x13versioning_behavior\x18\x10 \x01(\x0e\x32).temporal.api.enums.v1.VersioningBehavior\x12O\n\x12\x64\x65ployment_options\x18\x11 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12\x1b\n\x13worker_instance_key\x18\x13 \x01(\t\x12!\n\x19worker_control_task_queue\x18\x14 \x01(\t\x12\x13\n\x0bpage_number\x18\x15 \x01(\x05\x12\x19\n\x11intermediate_page\x18\x16 \x01(\x08\x1a_\n\x11QueryResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32*.temporal.api.query.v1.WorkflowQueryResult:\x02\x38\x01\x1a\x45\n\x0c\x43\x61pabilities\x12\x35\n-discard_speculative_workflow_task_with_events\x18\x01 \x01(\x08"\xf5\x01\n$RespondWorkflowTaskCompletedResponse\x12U\n\rworkflow_task\x18\x01 \x01(\x0b\x32>.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse\x12V\n\x0e\x61\x63tivity_tasks\x18\x02 \x03(\x0b\x32>.temporal.api.workflowservice.v1.PollActivityTaskQueueResponse\x12\x1e\n\x16reset_history_event_id\x18\x03 \x01(\x03"\x8d\x04\n RespondWorkflowTaskFailedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12=\n\x05\x63\x61use\x18\x02 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowTaskFailedCause\x12\x31\n\x07\x66\x61ilure\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x1b\n\x0f\x62inary_checksum\x18\x05 \x01(\tB\x02\x18\x01\x12\x11\n\tnamespace\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x0b \x01(\t\x12\x33\n\x08messages\x18\x07 \x03(\x0b\x32!.temporal.api.protocol.v1.Message\x12\x46\n\x0eworker_version\x18\x08 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\ndeployment\x18\t \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\n \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions"#\n!RespondWorkflowTaskFailedResponse"\xe6\x03\n\x1cPollActivityTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x17\n\x0fpoller_group_id\x18\n \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x1b\n\x13worker_instance_key\x18\x08 \x01(\t\x12!\n\x19worker_control_task_queue\x18\t \x01(\t\x12I\n\x13task_queue_metadata\x18\x04 \x01(\x0b\x32,.temporal.api.taskqueue.v1.TaskQueueMetadata\x12Z\n\x1bworker_version_capabilities\x18\x05 \x01(\x0b\x32\x31.temporal.api.common.v1.WorkerVersionCapabilitiesB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x06 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptionsJ\x04\x08\x07\x10\x08R\x10worker_heartbeat"\x9d\t\n\x1dPollActivityTaskQueueResponse\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x1a\n\x12workflow_namespace\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x45\n\x12workflow_execution\x18\x04 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12;\n\ractivity_type\x18\x05 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12\x13\n\x0b\x61\x63tivity_id\x18\x06 \x01(\t\x12.\n\x06header\x18\x07 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12/\n\x05input\x18\x08 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12;\n\x11heartbeat_details\x18\t \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x32\n\x0escheduled_time\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x42\n\x1e\x63urrent_attempt_scheduled_time\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0cstarted_time\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\r \x01(\x05\x12<\n\x19schedule_to_close_timeout\x18\x0e \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\x0f \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x10 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x11 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12Q\n\x17poller_scaling_decision\x18\x12 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.PollerScalingDecision\x12\x32\n\x08priority\x18\x13 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12\x17\n\x0f\x61\x63tivity_run_id\x18\x14 \x01(\t\x12J\n\x12poller_group_infos\x18\x15 \x03(\x0b\x32*.temporal.api.taskqueue.v1.PollerGroupInfoB\x02\x18\x01\x12G\n\x12poller_groups_info\x18\x16 \x01(\x0b\x32+.temporal.api.taskqueue.v1.PollerGroupsInfo"\xa5\x01\n"RecordActivityTaskHeartbeatRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x31\n\x07\x64\x65tails\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tnamespace\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\x05 \x01(\t"p\n#RecordActivityTaskHeartbeatResponse\x12\x18\n\x10\x63\x61ncel_requested\x18\x01 \x01(\x08\x12\x17\n\x0f\x61\x63tivity_paused\x18\x02 \x01(\x08\x12\x16\n\x0e\x61\x63tivity_reset\x18\x03 \x01(\x08"\xcf\x01\n&RecordActivityTaskHeartbeatByIdRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12\x31\n\x07\x64\x65tails\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x07 \x01(\t"t\n\'RecordActivityTaskHeartbeatByIdResponse\x12\x18\n\x10\x63\x61ncel_requested\x18\x01 \x01(\x08\x12\x17\n\x0f\x61\x63tivity_paused\x18\x02 \x01(\x08\x12\x16\n\x0e\x61\x63tivity_reset\x18\x03 \x01(\x08"\xfe\x02\n#RespondActivityTaskCompletedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x30\n\x06result\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tnamespace\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\x08 \x01(\t\x12\x46\n\x0eworker_version\x18\x05 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\ndeployment\x18\x06 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x07 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions"&\n$RespondActivityTaskCompletedResponse"\xcf\x01\n\'RespondActivityTaskCompletedByIdRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12\x30\n\x06result\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x07 \x01(\t"*\n(RespondActivityTaskCompletedByIdResponse"\xfd\x03\n RespondActivityTaskFailedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tnamespace\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\t \x01(\t\x12@\n\x16last_heartbeat_details\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x46\n\x0eworker_version\x18\x06 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\ndeployment\x18\x07 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x08 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12=\n\x05\x63\x61use\x18\n \x01(\x0e\x32..temporal.api.enums.v1.ActivityTaskFailedCause"W\n!RespondActivityTaskFailedResponse\x12\x32\n\x08\x66\x61ilures\x18\x01 \x03(\x0b\x32 .temporal.api.failure.v1.Failure"\xce\x02\n$RespondActivityTaskFailedByIdRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x10\n\x08identity\x18\x06 \x01(\t\x12@\n\x16last_heartbeat_details\x18\x07 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x13\n\x0bresource_id\x18\x08 \x01(\t\x12=\n\x05\x63\x61use\x18\t \x01(\x0e\x32..temporal.api.enums.v1.ActivityTaskFailedCause"[\n%RespondActivityTaskFailedByIdResponse\x12\x32\n\x08\x66\x61ilures\x18\x01 \x03(\x0b\x32 .temporal.api.failure.v1.Failure"\xfe\x02\n"RespondActivityTaskCanceledRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x31\n\x07\x64\x65tails\x18\x02 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tnamespace\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\x08 \x01(\t\x12\x46\n\x0eworker_version\x18\x05 \x01(\x0b\x32*.temporal.api.common.v1.WorkerVersionStampB\x02\x18\x01\x12>\n\ndeployment\x18\x06 \x01(\x0b\x32&.temporal.api.deployment.v1.DeploymentB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x07 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions"%\n#RespondActivityTaskCanceledResponse"\xa0\x02\n&RespondActivityTaskCanceledByIdRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12\x31\n\x07\x64\x65tails\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x06 \x01(\t\x12O\n\x12\x64\x65ployment_options\x18\x07 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12\x13\n\x0bresource_id\x18\x08 \x01(\t")\n\'RespondActivityTaskCanceledByIdResponse"\x84\x02\n%RequestCancelWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t\x12+\n\x05links\x18\x07 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link"(\n&RequestCancelWorkflowExecutionResponse"\xde\x02\n\x1eSignalWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x13\n\x0bsignal_name\x18\x03 \x01(\t\x12/\n\x05input\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x13\n\x07\x63ontrol\x18\x07 \x01(\tB\x02\x18\x01\x12.\n\x06header\x18\x08 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12+\n\x05links\x18\n \x03(\x0b\x32\x1c.temporal.api.common.v1.LinkJ\x04\x08\t\x10\n"M\n\x1fSignalWorkflowExecutionResponse\x12*\n\x04link\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xc7\x01\n\x14NotifyChannelRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12@\n\x0cnotification\x18\x02 \x01(\x0b\x32*.temporal.api.notification.v1.Notification\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x34\n\texecution\x18\x05 \x01(\x0b\x32!.temporal.api.common.v1.Execution"/\n\x15NotifyChannelResponse\x12\x16\n\x0elistener_count\x18\x01 \x01(\x05"\xd4\x01\n\x1eRegisterChannelListenerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0f\n\x07\x63hannel\x18\x02 \x01(\t\x12\x32\n\x08\x63\x61llback\x18\x03 \x01(\x0b\x32 .temporal.api.common.v1.Callback\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x34\n\texecution\x18\x06 \x01(\x0b\x32!.temporal.api.common.v1.Execution"6\n\x1fRegisterChannelListenerResponse\x12\x13\n\x0blistener_id\x18\x01 \x01(\t"\xa3\x01\n UnregisterChannelListenerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0f\n\x07\x63hannel\x18\x02 \x01(\t\x12\x13\n\x0blistener_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x34\n\texecution\x18\x05 \x01(\x0b\x32!.temporal.api.common.v1.Execution"#\n!UnregisterChannelListenerResponse"\xc9\x01\n\x12PollChannelRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0f\n\x07\x63hannel\x18\x02 \x01(\t\x12\x15\n\rafter_counter\x18\x03 \x01(\x03\x12\'\n\x04wait\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x19\n\x11max_notifications\x18\x05 \x01(\x05\x12\x34\n\texecution\x18\x06 \x01(\x0b\x32!.temporal.api.common.v1.Execution"X\n\x13PollChannelResponse\x12\x41\n\rnotifications\x18\x01 \x03(\x0b\x32*.temporal.api.notification.v1.Notification"r\n\x16\x44\x65scribeChannelRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0f\n\x07\x63hannel\x18\x02 \x01(\t\x12\x34\n\texecution\x18\x03 \x01(\x0b\x32!.temporal.api.common.v1.Execution"\x9e\x02\n\x17\x44\x65scribeChannelResponse\x12@\n\tlisteners\x18\x01 \x03(\x0b\x32-.temporal.api.notification.v1.ChannelListener\x12:\n\x06latest\x18\x02 \x01(\x0b\x32*.temporal.api.notification.v1.Notification\x12\x16\n\x0eretained_count\x18\x03 \x01(\x05\x12\x37\n\x04kind\x18\x04 \x01(\x0e\x32).temporal.api.notification.v1.ChannelKind\x12\x34\n\tlinked_to\x18\x05 \x01(\x0b\x32!.temporal.api.common.v1.Execution"\xbb\n\n\'SignalWithStartWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12;\n\rworkflow_type\x18\x03 \x01(\x0b\x32$.temporal.api.common.v1.WorkflowType\x12\x38\n\ntask_queue\x18\x04 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12/\n\x05input\x18\x05 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12=\n\x1aworkflow_execution_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x10\n\x08identity\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t\x12N\n\x18workflow_id_reuse_policy\x18\x0b \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12T\n\x1bworkflow_id_conflict_policy\x18\x16 \x01(\x0e\x32/.temporal.api.enums.v1.WorkflowIdConflictPolicy\x12\x13\n\x0bsignal_name\x18\x0c \x01(\t\x12\x36\n\x0csignal_input\x18\r \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x13\n\x07\x63ontrol\x18\x0e \x01(\tB\x02\x18\x01\x12\x39\n\x0cretry_policy\x18\x0f \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x10 \x01(\t\x12*\n\x04memo\x18\x11 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x12 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\x06header\x18\x13 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x37\n\x14workflow_start_delay\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\ruser_metadata\x18\x17 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12+\n\x05links\x18\x18 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x12I\n\x13versioning_override\x18\x19 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x12\x32\n\x08priority\x18\x1a \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12H\n\x14time_skipping_config\x18\x1b \x01(\x0b\x32*.temporal.api.common.v1.TimeSkippingConfigJ\x04\x08\x15\x10\x16"\x9e\x01\n(SignalWithStartWorkflowExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x04 \x01(\t\x12\x0f\n\x07started\x18\x02 \x01(\x08\x12\x31\n\x0bsignal_link\x18\x03 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xc1\x03\n\x1dResetWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12%\n\x1dworkflow_task_finish_event_id\x18\x04 \x01(\x03\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12G\n\x12reset_reapply_type\x18\x06 \x01(\x0e\x32\'.temporal.api.enums.v1.ResetReapplyTypeB\x02\x18\x01\x12S\n\x1breset_reapply_exclude_types\x18\x07 \x03(\x0e\x32..temporal.api.enums.v1.ResetReapplyExcludeType\x12K\n\x15post_reset_operations\x18\x08 \x03(\x0b\x32,.temporal.api.workflow.v1.PostResetOperation\x12\x10\n\x08identity\x18\t \x01(\t"0\n\x1eResetWorkflowExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\x9f\x02\n!TerminateWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x31\n\x07\x64\x65tails\x18\x04 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x06 \x01(\t\x12+\n\x05links\x18\x07 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link"$\n"TerminateWorkflowExecutionResponse"z\n\x1e\x44\x65leteWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution"!\n\x1f\x44\x65leteWorkflowExecutionResponse"\xc9\x02\n!ListOpenWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x19\n\x11maximum_page_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\x42\n\x11start_time_filter\x18\x04 \x01(\x0b\x32\'.temporal.api.filter.v1.StartTimeFilter\x12K\n\x10\x65xecution_filter\x18\x05 \x01(\x0b\x32/.temporal.api.filter.v1.WorkflowExecutionFilterH\x00\x12\x41\n\x0btype_filter\x18\x06 \x01(\x0b\x32*.temporal.api.filter.v1.WorkflowTypeFilterH\x00\x42\t\n\x07\x66ilters"\x82\x01\n"ListOpenWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\x8a\x03\n#ListClosedWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x19\n\x11maximum_page_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\x42\n\x11start_time_filter\x18\x04 \x01(\x0b\x32\'.temporal.api.filter.v1.StartTimeFilter\x12K\n\x10\x65xecution_filter\x18\x05 \x01(\x0b\x32/.temporal.api.filter.v1.WorkflowExecutionFilterH\x00\x12\x41\n\x0btype_filter\x18\x06 \x01(\x0b\x32*.temporal.api.filter.v1.WorkflowTypeFilterH\x00\x12=\n\rstatus_filter\x18\x07 \x01(\x0b\x32$.temporal.api.filter.v1.StatusFilterH\x00\x42\t\n\x07\x66ilters"\x84\x01\n$ListClosedWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"m\n\x1dListWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"~\n\x1eListWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"u\n%ListArchivedWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"\x86\x01\n&ListArchivedWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"m\n\x1dScanWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"~\n\x1eScanWorkflowExecutionsResponse\x12\x43\n\nexecutions\x18\x01 \x03(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"B\n\x1e\x43ountWorkflowExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t"\xed\x01\n\x1f\x43ountWorkflowExecutionsResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12\x61\n\x06groups\x18\x02 \x03(\x0b\x32Q.temporal.api.workflowservice.v1.CountWorkflowExecutionsResponse.AggregationGroup\x1aX\n\x10\x41ggregationGroup\x12\x35\n\x0cgroup_values\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\r\n\x05\x63ount\x18\x02 \x01(\x03"\x1c\n\x1aGetSearchAttributesRequest"\xc9\x01\n\x1bGetSearchAttributesResponse\x12T\n\x04keys\x18\x01 \x03(\x0b\x32\x46.temporal.api.workflowservice.v1.GetSearchAttributesResponse.KeysEntry\x1aT\n\tKeysEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x36\n\x05value\x18\x02 \x01(\x0e\x32\'.temporal.api.enums.v1.IndexedValueType:\x02\x38\x01"\xe9\x02\n RespondQueryTaskCompletedRequest\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12>\n\x0e\x63ompleted_type\x18\x02 \x01(\x0e\x32&.temporal.api.enums.v1.QueryResultType\x12\x36\n\x0cquery_result\x18\x03 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x15\n\rerror_message\x18\x04 \x01(\t\x12\x11\n\tnamespace\x18\x06 \x01(\t\x12\x31\n\x07\x66\x61ilure\x18\x07 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12=\n\x05\x63\x61use\x18\x08 \x01(\x0e\x32..temporal.api.enums.v1.WorkflowTaskFailedCause\x12\x17\n\x0fpoller_group_id\x18\t \x01(\tJ\x04\x08\x05\x10\x06"#\n!RespondQueryTaskCompletedResponse"n\n\x1bResetStickyTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution"\x1e\n\x1cResetStickyTaskQueueResponse"\x9b\x02\n\x15ShutdownWorkerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x19\n\x11sticky_task_queue\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x41\n\x10worker_heartbeat\x18\x05 \x01(\x0b\x32\'.temporal.api.worker.v1.WorkerHeartbeat\x12\x1b\n\x13worker_instance_key\x18\x06 \x01(\t\x12\x12\n\ntask_queue\x18\x07 \x01(\t\x12>\n\x10task_queue_types\x18\x08 \x03(\x0e\x32$.temporal.api.enums.v1.TaskQueueType"\x18\n\x16ShutdownWorkerResponse"\xe9\x01\n\x14QueryWorkflowRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x33\n\x05query\x18\x03 \x01(\x0b\x32$.temporal.api.query.v1.WorkflowQuery\x12K\n\x16query_reject_condition\x18\x04 \x01(\x0e\x32+.temporal.api.enums.v1.QueryRejectCondition"\xb9\x01\n\x15QueryWorkflowResponse\x12\x36\n\x0cquery_result\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12<\n\x0equery_rejected\x18\x02 \x01(\x0b\x32$.temporal.api.query.v1.QueryRejected\x12*\n\x04link\x18\x03 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"s\n DescribeWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution"\xeb\x05\n!DescribeWorkflowExecutionResponse\x12K\n\x10\x65xecution_config\x18\x01 \x01(\x0b\x32\x31.temporal.api.workflow.v1.WorkflowExecutionConfig\x12P\n\x17workflow_execution_info\x18\x02 \x01(\x0b\x32/.temporal.api.workflow.v1.WorkflowExecutionInfo\x12I\n\x12pending_activities\x18\x03 \x03(\x0b\x32-.temporal.api.workflow.v1.PendingActivityInfo\x12M\n\x10pending_children\x18\x04 \x03(\x0b\x32\x33.temporal.api.workflow.v1.PendingChildExecutionInfo\x12P\n\x15pending_workflow_task\x18\x05 \x01(\x0b\x32\x31.temporal.api.workflow.v1.PendingWorkflowTaskInfo\x12\x39\n\tcallbacks\x18\x06 \x03(\x0b\x32&.temporal.api.workflow.v1.CallbackInfo\x12U\n\x18pending_nexus_operations\x18\x07 \x03(\x0b\x32\x33.temporal.api.workflow.v1.PendingNexusOperationInfo\x12W\n\x16workflow_extended_info\x18\x08 \x01(\x0b\x32\x37.temporal.api.workflow.v1.WorkflowExecutionExtendedInfo\x12P\n\x15\x63hannel_subscriptions\x18\t \x03(\x0b\x32\x31.temporal.api.workflow.v1.ChannelSubscriptionInfo"\x90\x04\n\x18\x44\x65scribeTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12=\n\x0ftask_queue_type\x18\x03 \x01(\x0e\x32$.temporal.api.enums.v1.TaskQueueType\x12\x14\n\x0creport_stats\x18\x08 \x01(\x08\x12\x15\n\rreport_config\x18\x0b \x01(\x08\x12%\n\x19include_task_queue_status\x18\x04 \x01(\x08\x42\x02\x18\x01\x12\x42\n\x08\x61pi_mode\x18\x05 \x01(\x0e\x32,.temporal.api.enums.v1.DescribeTaskQueueModeB\x02\x18\x01\x12J\n\x08versions\x18\x06 \x01(\x0b\x32\x34.temporal.api.taskqueue.v1.TaskQueueVersionSelectionB\x02\x18\x01\x12\x42\n\x10task_queue_types\x18\x07 \x03(\x0e\x32$.temporal.api.enums.v1.TaskQueueTypeB\x02\x18\x01\x12\x1a\n\x0ereport_pollers\x18\t \x01(\x08\x42\x02\x18\x01\x12$\n\x18report_task_reachability\x18\n \x01(\x08\x42\x02\x18\x01"\xec\x07\n\x19\x44\x65scribeTaskQueueResponse\x12\x36\n\x07pollers\x18\x01 \x03(\x0b\x32%.temporal.api.taskqueue.v1.PollerInfo\x12\x38\n\x05stats\x18\x05 \x01(\x0b\x32).temporal.api.taskqueue.v1.TaskQueueStats\x12q\n\x15stats_by_priority_key\x18\x08 \x03(\x0b\x32R.temporal.api.workflowservice.v1.DescribeTaskQueueResponse.StatsByPriorityKeyEntry\x12K\n\x0fversioning_info\x18\x04 \x01(\x0b\x32\x32.temporal.api.taskqueue.v1.TaskQueueVersioningInfo\x12:\n\x06\x63onfig\x18\x06 \x01(\x0b\x32*.temporal.api.taskqueue.v1.TaskQueueConfig\x12k\n\x14\x65\x66\x66\x65\x63tive_rate_limit\x18\x07 \x01(\x0b\x32M.temporal.api.workflowservice.v1.DescribeTaskQueueResponse.EffectiveRateLimit\x12I\n\x11task_queue_status\x18\x02 \x01(\x0b\x32*.temporal.api.taskqueue.v1.TaskQueueStatusB\x02\x18\x01\x12g\n\rversions_info\x18\x03 \x03(\x0b\x32L.temporal.api.workflowservice.v1.DescribeTaskQueueResponse.VersionsInfoEntryB\x02\x18\x01\x1a\x64\n\x17StatsByPriorityKeyEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32).temporal.api.taskqueue.v1.TaskQueueStats:\x02\x38\x01\x1at\n\x12\x45\x66\x66\x65\x63tiveRateLimit\x12\x1b\n\x13requests_per_second\x18\x01 \x01(\x02\x12\x41\n\x11rate_limit_source\x18\x02 \x01(\x0e\x32&.temporal.api.enums.v1.RateLimitSource\x1a\x64\n\x11VersionsInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12>\n\x05value\x18\x02 \x01(\x0b\x32/.temporal.api.taskqueue.v1.TaskQueueVersionInfo:\x02\x38\x01"\x17\n\x15GetClusterInfoRequest"\xd1\x03\n\x16GetClusterInfoResponse\x12h\n\x11supported_clients\x18\x01 \x03(\x0b\x32M.temporal.api.workflowservice.v1.GetClusterInfoResponse.SupportedClientsEntry\x12\x16\n\x0eserver_version\x18\x02 \x01(\t\x12\x12\n\ncluster_id\x18\x03 \x01(\t\x12:\n\x0cversion_info\x18\x04 \x01(\x0b\x32$.temporal.api.version.v1.VersionInfo\x12\x14\n\x0c\x63luster_name\x18\x05 \x01(\t\x12\x1b\n\x13history_shard_count\x18\x06 \x01(\x05\x12\x19\n\x11persistence_store\x18\x07 \x01(\t\x12\x18\n\x10visibility_store\x18\x08 \x01(\t\x12 \n\x18initial_failover_version\x18\t \x01(\x03\x12"\n\x1a\x66\x61ilover_version_increment\x18\n \x01(\x03\x1a\x37\n\x15SupportedClientsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"\x16\n\x14GetSystemInfoRequest"\xc1\x04\n\x15GetSystemInfoResponse\x12\x16\n\x0eserver_version\x18\x01 \x01(\t\x12Y\n\x0c\x63\x61pabilities\x18\x02 \x01(\x0b\x32\x43.temporal.api.workflowservice.v1.GetSystemInfoResponse.Capabilities\x1a\xb4\x03\n\x0c\x43\x61pabilities\x12\x1f\n\x17signal_and_query_header\x18\x01 \x01(\x08\x12&\n\x1einternal_error_differentiation\x18\x02 \x01(\x08\x12*\n"activity_failure_include_heartbeat\x18\x03 \x01(\x08\x12\x1a\n\x12supports_schedules\x18\x04 \x01(\x08\x12"\n\x1a\x65ncoded_failure_attributes\x18\x05 \x01(\x08\x12!\n\x19\x62uild_id_based_versioning\x18\x06 \x01(\x08\x12\x13\n\x0bupsert_memo\x18\x07 \x01(\x08\x12\x1c\n\x14\x65\x61ger_workflow_start\x18\x08 \x01(\x08\x12\x14\n\x0csdk_metadata\x18\t \x01(\x08\x12\'\n\x1f\x63ount_group_by_execution_status\x18\n \x01(\x08\x12\r\n\x05nexus\x18\x0b \x01(\x08\x12!\n\x19server_scaled_deployments\x18\x0c \x01(\x08\x12(\n server_scaled_provider_cloud_run\x18\r \x01(\x08"m\n\x1eListTaskQueuePartitionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x02 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue"\xdf\x01\n\x1fListTaskQueuePartitionsResponse\x12]\n\x1e\x61\x63tivity_task_queue_partitions\x18\x01 \x03(\x0b\x32\x35.temporal.api.taskqueue.v1.TaskQueuePartitionMetadata\x12]\n\x1eworkflow_task_queue_partitions\x18\x02 \x03(\x0b\x32\x35.temporal.api.taskqueue.v1.TaskQueuePartitionMetadata"\xcc\x02\n\x15\x43reateScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12\x34\n\x08schedule\x18\x03 \x01(\x0b\x32".temporal.api.schedule.v1.Schedule\x12>\n\rinitial_patch\x18\x04 \x01(\x0b\x32\'.temporal.api.schedule.v1.SchedulePatch\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12*\n\x04memo\x18\x07 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"0\n\x16\x43reateScheduleResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c"A\n\x17\x44\x65scribeScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t"\x8f\x02\n\x18\x44\x65scribeScheduleResponse\x12\x34\n\x08schedule\x18\x01 \x01(\x0b\x32".temporal.api.schedule.v1.Schedule\x12\x34\n\x04info\x18\x02 \x01(\x0b\x32&.temporal.api.schedule.v1.ScheduleInfo\x12*\n\x04memo\x18\x03 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x04 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x16\n\x0e\x63onflict_token\x18\x05 \x01(\x0c"\xa4\x02\n\x15UpdateScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12\x34\n\x08schedule\x18\x03 \x01(\x0b\x32".temporal.api.schedule.v1.Schedule\x12\x16\n\x0e\x63onflict_token\x18\x04 \x01(\x0c\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x43\n\x11search_attributes\x18\x07 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12*\n\x04memo\x18\x08 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\x18\n\x16UpdateScheduleResponse"\x9c\x01\n\x14PatchScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12\x36\n\x05patch\x18\x03 \x01(\x0b\x32\'.temporal.api.schedule.v1.SchedulePatch\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t"\x17\n\x15PatchScheduleResponse"\xa8\x01\n ListScheduleMatchingTimesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12.\n\nstart_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"S\n!ListScheduleMatchingTimesResponse\x12.\n\nstart_time\x18\x01 \x03(\x0b\x32\x1a.google.protobuf.Timestamp"Q\n\x15\x44\x65leteScheduleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bschedule_id\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t"\x18\n\x16\x44\x65leteScheduleResponse"l\n\x14ListSchedulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x19\n\x11maximum_page_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"p\n\x15ListSchedulesResponse\x12>\n\tschedules\x18\x01 \x03(\x0b\x32+.temporal.api.schedule.v1.ScheduleListEntry\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"9\n\x15\x43ountSchedulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t"\xdb\x01\n\x16\x43ountSchedulesResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12X\n\x06groups\x18\x02 \x03(\x0b\x32H.temporal.api.workflowservice.v1.CountSchedulesResponse.AggregationGroup\x1aX\n\x10\x41ggregationGroup\x12\x35\n\x0cgroup_values\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\r\n\x05\x63ount\x18\x02 \x01(\x03"\x86\x05\n\'UpdateWorkerBuildIdCompatibilityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12-\n#add_new_build_id_in_new_default_set\x18\x03 \x01(\tH\x00\x12\x87\x01\n\x1b\x61\x64\x64_new_compatible_build_id\x18\x04 \x01(\x0b\x32`.temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest.AddNewCompatibleVersionH\x00\x12!\n\x17promote_set_by_build_id\x18\x05 \x01(\tH\x00\x12%\n\x1bpromote_build_id_within_set\x18\x06 \x01(\tH\x00\x12h\n\nmerge_sets\x18\x07 \x01(\x0b\x32R.temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest.MergeSetsH\x00\x1ao\n\x17\x41\x64\x64NewCompatibleVersion\x12\x14\n\x0cnew_build_id\x18\x01 \x01(\t\x12$\n\x1c\x65xisting_compatible_build_id\x18\x02 \x01(\t\x12\x18\n\x10make_set_default\x18\x03 \x01(\x08\x1aI\n\tMergeSets\x12\x1c\n\x14primary_set_build_id\x18\x01 \x01(\t\x12\x1e\n\x16secondary_set_build_id\x18\x02 \x01(\tB\x0b\n\toperation"@\n(UpdateWorkerBuildIdCompatibilityResponseJ\x04\x08\x01\x10\x02R\x0eversion_set_id"_\n$GetWorkerBuildIdCompatibilityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x10\n\x08max_sets\x18\x03 \x01(\x05"t\n%GetWorkerBuildIdCompatibilityResponse\x12K\n\x12major_version_sets\x18\x01 \x03(\x0b\x32/.temporal.api.taskqueue.v1.CompatibleVersionSet"\xb5\r\n"UpdateWorkerVersioningRulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x16\n\x0e\x63onflict_token\x18\x03 \x01(\x0c\x12\x81\x01\n\x16insert_assignment_rule\x18\x04 \x01(\x0b\x32_.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.InsertBuildIdAssignmentRuleH\x00\x12\x83\x01\n\x17replace_assignment_rule\x18\x05 \x01(\x0b\x32`.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.ReplaceBuildIdAssignmentRuleH\x00\x12\x81\x01\n\x16\x64\x65lete_assignment_rule\x18\x06 \x01(\x0b\x32_.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.DeleteBuildIdAssignmentRuleH\x00\x12\x8c\x01\n\x1c\x61\x64\x64_compatible_redirect_rule\x18\x07 \x01(\x0b\x32\x64.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.AddCompatibleBuildIdRedirectRuleH\x00\x12\x94\x01\n replace_compatible_redirect_rule\x18\x08 \x01(\x0b\x32h.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.ReplaceCompatibleBuildIdRedirectRuleH\x00\x12\x92\x01\n\x1f\x64\x65lete_compatible_redirect_rule\x18\t \x01(\x0b\x32g.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.DeleteCompatibleBuildIdRedirectRuleH\x00\x12l\n\x0f\x63ommit_build_id\x18\n \x01(\x0b\x32Q.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest.CommitBuildIdH\x00\x1aq\n\x1bInsertBuildIdAssignmentRule\x12\x12\n\nrule_index\x18\x01 \x01(\x05\x12>\n\x04rule\x18\x02 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.BuildIdAssignmentRule\x1a\x81\x01\n\x1cReplaceBuildIdAssignmentRule\x12\x12\n\nrule_index\x18\x01 \x01(\x05\x12>\n\x04rule\x18\x02 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.BuildIdAssignmentRule\x12\r\n\x05\x66orce\x18\x03 \x01(\x08\x1a@\n\x1b\x44\x65leteBuildIdAssignmentRule\x12\x12\n\nrule_index\x18\x01 \x01(\x05\x12\r\n\x05\x66orce\x18\x02 \x01(\x08\x1aj\n AddCompatibleBuildIdRedirectRule\x12\x46\n\x04rule\x18\x01 \x01(\x0b\x32\x38.temporal.api.taskqueue.v1.CompatibleBuildIdRedirectRule\x1an\n$ReplaceCompatibleBuildIdRedirectRule\x12\x46\n\x04rule\x18\x01 \x01(\x0b\x32\x38.temporal.api.taskqueue.v1.CompatibleBuildIdRedirectRule\x1a>\n#DeleteCompatibleBuildIdRedirectRule\x12\x17\n\x0fsource_build_id\x18\x01 \x01(\t\x1a\x37\n\rCommitBuildId\x12\x17\n\x0ftarget_build_id\x18\x01 \x01(\t\x12\r\n\x05\x66orce\x18\x02 \x01(\x08\x42\x0b\n\toperation"\xfc\x01\n#UpdateWorkerVersioningRulesResponse\x12U\n\x10\x61ssignment_rules\x18\x01 \x03(\x0b\x32;.temporal.api.taskqueue.v1.TimestampedBuildIdAssignmentRule\x12\x66\n\x19\x63ompatible_redirect_rules\x18\x02 \x03(\x0b\x32\x43.temporal.api.taskqueue.v1.TimestampedCompatibleBuildIdRedirectRule\x12\x16\n\x0e\x63onflict_token\x18\x03 \x01(\x0c"H\n\x1fGetWorkerVersioningRulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t"\xf9\x01\n GetWorkerVersioningRulesResponse\x12U\n\x10\x61ssignment_rules\x18\x01 \x03(\x0b\x32;.temporal.api.taskqueue.v1.TimestampedBuildIdAssignmentRule\x12\x66\n\x19\x63ompatible_redirect_rules\x18\x02 \x03(\x0b\x32\x43.temporal.api.taskqueue.v1.TimestampedCompatibleBuildIdRedirectRule\x12\x16\n\x0e\x63onflict_token\x18\x03 \x01(\x0c"\x9c\x01\n GetWorkerTaskReachabilityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tbuild_ids\x18\x02 \x03(\t\x12\x13\n\x0btask_queues\x18\x03 \x03(\t\x12=\n\x0creachability\x18\x04 \x01(\x0e\x32\'.temporal.api.enums.v1.TaskReachability"r\n!GetWorkerTaskReachabilityResponse\x12M\n\x15\x62uild_id_reachability\x18\x01 \x03(\x0b\x32..temporal.api.taskqueue.v1.BuildIdReachability"\x85\x02\n\x1eUpdateWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x1e\n\x16\x66irst_execution_run_id\x18\x03 \x01(\t\x12\x37\n\x0bwait_policy\x18\x04 \x01(\x0b\x32".temporal.api.update.v1.WaitPolicy\x12\x30\n\x07request\x18\x05 \x01(\x0b\x32\x1f.temporal.api.update.v1.Request"\x83\x02\n\x1fUpdateWorkflowExecutionResponse\x12\x35\n\nupdate_ref\x18\x01 \x01(\x0b\x32!.temporal.api.update.v1.UpdateRef\x12\x30\n\x07outcome\x18\x02 \x01(\x0b\x32\x1f.temporal.api.update.v1.Outcome\x12K\n\x05stage\x18\x03 \x01(\x0e\x32<.temporal.api.enums.v1.UpdateWorkflowExecutionLifecycleStage\x12*\n\x04link\x18\x04 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xd6\n\n\x1aStartBatchOperationRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x18\n\x10visibility_query\x18\x02 \x01(\t\x12\x0e\n\x06job_id\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x41\n\nexecutions\x18\x05 \x03(\x0b\x32).temporal.api.common.v1.WorkflowExecutionB\x02\x18\x01\x12<\n\x11target_executions\x18\x16 \x03(\x0b\x32!.temporal.api.common.v1.Execution\x12!\n\x19max_operations_per_second\x18\x06 \x01(\x02\x12Q\n\x15termination_operation\x18\n \x01(\x0b\x32\x30.temporal.api.batch.v1.BatchOperationTerminationH\x00\x12G\n\x10signal_operation\x18\x0b \x01(\x0b\x32+.temporal.api.batch.v1.BatchOperationSignalH\x00\x12S\n\x16\x63\x61ncellation_operation\x18\x0c \x01(\x0b\x32\x31.temporal.api.batch.v1.BatchOperationCancellationH\x00\x12K\n\x12\x64\x65letion_operation\x18\r \x01(\x0b\x32-.temporal.api.batch.v1.BatchOperationDeletionH\x00\x12\x45\n\x0freset_operation\x18\x0e \x01(\x0b\x32*.temporal.api.batch.v1.BatchOperationResetH\x00\x12p\n!update_workflow_options_operation\x18\x0f \x01(\x0b\x32\x43.temporal.api.batch.v1.BatchOperationUpdateWorkflowExecutionOptionsH\x00\x12^\n\x1cunpause_activities_operation\x18\x10 \x01(\x0b\x32\x36.temporal.api.batch.v1.BatchOperationUnpauseActivitiesH\x00\x12Z\n\x1areset_activities_operation\x18\x11 \x01(\x0b\x32\x34.temporal.api.batch.v1.BatchOperationResetActivitiesH\x00\x12g\n!update_activity_options_operation\x18\x12 \x01(\x0b\x32:.temporal.api.batch.v1.BatchOperationUpdateActivityOptionsH\x00\x12\\\n\x1b\x63\x61ncel_activities_operation\x18\x13 \x01(\x0b\x32\x35.temporal.api.batch.v1.BatchOperationCancelActivitiesH\x00\x12\x62\n\x1eterminate_activities_operation\x18\x14 \x01(\x0b\x32\x38.temporal.api.batch.v1.BatchOperationTerminateActivitiesH\x00\x12\\\n\x1b\x64\x65lete_activities_operation\x18\x15 \x01(\x0b\x32\x35.temporal.api.batch.v1.BatchOperationDeleteActivitiesH\x00\x42\x0b\n\toperation"\x1d\n\x1bStartBatchOperationResponse"`\n\x19StopBatchOperationRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t"\x1c\n\x1aStopBatchOperationResponse"B\n\x1d\x44\x65scribeBatchOperationRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t"\xd8\x03\n\x1e\x44\x65scribeBatchOperationResponse\x12\x41\n\x0eoperation_type\x18\x01 \x01(\x0e\x32).temporal.api.enums.v1.BatchOperationType\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12\x39\n\x05state\x18\x03 \x01(\x0e\x32*.temporal.api.enums.v1.BatchOperationState\x12.\n\nstart_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nclose_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x15total_operation_count\x18\x06 \x01(\x03\x12 \n\x18\x63omplete_operation_count\x18\x07 \x01(\x03\x12\x1f\n\x17\x66\x61ilure_operation_count\x18\x08 \x01(\x03\x12\x10\n\x08identity\x18\t \x01(\t\x12\x0e\n\x06reason\x18\n \x01(\t\x12\r\n\x05query\x18\x0b \x01(\t\x12\x35\n\nexecutions\x18\x0c \x03(\x0b\x32!.temporal.api.common.v1.Execution"[\n\x1aListBatchOperationsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c"y\n\x1bListBatchOperationsResponse\x12\x41\n\x0eoperation_info\x18\x01 \x03(\x0b\x32).temporal.api.batch.v1.BatchOperationInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\xb9\x01\n"PollWorkflowExecutionUpdateRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x35\n\nupdate_ref\x18\x02 \x01(\x0b\x32!.temporal.api.update.v1.UpdateRef\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x37\n\x0bwait_policy\x18\x04 \x01(\x0b\x32".temporal.api.update.v1.WaitPolicy"\xdb\x01\n#PollWorkflowExecutionUpdateResponse\x12\x30\n\x07outcome\x18\x01 \x01(\x0b\x32\x1f.temporal.api.update.v1.Outcome\x12K\n\x05stage\x18\x02 \x01(\x0e\x32<.temporal.api.enums.v1.UpdateWorkflowExecutionLifecycleStage\x12\x35\n\nupdate_ref\x18\x03 \x01(\x0b\x32!.temporal.api.update.v1.UpdateRef"\xa0\x03\n\x19PollNexusTaskQueueRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\ntask_queue\x18\x03 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12\x17\n\x0fpoller_group_id\x18\t \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x1b\n\x13worker_instance_key\x18\x08 \x01(\t\x12Z\n\x1bworker_version_capabilities\x18\x04 \x01(\x0b\x32\x31.temporal.api.common.v1.WorkerVersionCapabilitiesB\x02\x18\x01\x12O\n\x12\x64\x65ployment_options\x18\x06 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentOptions\x12\x41\n\x10worker_heartbeat\x18\x07 \x03(\x0b\x32\'.temporal.api.worker.v1.WorkerHeartbeat"\xe2\x02\n\x1aPollNexusTaskQueueResponse\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12/\n\x07request\x18\x02 \x01(\x0b\x32\x1e.temporal.api.nexus.v1.Request\x12Q\n\x17poller_scaling_decision\x18\x03 \x01(\x0b\x32\x30.temporal.api.taskqueue.v1.PollerScalingDecision\x12\x17\n\x0fpoller_group_id\x18\x04 \x01(\t\x12J\n\x12poller_group_infos\x18\x05 \x03(\x0b\x32*.temporal.api.taskqueue.v1.PollerGroupInfoB\x02\x18\x01\x12G\n\x12poller_groups_info\x18\x06 \x01(\x0b\x32+.temporal.api.taskqueue.v1.PollerGroupsInfo"\xa7\x01\n RespondNexusTaskCompletedRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\ntask_token\x18\x03 \x01(\x0c\x12\x31\n\x08response\x18\x04 \x01(\x0b\x32\x1f.temporal.api.nexus.v1.Response\x12\x17\n\x0fpoller_group_id\x18\x05 \x01(\t"#\n!RespondNexusTaskCompletedResponse"\xdc\x01\n\x1dRespondNexusTaskFailedRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\ntask_token\x18\x03 \x01(\x0c\x12\x36\n\x05\x65rror\x18\x04 \x01(\x0b\x32#.temporal.api.nexus.v1.HandlerErrorB\x02\x18\x01\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x17\n\x0fpoller_group_id\x18\x06 \x01(\t" \n\x1eRespondNexusTaskFailedResponse"\xf4\x02\n\x1c\x45xecuteMultiOperationRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12[\n\noperations\x18\x02 \x03(\x0b\x32G.temporal.api.workflowservice.v1.ExecuteMultiOperationRequest.Operation\x12\x13\n\x0bresource_id\x18\x03 \x01(\t\x1a\xce\x01\n\tOperation\x12X\n\x0estart_workflow\x18\x01 \x01(\x0b\x32>.temporal.api.workflowservice.v1.StartWorkflowExecutionRequestH\x00\x12Z\n\x0fupdate_workflow\x18\x02 \x01(\x0b\x32?.temporal.api.workflowservice.v1.UpdateWorkflowExecutionRequestH\x00\x42\x0b\n\toperation"\xcc\x02\n\x1d\x45xecuteMultiOperationResponse\x12Z\n\tresponses\x18\x01 \x03(\x0b\x32G.temporal.api.workflowservice.v1.ExecuteMultiOperationResponse.Response\x1a\xce\x01\n\x08Response\x12Y\n\x0estart_workflow\x18\x01 \x01(\x0b\x32?.temporal.api.workflowservice.v1.StartWorkflowExecutionResponseH\x00\x12[\n\x0fupdate_workflow\x18\x02 \x01(\x0b\x32@.temporal.api.workflowservice.v1.UpdateWorkflowExecutionResponseH\x00\x42\n\n\x08response"\xd0\x02\n\x1cUpdateActivityOptionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x43\n\x10\x61\x63tivity_options\x18\x04 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions\x12/\n\x0bupdate_mask\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.FieldMask\x12\x0c\n\x02id\x18\x06 \x01(\tH\x00\x12\x0e\n\x04type\x18\x07 \x01(\tH\x00\x12\x13\n\tmatch_all\x18\t \x01(\x08H\x00\x12\x18\n\x10restore_original\x18\x08 \x01(\x08\x42\n\n\x08\x61\x63tivity"\xbf\x02\n%UpdateActivityExecutionOptionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x43\n\x10\x61\x63tivity_options\x18\x06 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions\x12/\n\x0bupdate_mask\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.FieldMask\x12\x18\n\x10restore_original\x18\x08 \x01(\x08\x12\x13\n\x0bresource_id\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t"d\n\x1dUpdateActivityOptionsResponse\x12\x43\n\x10\x61\x63tivity_options\x18\x01 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions"m\n&UpdateActivityExecutionOptionsResponse\x12\x43\n\x10\x61\x63tivity_options\x18\x01 \x01(\x0b\x32).temporal.api.activity.v1.ActivityOptions"\xc7\x01\n\x14PauseActivityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0c\n\x02id\x18\x04 \x01(\tH\x00\x12\x0e\n\x04type\x18\x05 \x01(\tH\x00\x12\x0e\n\x06reason\x18\x06 \x01(\t\x12\x12\n\nrequest_id\x18\x07 \x01(\tB\n\n\x08\x61\x63tivity"\xb7\x01\n\x1dPauseActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x07 \x01(\t\x12\x12\n\nrequest_id\x18\x08 \x01(\t"\x17\n\x15PauseActivityResponse" \n\x1ePauseActivityExecutionResponse"\x98\x02\n\x16UnpauseActivityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0c\n\x02id\x18\x04 \x01(\tH\x00\x12\x0e\n\x04type\x18\x05 \x01(\tH\x00\x12\x15\n\x0bunpause_all\x18\x06 \x01(\x08H\x00\x12\x16\n\x0ereset_attempts\x18\x07 \x01(\x08\x12\x17\n\x0freset_heartbeat\x18\x08 \x01(\x08\x12)\n\x06jitter\x18\t \x01(\x0b\x32\x19.google.protobuf.DurationB\n\n\x08\x61\x63tivity"\x91\x02\n\x1fUnpauseActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x08 \x01(\t\x12)\n\x06jitter\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bresource_id\x18\n \x01(\t\x12\x12\n\nrequest_id\x18\x0b \x01(\tJ\x04\x08\x06\x10\x07J\x04\x08\x07\x10\x08R\x0ereset_attemptsR\x0freset_heartbeat"\x19\n\x17UnpauseActivityResponse""\n UnpauseActivityExecutionResponse"\xb3\x02\n\x14ResetActivityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0c\n\x02id\x18\x04 \x01(\tH\x00\x12\x0e\n\x04type\x18\x05 \x01(\tH\x00\x12\x13\n\tmatch_all\x18\n \x01(\x08H\x00\x12\x17\n\x0freset_heartbeat\x18\x06 \x01(\x08\x12\x13\n\x0bkeep_paused\x18\x07 \x01(\x08\x12)\n\x06jitter\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12 \n\x18restore_original_options\x18\t \x01(\x08\x42\n\n\x08\x61\x63tivity"\xa2\x02\n\x1dResetActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x13\n\x0bkeep_paused\x18\x06 \x01(\x08\x12)\n\x06jitter\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12 \n\x18restore_original_options\x18\x08 \x01(\x08\x12\x13\n\x0bresource_id\x18\t \x01(\t\x12\x12\n\nrequest_id\x18\n \x01(\t\x12\x17\n\x0freset_heartbeat\x18\x0b \x01(\x08"\x17\n\x15ResetActivityResponse" \n\x1eResetActivityExecutionResponse"\x9c\x02\n%UpdateWorkflowExecutionOptionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12V\n\x1aworkflow_execution_options\x18\x03 \x01(\x0b\x32\x32.temporal.api.workflow.v1.WorkflowExecutionOptions\x12/\n\x0bupdate_mask\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.FieldMask\x12\x10\n\x08identity\x18\x05 \x01(\t"\xb1\x01\n&UpdateWorkflowExecutionOptionsResponse\x12V\n\x1aworkflow_execution_options\x18\x01 \x01(\x0b\x32\x32.temporal.api.workflow.v1.WorkflowExecutionOptions\x12/\n\x0bupdate_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"j\n\x19\x44\x65scribeDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12:\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.Deployment"a\n\x1a\x44\x65scribeDeploymentResponse\x12\x43\n\x0f\x64\x65ployment_info\x18\x01 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo"\xc2\x01\n&DescribeWorkerDeploymentVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x07version\x18\x02 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x03 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x1f\n\x17report_task_queue_stats\x18\x04 \x01(\x08"\x8c\x05\n\'DescribeWorkerDeploymentVersionResponse\x12_\n\x1eworker_deployment_version_info\x18\x01 \x01(\x0b\x32\x37.temporal.api.deployment.v1.WorkerDeploymentVersionInfo\x12v\n\x13version_task_queues\x18\x02 \x03(\x0b\x32Y.temporal.api.workflowservice.v1.DescribeWorkerDeploymentVersionResponse.VersionTaskQueue\x1a\x87\x03\n\x10VersionTaskQueue\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32$.temporal.api.enums.v1.TaskQueueType\x12\x38\n\x05stats\x18\x03 \x01(\x0b\x32).temporal.api.taskqueue.v1.TaskQueueStats\x12\x90\x01\n\x15stats_by_priority_key\x18\x04 \x03(\x0b\x32q.temporal.api.workflowservice.v1.DescribeWorkerDeploymentVersionResponse.VersionTaskQueue.StatsByPriorityKeyEntry\x1a\x64\n\x17StatsByPriorityKeyEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32).temporal.api.taskqueue.v1.TaskQueueStats:\x02\x38\x01"M\n\x1f\x44\x65scribeWorkerDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t"\x8c\x01\n DescribeWorkerDeploymentResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c\x12P\n\x16worker_deployment_info\x18\x02 \x01(\x0b\x32\x30.temporal.api.deployment.v1.WorkerDeploymentInfo"l\n\x16ListDeploymentsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\x13\n\x0bseries_name\x18\x04 \x01(\t"w\n\x17ListDeploymentsResponse\x12\x17\n\x0fnext_page_token\x18\x01 \x01(\x0c\x12\x43\n\x0b\x64\x65ployments\x18\x02 \x03(\x0b\x32..temporal.api.deployment.v1.DeploymentListInfo"\xcd\x01\n\x1bSetCurrentDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12:\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.Deployment\x12\x10\n\x08identity\x18\x03 \x01(\t\x12M\n\x0fupdate_metadata\x18\x04 \x01(\x0b\x32\x34.temporal.api.deployment.v1.UpdateDeploymentMetadata"\xb9\x01\n\x1cSetCurrentDeploymentResponse\x12K\n\x17\x63urrent_deployment_info\x18\x01 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo\x12L\n\x18previous_deployment_info\x18\x02 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo"\xe5\x01\n(SetWorkerDeploymentCurrentVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x13\n\x07version\x18\x03 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x07 \x01(\t\x12\x16\n\x0e\x63onflict_token\x18\x04 \x01(\x0c\x12\x10\n\x08identity\x18\x05 \x01(\t\x12"\n\x1aignore_missing_task_queues\x18\x06 \x01(\x08\x12\x18\n\x10\x61llow_no_pollers\x18\t \x01(\x08"\xbf\x01\n)SetWorkerDeploymentCurrentVersionResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c\x12\x1c\n\x10previous_version\x18\x02 \x01(\tB\x02\x18\x01\x12\\\n\x1bprevious_deployment_version\x18\x03 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersionB\x02\x18\x01"\xf9\x01\n(SetWorkerDeploymentRampingVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x13\n\x07version\x18\x03 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x08 \x01(\t\x12\x12\n\npercentage\x18\x04 \x01(\x02\x12\x16\n\x0e\x63onflict_token\x18\x05 \x01(\x0c\x12\x10\n\x08identity\x18\x06 \x01(\t\x12"\n\x1aignore_missing_task_queues\x18\x07 \x01(\x08\x12\x18\n\x10\x61llow_no_pollers\x18\n \x01(\x08"\xe0\x01\n)SetWorkerDeploymentRampingVersionResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c\x12\x1c\n\x10previous_version\x18\x02 \x01(\tB\x02\x18\x01\x12\\\n\x1bprevious_deployment_version\x18\x04 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersionB\x02\x18\x01\x12\x1f\n\x13previous_percentage\x18\x03 \x01(\x02\x42\x02\x18\x01"q\n\x1d\x43reateWorkerDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t"8\n\x1e\x43reateWorkerDeploymentResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c"]\n\x1cListWorkerDeploymentsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c"\x9f\x05\n\x1dListWorkerDeploymentsResponse\x12\x17\n\x0fnext_page_token\x18\x01 \x01(\x0c\x12r\n\x12worker_deployments\x18\x02 \x03(\x0b\x32V.temporal.api.workflowservice.v1.ListWorkerDeploymentsResponse.WorkerDeploymentSummary\x1a\xf0\x03\n\x17WorkerDeploymentSummary\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\x0b\x63reate_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x41\n\x0erouting_config\x18\x03 \x01(\x0b\x32).temporal.api.deployment.v1.RoutingConfig\x12o\n\x16latest_version_summary\x18\x04 \x01(\x0b\x32O.temporal.api.deployment.v1.WorkerDeploymentInfo.WorkerDeploymentVersionSummary\x12p\n\x17\x63urrent_version_summary\x18\x05 \x01(\x0b\x32O.temporal.api.deployment.v1.WorkerDeploymentInfo.WorkerDeploymentVersionSummary\x12p\n\x17ramping_version_summary\x18\x06 \x01(\x0b\x32O.temporal.api.deployment.v1.WorkerDeploymentInfo.WorkerDeploymentVersionSummary"\xf0\x01\n$CreateWorkerDeploymentVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12O\n\x12\x64\x65ployment_version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12>\n\x0e\x63ompute_config\x18\x04 \x01(\x0b\x32&.temporal.api.compute.v1.ComputeConfig\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t"\'\n%CreateWorkerDeploymentVersionResponse"\xc8\x01\n$DeleteWorkerDeploymentVersionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x07version\x18\x02 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x05 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x15\n\rskip_drainage\x18\x03 \x01(\x08\x12\x10\n\x08identity\x18\x04 \x01(\t"\'\n%DeleteWorkerDeploymentVersionResponse"]\n\x1d\x44\x65leteWorkerDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t" \n\x1e\x44\x65leteWorkerDeploymentResponse"\x84\x04\n1UpdateWorkerDeploymentVersionComputeConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12O\n\x12\x64\x65ployment_version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x99\x01\n\x1d\x63ompute_config_scaling_groups\x18\x06 \x03(\x0b\x32r.temporal.api.workflowservice.v1.UpdateWorkerDeploymentVersionComputeConfigRequest.ComputeConfigScalingGroupsEntry\x12,\n$remove_compute_config_scaling_groups\x18\x07 \x03(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x1a{\n\x1f\x43omputeConfigScalingGroupsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12G\n\x05value\x18\x02 \x01(\x0b\x32\x38.temporal.api.compute.v1.ComputeConfigScalingGroupUpdate:\x02\x38\x01"4\n2UpdateWorkerDeploymentVersionComputeConfigResponse"\xf4\x03\n3ValidateWorkerDeploymentVersionComputeConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12O\n\x12\x64\x65ployment_version\x18\x02 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12\x9b\x01\n\x1d\x63ompute_config_scaling_groups\x18\x06 \x03(\x0b\x32t.temporal.api.workflowservice.v1.ValidateWorkerDeploymentVersionComputeConfigRequest.ComputeConfigScalingGroupsEntry\x12,\n$remove_compute_config_scaling_groups\x18\x07 \x03(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x1a{\n\x1f\x43omputeConfigScalingGroupsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12G\n\x05value\x18\x02 \x01(\x0b\x32\x38.temporal.api.compute.v1.ComputeConfigScalingGroupUpdate:\x02\x38\x01"6\n4ValidateWorkerDeploymentVersionComputeConfigResponse"\xa2\x03\n,UpdateWorkerDeploymentVersionMetadataRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x07version\x18\x02 \x01(\tB\x02\x18\x01\x12O\n\x12\x64\x65ployment_version\x18\x05 \x01(\x0b\x32\x33.temporal.api.deployment.v1.WorkerDeploymentVersion\x12x\n\x0eupsert_entries\x18\x03 \x03(\x0b\x32`.temporal.api.workflowservice.v1.UpdateWorkerDeploymentVersionMetadataRequest.UpsertEntriesEntry\x12\x16\n\x0eremove_entries\x18\x04 \x03(\t\x12\x10\n\x08identity\x18\x06 \x01(\t\x1aU\n\x12UpsertEntriesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"n\n-UpdateWorkerDeploymentVersionMetadataResponse\x12=\n\x08metadata\x18\x01 \x01(\x0b\x32+.temporal.api.deployment.v1.VersionMetadata"\xbd\x01\n!SetWorkerDeploymentManagerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0f\x64\x65ployment_name\x18\x02 \x01(\t\x12\x1a\n\x10manager_identity\x18\x03 \x01(\tH\x00\x12\x0e\n\x04self\x18\x04 \x01(\x08H\x00\x12\x16\n\x0e\x63onflict_token\x18\x05 \x01(\x0c\x12\x10\n\x08identity\x18\x06 \x01(\tB\x16\n\x14new_manager_identity"c\n"SetWorkerDeploymentManagerResponse\x12\x16\n\x0e\x63onflict_token\x18\x01 \x01(\x0c\x12%\n\x19previous_manager_identity\x18\x02 \x01(\tB\x02\x18\x01"E\n\x1bGetCurrentDeploymentRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bseries_name\x18\x02 \x01(\t"k\n\x1cGetCurrentDeploymentResponse\x12K\n\x17\x63urrent_deployment_info\x18\x01 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo"q\n GetDeploymentReachabilityRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12:\n\ndeployment\x18\x02 \x01(\x0b\x32&.temporal.api.deployment.v1.Deployment"\xe3\x01\n!GetDeploymentReachabilityResponse\x12\x43\n\x0f\x64\x65ployment_info\x18\x01 \x01(\x0b\x32*.temporal.api.deployment.v1.DeploymentInfo\x12\x43\n\x0creachability\x18\x02 \x01(\x0e\x32-.temporal.api.enums.v1.DeploymentReachability\x12\x34\n\x10last_update_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\xb4\x01\n\x19\x43reateWorkflowRuleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x35\n\x04spec\x18\x02 \x01(\x0b\x32\'.temporal.api.rules.v1.WorkflowRuleSpec\x12\x12\n\nforce_scan\x18\x03 \x01(\x08\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x10\n\x08identity\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t"_\n\x1a\x43reateWorkflowRuleResponse\x12\x31\n\x04rule\x18\x01 \x01(\x0b\x32#.temporal.api.rules.v1.WorkflowRule\x12\x0e\n\x06job_id\x18\x02 \x01(\t"A\n\x1b\x44\x65scribeWorkflowRuleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0f\n\x07rule_id\x18\x02 \x01(\t"Q\n\x1c\x44\x65scribeWorkflowRuleResponse\x12\x31\n\x04rule\x18\x01 \x01(\x0b\x32#.temporal.api.rules.v1.WorkflowRule"?\n\x19\x44\x65leteWorkflowRuleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0f\n\x07rule_id\x18\x02 \x01(\t"\x1c\n\x1a\x44\x65leteWorkflowRuleResponse"F\n\x18ListWorkflowRulesRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"h\n\x19ListWorkflowRulesResponse\x12\x32\n\x05rules\x18\x01 \x03(\x0b\x32#.temporal.api.rules.v1.WorkflowRule\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\xce\x01\n\x1aTriggerWorkflowRuleRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12<\n\texecution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x0c\n\x02id\x18\x04 \x01(\tH\x00\x12\x37\n\x04spec\x18\x05 \x01(\x0b\x32\'.temporal.api.rules.v1.WorkflowRuleSpecH\x00\x12\x10\n\x08identity\x18\x03 \x01(\tB\x06\n\x04rule".\n\x1bTriggerWorkflowRuleResponse\x12\x0f\n\x07\x61pplied\x18\x01 \x01(\x08"\x9b\x01\n\x1cRecordWorkerHeartbeatRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x41\n\x10worker_heartbeat\x18\x03 \x03(\x0b\x32\'.temporal.api.worker.v1.WorkerHeartbeat\x12\x13\n\x0bresource_id\x18\x04 \x01(\t"\x1f\n\x1dRecordWorkerHeartbeatResponse"\x82\x01\n\x12ListWorkersRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t\x12\x1e\n\x16include_system_workers\x18\x05 \x01(\x08"\xa5\x01\n\x13ListWorkersResponse\x12<\n\x0cworkers_info\x18\x01 \x03(\x0b\x32".temporal.api.worker.v1.WorkerInfoB\x02\x18\x01\x12\x37\n\x07workers\x18\x03 \x03(\x0b\x32&.temporal.api.worker.v1.WorkerListInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\xd5\x05\n\x1cUpdateTaskQueueConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\ntask_queue\x18\x03 \x01(\t\x12=\n\x0ftask_queue_type\x18\x04 \x01(\x0e\x32$.temporal.api.enums.v1.TaskQueueType\x12n\n\x17update_queue_rate_limit\x18\x05 \x01(\x0b\x32M.temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest.RateLimitUpdate\x12}\n&update_fairness_key_rate_limit_default\x18\x06 \x01(\x0b\x32M.temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest.RateLimitUpdate\x12\x84\x01\n\x1dset_fairness_weight_overrides\x18\x07 \x03(\x0b\x32].temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest.SetFairnessWeightOverridesEntry\x12\'\n\x1funset_fairness_weight_overrides\x18\x08 \x03(\t\x1a[\n\x0fRateLimitUpdate\x12\x38\n\nrate_limit\x18\x01 \x01(\x0b\x32$.temporal.api.taskqueue.v1.RateLimit\x12\x0e\n\x06reason\x18\x02 \x01(\t\x1a\x41\n\x1fSetFairnessWeightOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01"[\n\x1dUpdateTaskQueueConfigResponse\x12:\n\x06\x63onfig\x18\x01 \x01(\x0b\x32*.temporal.api.taskqueue.v1.TaskQueueConfig"\x9e\x01\n\x18\x46\x65tchWorkerConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x38\n\x08selector\x18\x06 \x01(\x0b\x32&.temporal.api.common.v1.WorkerSelector\x12\x13\n\x0bresource_id\x18\x07 \x01(\t"U\n\x19\x46\x65tchWorkerConfigResponse\x12\x38\n\rworker_config\x18\x01 \x01(\x0b\x32!.temporal.api.sdk.v1.WorkerConfig"\x8a\x02\n\x19UpdateWorkerConfigRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x38\n\rworker_config\x18\x04 \x01(\x0b\x32!.temporal.api.sdk.v1.WorkerConfig\x12/\n\x0bupdate_mask\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.FieldMask\x12\x38\n\x08selector\x18\x06 \x01(\x0b\x32&.temporal.api.common.v1.WorkerSelector\x12\x13\n\x0bresource_id\x18\x07 \x01(\t"d\n\x1aUpdateWorkerConfigResponse\x12:\n\rworker_config\x18\x01 \x01(\x0b\x32!.temporal.api.sdk.v1.WorkerConfigH\x00\x42\n\n\x08response"G\n\x15\x44\x65scribeWorkerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x1b\n\x13worker_instance_key\x18\x02 \x01(\t"Q\n\x16\x44\x65scribeWorkerResponse\x12\x37\n\x0bworker_info\x18\x01 \x01(\x0b\x32".temporal.api.worker.v1.WorkerInfo"W\n\x13\x43ountWorkersRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x1e\n\x16include_system_workers\x18\x03 \x01(\x08"%\n\x14\x43ountWorkersResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03"\x8d\x01\n\x1dPauseWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x0e\n\x06reason\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t" \n\x1ePauseWorkflowExecutionResponse"\x8f\x01\n\x1fUnpauseWorkflowExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x0e\n\x06reason\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t""\n UnpauseWorkflowExecutionResponse"\x99\t\n\x1dStartActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x04 \x01(\t\x12;\n\ractivity_type\x18\x05 \x01(\x0b\x32$.temporal.api.common.v1.ActivityType\x12\x38\n\ntask_queue\x18\x06 \x01(\x0b\x32$.temporal.api.taskqueue.v1.TaskQueue\x12<\n\x19schedule_to_close_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12/\n\x05input\x18\x0c \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x45\n\x0fid_reuse_policy\x18\r \x01(\x0e\x32,.temporal.api.enums.v1.ActivityIdReusePolicy\x12K\n\x12id_conflict_policy\x18\x0e \x01(\x0e\x32/.temporal.api.enums.v1.ActivityIdConflictPolicy\x12\x43\n\x11search_attributes\x18\x0f \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\x06header\x18\x10 \x01(\x0b\x32\x1e.temporal.api.common.v1.Header\x12\x38\n\ruser_metadata\x18\x11 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x32\n\x08priority\x18\x12 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12>\n\x14\x63ompletion_callbacks\x18\x13 \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12+\n\x05links\x18\x14 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x12\x46\n\x13on_conflict_options\x18\x15 \x01(\x0b\x32).temporal.api.common.v1.OnConflictOptions\x12.\n\x0bstart_delay\x18\x16 \x01(\x0b\x32\x19.google.protobuf.Duration"m\n\x1eStartActivityExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0f\n\x07started\x18\x02 \x01(\x08\x12*\n\x04link\x18\x03 \x01(\x0b\x32\x1c.temporal.api.common.v1.Link"\xe4\x01\n DescribeActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x15\n\rinclude_input\x18\x04 \x01(\x08\x12\x17\n\x0finclude_outcome\x18\x05 \x01(\x08\x12\x17\n\x0flong_poll_token\x18\x06 \x01(\x0c\x12!\n\x19include_heartbeat_details\x18\x07 \x01(\x08\x12\x1c\n\x14include_last_failure\x18\x08 \x01(\x08"\xbc\x02\n!DescribeActivityExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12=\n\x04info\x18\x02 \x01(\x0b\x32/.temporal.api.activity.v1.ActivityExecutionInfo\x12/\n\x05input\x18\x03 \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x43\n\x07outcome\x18\x04 \x01(\x0b\x32\x32.temporal.api.activity.v1.ActivityExecutionOutcome\x12\x17\n\x0flong_poll_token\x18\x05 \x01(\x0c\x12\x39\n\tcallbacks\x18\x06 \x03(\x0b\x32&.temporal.api.activity.v1.CallbackInfo"V\n\x1cPollActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"t\n\x1dPollActivityExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x43\n\x07outcome\x18\x02 \x01(\x0b\x32\x32.temporal.api.activity.v1.ActivityExecutionOutcome"m\n\x1dListActivityExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"\x82\x01\n\x1eListActivityExecutionsResponse\x12G\n\nexecutions\x18\x01 \x03(\x0b\x32\x33.temporal.api.activity.v1.ActivityExecutionListInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"\x8e\x08\n#StartNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x12\n\nrequest_id\x18\x03 \x01(\t\x12\x14\n\x0coperation_id\x18\x04 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x05 \x01(\t\x12\x0f\n\x07service\x18\x06 \x01(\t\x12\x11\n\toperation\x18\x07 \x01(\t\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12.\n\x05input\x18\x0b \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12K\n\x0fid_reuse_policy\x18\x0c \x01(\x0e\x32\x32.temporal.api.enums.v1.NexusOperationIdReusePolicy\x12Q\n\x12id_conflict_policy\x18\r \x01(\x0e\x32\x35.temporal.api.enums.v1.NexusOperationIdConflictPolicy\x12N\n\x13on_conflict_options\x18\x11 \x01(\x0b\x32\x31.temporal.api.nexusoperation.v1.OnConflictOptions\x12\x43\n\x11search_attributes\x18\x0e \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12k\n\x0cnexus_header\x18\x0f \x03(\x0b\x32U.temporal.api.workflowservice.v1.StartNexusOperationExecutionRequest.NexusHeaderEntry\x12\x38\n\ruser_metadata\x18\x10 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12>\n\x14\x63ompletion_callbacks\x18\x12 \x03(\x0b\x32 .temporal.api.common.v1.Callback\x12+\n\x05links\x18\x13 \x03(\x0b\x32\x1c.temporal.api.common.v1.Link\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"G\n$StartNexusOperationExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0f\n\x07started\x18\x02 \x01(\x08"\xaa\x01\n&DescribeNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x15\n\rinclude_input\x18\x04 \x01(\x08\x12\x17\n\x0finclude_outcome\x18\x05 \x01(\x08\x12\x17\n\x0flong_poll_token\x18\x06 \x01(\x0c"\x83\x03\n\'DescribeNexusOperationExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12@\n\x04info\x18\x02 \x01(\x0b\x32\x32.temporal.api.nexus.v1.NexusOperationExecutionInfo\x12.\n\x05input\x18\x03 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x31\n\x06result\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x12\x33\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x17\n\x0flong_poll_token\x18\x06 \x01(\x0c\x12J\n\x14\x63ompletion_callbacks\x18\x07 \x03(\x0b\x32,.temporal.api.nexusoperation.v1.CallbackInfoB\t\n\x07outcome"\xa1\x01\n"PollNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x42\n\nwait_stage\x18\x04 \x01(\x0e\x32..temporal.api.enums.v1.NexusOperationWaitStage"\x85\x02\n#PollNexusOperationExecutionResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x42\n\nwait_stage\x18\x02 \x01(\x0e\x32..temporal.api.enums.v1.NexusOperationWaitStage\x12\x17\n\x0foperation_token\x18\x03 \x01(\t\x12\x31\n\x06result\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x12\x33\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07outcome"s\n#ListNexusOperationExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x17\n\x0fnext_page_token\x18\x03 \x01(\x0c\x12\r\n\x05query\x18\x04 \x01(\t"\x8b\x01\n$ListNexusOperationExecutionsResponse\x12J\n\noperations\x18\x01 \x03(\x0b\x32\x36.temporal.api.nexus.v1.NexusOperationExecutionListInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\x0c"B\n\x1e\x43ountActivityExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t"\xed\x01\n\x1f\x43ountActivityExecutionsResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12\x61\n\x06groups\x18\x02 \x03(\x0b\x32Q.temporal.api.workflowservice.v1.CountActivityExecutionsResponse.AggregationGroup\x1aX\n\x10\x41ggregationGroup\x12\x35\n\x0cgroup_values\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\r\n\x05\x63ount\x18\x02 \x01(\x03"H\n$CountNexusOperationExecutionsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t"\xf9\x01\n%CountNexusOperationExecutionsResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12g\n\x06groups\x18\x02 \x03(\x0b\x32W.temporal.api.workflowservice.v1.CountNexusOperationExecutionsResponse.AggregationGroup\x1aX\n\x10\x41ggregationGroup\x12\x35\n\x0cgroup_values\x18\x01 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\r\n\x05\x63ount\x18\x02 \x01(\x03"\x95\x01\n%RequestCancelActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t"(\n&RequestCancelActivityExecutionResponse"\x91\x01\n!TerminateActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t"$\n"TerminateActivityExecutionResponse"X\n\x1e\x44\x65leteActivityExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"!\n\x1f\x44\x65leteActivityExecutionResponse"\x9c\x01\n+RequestCancelNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t".\n,RequestCancelNexusOperationExecutionResponse"\x98\x01\n\'TerminateNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\x10\n\x08identity\x18\x04 \x01(\t\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x0e\n\x06reason\x18\x06 \x01(\t"*\n(TerminateNexusOperationExecutionResponse"_\n$DeleteNexusOperationExecutionRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x14\n\x0coperation_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"\'\n%DeleteNexusOperationExecutionResponse"\x9d\x01\n(PollWorkflowExecutionTimeSkippingRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x45\n\x12workflow_execution\x18\x02 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x17\n\x0f\x66\x61st_forward_id\x18\x03 \x01(\t"\xe8\x01\n)PollWorkflowExecutionTimeSkippingResponse\x12T\n\x1b\x66\x61st_forward_polling_result\x18\x01 \x01(\x0e\x32/.temporal.api.enums.v1.FastForwardPollingResult\x12\x15\n\rfailed_reason\x18\x02 \x01(\t\x12N\n\x11\x66\x61st_forward_info\x18\x03 \x01(\x0b\x32\x33.temporal.api.common.v1.TimeSkippingFastForwardInfoB\xbe\x01\n"io.temporal.api.workflowservice.v1B\x14RequestResponseProtoP\x01Z5go.temporal.io/api/workflowservice/v1;workflowservice\xaa\x02!Temporalio.Api.WorkflowService.V1\xea\x02$Temporalio::Api::WorkflowService::V1b\x06proto3' ) @@ -267,6 +276,24 @@ _SIGNALWORKFLOWEXECUTIONRESPONSE = DESCRIPTOR.message_types_by_name[ "SignalWorkflowExecutionResponse" ] +_NOTIFYCHANNELREQUEST = DESCRIPTOR.message_types_by_name["NotifyChannelRequest"] +_NOTIFYCHANNELRESPONSE = DESCRIPTOR.message_types_by_name["NotifyChannelResponse"] +_REGISTERCHANNELLISTENERREQUEST = DESCRIPTOR.message_types_by_name[ + "RegisterChannelListenerRequest" +] +_REGISTERCHANNELLISTENERRESPONSE = DESCRIPTOR.message_types_by_name[ + "RegisterChannelListenerResponse" +] +_UNREGISTERCHANNELLISTENERREQUEST = DESCRIPTOR.message_types_by_name[ + "UnregisterChannelListenerRequest" +] +_UNREGISTERCHANNELLISTENERRESPONSE = DESCRIPTOR.message_types_by_name[ + "UnregisterChannelListenerResponse" +] +_POLLCHANNELREQUEST = DESCRIPTOR.message_types_by_name["PollChannelRequest"] +_POLLCHANNELRESPONSE = DESCRIPTOR.message_types_by_name["PollChannelResponse"] +_DESCRIBECHANNELREQUEST = DESCRIPTOR.message_types_by_name["DescribeChannelRequest"] +_DESCRIBECHANNELRESPONSE = DESCRIPTOR.message_types_by_name["DescribeChannelResponse"] _SIGNALWITHSTARTWORKFLOWEXECUTIONREQUEST = DESCRIPTOR.message_types_by_name[ "SignalWithStartWorkflowExecutionRequest" ] @@ -1435,6 +1462,116 @@ ) _sym_db.RegisterMessage(SignalWorkflowExecutionResponse) +NotifyChannelRequest = _reflection.GeneratedProtocolMessageType( + "NotifyChannelRequest", + (_message.Message,), + { + "DESCRIPTOR": _NOTIFYCHANNELREQUEST, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.NotifyChannelRequest) + }, +) +_sym_db.RegisterMessage(NotifyChannelRequest) + +NotifyChannelResponse = _reflection.GeneratedProtocolMessageType( + "NotifyChannelResponse", + (_message.Message,), + { + "DESCRIPTOR": _NOTIFYCHANNELRESPONSE, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.NotifyChannelResponse) + }, +) +_sym_db.RegisterMessage(NotifyChannelResponse) + +RegisterChannelListenerRequest = _reflection.GeneratedProtocolMessageType( + "RegisterChannelListenerRequest", + (_message.Message,), + { + "DESCRIPTOR": _REGISTERCHANNELLISTENERREQUEST, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.RegisterChannelListenerRequest) + }, +) +_sym_db.RegisterMessage(RegisterChannelListenerRequest) + +RegisterChannelListenerResponse = _reflection.GeneratedProtocolMessageType( + "RegisterChannelListenerResponse", + (_message.Message,), + { + "DESCRIPTOR": _REGISTERCHANNELLISTENERRESPONSE, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.RegisterChannelListenerResponse) + }, +) +_sym_db.RegisterMessage(RegisterChannelListenerResponse) + +UnregisterChannelListenerRequest = _reflection.GeneratedProtocolMessageType( + "UnregisterChannelListenerRequest", + (_message.Message,), + { + "DESCRIPTOR": _UNREGISTERCHANNELLISTENERREQUEST, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.UnregisterChannelListenerRequest) + }, +) +_sym_db.RegisterMessage(UnregisterChannelListenerRequest) + +UnregisterChannelListenerResponse = _reflection.GeneratedProtocolMessageType( + "UnregisterChannelListenerResponse", + (_message.Message,), + { + "DESCRIPTOR": _UNREGISTERCHANNELLISTENERRESPONSE, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.UnregisterChannelListenerResponse) + }, +) +_sym_db.RegisterMessage(UnregisterChannelListenerResponse) + +PollChannelRequest = _reflection.GeneratedProtocolMessageType( + "PollChannelRequest", + (_message.Message,), + { + "DESCRIPTOR": _POLLCHANNELREQUEST, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.PollChannelRequest) + }, +) +_sym_db.RegisterMessage(PollChannelRequest) + +PollChannelResponse = _reflection.GeneratedProtocolMessageType( + "PollChannelResponse", + (_message.Message,), + { + "DESCRIPTOR": _POLLCHANNELRESPONSE, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.PollChannelResponse) + }, +) +_sym_db.RegisterMessage(PollChannelResponse) + +DescribeChannelRequest = _reflection.GeneratedProtocolMessageType( + "DescribeChannelRequest", + (_message.Message,), + { + "DESCRIPTOR": _DESCRIBECHANNELREQUEST, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.DescribeChannelRequest) + }, +) +_sym_db.RegisterMessage(DescribeChannelRequest) + +DescribeChannelResponse = _reflection.GeneratedProtocolMessageType( + "DescribeChannelResponse", + (_message.Message,), + { + "DESCRIPTOR": _DESCRIBECHANNELRESPONSE, + "__module__": "temporalio.api.workflowservice.v1.request_response_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.workflowservice.v1.DescribeChannelResponse) + }, +) +_sym_db.RegisterMessage(DescribeChannelResponse) + SignalWithStartWorkflowExecutionRequest = _reflection.GeneratedProtocolMessageType( "SignalWithStartWorkflowExecutionRequest", (_message.Message,), @@ -4214,578 +4351,598 @@ _UPDATETASKQUEUECONFIGREQUEST_SETFAIRNESSWEIGHTOVERRIDESENTRY._serialized_options = b"8\001" _STARTNEXUSOPERATIONEXECUTIONREQUEST_NEXUSHEADERENTRY._options = None _STARTNEXUSOPERATIONEXECUTIONREQUEST_NEXUSHEADERENTRY._serialized_options = b"8\001" - _REGISTERNAMESPACEREQUEST._serialized_start = 1646 - _REGISTERNAMESPACEREQUEST._serialized_end = 2294 - _REGISTERNAMESPACEREQUEST_DATAENTRY._serialized_start = 2251 - _REGISTERNAMESPACEREQUEST_DATAENTRY._serialized_end = 2294 - _REGISTERNAMESPACERESPONSE._serialized_start = 2296 - _REGISTERNAMESPACERESPONSE._serialized_end = 2323 - _LISTNAMESPACESREQUEST._serialized_start = 2326 - _LISTNAMESPACESREQUEST._serialized_end = 2463 - _LISTNAMESPACESRESPONSE._serialized_start = 2466 - _LISTNAMESPACESRESPONSE._serialized_end = 2595 - _DESCRIBENAMESPACEREQUEST._serialized_start = 2597 - _DESCRIBENAMESPACEREQUEST._serialized_end = 2680 - _DESCRIBENAMESPACERESPONSE._serialized_start = 2683 - _DESCRIBENAMESPACERESPONSE._serialized_end = 3196 - _UPDATENAMESPACEREQUEST._serialized_start = 3199 - _UPDATENAMESPACEREQUEST._serialized_end = 3534 - _UPDATENAMESPACERESPONSE._serialized_start = 3537 - _UPDATENAMESPACERESPONSE._serialized_end = 3828 - _DEPRECATENAMESPACEREQUEST._serialized_start = 3830 - _DEPRECATENAMESPACEREQUEST._serialized_end = 3900 - _DEPRECATENAMESPACERESPONSE._serialized_start = 3902 - _DEPRECATENAMESPACERESPONSE._serialized_end = 3930 - _STARTWORKFLOWEXECUTIONREQUEST._serialized_start = 3933 - _STARTWORKFLOWEXECUTIONREQUEST._serialized_end = 5550 - _STARTWORKFLOWEXECUTIONRESPONSE._serialized_start = 5553 - _STARTWORKFLOWEXECUTIONRESPONSE._serialized_end = 5851 - _GETWORKFLOWEXECUTIONHISTORYREQUEST._serialized_start = 5854 - _GETWORKFLOWEXECUTIONHISTORYREQUEST._serialized_end = 6152 - _GETWORKFLOWEXECUTIONHISTORYRESPONSE._serialized_start = 6155 - _GETWORKFLOWEXECUTIONHISTORYRESPONSE._serialized_end = 6341 - _GETWORKFLOWEXECUTIONHISTORYREVERSEREQUEST._serialized_start = 6344 - _GETWORKFLOWEXECUTIONHISTORYREVERSEREQUEST._serialized_end = 6520 - _GETWORKFLOWEXECUTIONHISTORYREVERSERESPONSE._serialized_start = 6522 - _GETWORKFLOWEXECUTIONHISTORYREVERSERESPONSE._serialized_end = 6642 - _POLLWORKFLOWTASKQUEUEREQUEST._serialized_start = 6645 - _POLLWORKFLOWTASKQUEUEREQUEST._serialized_end = 7085 - _POLLWORKFLOWTASKQUEUERESPONSE._serialized_start = 7088 - _POLLWORKFLOWTASKQUEUERESPONSE._serialized_end = 8175 - _POLLWORKFLOWTASKQUEUERESPONSE_QUERIESENTRY._serialized_start = 8091 - _POLLWORKFLOWTASKQUEUERESPONSE_QUERIESENTRY._serialized_end = 8175 - _RESPONDWORKFLOWTASKCOMPLETEDREQUEST._serialized_start = 8178 - _RESPONDWORKFLOWTASKCOMPLETEDREQUEST._serialized_end = 9516 - _RESPONDWORKFLOWTASKCOMPLETEDREQUEST_QUERYRESULTSENTRY._serialized_start = 9350 - _RESPONDWORKFLOWTASKCOMPLETEDREQUEST_QUERYRESULTSENTRY._serialized_end = 9445 - _RESPONDWORKFLOWTASKCOMPLETEDREQUEST_CAPABILITIES._serialized_start = 9447 - _RESPONDWORKFLOWTASKCOMPLETEDREQUEST_CAPABILITIES._serialized_end = 9516 - _RESPONDWORKFLOWTASKCOMPLETEDRESPONSE._serialized_start = 9519 - _RESPONDWORKFLOWTASKCOMPLETEDRESPONSE._serialized_end = 9764 - _RESPONDWORKFLOWTASKFAILEDREQUEST._serialized_start = 9767 - _RESPONDWORKFLOWTASKFAILEDREQUEST._serialized_end = 10292 - _RESPONDWORKFLOWTASKFAILEDRESPONSE._serialized_start = 10294 - _RESPONDWORKFLOWTASKFAILEDRESPONSE._serialized_end = 10329 - _POLLACTIVITYTASKQUEUEREQUEST._serialized_start = 10332 - _POLLACTIVITYTASKQUEUEREQUEST._serialized_end = 10818 - _POLLACTIVITYTASKQUEUERESPONSE._serialized_start = 10821 - _POLLACTIVITYTASKQUEUERESPONSE._serialized_end = 12002 - _RECORDACTIVITYTASKHEARTBEATREQUEST._serialized_start = 12005 - _RECORDACTIVITYTASKHEARTBEATREQUEST._serialized_end = 12170 - _RECORDACTIVITYTASKHEARTBEATRESPONSE._serialized_start = 12172 - _RECORDACTIVITYTASKHEARTBEATRESPONSE._serialized_end = 12284 - _RECORDACTIVITYTASKHEARTBEATBYIDREQUEST._serialized_start = 12287 - _RECORDACTIVITYTASKHEARTBEATBYIDREQUEST._serialized_end = 12494 - _RECORDACTIVITYTASKHEARTBEATBYIDRESPONSE._serialized_start = 12496 - _RECORDACTIVITYTASKHEARTBEATBYIDRESPONSE._serialized_end = 12612 - _RESPONDACTIVITYTASKCOMPLETEDREQUEST._serialized_start = 12615 - _RESPONDACTIVITYTASKCOMPLETEDREQUEST._serialized_end = 12997 - _RESPONDACTIVITYTASKCOMPLETEDRESPONSE._serialized_start = 12999 - _RESPONDACTIVITYTASKCOMPLETEDRESPONSE._serialized_end = 13037 - _RESPONDACTIVITYTASKCOMPLETEDBYIDREQUEST._serialized_start = 13040 - _RESPONDACTIVITYTASKCOMPLETEDBYIDREQUEST._serialized_end = 13247 - _RESPONDACTIVITYTASKCOMPLETEDBYIDRESPONSE._serialized_start = 13249 - _RESPONDACTIVITYTASKCOMPLETEDBYIDRESPONSE._serialized_end = 13291 - _RESPONDACTIVITYTASKFAILEDREQUEST._serialized_start = 13294 - _RESPONDACTIVITYTASKFAILEDREQUEST._serialized_end = 13740 - _RESPONDACTIVITYTASKFAILEDRESPONSE._serialized_start = 13742 - _RESPONDACTIVITYTASKFAILEDRESPONSE._serialized_end = 13829 - _RESPONDACTIVITYTASKFAILEDBYIDREQUEST._serialized_start = 13832 - _RESPONDACTIVITYTASKFAILEDBYIDREQUEST._serialized_end = 14103 - _RESPONDACTIVITYTASKFAILEDBYIDRESPONSE._serialized_start = 14105 - _RESPONDACTIVITYTASKFAILEDBYIDRESPONSE._serialized_end = 14196 - _RESPONDACTIVITYTASKCANCELEDREQUEST._serialized_start = 14199 - _RESPONDACTIVITYTASKCANCELEDREQUEST._serialized_end = 14581 - _RESPONDACTIVITYTASKCANCELEDRESPONSE._serialized_start = 14583 - _RESPONDACTIVITYTASKCANCELEDRESPONSE._serialized_end = 14620 - _RESPONDACTIVITYTASKCANCELEDBYIDREQUEST._serialized_start = 14623 - _RESPONDACTIVITYTASKCANCELEDBYIDREQUEST._serialized_end = 14911 - _RESPONDACTIVITYTASKCANCELEDBYIDRESPONSE._serialized_start = 14913 - _RESPONDACTIVITYTASKCANCELEDBYIDRESPONSE._serialized_end = 14954 - _REQUESTCANCELWORKFLOWEXECUTIONREQUEST._serialized_start = 14957 - _REQUESTCANCELWORKFLOWEXECUTIONREQUEST._serialized_end = 15217 - _REQUESTCANCELWORKFLOWEXECUTIONRESPONSE._serialized_start = 15219 - _REQUESTCANCELWORKFLOWEXECUTIONRESPONSE._serialized_end = 15259 - _SIGNALWORKFLOWEXECUTIONREQUEST._serialized_start = 15262 - _SIGNALWORKFLOWEXECUTIONREQUEST._serialized_end = 15612 - _SIGNALWORKFLOWEXECUTIONRESPONSE._serialized_start = 15614 - _SIGNALWORKFLOWEXECUTIONRESPONSE._serialized_end = 15691 - _SIGNALWITHSTARTWORKFLOWEXECUTIONREQUEST._serialized_start = 15694 - _SIGNALWITHSTARTWORKFLOWEXECUTIONREQUEST._serialized_end = 17033 - _SIGNALWITHSTARTWORKFLOWEXECUTIONRESPONSE._serialized_start = 17036 - _SIGNALWITHSTARTWORKFLOWEXECUTIONRESPONSE._serialized_end = 17194 - _RESETWORKFLOWEXECUTIONREQUEST._serialized_start = 17197 - _RESETWORKFLOWEXECUTIONREQUEST._serialized_end = 17646 - _RESETWORKFLOWEXECUTIONRESPONSE._serialized_start = 17648 - _RESETWORKFLOWEXECUTIONRESPONSE._serialized_end = 17696 - _TERMINATEWORKFLOWEXECUTIONREQUEST._serialized_start = 17699 - _TERMINATEWORKFLOWEXECUTIONREQUEST._serialized_end = 17986 - _TERMINATEWORKFLOWEXECUTIONRESPONSE._serialized_start = 17988 - _TERMINATEWORKFLOWEXECUTIONRESPONSE._serialized_end = 18024 - _DELETEWORKFLOWEXECUTIONREQUEST._serialized_start = 18026 - _DELETEWORKFLOWEXECUTIONREQUEST._serialized_end = 18148 - _DELETEWORKFLOWEXECUTIONRESPONSE._serialized_start = 18150 - _DELETEWORKFLOWEXECUTIONRESPONSE._serialized_end = 18183 - _LISTOPENWORKFLOWEXECUTIONSREQUEST._serialized_start = 18186 - _LISTOPENWORKFLOWEXECUTIONSREQUEST._serialized_end = 18515 - _LISTOPENWORKFLOWEXECUTIONSRESPONSE._serialized_start = 18518 - _LISTOPENWORKFLOWEXECUTIONSRESPONSE._serialized_end = 18648 - _LISTCLOSEDWORKFLOWEXECUTIONSREQUEST._serialized_start = 18651 - _LISTCLOSEDWORKFLOWEXECUTIONSREQUEST._serialized_end = 19045 - _LISTCLOSEDWORKFLOWEXECUTIONSRESPONSE._serialized_start = 19048 - _LISTCLOSEDWORKFLOWEXECUTIONSRESPONSE._serialized_end = 19180 - _LISTWORKFLOWEXECUTIONSREQUEST._serialized_start = 19182 - _LISTWORKFLOWEXECUTIONSREQUEST._serialized_end = 19291 - _LISTWORKFLOWEXECUTIONSRESPONSE._serialized_start = 19293 - _LISTWORKFLOWEXECUTIONSRESPONSE._serialized_end = 19419 - _LISTARCHIVEDWORKFLOWEXECUTIONSREQUEST._serialized_start = 19421 - _LISTARCHIVEDWORKFLOWEXECUTIONSREQUEST._serialized_end = 19538 - _LISTARCHIVEDWORKFLOWEXECUTIONSRESPONSE._serialized_start = 19541 - _LISTARCHIVEDWORKFLOWEXECUTIONSRESPONSE._serialized_end = 19675 - _SCANWORKFLOWEXECUTIONSREQUEST._serialized_start = 19677 - _SCANWORKFLOWEXECUTIONSREQUEST._serialized_end = 19786 - _SCANWORKFLOWEXECUTIONSRESPONSE._serialized_start = 19788 - _SCANWORKFLOWEXECUTIONSRESPONSE._serialized_end = 19914 - _COUNTWORKFLOWEXECUTIONSREQUEST._serialized_start = 19916 - _COUNTWORKFLOWEXECUTIONSREQUEST._serialized_end = 19982 - _COUNTWORKFLOWEXECUTIONSRESPONSE._serialized_start = 19985 - _COUNTWORKFLOWEXECUTIONSRESPONSE._serialized_end = 20222 - _COUNTWORKFLOWEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_start = 20134 - _COUNTWORKFLOWEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_end = 20222 - _GETSEARCHATTRIBUTESREQUEST._serialized_start = 20224 - _GETSEARCHATTRIBUTESREQUEST._serialized_end = 20252 - _GETSEARCHATTRIBUTESRESPONSE._serialized_start = 20255 - _GETSEARCHATTRIBUTESRESPONSE._serialized_end = 20456 - _GETSEARCHATTRIBUTESRESPONSE_KEYSENTRY._serialized_start = 20372 - _GETSEARCHATTRIBUTESRESPONSE_KEYSENTRY._serialized_end = 20456 - _RESPONDQUERYTASKCOMPLETEDREQUEST._serialized_start = 20459 - _RESPONDQUERYTASKCOMPLETEDREQUEST._serialized_end = 20820 - _RESPONDQUERYTASKCOMPLETEDRESPONSE._serialized_start = 20822 - _RESPONDQUERYTASKCOMPLETEDRESPONSE._serialized_end = 20857 - _RESETSTICKYTASKQUEUEREQUEST._serialized_start = 20859 - _RESETSTICKYTASKQUEUEREQUEST._serialized_end = 20969 - _RESETSTICKYTASKQUEUERESPONSE._serialized_start = 20971 - _RESETSTICKYTASKQUEUERESPONSE._serialized_end = 21001 - _SHUTDOWNWORKERREQUEST._serialized_start = 21004 - _SHUTDOWNWORKERREQUEST._serialized_end = 21287 - _SHUTDOWNWORKERRESPONSE._serialized_start = 21289 - _SHUTDOWNWORKERRESPONSE._serialized_end = 21313 - _QUERYWORKFLOWREQUEST._serialized_start = 21316 - _QUERYWORKFLOWREQUEST._serialized_end = 21549 - _QUERYWORKFLOWRESPONSE._serialized_start = 21552 - _QUERYWORKFLOWRESPONSE._serialized_end = 21737 - _DESCRIBEWORKFLOWEXECUTIONREQUEST._serialized_start = 21739 - _DESCRIBEWORKFLOWEXECUTIONREQUEST._serialized_end = 21854 - _DESCRIBEWORKFLOWEXECUTIONRESPONSE._serialized_start = 21857 - _DESCRIBEWORKFLOWEXECUTIONRESPONSE._serialized_end = 22522 - _DESCRIBETASKQUEUEREQUEST._serialized_start = 22525 - _DESCRIBETASKQUEUEREQUEST._serialized_end = 23053 - _DESCRIBETASKQUEUERESPONSE._serialized_start = 23056 - _DESCRIBETASKQUEUERESPONSE._serialized_end = 24060 - _DESCRIBETASKQUEUERESPONSE_STATSBYPRIORITYKEYENTRY._serialized_start = 23740 - _DESCRIBETASKQUEUERESPONSE_STATSBYPRIORITYKEYENTRY._serialized_end = 23840 - _DESCRIBETASKQUEUERESPONSE_EFFECTIVERATELIMIT._serialized_start = 23842 - _DESCRIBETASKQUEUERESPONSE_EFFECTIVERATELIMIT._serialized_end = 23958 - _DESCRIBETASKQUEUERESPONSE_VERSIONSINFOENTRY._serialized_start = 23960 - _DESCRIBETASKQUEUERESPONSE_VERSIONSINFOENTRY._serialized_end = 24060 - _GETCLUSTERINFOREQUEST._serialized_start = 24062 - _GETCLUSTERINFOREQUEST._serialized_end = 24085 - _GETCLUSTERINFORESPONSE._serialized_start = 24088 - _GETCLUSTERINFORESPONSE._serialized_end = 24553 - _GETCLUSTERINFORESPONSE_SUPPORTEDCLIENTSENTRY._serialized_start = 24498 - _GETCLUSTERINFORESPONSE_SUPPORTEDCLIENTSENTRY._serialized_end = 24553 - _GETSYSTEMINFOREQUEST._serialized_start = 24555 - _GETSYSTEMINFOREQUEST._serialized_end = 24577 - _GETSYSTEMINFORESPONSE._serialized_start = 24580 - _GETSYSTEMINFORESPONSE._serialized_end = 25157 - _GETSYSTEMINFORESPONSE_CAPABILITIES._serialized_start = 24721 - _GETSYSTEMINFORESPONSE_CAPABILITIES._serialized_end = 25157 - _LISTTASKQUEUEPARTITIONSREQUEST._serialized_start = 25159 - _LISTTASKQUEUEPARTITIONSREQUEST._serialized_end = 25268 - _LISTTASKQUEUEPARTITIONSRESPONSE._serialized_start = 25271 - _LISTTASKQUEUEPARTITIONSRESPONSE._serialized_end = 25494 - _CREATESCHEDULEREQUEST._serialized_start = 25497 - _CREATESCHEDULEREQUEST._serialized_end = 25829 - _CREATESCHEDULERESPONSE._serialized_start = 25831 - _CREATESCHEDULERESPONSE._serialized_end = 25879 - _DESCRIBESCHEDULEREQUEST._serialized_start = 25881 - _DESCRIBESCHEDULEREQUEST._serialized_end = 25946 - _DESCRIBESCHEDULERESPONSE._serialized_start = 25949 - _DESCRIBESCHEDULERESPONSE._serialized_end = 26220 - _UPDATESCHEDULEREQUEST._serialized_start = 26223 - _UPDATESCHEDULEREQUEST._serialized_end = 26515 - _UPDATESCHEDULERESPONSE._serialized_start = 26517 - _UPDATESCHEDULERESPONSE._serialized_end = 26541 - _PATCHSCHEDULEREQUEST._serialized_start = 26544 - _PATCHSCHEDULEREQUEST._serialized_end = 26700 - _PATCHSCHEDULERESPONSE._serialized_start = 26702 - _PATCHSCHEDULERESPONSE._serialized_end = 26725 - _LISTSCHEDULEMATCHINGTIMESREQUEST._serialized_start = 26728 - _LISTSCHEDULEMATCHINGTIMESREQUEST._serialized_end = 26896 - _LISTSCHEDULEMATCHINGTIMESRESPONSE._serialized_start = 26898 - _LISTSCHEDULEMATCHINGTIMESRESPONSE._serialized_end = 26981 - _DELETESCHEDULEREQUEST._serialized_start = 26983 - _DELETESCHEDULEREQUEST._serialized_end = 27064 - _DELETESCHEDULERESPONSE._serialized_start = 27066 - _DELETESCHEDULERESPONSE._serialized_end = 27090 - _LISTSCHEDULESREQUEST._serialized_start = 27092 - _LISTSCHEDULESREQUEST._serialized_end = 27200 - _LISTSCHEDULESRESPONSE._serialized_start = 27202 - _LISTSCHEDULESRESPONSE._serialized_end = 27314 - _COUNTSCHEDULESREQUEST._serialized_start = 27316 - _COUNTSCHEDULESREQUEST._serialized_end = 27373 - _COUNTSCHEDULESRESPONSE._serialized_start = 27376 - _COUNTSCHEDULESRESPONSE._serialized_end = 27595 - _COUNTSCHEDULESRESPONSE_AGGREGATIONGROUP._serialized_start = 20134 - _COUNTSCHEDULESRESPONSE_AGGREGATIONGROUP._serialized_end = 20222 - _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST._serialized_start = 27598 - _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST._serialized_end = 28244 - _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST_ADDNEWCOMPATIBLEVERSION._serialized_start = 28045 + _REGISTERNAMESPACEREQUEST._serialized_start = 1774 + _REGISTERNAMESPACEREQUEST._serialized_end = 2422 + _REGISTERNAMESPACEREQUEST_DATAENTRY._serialized_start = 2379 + _REGISTERNAMESPACEREQUEST_DATAENTRY._serialized_end = 2422 + _REGISTERNAMESPACERESPONSE._serialized_start = 2424 + _REGISTERNAMESPACERESPONSE._serialized_end = 2451 + _LISTNAMESPACESREQUEST._serialized_start = 2454 + _LISTNAMESPACESREQUEST._serialized_end = 2591 + _LISTNAMESPACESRESPONSE._serialized_start = 2594 + _LISTNAMESPACESRESPONSE._serialized_end = 2723 + _DESCRIBENAMESPACEREQUEST._serialized_start = 2725 + _DESCRIBENAMESPACEREQUEST._serialized_end = 2808 + _DESCRIBENAMESPACERESPONSE._serialized_start = 2811 + _DESCRIBENAMESPACERESPONSE._serialized_end = 3324 + _UPDATENAMESPACEREQUEST._serialized_start = 3327 + _UPDATENAMESPACEREQUEST._serialized_end = 3662 + _UPDATENAMESPACERESPONSE._serialized_start = 3665 + _UPDATENAMESPACERESPONSE._serialized_end = 3956 + _DEPRECATENAMESPACEREQUEST._serialized_start = 3958 + _DEPRECATENAMESPACEREQUEST._serialized_end = 4028 + _DEPRECATENAMESPACERESPONSE._serialized_start = 4030 + _DEPRECATENAMESPACERESPONSE._serialized_end = 4058 + _STARTWORKFLOWEXECUTIONREQUEST._serialized_start = 4061 + _STARTWORKFLOWEXECUTIONREQUEST._serialized_end = 5678 + _STARTWORKFLOWEXECUTIONRESPONSE._serialized_start = 5681 + _STARTWORKFLOWEXECUTIONRESPONSE._serialized_end = 5979 + _GETWORKFLOWEXECUTIONHISTORYREQUEST._serialized_start = 5982 + _GETWORKFLOWEXECUTIONHISTORYREQUEST._serialized_end = 6280 + _GETWORKFLOWEXECUTIONHISTORYRESPONSE._serialized_start = 6283 + _GETWORKFLOWEXECUTIONHISTORYRESPONSE._serialized_end = 6469 + _GETWORKFLOWEXECUTIONHISTORYREVERSEREQUEST._serialized_start = 6472 + _GETWORKFLOWEXECUTIONHISTORYREVERSEREQUEST._serialized_end = 6648 + _GETWORKFLOWEXECUTIONHISTORYREVERSERESPONSE._serialized_start = 6650 + _GETWORKFLOWEXECUTIONHISTORYREVERSERESPONSE._serialized_end = 6770 + _POLLWORKFLOWTASKQUEUEREQUEST._serialized_start = 6773 + _POLLWORKFLOWTASKQUEUEREQUEST._serialized_end = 7213 + _POLLWORKFLOWTASKQUEUERESPONSE._serialized_start = 7216 + _POLLWORKFLOWTASKQUEUERESPONSE._serialized_end = 8369 + _POLLWORKFLOWTASKQUEUERESPONSE_QUERIESENTRY._serialized_start = 8279 + _POLLWORKFLOWTASKQUEUERESPONSE_QUERIESENTRY._serialized_end = 8363 + _RESPONDWORKFLOWTASKCOMPLETEDREQUEST._serialized_start = 8372 + _RESPONDWORKFLOWTASKCOMPLETEDREQUEST._serialized_end = 9710 + _RESPONDWORKFLOWTASKCOMPLETEDREQUEST_QUERYRESULTSENTRY._serialized_start = 9544 + _RESPONDWORKFLOWTASKCOMPLETEDREQUEST_QUERYRESULTSENTRY._serialized_end = 9639 + _RESPONDWORKFLOWTASKCOMPLETEDREQUEST_CAPABILITIES._serialized_start = 9641 + _RESPONDWORKFLOWTASKCOMPLETEDREQUEST_CAPABILITIES._serialized_end = 9710 + _RESPONDWORKFLOWTASKCOMPLETEDRESPONSE._serialized_start = 9713 + _RESPONDWORKFLOWTASKCOMPLETEDRESPONSE._serialized_end = 9958 + _RESPONDWORKFLOWTASKFAILEDREQUEST._serialized_start = 9961 + _RESPONDWORKFLOWTASKFAILEDREQUEST._serialized_end = 10486 + _RESPONDWORKFLOWTASKFAILEDRESPONSE._serialized_start = 10488 + _RESPONDWORKFLOWTASKFAILEDRESPONSE._serialized_end = 10523 + _POLLACTIVITYTASKQUEUEREQUEST._serialized_start = 10526 + _POLLACTIVITYTASKQUEUEREQUEST._serialized_end = 11012 + _POLLACTIVITYTASKQUEUERESPONSE._serialized_start = 11015 + _POLLACTIVITYTASKQUEUERESPONSE._serialized_end = 12196 + _RECORDACTIVITYTASKHEARTBEATREQUEST._serialized_start = 12199 + _RECORDACTIVITYTASKHEARTBEATREQUEST._serialized_end = 12364 + _RECORDACTIVITYTASKHEARTBEATRESPONSE._serialized_start = 12366 + _RECORDACTIVITYTASKHEARTBEATRESPONSE._serialized_end = 12478 + _RECORDACTIVITYTASKHEARTBEATBYIDREQUEST._serialized_start = 12481 + _RECORDACTIVITYTASKHEARTBEATBYIDREQUEST._serialized_end = 12688 + _RECORDACTIVITYTASKHEARTBEATBYIDRESPONSE._serialized_start = 12690 + _RECORDACTIVITYTASKHEARTBEATBYIDRESPONSE._serialized_end = 12806 + _RESPONDACTIVITYTASKCOMPLETEDREQUEST._serialized_start = 12809 + _RESPONDACTIVITYTASKCOMPLETEDREQUEST._serialized_end = 13191 + _RESPONDACTIVITYTASKCOMPLETEDRESPONSE._serialized_start = 13193 + _RESPONDACTIVITYTASKCOMPLETEDRESPONSE._serialized_end = 13231 + _RESPONDACTIVITYTASKCOMPLETEDBYIDREQUEST._serialized_start = 13234 + _RESPONDACTIVITYTASKCOMPLETEDBYIDREQUEST._serialized_end = 13441 + _RESPONDACTIVITYTASKCOMPLETEDBYIDRESPONSE._serialized_start = 13443 + _RESPONDACTIVITYTASKCOMPLETEDBYIDRESPONSE._serialized_end = 13485 + _RESPONDACTIVITYTASKFAILEDREQUEST._serialized_start = 13488 + _RESPONDACTIVITYTASKFAILEDREQUEST._serialized_end = 13997 + _RESPONDACTIVITYTASKFAILEDRESPONSE._serialized_start = 13999 + _RESPONDACTIVITYTASKFAILEDRESPONSE._serialized_end = 14086 + _RESPONDACTIVITYTASKFAILEDBYIDREQUEST._serialized_start = 14089 + _RESPONDACTIVITYTASKFAILEDBYIDREQUEST._serialized_end = 14423 + _RESPONDACTIVITYTASKFAILEDBYIDRESPONSE._serialized_start = 14425 + _RESPONDACTIVITYTASKFAILEDBYIDRESPONSE._serialized_end = 14516 + _RESPONDACTIVITYTASKCANCELEDREQUEST._serialized_start = 14519 + _RESPONDACTIVITYTASKCANCELEDREQUEST._serialized_end = 14901 + _RESPONDACTIVITYTASKCANCELEDRESPONSE._serialized_start = 14903 + _RESPONDACTIVITYTASKCANCELEDRESPONSE._serialized_end = 14940 + _RESPONDACTIVITYTASKCANCELEDBYIDREQUEST._serialized_start = 14943 + _RESPONDACTIVITYTASKCANCELEDBYIDREQUEST._serialized_end = 15231 + _RESPONDACTIVITYTASKCANCELEDBYIDRESPONSE._serialized_start = 15233 + _RESPONDACTIVITYTASKCANCELEDBYIDRESPONSE._serialized_end = 15274 + _REQUESTCANCELWORKFLOWEXECUTIONREQUEST._serialized_start = 15277 + _REQUESTCANCELWORKFLOWEXECUTIONREQUEST._serialized_end = 15537 + _REQUESTCANCELWORKFLOWEXECUTIONRESPONSE._serialized_start = 15539 + _REQUESTCANCELWORKFLOWEXECUTIONRESPONSE._serialized_end = 15579 + _SIGNALWORKFLOWEXECUTIONREQUEST._serialized_start = 15582 + _SIGNALWORKFLOWEXECUTIONREQUEST._serialized_end = 15932 + _SIGNALWORKFLOWEXECUTIONRESPONSE._serialized_start = 15934 + _SIGNALWORKFLOWEXECUTIONRESPONSE._serialized_end = 16011 + _NOTIFYCHANNELREQUEST._serialized_start = 16014 + _NOTIFYCHANNELREQUEST._serialized_end = 16213 + _NOTIFYCHANNELRESPONSE._serialized_start = 16215 + _NOTIFYCHANNELRESPONSE._serialized_end = 16262 + _REGISTERCHANNELLISTENERREQUEST._serialized_start = 16265 + _REGISTERCHANNELLISTENERREQUEST._serialized_end = 16477 + _REGISTERCHANNELLISTENERRESPONSE._serialized_start = 16479 + _REGISTERCHANNELLISTENERRESPONSE._serialized_end = 16533 + _UNREGISTERCHANNELLISTENERREQUEST._serialized_start = 16536 + _UNREGISTERCHANNELLISTENERREQUEST._serialized_end = 16699 + _UNREGISTERCHANNELLISTENERRESPONSE._serialized_start = 16701 + _UNREGISTERCHANNELLISTENERRESPONSE._serialized_end = 16736 + _POLLCHANNELREQUEST._serialized_start = 16739 + _POLLCHANNELREQUEST._serialized_end = 16940 + _POLLCHANNELRESPONSE._serialized_start = 16942 + _POLLCHANNELRESPONSE._serialized_end = 17030 + _DESCRIBECHANNELREQUEST._serialized_start = 17032 + _DESCRIBECHANNELREQUEST._serialized_end = 17146 + _DESCRIBECHANNELRESPONSE._serialized_start = 17149 + _DESCRIBECHANNELRESPONSE._serialized_end = 17435 + _SIGNALWITHSTARTWORKFLOWEXECUTIONREQUEST._serialized_start = 17438 + _SIGNALWITHSTARTWORKFLOWEXECUTIONREQUEST._serialized_end = 18777 + _SIGNALWITHSTARTWORKFLOWEXECUTIONRESPONSE._serialized_start = 18780 + _SIGNALWITHSTARTWORKFLOWEXECUTIONRESPONSE._serialized_end = 18938 + _RESETWORKFLOWEXECUTIONREQUEST._serialized_start = 18941 + _RESETWORKFLOWEXECUTIONREQUEST._serialized_end = 19390 + _RESETWORKFLOWEXECUTIONRESPONSE._serialized_start = 19392 + _RESETWORKFLOWEXECUTIONRESPONSE._serialized_end = 19440 + _TERMINATEWORKFLOWEXECUTIONREQUEST._serialized_start = 19443 + _TERMINATEWORKFLOWEXECUTIONREQUEST._serialized_end = 19730 + _TERMINATEWORKFLOWEXECUTIONRESPONSE._serialized_start = 19732 + _TERMINATEWORKFLOWEXECUTIONRESPONSE._serialized_end = 19768 + _DELETEWORKFLOWEXECUTIONREQUEST._serialized_start = 19770 + _DELETEWORKFLOWEXECUTIONREQUEST._serialized_end = 19892 + _DELETEWORKFLOWEXECUTIONRESPONSE._serialized_start = 19894 + _DELETEWORKFLOWEXECUTIONRESPONSE._serialized_end = 19927 + _LISTOPENWORKFLOWEXECUTIONSREQUEST._serialized_start = 19930 + _LISTOPENWORKFLOWEXECUTIONSREQUEST._serialized_end = 20259 + _LISTOPENWORKFLOWEXECUTIONSRESPONSE._serialized_start = 20262 + _LISTOPENWORKFLOWEXECUTIONSRESPONSE._serialized_end = 20392 + _LISTCLOSEDWORKFLOWEXECUTIONSREQUEST._serialized_start = 20395 + _LISTCLOSEDWORKFLOWEXECUTIONSREQUEST._serialized_end = 20789 + _LISTCLOSEDWORKFLOWEXECUTIONSRESPONSE._serialized_start = 20792 + _LISTCLOSEDWORKFLOWEXECUTIONSRESPONSE._serialized_end = 20924 + _LISTWORKFLOWEXECUTIONSREQUEST._serialized_start = 20926 + _LISTWORKFLOWEXECUTIONSREQUEST._serialized_end = 21035 + _LISTWORKFLOWEXECUTIONSRESPONSE._serialized_start = 21037 + _LISTWORKFLOWEXECUTIONSRESPONSE._serialized_end = 21163 + _LISTARCHIVEDWORKFLOWEXECUTIONSREQUEST._serialized_start = 21165 + _LISTARCHIVEDWORKFLOWEXECUTIONSREQUEST._serialized_end = 21282 + _LISTARCHIVEDWORKFLOWEXECUTIONSRESPONSE._serialized_start = 21285 + _LISTARCHIVEDWORKFLOWEXECUTIONSRESPONSE._serialized_end = 21419 + _SCANWORKFLOWEXECUTIONSREQUEST._serialized_start = 21421 + _SCANWORKFLOWEXECUTIONSREQUEST._serialized_end = 21530 + _SCANWORKFLOWEXECUTIONSRESPONSE._serialized_start = 21532 + _SCANWORKFLOWEXECUTIONSRESPONSE._serialized_end = 21658 + _COUNTWORKFLOWEXECUTIONSREQUEST._serialized_start = 21660 + _COUNTWORKFLOWEXECUTIONSREQUEST._serialized_end = 21726 + _COUNTWORKFLOWEXECUTIONSRESPONSE._serialized_start = 21729 + _COUNTWORKFLOWEXECUTIONSRESPONSE._serialized_end = 21966 + _COUNTWORKFLOWEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_start = 21878 + _COUNTWORKFLOWEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_end = 21966 + _GETSEARCHATTRIBUTESREQUEST._serialized_start = 21968 + _GETSEARCHATTRIBUTESREQUEST._serialized_end = 21996 + _GETSEARCHATTRIBUTESRESPONSE._serialized_start = 21999 + _GETSEARCHATTRIBUTESRESPONSE._serialized_end = 22200 + _GETSEARCHATTRIBUTESRESPONSE_KEYSENTRY._serialized_start = 22116 + _GETSEARCHATTRIBUTESRESPONSE_KEYSENTRY._serialized_end = 22200 + _RESPONDQUERYTASKCOMPLETEDREQUEST._serialized_start = 22203 + _RESPONDQUERYTASKCOMPLETEDREQUEST._serialized_end = 22564 + _RESPONDQUERYTASKCOMPLETEDRESPONSE._serialized_start = 22566 + _RESPONDQUERYTASKCOMPLETEDRESPONSE._serialized_end = 22601 + _RESETSTICKYTASKQUEUEREQUEST._serialized_start = 22603 + _RESETSTICKYTASKQUEUEREQUEST._serialized_end = 22713 + _RESETSTICKYTASKQUEUERESPONSE._serialized_start = 22715 + _RESETSTICKYTASKQUEUERESPONSE._serialized_end = 22745 + _SHUTDOWNWORKERREQUEST._serialized_start = 22748 + _SHUTDOWNWORKERREQUEST._serialized_end = 23031 + _SHUTDOWNWORKERRESPONSE._serialized_start = 23033 + _SHUTDOWNWORKERRESPONSE._serialized_end = 23057 + _QUERYWORKFLOWREQUEST._serialized_start = 23060 + _QUERYWORKFLOWREQUEST._serialized_end = 23293 + _QUERYWORKFLOWRESPONSE._serialized_start = 23296 + _QUERYWORKFLOWRESPONSE._serialized_end = 23481 + _DESCRIBEWORKFLOWEXECUTIONREQUEST._serialized_start = 23483 + _DESCRIBEWORKFLOWEXECUTIONREQUEST._serialized_end = 23598 + _DESCRIBEWORKFLOWEXECUTIONRESPONSE._serialized_start = 23601 + _DESCRIBEWORKFLOWEXECUTIONRESPONSE._serialized_end = 24348 + _DESCRIBETASKQUEUEREQUEST._serialized_start = 24351 + _DESCRIBETASKQUEUEREQUEST._serialized_end = 24879 + _DESCRIBETASKQUEUERESPONSE._serialized_start = 24882 + _DESCRIBETASKQUEUERESPONSE._serialized_end = 25886 + _DESCRIBETASKQUEUERESPONSE_STATSBYPRIORITYKEYENTRY._serialized_start = 25566 + _DESCRIBETASKQUEUERESPONSE_STATSBYPRIORITYKEYENTRY._serialized_end = 25666 + _DESCRIBETASKQUEUERESPONSE_EFFECTIVERATELIMIT._serialized_start = 25668 + _DESCRIBETASKQUEUERESPONSE_EFFECTIVERATELIMIT._serialized_end = 25784 + _DESCRIBETASKQUEUERESPONSE_VERSIONSINFOENTRY._serialized_start = 25786 + _DESCRIBETASKQUEUERESPONSE_VERSIONSINFOENTRY._serialized_end = 25886 + _GETCLUSTERINFOREQUEST._serialized_start = 25888 + _GETCLUSTERINFOREQUEST._serialized_end = 25911 + _GETCLUSTERINFORESPONSE._serialized_start = 25914 + _GETCLUSTERINFORESPONSE._serialized_end = 26379 + _GETCLUSTERINFORESPONSE_SUPPORTEDCLIENTSENTRY._serialized_start = 26324 + _GETCLUSTERINFORESPONSE_SUPPORTEDCLIENTSENTRY._serialized_end = 26379 + _GETSYSTEMINFOREQUEST._serialized_start = 26381 + _GETSYSTEMINFOREQUEST._serialized_end = 26403 + _GETSYSTEMINFORESPONSE._serialized_start = 26406 + _GETSYSTEMINFORESPONSE._serialized_end = 26983 + _GETSYSTEMINFORESPONSE_CAPABILITIES._serialized_start = 26547 + _GETSYSTEMINFORESPONSE_CAPABILITIES._serialized_end = 26983 + _LISTTASKQUEUEPARTITIONSREQUEST._serialized_start = 26985 + _LISTTASKQUEUEPARTITIONSREQUEST._serialized_end = 27094 + _LISTTASKQUEUEPARTITIONSRESPONSE._serialized_start = 27097 + _LISTTASKQUEUEPARTITIONSRESPONSE._serialized_end = 27320 + _CREATESCHEDULEREQUEST._serialized_start = 27323 + _CREATESCHEDULEREQUEST._serialized_end = 27655 + _CREATESCHEDULERESPONSE._serialized_start = 27657 + _CREATESCHEDULERESPONSE._serialized_end = 27705 + _DESCRIBESCHEDULEREQUEST._serialized_start = 27707 + _DESCRIBESCHEDULEREQUEST._serialized_end = 27772 + _DESCRIBESCHEDULERESPONSE._serialized_start = 27775 + _DESCRIBESCHEDULERESPONSE._serialized_end = 28046 + _UPDATESCHEDULEREQUEST._serialized_start = 28049 + _UPDATESCHEDULEREQUEST._serialized_end = 28341 + _UPDATESCHEDULERESPONSE._serialized_start = 28343 + _UPDATESCHEDULERESPONSE._serialized_end = 28367 + _PATCHSCHEDULEREQUEST._serialized_start = 28370 + _PATCHSCHEDULEREQUEST._serialized_end = 28526 + _PATCHSCHEDULERESPONSE._serialized_start = 28528 + _PATCHSCHEDULERESPONSE._serialized_end = 28551 + _LISTSCHEDULEMATCHINGTIMESREQUEST._serialized_start = 28554 + _LISTSCHEDULEMATCHINGTIMESREQUEST._serialized_end = 28722 + _LISTSCHEDULEMATCHINGTIMESRESPONSE._serialized_start = 28724 + _LISTSCHEDULEMATCHINGTIMESRESPONSE._serialized_end = 28807 + _DELETESCHEDULEREQUEST._serialized_start = 28809 + _DELETESCHEDULEREQUEST._serialized_end = 28890 + _DELETESCHEDULERESPONSE._serialized_start = 28892 + _DELETESCHEDULERESPONSE._serialized_end = 28916 + _LISTSCHEDULESREQUEST._serialized_start = 28918 + _LISTSCHEDULESREQUEST._serialized_end = 29026 + _LISTSCHEDULESRESPONSE._serialized_start = 29028 + _LISTSCHEDULESRESPONSE._serialized_end = 29140 + _COUNTSCHEDULESREQUEST._serialized_start = 29142 + _COUNTSCHEDULESREQUEST._serialized_end = 29199 + _COUNTSCHEDULESRESPONSE._serialized_start = 29202 + _COUNTSCHEDULESRESPONSE._serialized_end = 29421 + _COUNTSCHEDULESRESPONSE_AGGREGATIONGROUP._serialized_start = 21878 + _COUNTSCHEDULESRESPONSE_AGGREGATIONGROUP._serialized_end = 21966 + _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST._serialized_start = 29424 + _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST._serialized_end = 30070 + _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST_ADDNEWCOMPATIBLEVERSION._serialized_start = 29871 _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST_ADDNEWCOMPATIBLEVERSION._serialized_end = ( - 28156 + 29982 ) - _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST_MERGESETS._serialized_start = 28158 - _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST_MERGESETS._serialized_end = 28231 - _UPDATEWORKERBUILDIDCOMPATIBILITYRESPONSE._serialized_start = 28246 - _UPDATEWORKERBUILDIDCOMPATIBILITYRESPONSE._serialized_end = 28310 - _GETWORKERBUILDIDCOMPATIBILITYREQUEST._serialized_start = 28312 - _GETWORKERBUILDIDCOMPATIBILITYREQUEST._serialized_end = 28407 - _GETWORKERBUILDIDCOMPATIBILITYRESPONSE._serialized_start = 28409 - _GETWORKERBUILDIDCOMPATIBILITYRESPONSE._serialized_end = 28525 - _UPDATEWORKERVERSIONINGRULESREQUEST._serialized_start = 28528 - _UPDATEWORKERVERSIONINGRULESREQUEST._serialized_end = 30245 - _UPDATEWORKERVERSIONINGRULESREQUEST_INSERTBUILDIDASSIGNMENTRULE._serialized_start = 29580 + _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST_MERGESETS._serialized_start = 29984 + _UPDATEWORKERBUILDIDCOMPATIBILITYREQUEST_MERGESETS._serialized_end = 30057 + _UPDATEWORKERBUILDIDCOMPATIBILITYRESPONSE._serialized_start = 30072 + _UPDATEWORKERBUILDIDCOMPATIBILITYRESPONSE._serialized_end = 30136 + _GETWORKERBUILDIDCOMPATIBILITYREQUEST._serialized_start = 30138 + _GETWORKERBUILDIDCOMPATIBILITYREQUEST._serialized_end = 30233 + _GETWORKERBUILDIDCOMPATIBILITYRESPONSE._serialized_start = 30235 + _GETWORKERBUILDIDCOMPATIBILITYRESPONSE._serialized_end = 30351 + _UPDATEWORKERVERSIONINGRULESREQUEST._serialized_start = 30354 + _UPDATEWORKERVERSIONINGRULESREQUEST._serialized_end = 32071 + _UPDATEWORKERVERSIONINGRULESREQUEST_INSERTBUILDIDASSIGNMENTRULE._serialized_start = 31406 _UPDATEWORKERVERSIONINGRULESREQUEST_INSERTBUILDIDASSIGNMENTRULE._serialized_end = ( - 29693 + 31519 ) - _UPDATEWORKERVERSIONINGRULESREQUEST_REPLACEBUILDIDASSIGNMENTRULE._serialized_start = 29696 + _UPDATEWORKERVERSIONINGRULESREQUEST_REPLACEBUILDIDASSIGNMENTRULE._serialized_start = 31522 _UPDATEWORKERVERSIONINGRULESREQUEST_REPLACEBUILDIDASSIGNMENTRULE._serialized_end = ( - 29825 + 31651 ) - _UPDATEWORKERVERSIONINGRULESREQUEST_DELETEBUILDIDASSIGNMENTRULE._serialized_start = 29827 + _UPDATEWORKERVERSIONINGRULESREQUEST_DELETEBUILDIDASSIGNMENTRULE._serialized_start = 31653 _UPDATEWORKERVERSIONINGRULESREQUEST_DELETEBUILDIDASSIGNMENTRULE._serialized_end = ( - 29891 + 31717 ) - _UPDATEWORKERVERSIONINGRULESREQUEST_ADDCOMPATIBLEBUILDIDREDIRECTRULE._serialized_start = 29893 - _UPDATEWORKERVERSIONINGRULESREQUEST_ADDCOMPATIBLEBUILDIDREDIRECTRULE._serialized_end = 29999 - _UPDATEWORKERVERSIONINGRULESREQUEST_REPLACECOMPATIBLEBUILDIDREDIRECTRULE._serialized_start = 30001 - _UPDATEWORKERVERSIONINGRULESREQUEST_REPLACECOMPATIBLEBUILDIDREDIRECTRULE._serialized_end = 30111 - _UPDATEWORKERVERSIONINGRULESREQUEST_DELETECOMPATIBLEBUILDIDREDIRECTRULE._serialized_start = 30113 - _UPDATEWORKERVERSIONINGRULESREQUEST_DELETECOMPATIBLEBUILDIDREDIRECTRULE._serialized_end = 30175 - _UPDATEWORKERVERSIONINGRULESREQUEST_COMMITBUILDID._serialized_start = 30177 - _UPDATEWORKERVERSIONINGRULESREQUEST_COMMITBUILDID._serialized_end = 30232 - _UPDATEWORKERVERSIONINGRULESRESPONSE._serialized_start = 30248 - _UPDATEWORKERVERSIONINGRULESRESPONSE._serialized_end = 30500 - _GETWORKERVERSIONINGRULESREQUEST._serialized_start = 30502 - _GETWORKERVERSIONINGRULESREQUEST._serialized_end = 30574 - _GETWORKERVERSIONINGRULESRESPONSE._serialized_start = 30577 - _GETWORKERVERSIONINGRULESRESPONSE._serialized_end = 30826 - _GETWORKERTASKREACHABILITYREQUEST._serialized_start = 30829 - _GETWORKERTASKREACHABILITYREQUEST._serialized_end = 30985 - _GETWORKERTASKREACHABILITYRESPONSE._serialized_start = 30987 - _GETWORKERTASKREACHABILITYRESPONSE._serialized_end = 31101 - _UPDATEWORKFLOWEXECUTIONREQUEST._serialized_start = 31104 - _UPDATEWORKFLOWEXECUTIONREQUEST._serialized_end = 31365 - _UPDATEWORKFLOWEXECUTIONRESPONSE._serialized_start = 31368 - _UPDATEWORKFLOWEXECUTIONRESPONSE._serialized_end = 31627 - _STARTBATCHOPERATIONREQUEST._serialized_start = 31630 - _STARTBATCHOPERATIONREQUEST._serialized_end = 32996 - _STARTBATCHOPERATIONRESPONSE._serialized_start = 32998 - _STARTBATCHOPERATIONRESPONSE._serialized_end = 33027 - _STOPBATCHOPERATIONREQUEST._serialized_start = 33029 - _STOPBATCHOPERATIONREQUEST._serialized_end = 33125 - _STOPBATCHOPERATIONRESPONSE._serialized_start = 33127 - _STOPBATCHOPERATIONRESPONSE._serialized_end = 33155 - _DESCRIBEBATCHOPERATIONREQUEST._serialized_start = 33157 - _DESCRIBEBATCHOPERATIONREQUEST._serialized_end = 33223 - _DESCRIBEBATCHOPERATIONRESPONSE._serialized_start = 33226 - _DESCRIBEBATCHOPERATIONRESPONSE._serialized_end = 33698 - _LISTBATCHOPERATIONSREQUEST._serialized_start = 33700 - _LISTBATCHOPERATIONSREQUEST._serialized_end = 33791 - _LISTBATCHOPERATIONSRESPONSE._serialized_start = 33793 - _LISTBATCHOPERATIONSRESPONSE._serialized_end = 33914 - _POLLWORKFLOWEXECUTIONUPDATEREQUEST._serialized_start = 33917 - _POLLWORKFLOWEXECUTIONUPDATEREQUEST._serialized_end = 34102 - _POLLWORKFLOWEXECUTIONUPDATERESPONSE._serialized_start = 34105 - _POLLWORKFLOWEXECUTIONUPDATERESPONSE._serialized_end = 34324 - _POLLNEXUSTASKQUEUEREQUEST._serialized_start = 34327 - _POLLNEXUSTASKQUEUEREQUEST._serialized_end = 34743 - _POLLNEXUSTASKQUEUERESPONSE._serialized_start = 34746 - _POLLNEXUSTASKQUEUERESPONSE._serialized_end = 35100 - _RESPONDNEXUSTASKCOMPLETEDREQUEST._serialized_start = 35103 - _RESPONDNEXUSTASKCOMPLETEDREQUEST._serialized_end = 35270 - _RESPONDNEXUSTASKCOMPLETEDRESPONSE._serialized_start = 35272 - _RESPONDNEXUSTASKCOMPLETEDRESPONSE._serialized_end = 35307 - _RESPONDNEXUSTASKFAILEDREQUEST._serialized_start = 35310 - _RESPONDNEXUSTASKFAILEDREQUEST._serialized_end = 35530 - _RESPONDNEXUSTASKFAILEDRESPONSE._serialized_start = 35532 - _RESPONDNEXUSTASKFAILEDRESPONSE._serialized_end = 35564 - _EXECUTEMULTIOPERATIONREQUEST._serialized_start = 35567 - _EXECUTEMULTIOPERATIONREQUEST._serialized_end = 35939 - _EXECUTEMULTIOPERATIONREQUEST_OPERATION._serialized_start = 35733 - _EXECUTEMULTIOPERATIONREQUEST_OPERATION._serialized_end = 35939 - _EXECUTEMULTIOPERATIONRESPONSE._serialized_start = 35942 - _EXECUTEMULTIOPERATIONRESPONSE._serialized_end = 36274 - _EXECUTEMULTIOPERATIONRESPONSE_RESPONSE._serialized_start = 36068 - _EXECUTEMULTIOPERATIONRESPONSE_RESPONSE._serialized_end = 36274 - _UPDATEACTIVITYOPTIONSREQUEST._serialized_start = 36277 - _UPDATEACTIVITYOPTIONSREQUEST._serialized_end = 36613 - _UPDATEACTIVITYEXECUTIONOPTIONSREQUEST._serialized_start = 36616 - _UPDATEACTIVITYEXECUTIONOPTIONSREQUEST._serialized_end = 36935 - _UPDATEACTIVITYOPTIONSRESPONSE._serialized_start = 36937 - _UPDATEACTIVITYOPTIONSRESPONSE._serialized_end = 37037 - _UPDATEACTIVITYEXECUTIONOPTIONSRESPONSE._serialized_start = 37039 - _UPDATEACTIVITYEXECUTIONOPTIONSRESPONSE._serialized_end = 37148 - _PAUSEACTIVITYREQUEST._serialized_start = 37151 - _PAUSEACTIVITYREQUEST._serialized_end = 37350 - _PAUSEACTIVITYEXECUTIONREQUEST._serialized_start = 37353 - _PAUSEACTIVITYEXECUTIONREQUEST._serialized_end = 37536 - _PAUSEACTIVITYRESPONSE._serialized_start = 37538 - _PAUSEACTIVITYRESPONSE._serialized_end = 37561 - _PAUSEACTIVITYEXECUTIONRESPONSE._serialized_start = 37563 - _PAUSEACTIVITYEXECUTIONRESPONSE._serialized_end = 37595 - _UNPAUSEACTIVITYREQUEST._serialized_start = 37598 - _UNPAUSEACTIVITYREQUEST._serialized_end = 37878 - _UNPAUSEACTIVITYEXECUTIONREQUEST._serialized_start = 37881 - _UNPAUSEACTIVITYEXECUTIONREQUEST._serialized_end = 38154 - _UNPAUSEACTIVITYRESPONSE._serialized_start = 38156 - _UNPAUSEACTIVITYRESPONSE._serialized_end = 38181 - _UNPAUSEACTIVITYEXECUTIONRESPONSE._serialized_start = 38183 - _UNPAUSEACTIVITYEXECUTIONRESPONSE._serialized_end = 38217 - _RESETACTIVITYREQUEST._serialized_start = 38220 - _RESETACTIVITYREQUEST._serialized_end = 38527 - _RESETACTIVITYEXECUTIONREQUEST._serialized_start = 38530 - _RESETACTIVITYEXECUTIONREQUEST._serialized_end = 38820 - _RESETACTIVITYRESPONSE._serialized_start = 38822 - _RESETACTIVITYRESPONSE._serialized_end = 38845 - _RESETACTIVITYEXECUTIONRESPONSE._serialized_start = 38847 - _RESETACTIVITYEXECUTIONRESPONSE._serialized_end = 38879 - _UPDATEWORKFLOWEXECUTIONOPTIONSREQUEST._serialized_start = 38882 - _UPDATEWORKFLOWEXECUTIONOPTIONSREQUEST._serialized_end = 39166 - _UPDATEWORKFLOWEXECUTIONOPTIONSRESPONSE._serialized_start = 39169 - _UPDATEWORKFLOWEXECUTIONOPTIONSRESPONSE._serialized_end = 39346 - _DESCRIBEDEPLOYMENTREQUEST._serialized_start = 39348 - _DESCRIBEDEPLOYMENTREQUEST._serialized_end = 39454 - _DESCRIBEDEPLOYMENTRESPONSE._serialized_start = 39456 - _DESCRIBEDEPLOYMENTRESPONSE._serialized_end = 39553 - _DESCRIBEWORKERDEPLOYMENTVERSIONREQUEST._serialized_start = 39556 - _DESCRIBEWORKERDEPLOYMENTVERSIONREQUEST._serialized_end = 39750 - _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_start = 39753 - _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_end = 40405 - _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE_VERSIONTASKQUEUE._serialized_start = 40014 - _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE_VERSIONTASKQUEUE._serialized_end = 40405 - _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE_VERSIONTASKQUEUE_STATSBYPRIORITYKEYENTRY._serialized_start = 23740 - _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE_VERSIONTASKQUEUE_STATSBYPRIORITYKEYENTRY._serialized_end = 23840 - _DESCRIBEWORKERDEPLOYMENTREQUEST._serialized_start = 40407 - _DESCRIBEWORKERDEPLOYMENTREQUEST._serialized_end = 40484 - _DESCRIBEWORKERDEPLOYMENTRESPONSE._serialized_start = 40487 - _DESCRIBEWORKERDEPLOYMENTRESPONSE._serialized_end = 40627 - _LISTDEPLOYMENTSREQUEST._serialized_start = 40629 - _LISTDEPLOYMENTSREQUEST._serialized_end = 40737 - _LISTDEPLOYMENTSRESPONSE._serialized_start = 40739 - _LISTDEPLOYMENTSRESPONSE._serialized_end = 40858 - _SETCURRENTDEPLOYMENTREQUEST._serialized_start = 40861 - _SETCURRENTDEPLOYMENTREQUEST._serialized_end = 41066 - _SETCURRENTDEPLOYMENTRESPONSE._serialized_start = 41069 - _SETCURRENTDEPLOYMENTRESPONSE._serialized_end = 41254 - _SETWORKERDEPLOYMENTCURRENTVERSIONREQUEST._serialized_start = 41257 - _SETWORKERDEPLOYMENTCURRENTVERSIONREQUEST._serialized_end = 41486 - _SETWORKERDEPLOYMENTCURRENTVERSIONRESPONSE._serialized_start = 41489 - _SETWORKERDEPLOYMENTCURRENTVERSIONRESPONSE._serialized_end = 41680 - _SETWORKERDEPLOYMENTRAMPINGVERSIONREQUEST._serialized_start = 41683 - _SETWORKERDEPLOYMENTRAMPINGVERSIONREQUEST._serialized_end = 41932 - _SETWORKERDEPLOYMENTRAMPINGVERSIONRESPONSE._serialized_start = 41935 - _SETWORKERDEPLOYMENTRAMPINGVERSIONRESPONSE._serialized_end = 42159 - _CREATEWORKERDEPLOYMENTREQUEST._serialized_start = 42161 - _CREATEWORKERDEPLOYMENTREQUEST._serialized_end = 42274 - _CREATEWORKERDEPLOYMENTRESPONSE._serialized_start = 42276 - _CREATEWORKERDEPLOYMENTRESPONSE._serialized_end = 42332 - _LISTWORKERDEPLOYMENTSREQUEST._serialized_start = 42334 - _LISTWORKERDEPLOYMENTSREQUEST._serialized_end = 42427 - _LISTWORKERDEPLOYMENTSRESPONSE._serialized_start = 42430 - _LISTWORKERDEPLOYMENTSRESPONSE._serialized_end = 43101 - _LISTWORKERDEPLOYMENTSRESPONSE_WORKERDEPLOYMENTSUMMARY._serialized_start = 42605 - _LISTWORKERDEPLOYMENTSRESPONSE_WORKERDEPLOYMENTSUMMARY._serialized_end = 43101 - _CREATEWORKERDEPLOYMENTVERSIONREQUEST._serialized_start = 43104 - _CREATEWORKERDEPLOYMENTVERSIONREQUEST._serialized_end = 43344 - _CREATEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_start = 43346 - _CREATEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_end = 43385 - _DELETEWORKERDEPLOYMENTVERSIONREQUEST._serialized_start = 43388 - _DELETEWORKERDEPLOYMENTVERSIONREQUEST._serialized_end = 43588 - _DELETEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_start = 43590 - _DELETEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_end = 43629 - _DELETEWORKERDEPLOYMENTREQUEST._serialized_start = 43631 - _DELETEWORKERDEPLOYMENTREQUEST._serialized_end = 43724 - _DELETEWORKERDEPLOYMENTRESPONSE._serialized_start = 43726 - _DELETEWORKERDEPLOYMENTRESPONSE._serialized_end = 43758 - _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST._serialized_start = 43761 - _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST._serialized_end = 44277 - _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST_COMPUTECONFIGSCALINGGROUPSENTRY._serialized_start = 44154 - _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST_COMPUTECONFIGSCALINGGROUPSENTRY._serialized_end = 44277 - _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGRESPONSE._serialized_start = 44279 - _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGRESPONSE._serialized_end = 44331 - _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST._serialized_start = 44334 - _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST._serialized_end = 44834 - _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST_COMPUTECONFIGSCALINGGROUPSENTRY._serialized_start = 44154 - _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST_COMPUTECONFIGSCALINGGROUPSENTRY._serialized_end = 44277 - _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGRESPONSE._serialized_start = 44836 - _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGRESPONSE._serialized_end = 44890 - _UPDATEWORKERDEPLOYMENTVERSIONMETADATAREQUEST._serialized_start = 44893 - _UPDATEWORKERDEPLOYMENTVERSIONMETADATAREQUEST._serialized_end = 45311 - _UPDATEWORKERDEPLOYMENTVERSIONMETADATAREQUEST_UPSERTENTRIESENTRY._serialized_start = 45226 + _UPDATEWORKERVERSIONINGRULESREQUEST_ADDCOMPATIBLEBUILDIDREDIRECTRULE._serialized_start = 31719 + _UPDATEWORKERVERSIONINGRULESREQUEST_ADDCOMPATIBLEBUILDIDREDIRECTRULE._serialized_end = 31825 + _UPDATEWORKERVERSIONINGRULESREQUEST_REPLACECOMPATIBLEBUILDIDREDIRECTRULE._serialized_start = 31827 + _UPDATEWORKERVERSIONINGRULESREQUEST_REPLACECOMPATIBLEBUILDIDREDIRECTRULE._serialized_end = 31937 + _UPDATEWORKERVERSIONINGRULESREQUEST_DELETECOMPATIBLEBUILDIDREDIRECTRULE._serialized_start = 31939 + _UPDATEWORKERVERSIONINGRULESREQUEST_DELETECOMPATIBLEBUILDIDREDIRECTRULE._serialized_end = 32001 + _UPDATEWORKERVERSIONINGRULESREQUEST_COMMITBUILDID._serialized_start = 32003 + _UPDATEWORKERVERSIONINGRULESREQUEST_COMMITBUILDID._serialized_end = 32058 + _UPDATEWORKERVERSIONINGRULESRESPONSE._serialized_start = 32074 + _UPDATEWORKERVERSIONINGRULESRESPONSE._serialized_end = 32326 + _GETWORKERVERSIONINGRULESREQUEST._serialized_start = 32328 + _GETWORKERVERSIONINGRULESREQUEST._serialized_end = 32400 + _GETWORKERVERSIONINGRULESRESPONSE._serialized_start = 32403 + _GETWORKERVERSIONINGRULESRESPONSE._serialized_end = 32652 + _GETWORKERTASKREACHABILITYREQUEST._serialized_start = 32655 + _GETWORKERTASKREACHABILITYREQUEST._serialized_end = 32811 + _GETWORKERTASKREACHABILITYRESPONSE._serialized_start = 32813 + _GETWORKERTASKREACHABILITYRESPONSE._serialized_end = 32927 + _UPDATEWORKFLOWEXECUTIONREQUEST._serialized_start = 32930 + _UPDATEWORKFLOWEXECUTIONREQUEST._serialized_end = 33191 + _UPDATEWORKFLOWEXECUTIONRESPONSE._serialized_start = 33194 + _UPDATEWORKFLOWEXECUTIONRESPONSE._serialized_end = 33453 + _STARTBATCHOPERATIONREQUEST._serialized_start = 33456 + _STARTBATCHOPERATIONREQUEST._serialized_end = 34822 + _STARTBATCHOPERATIONRESPONSE._serialized_start = 34824 + _STARTBATCHOPERATIONRESPONSE._serialized_end = 34853 + _STOPBATCHOPERATIONREQUEST._serialized_start = 34855 + _STOPBATCHOPERATIONREQUEST._serialized_end = 34951 + _STOPBATCHOPERATIONRESPONSE._serialized_start = 34953 + _STOPBATCHOPERATIONRESPONSE._serialized_end = 34981 + _DESCRIBEBATCHOPERATIONREQUEST._serialized_start = 34983 + _DESCRIBEBATCHOPERATIONREQUEST._serialized_end = 35049 + _DESCRIBEBATCHOPERATIONRESPONSE._serialized_start = 35052 + _DESCRIBEBATCHOPERATIONRESPONSE._serialized_end = 35524 + _LISTBATCHOPERATIONSREQUEST._serialized_start = 35526 + _LISTBATCHOPERATIONSREQUEST._serialized_end = 35617 + _LISTBATCHOPERATIONSRESPONSE._serialized_start = 35619 + _LISTBATCHOPERATIONSRESPONSE._serialized_end = 35740 + _POLLWORKFLOWEXECUTIONUPDATEREQUEST._serialized_start = 35743 + _POLLWORKFLOWEXECUTIONUPDATEREQUEST._serialized_end = 35928 + _POLLWORKFLOWEXECUTIONUPDATERESPONSE._serialized_start = 35931 + _POLLWORKFLOWEXECUTIONUPDATERESPONSE._serialized_end = 36150 + _POLLNEXUSTASKQUEUEREQUEST._serialized_start = 36153 + _POLLNEXUSTASKQUEUEREQUEST._serialized_end = 36569 + _POLLNEXUSTASKQUEUERESPONSE._serialized_start = 36572 + _POLLNEXUSTASKQUEUERESPONSE._serialized_end = 36926 + _RESPONDNEXUSTASKCOMPLETEDREQUEST._serialized_start = 36929 + _RESPONDNEXUSTASKCOMPLETEDREQUEST._serialized_end = 37096 + _RESPONDNEXUSTASKCOMPLETEDRESPONSE._serialized_start = 37098 + _RESPONDNEXUSTASKCOMPLETEDRESPONSE._serialized_end = 37133 + _RESPONDNEXUSTASKFAILEDREQUEST._serialized_start = 37136 + _RESPONDNEXUSTASKFAILEDREQUEST._serialized_end = 37356 + _RESPONDNEXUSTASKFAILEDRESPONSE._serialized_start = 37358 + _RESPONDNEXUSTASKFAILEDRESPONSE._serialized_end = 37390 + _EXECUTEMULTIOPERATIONREQUEST._serialized_start = 37393 + _EXECUTEMULTIOPERATIONREQUEST._serialized_end = 37765 + _EXECUTEMULTIOPERATIONREQUEST_OPERATION._serialized_start = 37559 + _EXECUTEMULTIOPERATIONREQUEST_OPERATION._serialized_end = 37765 + _EXECUTEMULTIOPERATIONRESPONSE._serialized_start = 37768 + _EXECUTEMULTIOPERATIONRESPONSE._serialized_end = 38100 + _EXECUTEMULTIOPERATIONRESPONSE_RESPONSE._serialized_start = 37894 + _EXECUTEMULTIOPERATIONRESPONSE_RESPONSE._serialized_end = 38100 + _UPDATEACTIVITYOPTIONSREQUEST._serialized_start = 38103 + _UPDATEACTIVITYOPTIONSREQUEST._serialized_end = 38439 + _UPDATEACTIVITYEXECUTIONOPTIONSREQUEST._serialized_start = 38442 + _UPDATEACTIVITYEXECUTIONOPTIONSREQUEST._serialized_end = 38761 + _UPDATEACTIVITYOPTIONSRESPONSE._serialized_start = 38763 + _UPDATEACTIVITYOPTIONSRESPONSE._serialized_end = 38863 + _UPDATEACTIVITYEXECUTIONOPTIONSRESPONSE._serialized_start = 38865 + _UPDATEACTIVITYEXECUTIONOPTIONSRESPONSE._serialized_end = 38974 + _PAUSEACTIVITYREQUEST._serialized_start = 38977 + _PAUSEACTIVITYREQUEST._serialized_end = 39176 + _PAUSEACTIVITYEXECUTIONREQUEST._serialized_start = 39179 + _PAUSEACTIVITYEXECUTIONREQUEST._serialized_end = 39362 + _PAUSEACTIVITYRESPONSE._serialized_start = 39364 + _PAUSEACTIVITYRESPONSE._serialized_end = 39387 + _PAUSEACTIVITYEXECUTIONRESPONSE._serialized_start = 39389 + _PAUSEACTIVITYEXECUTIONRESPONSE._serialized_end = 39421 + _UNPAUSEACTIVITYREQUEST._serialized_start = 39424 + _UNPAUSEACTIVITYREQUEST._serialized_end = 39704 + _UNPAUSEACTIVITYEXECUTIONREQUEST._serialized_start = 39707 + _UNPAUSEACTIVITYEXECUTIONREQUEST._serialized_end = 39980 + _UNPAUSEACTIVITYRESPONSE._serialized_start = 39982 + _UNPAUSEACTIVITYRESPONSE._serialized_end = 40007 + _UNPAUSEACTIVITYEXECUTIONRESPONSE._serialized_start = 40009 + _UNPAUSEACTIVITYEXECUTIONRESPONSE._serialized_end = 40043 + _RESETACTIVITYREQUEST._serialized_start = 40046 + _RESETACTIVITYREQUEST._serialized_end = 40353 + _RESETACTIVITYEXECUTIONREQUEST._serialized_start = 40356 + _RESETACTIVITYEXECUTIONREQUEST._serialized_end = 40646 + _RESETACTIVITYRESPONSE._serialized_start = 40648 + _RESETACTIVITYRESPONSE._serialized_end = 40671 + _RESETACTIVITYEXECUTIONRESPONSE._serialized_start = 40673 + _RESETACTIVITYEXECUTIONRESPONSE._serialized_end = 40705 + _UPDATEWORKFLOWEXECUTIONOPTIONSREQUEST._serialized_start = 40708 + _UPDATEWORKFLOWEXECUTIONOPTIONSREQUEST._serialized_end = 40992 + _UPDATEWORKFLOWEXECUTIONOPTIONSRESPONSE._serialized_start = 40995 + _UPDATEWORKFLOWEXECUTIONOPTIONSRESPONSE._serialized_end = 41172 + _DESCRIBEDEPLOYMENTREQUEST._serialized_start = 41174 + _DESCRIBEDEPLOYMENTREQUEST._serialized_end = 41280 + _DESCRIBEDEPLOYMENTRESPONSE._serialized_start = 41282 + _DESCRIBEDEPLOYMENTRESPONSE._serialized_end = 41379 + _DESCRIBEWORKERDEPLOYMENTVERSIONREQUEST._serialized_start = 41382 + _DESCRIBEWORKERDEPLOYMENTVERSIONREQUEST._serialized_end = 41576 + _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_start = 41579 + _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_end = 42231 + _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE_VERSIONTASKQUEUE._serialized_start = 41840 + _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE_VERSIONTASKQUEUE._serialized_end = 42231 + _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE_VERSIONTASKQUEUE_STATSBYPRIORITYKEYENTRY._serialized_start = 25566 + _DESCRIBEWORKERDEPLOYMENTVERSIONRESPONSE_VERSIONTASKQUEUE_STATSBYPRIORITYKEYENTRY._serialized_end = 25666 + _DESCRIBEWORKERDEPLOYMENTREQUEST._serialized_start = 42233 + _DESCRIBEWORKERDEPLOYMENTREQUEST._serialized_end = 42310 + _DESCRIBEWORKERDEPLOYMENTRESPONSE._serialized_start = 42313 + _DESCRIBEWORKERDEPLOYMENTRESPONSE._serialized_end = 42453 + _LISTDEPLOYMENTSREQUEST._serialized_start = 42455 + _LISTDEPLOYMENTSREQUEST._serialized_end = 42563 + _LISTDEPLOYMENTSRESPONSE._serialized_start = 42565 + _LISTDEPLOYMENTSRESPONSE._serialized_end = 42684 + _SETCURRENTDEPLOYMENTREQUEST._serialized_start = 42687 + _SETCURRENTDEPLOYMENTREQUEST._serialized_end = 42892 + _SETCURRENTDEPLOYMENTRESPONSE._serialized_start = 42895 + _SETCURRENTDEPLOYMENTRESPONSE._serialized_end = 43080 + _SETWORKERDEPLOYMENTCURRENTVERSIONREQUEST._serialized_start = 43083 + _SETWORKERDEPLOYMENTCURRENTVERSIONREQUEST._serialized_end = 43312 + _SETWORKERDEPLOYMENTCURRENTVERSIONRESPONSE._serialized_start = 43315 + _SETWORKERDEPLOYMENTCURRENTVERSIONRESPONSE._serialized_end = 43506 + _SETWORKERDEPLOYMENTRAMPINGVERSIONREQUEST._serialized_start = 43509 + _SETWORKERDEPLOYMENTRAMPINGVERSIONREQUEST._serialized_end = 43758 + _SETWORKERDEPLOYMENTRAMPINGVERSIONRESPONSE._serialized_start = 43761 + _SETWORKERDEPLOYMENTRAMPINGVERSIONRESPONSE._serialized_end = 43985 + _CREATEWORKERDEPLOYMENTREQUEST._serialized_start = 43987 + _CREATEWORKERDEPLOYMENTREQUEST._serialized_end = 44100 + _CREATEWORKERDEPLOYMENTRESPONSE._serialized_start = 44102 + _CREATEWORKERDEPLOYMENTRESPONSE._serialized_end = 44158 + _LISTWORKERDEPLOYMENTSREQUEST._serialized_start = 44160 + _LISTWORKERDEPLOYMENTSREQUEST._serialized_end = 44253 + _LISTWORKERDEPLOYMENTSRESPONSE._serialized_start = 44256 + _LISTWORKERDEPLOYMENTSRESPONSE._serialized_end = 44927 + _LISTWORKERDEPLOYMENTSRESPONSE_WORKERDEPLOYMENTSUMMARY._serialized_start = 44431 + _LISTWORKERDEPLOYMENTSRESPONSE_WORKERDEPLOYMENTSUMMARY._serialized_end = 44927 + _CREATEWORKERDEPLOYMENTVERSIONREQUEST._serialized_start = 44930 + _CREATEWORKERDEPLOYMENTVERSIONREQUEST._serialized_end = 45170 + _CREATEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_start = 45172 + _CREATEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_end = 45211 + _DELETEWORKERDEPLOYMENTVERSIONREQUEST._serialized_start = 45214 + _DELETEWORKERDEPLOYMENTVERSIONREQUEST._serialized_end = 45414 + _DELETEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_start = 45416 + _DELETEWORKERDEPLOYMENTVERSIONRESPONSE._serialized_end = 45455 + _DELETEWORKERDEPLOYMENTREQUEST._serialized_start = 45457 + _DELETEWORKERDEPLOYMENTREQUEST._serialized_end = 45550 + _DELETEWORKERDEPLOYMENTRESPONSE._serialized_start = 45552 + _DELETEWORKERDEPLOYMENTRESPONSE._serialized_end = 45584 + _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST._serialized_start = 45587 + _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST._serialized_end = 46103 + _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST_COMPUTECONFIGSCALINGGROUPSENTRY._serialized_start = 45980 + _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST_COMPUTECONFIGSCALINGGROUPSENTRY._serialized_end = 46103 + _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGRESPONSE._serialized_start = 46105 + _UPDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGRESPONSE._serialized_end = 46157 + _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST._serialized_start = 46160 + _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST._serialized_end = 46660 + _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST_COMPUTECONFIGSCALINGGROUPSENTRY._serialized_start = 45980 + _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGREQUEST_COMPUTECONFIGSCALINGGROUPSENTRY._serialized_end = 46103 + _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGRESPONSE._serialized_start = 46662 + _VALIDATEWORKERDEPLOYMENTVERSIONCOMPUTECONFIGRESPONSE._serialized_end = 46716 + _UPDATEWORKERDEPLOYMENTVERSIONMETADATAREQUEST._serialized_start = 46719 + _UPDATEWORKERDEPLOYMENTVERSIONMETADATAREQUEST._serialized_end = 47137 + _UPDATEWORKERDEPLOYMENTVERSIONMETADATAREQUEST_UPSERTENTRIESENTRY._serialized_start = 47052 _UPDATEWORKERDEPLOYMENTVERSIONMETADATAREQUEST_UPSERTENTRIESENTRY._serialized_end = ( - 45311 + 47137 ) - _UPDATEWORKERDEPLOYMENTVERSIONMETADATARESPONSE._serialized_start = 45313 - _UPDATEWORKERDEPLOYMENTVERSIONMETADATARESPONSE._serialized_end = 45423 - _SETWORKERDEPLOYMENTMANAGERREQUEST._serialized_start = 45426 - _SETWORKERDEPLOYMENTMANAGERREQUEST._serialized_end = 45615 - _SETWORKERDEPLOYMENTMANAGERRESPONSE._serialized_start = 45617 - _SETWORKERDEPLOYMENTMANAGERRESPONSE._serialized_end = 45716 - _GETCURRENTDEPLOYMENTREQUEST._serialized_start = 45718 - _GETCURRENTDEPLOYMENTREQUEST._serialized_end = 45787 - _GETCURRENTDEPLOYMENTRESPONSE._serialized_start = 45789 - _GETCURRENTDEPLOYMENTRESPONSE._serialized_end = 45896 - _GETDEPLOYMENTREACHABILITYREQUEST._serialized_start = 45898 - _GETDEPLOYMENTREACHABILITYREQUEST._serialized_end = 46011 - _GETDEPLOYMENTREACHABILITYRESPONSE._serialized_start = 46014 - _GETDEPLOYMENTREACHABILITYRESPONSE._serialized_end = 46241 - _CREATEWORKFLOWRULEREQUEST._serialized_start = 46244 - _CREATEWORKFLOWRULEREQUEST._serialized_end = 46424 - _CREATEWORKFLOWRULERESPONSE._serialized_start = 46426 - _CREATEWORKFLOWRULERESPONSE._serialized_end = 46521 - _DESCRIBEWORKFLOWRULEREQUEST._serialized_start = 46523 - _DESCRIBEWORKFLOWRULEREQUEST._serialized_end = 46588 - _DESCRIBEWORKFLOWRULERESPONSE._serialized_start = 46590 - _DESCRIBEWORKFLOWRULERESPONSE._serialized_end = 46671 - _DELETEWORKFLOWRULEREQUEST._serialized_start = 46673 - _DELETEWORKFLOWRULEREQUEST._serialized_end = 46736 - _DELETEWORKFLOWRULERESPONSE._serialized_start = 46738 - _DELETEWORKFLOWRULERESPONSE._serialized_end = 46766 - _LISTWORKFLOWRULESREQUEST._serialized_start = 46768 - _LISTWORKFLOWRULESREQUEST._serialized_end = 46838 - _LISTWORKFLOWRULESRESPONSE._serialized_start = 46840 - _LISTWORKFLOWRULESRESPONSE._serialized_end = 46944 - _TRIGGERWORKFLOWRULEREQUEST._serialized_start = 46947 - _TRIGGERWORKFLOWRULEREQUEST._serialized_end = 47153 - _TRIGGERWORKFLOWRULERESPONSE._serialized_start = 47155 - _TRIGGERWORKFLOWRULERESPONSE._serialized_end = 47201 - _RECORDWORKERHEARTBEATREQUEST._serialized_start = 47204 - _RECORDWORKERHEARTBEATREQUEST._serialized_end = 47359 - _RECORDWORKERHEARTBEATRESPONSE._serialized_start = 47361 - _RECORDWORKERHEARTBEATRESPONSE._serialized_end = 47392 - _LISTWORKERSREQUEST._serialized_start = 47395 - _LISTWORKERSREQUEST._serialized_end = 47525 - _LISTWORKERSRESPONSE._serialized_start = 47528 - _LISTWORKERSRESPONSE._serialized_end = 47693 - _UPDATETASKQUEUECONFIGREQUEST._serialized_start = 47696 - _UPDATETASKQUEUECONFIGREQUEST._serialized_end = 48421 - _UPDATETASKQUEUECONFIGREQUEST_RATELIMITUPDATE._serialized_start = 48263 - _UPDATETASKQUEUECONFIGREQUEST_RATELIMITUPDATE._serialized_end = 48354 + _UPDATEWORKERDEPLOYMENTVERSIONMETADATARESPONSE._serialized_start = 47139 + _UPDATEWORKERDEPLOYMENTVERSIONMETADATARESPONSE._serialized_end = 47249 + _SETWORKERDEPLOYMENTMANAGERREQUEST._serialized_start = 47252 + _SETWORKERDEPLOYMENTMANAGERREQUEST._serialized_end = 47441 + _SETWORKERDEPLOYMENTMANAGERRESPONSE._serialized_start = 47443 + _SETWORKERDEPLOYMENTMANAGERRESPONSE._serialized_end = 47542 + _GETCURRENTDEPLOYMENTREQUEST._serialized_start = 47544 + _GETCURRENTDEPLOYMENTREQUEST._serialized_end = 47613 + _GETCURRENTDEPLOYMENTRESPONSE._serialized_start = 47615 + _GETCURRENTDEPLOYMENTRESPONSE._serialized_end = 47722 + _GETDEPLOYMENTREACHABILITYREQUEST._serialized_start = 47724 + _GETDEPLOYMENTREACHABILITYREQUEST._serialized_end = 47837 + _GETDEPLOYMENTREACHABILITYRESPONSE._serialized_start = 47840 + _GETDEPLOYMENTREACHABILITYRESPONSE._serialized_end = 48067 + _CREATEWORKFLOWRULEREQUEST._serialized_start = 48070 + _CREATEWORKFLOWRULEREQUEST._serialized_end = 48250 + _CREATEWORKFLOWRULERESPONSE._serialized_start = 48252 + _CREATEWORKFLOWRULERESPONSE._serialized_end = 48347 + _DESCRIBEWORKFLOWRULEREQUEST._serialized_start = 48349 + _DESCRIBEWORKFLOWRULEREQUEST._serialized_end = 48414 + _DESCRIBEWORKFLOWRULERESPONSE._serialized_start = 48416 + _DESCRIBEWORKFLOWRULERESPONSE._serialized_end = 48497 + _DELETEWORKFLOWRULEREQUEST._serialized_start = 48499 + _DELETEWORKFLOWRULEREQUEST._serialized_end = 48562 + _DELETEWORKFLOWRULERESPONSE._serialized_start = 48564 + _DELETEWORKFLOWRULERESPONSE._serialized_end = 48592 + _LISTWORKFLOWRULESREQUEST._serialized_start = 48594 + _LISTWORKFLOWRULESREQUEST._serialized_end = 48664 + _LISTWORKFLOWRULESRESPONSE._serialized_start = 48666 + _LISTWORKFLOWRULESRESPONSE._serialized_end = 48770 + _TRIGGERWORKFLOWRULEREQUEST._serialized_start = 48773 + _TRIGGERWORKFLOWRULEREQUEST._serialized_end = 48979 + _TRIGGERWORKFLOWRULERESPONSE._serialized_start = 48981 + _TRIGGERWORKFLOWRULERESPONSE._serialized_end = 49027 + _RECORDWORKERHEARTBEATREQUEST._serialized_start = 49030 + _RECORDWORKERHEARTBEATREQUEST._serialized_end = 49185 + _RECORDWORKERHEARTBEATRESPONSE._serialized_start = 49187 + _RECORDWORKERHEARTBEATRESPONSE._serialized_end = 49218 + _LISTWORKERSREQUEST._serialized_start = 49221 + _LISTWORKERSREQUEST._serialized_end = 49351 + _LISTWORKERSRESPONSE._serialized_start = 49354 + _LISTWORKERSRESPONSE._serialized_end = 49519 + _UPDATETASKQUEUECONFIGREQUEST._serialized_start = 49522 + _UPDATETASKQUEUECONFIGREQUEST._serialized_end = 50247 + _UPDATETASKQUEUECONFIGREQUEST_RATELIMITUPDATE._serialized_start = 50089 + _UPDATETASKQUEUECONFIGREQUEST_RATELIMITUPDATE._serialized_end = 50180 _UPDATETASKQUEUECONFIGREQUEST_SETFAIRNESSWEIGHTOVERRIDESENTRY._serialized_start = ( - 48356 + 50182 ) _UPDATETASKQUEUECONFIGREQUEST_SETFAIRNESSWEIGHTOVERRIDESENTRY._serialized_end = ( - 48421 + 50247 ) - _UPDATETASKQUEUECONFIGRESPONSE._serialized_start = 48423 - _UPDATETASKQUEUECONFIGRESPONSE._serialized_end = 48514 - _FETCHWORKERCONFIGREQUEST._serialized_start = 48517 - _FETCHWORKERCONFIGREQUEST._serialized_end = 48675 - _FETCHWORKERCONFIGRESPONSE._serialized_start = 48677 - _FETCHWORKERCONFIGRESPONSE._serialized_end = 48762 - _UPDATEWORKERCONFIGREQUEST._serialized_start = 48765 - _UPDATEWORKERCONFIGREQUEST._serialized_end = 49031 - _UPDATEWORKERCONFIGRESPONSE._serialized_start = 49033 - _UPDATEWORKERCONFIGRESPONSE._serialized_end = 49133 - _DESCRIBEWORKERREQUEST._serialized_start = 49135 - _DESCRIBEWORKERREQUEST._serialized_end = 49206 - _DESCRIBEWORKERRESPONSE._serialized_start = 49208 - _DESCRIBEWORKERRESPONSE._serialized_end = 49289 - _COUNTWORKERSREQUEST._serialized_start = 49291 - _COUNTWORKERSREQUEST._serialized_end = 49378 - _COUNTWORKERSRESPONSE._serialized_start = 49380 - _COUNTWORKERSRESPONSE._serialized_end = 49417 - _PAUSEWORKFLOWEXECUTIONREQUEST._serialized_start = 49420 - _PAUSEWORKFLOWEXECUTIONREQUEST._serialized_end = 49561 - _PAUSEWORKFLOWEXECUTIONRESPONSE._serialized_start = 49563 - _PAUSEWORKFLOWEXECUTIONRESPONSE._serialized_end = 49595 - _UNPAUSEWORKFLOWEXECUTIONREQUEST._serialized_start = 49598 - _UNPAUSEWORKFLOWEXECUTIONREQUEST._serialized_end = 49741 - _UNPAUSEWORKFLOWEXECUTIONRESPONSE._serialized_start = 49743 - _UNPAUSEWORKFLOWEXECUTIONRESPONSE._serialized_end = 49777 - _STARTACTIVITYEXECUTIONREQUEST._serialized_start = 49780 - _STARTACTIVITYEXECUTIONREQUEST._serialized_end = 50957 - _STARTACTIVITYEXECUTIONRESPONSE._serialized_start = 50959 - _STARTACTIVITYEXECUTIONRESPONSE._serialized_end = 51068 - _DESCRIBEACTIVITYEXECUTIONREQUEST._serialized_start = 51071 - _DESCRIBEACTIVITYEXECUTIONREQUEST._serialized_end = 51299 - _DESCRIBEACTIVITYEXECUTIONRESPONSE._serialized_start = 51302 - _DESCRIBEACTIVITYEXECUTIONRESPONSE._serialized_end = 51618 - _POLLACTIVITYEXECUTIONREQUEST._serialized_start = 51620 - _POLLACTIVITYEXECUTIONREQUEST._serialized_end = 51706 - _POLLACTIVITYEXECUTIONRESPONSE._serialized_start = 51708 - _POLLACTIVITYEXECUTIONRESPONSE._serialized_end = 51824 - _LISTACTIVITYEXECUTIONSREQUEST._serialized_start = 51826 - _LISTACTIVITYEXECUTIONSREQUEST._serialized_end = 51935 - _LISTACTIVITYEXECUTIONSRESPONSE._serialized_start = 51938 - _LISTACTIVITYEXECUTIONSRESPONSE._serialized_end = 52068 - _STARTNEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 52071 - _STARTNEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 52920 - _STARTNEXUSOPERATIONEXECUTIONREQUEST_NEXUSHEADERENTRY._serialized_start = 52870 - _STARTNEXUSOPERATIONEXECUTIONREQUEST_NEXUSHEADERENTRY._serialized_end = 52920 - _STARTNEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 52922 - _STARTNEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 52993 - _DESCRIBENEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 52996 - _DESCRIBENEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 53166 - _DESCRIBENEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 53169 - _DESCRIBENEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 53480 - _POLLNEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 53483 - _POLLNEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 53644 - _POLLNEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 53647 - _POLLNEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 53908 - _LISTNEXUSOPERATIONEXECUTIONSREQUEST._serialized_start = 53910 - _LISTNEXUSOPERATIONEXECUTIONSREQUEST._serialized_end = 54025 - _LISTNEXUSOPERATIONEXECUTIONSRESPONSE._serialized_start = 54028 - _LISTNEXUSOPERATIONEXECUTIONSRESPONSE._serialized_end = 54167 - _COUNTACTIVITYEXECUTIONSREQUEST._serialized_start = 54169 - _COUNTACTIVITYEXECUTIONSREQUEST._serialized_end = 54235 - _COUNTACTIVITYEXECUTIONSRESPONSE._serialized_start = 54238 - _COUNTACTIVITYEXECUTIONSRESPONSE._serialized_end = 54475 - _COUNTACTIVITYEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_start = 20134 - _COUNTACTIVITYEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_end = 20222 - _COUNTNEXUSOPERATIONEXECUTIONSREQUEST._serialized_start = 54477 - _COUNTNEXUSOPERATIONEXECUTIONSREQUEST._serialized_end = 54549 - _COUNTNEXUSOPERATIONEXECUTIONSRESPONSE._serialized_start = 54552 - _COUNTNEXUSOPERATIONEXECUTIONSRESPONSE._serialized_end = 54801 - _COUNTNEXUSOPERATIONEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_start = 20134 - _COUNTNEXUSOPERATIONEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_end = 20222 - _REQUESTCANCELACTIVITYEXECUTIONREQUEST._serialized_start = 54804 - _REQUESTCANCELACTIVITYEXECUTIONREQUEST._serialized_end = 54953 - _REQUESTCANCELACTIVITYEXECUTIONRESPONSE._serialized_start = 54955 - _REQUESTCANCELACTIVITYEXECUTIONRESPONSE._serialized_end = 54995 - _TERMINATEACTIVITYEXECUTIONREQUEST._serialized_start = 54998 - _TERMINATEACTIVITYEXECUTIONREQUEST._serialized_end = 55143 - _TERMINATEACTIVITYEXECUTIONRESPONSE._serialized_start = 55145 - _TERMINATEACTIVITYEXECUTIONRESPONSE._serialized_end = 55181 - _DELETEACTIVITYEXECUTIONREQUEST._serialized_start = 55183 - _DELETEACTIVITYEXECUTIONREQUEST._serialized_end = 55271 - _DELETEACTIVITYEXECUTIONRESPONSE._serialized_start = 55273 - _DELETEACTIVITYEXECUTIONRESPONSE._serialized_end = 55306 - _REQUESTCANCELNEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 55309 - _REQUESTCANCELNEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 55465 - _REQUESTCANCELNEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 55467 - _REQUESTCANCELNEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 55513 - _TERMINATENEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 55516 - _TERMINATENEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 55668 - _TERMINATENEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 55670 - _TERMINATENEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 55712 - _DELETENEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 55714 - _DELETENEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 55809 - _DELETENEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 55811 - _DELETENEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 55850 - _POLLWORKFLOWEXECUTIONTIMESKIPPINGREQUEST._serialized_start = 55853 - _POLLWORKFLOWEXECUTIONTIMESKIPPINGREQUEST._serialized_end = 56010 - _POLLWORKFLOWEXECUTIONTIMESKIPPINGRESPONSE._serialized_start = 56013 - _POLLWORKFLOWEXECUTIONTIMESKIPPINGRESPONSE._serialized_end = 56245 + _UPDATETASKQUEUECONFIGRESPONSE._serialized_start = 50249 + _UPDATETASKQUEUECONFIGRESPONSE._serialized_end = 50340 + _FETCHWORKERCONFIGREQUEST._serialized_start = 50343 + _FETCHWORKERCONFIGREQUEST._serialized_end = 50501 + _FETCHWORKERCONFIGRESPONSE._serialized_start = 50503 + _FETCHWORKERCONFIGRESPONSE._serialized_end = 50588 + _UPDATEWORKERCONFIGREQUEST._serialized_start = 50591 + _UPDATEWORKERCONFIGREQUEST._serialized_end = 50857 + _UPDATEWORKERCONFIGRESPONSE._serialized_start = 50859 + _UPDATEWORKERCONFIGRESPONSE._serialized_end = 50959 + _DESCRIBEWORKERREQUEST._serialized_start = 50961 + _DESCRIBEWORKERREQUEST._serialized_end = 51032 + _DESCRIBEWORKERRESPONSE._serialized_start = 51034 + _DESCRIBEWORKERRESPONSE._serialized_end = 51115 + _COUNTWORKERSREQUEST._serialized_start = 51117 + _COUNTWORKERSREQUEST._serialized_end = 51204 + _COUNTWORKERSRESPONSE._serialized_start = 51206 + _COUNTWORKERSRESPONSE._serialized_end = 51243 + _PAUSEWORKFLOWEXECUTIONREQUEST._serialized_start = 51246 + _PAUSEWORKFLOWEXECUTIONREQUEST._serialized_end = 51387 + _PAUSEWORKFLOWEXECUTIONRESPONSE._serialized_start = 51389 + _PAUSEWORKFLOWEXECUTIONRESPONSE._serialized_end = 51421 + _UNPAUSEWORKFLOWEXECUTIONREQUEST._serialized_start = 51424 + _UNPAUSEWORKFLOWEXECUTIONREQUEST._serialized_end = 51567 + _UNPAUSEWORKFLOWEXECUTIONRESPONSE._serialized_start = 51569 + _UNPAUSEWORKFLOWEXECUTIONRESPONSE._serialized_end = 51603 + _STARTACTIVITYEXECUTIONREQUEST._serialized_start = 51606 + _STARTACTIVITYEXECUTIONREQUEST._serialized_end = 52783 + _STARTACTIVITYEXECUTIONRESPONSE._serialized_start = 52785 + _STARTACTIVITYEXECUTIONRESPONSE._serialized_end = 52894 + _DESCRIBEACTIVITYEXECUTIONREQUEST._serialized_start = 52897 + _DESCRIBEACTIVITYEXECUTIONREQUEST._serialized_end = 53125 + _DESCRIBEACTIVITYEXECUTIONRESPONSE._serialized_start = 53128 + _DESCRIBEACTIVITYEXECUTIONRESPONSE._serialized_end = 53444 + _POLLACTIVITYEXECUTIONREQUEST._serialized_start = 53446 + _POLLACTIVITYEXECUTIONREQUEST._serialized_end = 53532 + _POLLACTIVITYEXECUTIONRESPONSE._serialized_start = 53534 + _POLLACTIVITYEXECUTIONRESPONSE._serialized_end = 53650 + _LISTACTIVITYEXECUTIONSREQUEST._serialized_start = 53652 + _LISTACTIVITYEXECUTIONSREQUEST._serialized_end = 53761 + _LISTACTIVITYEXECUTIONSRESPONSE._serialized_start = 53764 + _LISTACTIVITYEXECUTIONSRESPONSE._serialized_end = 53894 + _STARTNEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 53897 + _STARTNEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 54935 + _STARTNEXUSOPERATIONEXECUTIONREQUEST_NEXUSHEADERENTRY._serialized_start = 54885 + _STARTNEXUSOPERATIONEXECUTIONREQUEST_NEXUSHEADERENTRY._serialized_end = 54935 + _STARTNEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 54937 + _STARTNEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 55008 + _DESCRIBENEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 55011 + _DESCRIBENEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 55181 + _DESCRIBENEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 55184 + _DESCRIBENEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 55571 + _POLLNEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 55574 + _POLLNEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 55735 + _POLLNEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 55738 + _POLLNEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 55999 + _LISTNEXUSOPERATIONEXECUTIONSREQUEST._serialized_start = 56001 + _LISTNEXUSOPERATIONEXECUTIONSREQUEST._serialized_end = 56116 + _LISTNEXUSOPERATIONEXECUTIONSRESPONSE._serialized_start = 56119 + _LISTNEXUSOPERATIONEXECUTIONSRESPONSE._serialized_end = 56258 + _COUNTACTIVITYEXECUTIONSREQUEST._serialized_start = 56260 + _COUNTACTIVITYEXECUTIONSREQUEST._serialized_end = 56326 + _COUNTACTIVITYEXECUTIONSRESPONSE._serialized_start = 56329 + _COUNTACTIVITYEXECUTIONSRESPONSE._serialized_end = 56566 + _COUNTACTIVITYEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_start = 21878 + _COUNTACTIVITYEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_end = 21966 + _COUNTNEXUSOPERATIONEXECUTIONSREQUEST._serialized_start = 56568 + _COUNTNEXUSOPERATIONEXECUTIONSREQUEST._serialized_end = 56640 + _COUNTNEXUSOPERATIONEXECUTIONSRESPONSE._serialized_start = 56643 + _COUNTNEXUSOPERATIONEXECUTIONSRESPONSE._serialized_end = 56892 + _COUNTNEXUSOPERATIONEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_start = 21878 + _COUNTNEXUSOPERATIONEXECUTIONSRESPONSE_AGGREGATIONGROUP._serialized_end = 21966 + _REQUESTCANCELACTIVITYEXECUTIONREQUEST._serialized_start = 56895 + _REQUESTCANCELACTIVITYEXECUTIONREQUEST._serialized_end = 57044 + _REQUESTCANCELACTIVITYEXECUTIONRESPONSE._serialized_start = 57046 + _REQUESTCANCELACTIVITYEXECUTIONRESPONSE._serialized_end = 57086 + _TERMINATEACTIVITYEXECUTIONREQUEST._serialized_start = 57089 + _TERMINATEACTIVITYEXECUTIONREQUEST._serialized_end = 57234 + _TERMINATEACTIVITYEXECUTIONRESPONSE._serialized_start = 57236 + _TERMINATEACTIVITYEXECUTIONRESPONSE._serialized_end = 57272 + _DELETEACTIVITYEXECUTIONREQUEST._serialized_start = 57274 + _DELETEACTIVITYEXECUTIONREQUEST._serialized_end = 57362 + _DELETEACTIVITYEXECUTIONRESPONSE._serialized_start = 57364 + _DELETEACTIVITYEXECUTIONRESPONSE._serialized_end = 57397 + _REQUESTCANCELNEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 57400 + _REQUESTCANCELNEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 57556 + _REQUESTCANCELNEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 57558 + _REQUESTCANCELNEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 57604 + _TERMINATENEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 57607 + _TERMINATENEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 57759 + _TERMINATENEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 57761 + _TERMINATENEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 57803 + _DELETENEXUSOPERATIONEXECUTIONREQUEST._serialized_start = 57805 + _DELETENEXUSOPERATIONEXECUTIONREQUEST._serialized_end = 57900 + _DELETENEXUSOPERATIONEXECUTIONRESPONSE._serialized_start = 57902 + _DELETENEXUSOPERATIONEXECUTIONRESPONSE._serialized_end = 57941 + _POLLWORKFLOWEXECUTIONTIMESKIPPINGREQUEST._serialized_start = 57944 + _POLLWORKFLOWEXECUTIONTIMESKIPPINGREQUEST._serialized_end = 58101 + _POLLWORKFLOWEXECUTIONTIMESKIPPINGRESPONSE._serialized_start = 58104 + _POLLWORKFLOWEXECUTIONTIMESKIPPINGRESPONSE._serialized_end = 58336 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/workflowservice/v1/request_response_pb2.pyi b/temporalio/api/workflowservice/v1/request_response_pb2.pyi index eb92437164..8e463e8f99 100644 --- a/temporalio/api/workflowservice/v1/request_response_pb2.pyi +++ b/temporalio/api/workflowservice/v1/request_response_pb2.pyi @@ -38,6 +38,8 @@ import temporalio.api.filter.v1.message_pb2 import temporalio.api.history.v1.message_pb2 import temporalio.api.namespace.v1.message_pb2 import temporalio.api.nexus.v1.message_pb2 +import temporalio.api.nexusoperation.v1.message_pb2 +import temporalio.api.notification.v1.message_pb2 import temporalio.api.protocol.v1.message_pb2 import temporalio.api.query.v1.message_pb2 import temporalio.api.replication.v1.message_pb2 @@ -46,6 +48,7 @@ import temporalio.api.schedule.v1.message_pb2 import temporalio.api.sdk.v1.task_complete_metadata_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 import temporalio.api.sdk.v1.worker_config_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.taskqueue.v1.message_pb2 import temporalio.api.update.v1.message_pb2 import temporalio.api.version.v1.message_pb2 @@ -1278,6 +1281,7 @@ class PollWorkflowTaskQueueResponse(google.protobuf.message.Message): POLLER_GROUP_ID_FIELD_NUMBER: builtins.int POLLER_GROUP_INFOS_FIELD_NUMBER: builtins.int POLLER_GROUPS_INFO_FIELD_NUMBER: builtins.int + STREAM_SLICES_FIELD_NUMBER: builtins.int task_token: builtins.bytes """A unique identifier for this task""" @property @@ -1392,6 +1396,15 @@ class PollWorkflowTaskQueueResponse(google.protobuf.message.Message): 3. If every group has some pending polls, assign the next poll to a group randomly according to the weights. """ + @property + def stream_slices( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamSlice + ]: + """Stream data attached to this task. Delivered out of band so the payloads + never enter History; only the offset ranges are recorded there. + """ def __init__( self, *, @@ -1427,6 +1440,10 @@ class PollWorkflowTaskQueueResponse(google.protobuf.message.Message): | None = ..., poller_groups_info: temporalio.api.taskqueue.v1.message_pb2.PollerGroupsInfo | None = ..., + stream_slices: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamSlice + ] + | None = ..., ) -> None: ... def HasField( self, @@ -1484,6 +1501,8 @@ class PollWorkflowTaskQueueResponse(google.protobuf.message.Message): b"started_event_id", "started_time", b"started_time", + "stream_slices", + b"stream_slices", "task_token", b"task_token", "workflow_execution", @@ -2695,6 +2714,7 @@ class RespondActivityTaskFailedRequest(google.protobuf.message.Message): WORKER_VERSION_FIELD_NUMBER: builtins.int DEPLOYMENT_FIELD_NUMBER: builtins.int DEPLOYMENT_OPTIONS_FIELD_NUMBER: builtins.int + CAUSE_FIELD_NUMBER: builtins.int task_token: builtins.bytes """The task token as received in `PollActivityTaskQueueResponse`""" @property @@ -2726,6 +2746,8 @@ class RespondActivityTaskFailedRequest(google.protobuf.message.Message): self, ) -> temporalio.api.deployment.v1.message_pb2.WorkerDeploymentOptions: """Worker deployment options that user has set in the worker.""" + cause: temporalio.api.enums.v1.failed_cause_pb2.ActivityTaskFailedCause.ValueType + """Why did the task fail? When unset, the failure is treated as an unspecified activity failure.""" def __init__( self, *, @@ -2741,6 +2763,7 @@ class RespondActivityTaskFailedRequest(google.protobuf.message.Message): deployment: temporalio.api.deployment.v1.message_pb2.Deployment | None = ..., deployment_options: temporalio.api.deployment.v1.message_pb2.WorkerDeploymentOptions | None = ..., + cause: temporalio.api.enums.v1.failed_cause_pb2.ActivityTaskFailedCause.ValueType = ..., ) -> None: ... def HasField( self, @@ -2760,6 +2783,8 @@ class RespondActivityTaskFailedRequest(google.protobuf.message.Message): def ClearField( self, field_name: typing_extensions.Literal[ + "cause", + b"cause", "deployment", b"deployment", "deployment_options", @@ -2821,6 +2846,7 @@ class RespondActivityTaskFailedByIdRequest(google.protobuf.message.Message): IDENTITY_FIELD_NUMBER: builtins.int LAST_HEARTBEAT_DETAILS_FIELD_NUMBER: builtins.int RESOURCE_ID_FIELD_NUMBER: builtins.int + CAUSE_FIELD_NUMBER: builtins.int namespace: builtins.str """Namespace of the workflow which scheduled this activity""" workflow_id: builtins.str @@ -2841,6 +2867,10 @@ class RespondActivityTaskFailedByIdRequest(google.protobuf.message.Message): """Additional details to be stored as last activity heartbeat""" resource_id: builtins.str """Resource ID for routing. Contains "workflow:workflow_id" or "activity:activity_id" for standalone activities.""" + cause: temporalio.api.enums.v1.failed_cause_pb2.ActivityTaskFailedCause.ValueType + """Why did the activity task fail? Optional; when unset the failure is treated as a normal + activity failure. See the type's doc for more. + """ def __init__( self, *, @@ -2853,6 +2883,7 @@ class RespondActivityTaskFailedByIdRequest(google.protobuf.message.Message): last_heartbeat_details: temporalio.api.common.v1.message_pb2.Payloads | None = ..., resource_id: builtins.str = ..., + cause: temporalio.api.enums.v1.failed_cause_pb2.ActivityTaskFailedCause.ValueType = ..., ) -> None: ... def HasField( self, @@ -2865,6 +2896,8 @@ class RespondActivityTaskFailedByIdRequest(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "activity_id", b"activity_id", + "cause", + b"cause", "failure", b"failure", "identity", @@ -3301,6 +3334,411 @@ class SignalWorkflowExecutionResponse(google.protobuf.message.Message): global___SignalWorkflowExecutionResponse = SignalWorkflowExecutionResponse +class NotifyChannelRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + NOTIFICATION_FIELD_NUMBER: builtins.int + IDENTITY_FIELD_NUMBER: builtins.int + REQUEST_ID_FIELD_NUMBER: builtins.int + EXECUTION_FIELD_NUMBER: builtins.int + namespace: builtins.str + @property + def notification( + self, + ) -> temporalio.api.notification.v1.message_pb2.Notification: ... + identity: builtins.str + """The identity of the caller, for audit, metrics and logs. It is not copied + into the notification. A writer that wants the consumer to see who wrote + puts that in the notification's `metadata`. + """ + request_id: builtins.str + """Used to de-dupe a retried notification.""" + @property + def execution(self) -> temporalio.api.common.v1.message_pb2.Execution: + """When set, the call addresses the channel linked to this execution. + `run_id` is optional and resolves to the current run of a workflow chain, + as a Signal does. When unset, the call addresses the independent channel + of that name. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + notification: temporalio.api.notification.v1.message_pb2.Notification + | None = ..., + identity: builtins.str = ..., + request_id: builtins.str = ..., + execution: temporalio.api.common.v1.message_pb2.Execution | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "execution", b"execution", "notification", b"notification" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "execution", + b"execution", + "identity", + b"identity", + "namespace", + b"namespace", + "notification", + b"notification", + "request_id", + b"request_id", + ], + ) -> None: ... + +global___NotifyChannelRequest = NotifyChannelRequest + +class NotifyChannelResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LISTENER_COUNT_FIELD_NUMBER: builtins.int + listener_count: builtins.int + """Listeners registered when the notification was accepted. Zero means the + notification was retained for pollers and woke nobody. + """ + def __init__( + self, + *, + listener_count: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["listener_count", b"listener_count"] + ) -> None: ... + +global___NotifyChannelResponse = NotifyChannelResponse + +class RegisterChannelListenerRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + CHANNEL_FIELD_NUMBER: builtins.int + CALLBACK_FIELD_NUMBER: builtins.int + REQUEST_ID_FIELD_NUMBER: builtins.int + IDENTITY_FIELD_NUMBER: builtins.int + EXECUTION_FIELD_NUMBER: builtins.int + namespace: builtins.str + channel: builtins.str + @property + def callback(self) -> temporalio.api.common.v1.message_pb2.Callback: + """Invoked with each notification on the channel.""" + request_id: builtins.str + """Used to de-dupe a retried registration.""" + identity: builtins.str + """The identity of the caller, for metrics and logs.""" + @property + def execution(self) -> temporalio.api.common.v1.message_pb2.Execution: + """When set, the call addresses the channel linked to this execution. + `run_id` is optional and resolves to the current run of a workflow chain, + as a Signal does. When unset, the call addresses the independent channel + of that name. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + channel: builtins.str = ..., + callback: temporalio.api.common.v1.message_pb2.Callback | None = ..., + request_id: builtins.str = ..., + identity: builtins.str = ..., + execution: temporalio.api.common.v1.message_pb2.Execution | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "callback", b"callback", "execution", b"execution" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "callback", + b"callback", + "channel", + b"channel", + "execution", + b"execution", + "identity", + b"identity", + "namespace", + b"namespace", + "request_id", + b"request_id", + ], + ) -> None: ... + +global___RegisterChannelListenerRequest = RegisterChannelListenerRequest + +class RegisterChannelListenerResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LISTENER_ID_FIELD_NUMBER: builtins.int + listener_id: builtins.str + def __init__( + self, + *, + listener_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["listener_id", b"listener_id"] + ) -> None: ... + +global___RegisterChannelListenerResponse = RegisterChannelListenerResponse + +class UnregisterChannelListenerRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + CHANNEL_FIELD_NUMBER: builtins.int + LISTENER_ID_FIELD_NUMBER: builtins.int + IDENTITY_FIELD_NUMBER: builtins.int + EXECUTION_FIELD_NUMBER: builtins.int + namespace: builtins.str + channel: builtins.str + listener_id: builtins.str + identity: builtins.str + """The identity of the caller, for metrics and logs.""" + @property + def execution(self) -> temporalio.api.common.v1.message_pb2.Execution: + """When set, the call addresses the channel linked to this execution. + `run_id` is optional and resolves to the current run of a workflow chain, + as a Signal does. When unset, the call addresses the independent channel + of that name. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + channel: builtins.str = ..., + listener_id: builtins.str = ..., + identity: builtins.str = ..., + execution: temporalio.api.common.v1.message_pb2.Execution | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["execution", b"execution"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "channel", + b"channel", + "execution", + b"execution", + "identity", + b"identity", + "listener_id", + b"listener_id", + "namespace", + b"namespace", + ], + ) -> None: ... + +global___UnregisterChannelListenerRequest = UnregisterChannelListenerRequest + +class UnregisterChannelListenerResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___UnregisterChannelListenerResponse = UnregisterChannelListenerResponse + +class PollChannelRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + CHANNEL_FIELD_NUMBER: builtins.int + AFTER_COUNTER_FIELD_NUMBER: builtins.int + WAIT_FIELD_NUMBER: builtins.int + MAX_NOTIFICATIONS_FIELD_NUMBER: builtins.int + EXECUTION_FIELD_NUMBER: builtins.int + namespace: builtins.str + channel: builtins.str + after_counter: builtins.int + """Only notifications with a counter above this one are returned. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "after" names the exclusive lower bound. --) + """ + @property + def wait(self) -> google.protobuf.duration_pb2.Duration: + """How long to wait for a notification when none is retained above + `after_counter`. + """ + max_notifications: builtins.int + """At most this many notifications are returned. Zero means the server's + default. + """ + @property + def execution(self) -> temporalio.api.common.v1.message_pb2.Execution: + """When set, the call addresses the channel linked to this execution. + `run_id` is optional and resolves to the current run of a workflow chain, + as a Signal does. When unset, the call addresses the independent channel + of that name. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + channel: builtins.str = ..., + after_counter: builtins.int = ..., + wait: google.protobuf.duration_pb2.Duration | None = ..., + max_notifications: builtins.int = ..., + execution: temporalio.api.common.v1.message_pb2.Execution | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "execution", b"execution", "wait", b"wait" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "after_counter", + b"after_counter", + "channel", + b"channel", + "execution", + b"execution", + "max_notifications", + b"max_notifications", + "namespace", + b"namespace", + "wait", + b"wait", + ], + ) -> None: ... + +global___PollChannelRequest = PollChannelRequest + +class PollChannelResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NOTIFICATIONS_FIELD_NUMBER: builtins.int + @property + def notifications( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.notification.v1.message_pb2.Notification + ]: ... + def __init__( + self, + *, + notifications: collections.abc.Iterable[ + temporalio.api.notification.v1.message_pb2.Notification + ] + | None = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["notifications", b"notifications"] + ) -> None: ... + +global___PollChannelResponse = PollChannelResponse + +class DescribeChannelRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAMESPACE_FIELD_NUMBER: builtins.int + CHANNEL_FIELD_NUMBER: builtins.int + EXECUTION_FIELD_NUMBER: builtins.int + namespace: builtins.str + channel: builtins.str + @property + def execution(self) -> temporalio.api.common.v1.message_pb2.Execution: + """When set, the call addresses the channel linked to this execution. + `run_id` is optional and resolves to the current run of a workflow chain, + as a Signal does. When unset, the call addresses the independent channel + of that name. + """ + def __init__( + self, + *, + namespace: builtins.str = ..., + channel: builtins.str = ..., + execution: temporalio.api.common.v1.message_pb2.Execution | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["execution", b"execution"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "channel", b"channel", "execution", b"execution", "namespace", b"namespace" + ], + ) -> None: ... + +global___DescribeChannelRequest = DescribeChannelRequest + +class DescribeChannelResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LISTENERS_FIELD_NUMBER: builtins.int + LATEST_FIELD_NUMBER: builtins.int + RETAINED_COUNT_FIELD_NUMBER: builtins.int + KIND_FIELD_NUMBER: builtins.int + LINKED_TO_FIELD_NUMBER: builtins.int + @property + def listeners( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.notification.v1.message_pb2.ChannelListener + ]: ... + @property + def latest(self) -> temporalio.api.notification.v1.message_pb2.Notification: + """The notification with the highest counter the channel retains.""" + retained_count: builtins.int + """How many notifications the channel retains for pollers.""" + kind: temporalio.api.notification.v1.message_pb2.ChannelKind.ValueType + @property + def linked_to(self) -> temporalio.api.common.v1.message_pb2.Execution: + """The owner of a linked channel and the run that holds it. Empty for an + independent channel. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "to" names the owner the channel is linked to. --) + """ + def __init__( + self, + *, + listeners: collections.abc.Iterable[ + temporalio.api.notification.v1.message_pb2.ChannelListener + ] + | None = ..., + latest: temporalio.api.notification.v1.message_pb2.Notification | None = ..., + retained_count: builtins.int = ..., + kind: temporalio.api.notification.v1.message_pb2.ChannelKind.ValueType = ..., + linked_to: temporalio.api.common.v1.message_pb2.Execution | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "latest", b"latest", "linked_to", b"linked_to" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "kind", + b"kind", + "latest", + b"latest", + "linked_to", + b"linked_to", + "listeners", + b"listeners", + "retained_count", + b"retained_count", + ], + ) -> None: ... + +global___DescribeChannelResponse = DescribeChannelResponse + class SignalWithStartWorkflowExecutionRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -4764,6 +5202,7 @@ class DescribeWorkflowExecutionResponse(google.protobuf.message.Message): CALLBACKS_FIELD_NUMBER: builtins.int PENDING_NEXUS_OPERATIONS_FIELD_NUMBER: builtins.int WORKFLOW_EXTENDED_INFO_FIELD_NUMBER: builtins.int + CHANNEL_SUBSCRIPTIONS_FIELD_NUMBER: builtins.int @property def execution_config( self, @@ -4804,6 +5243,15 @@ class DescribeWorkflowExecutionResponse(google.protobuf.message.Message): def workflow_extended_info( self, ) -> temporalio.api.workflow.v1.message_pb2.WorkflowExecutionExtendedInfo: ... + @property + def channel_subscriptions( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.workflow.v1.message_pb2.ChannelSubscriptionInfo + ]: + """The notification channels this run stands on: the independent channels it subscribed to and + the channels linked to it that hold any state. Empty when there are none. + """ def __init__( self, *, @@ -4831,6 +5279,10 @@ class DescribeWorkflowExecutionResponse(google.protobuf.message.Message): | None = ..., workflow_extended_info: temporalio.api.workflow.v1.message_pb2.WorkflowExecutionExtendedInfo | None = ..., + channel_subscriptions: collections.abc.Iterable[ + temporalio.api.workflow.v1.message_pb2.ChannelSubscriptionInfo + ] + | None = ..., ) -> None: ... def HasField( self, @@ -4850,6 +5302,8 @@ class DescribeWorkflowExecutionResponse(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "callbacks", b"callbacks", + "channel_subscriptions", + b"channel_subscriptions", "execution_config", b"execution_config", "pending_activities", @@ -12543,9 +12997,12 @@ class StartNexusOperationExecutionRequest(google.protobuf.message.Message): INPUT_FIELD_NUMBER: builtins.int ID_REUSE_POLICY_FIELD_NUMBER: builtins.int ID_CONFLICT_POLICY_FIELD_NUMBER: builtins.int + ON_CONFLICT_OPTIONS_FIELD_NUMBER: builtins.int SEARCH_ATTRIBUTES_FIELD_NUMBER: builtins.int NEXUS_HEADER_FIELD_NUMBER: builtins.int USER_METADATA_FIELD_NUMBER: builtins.int + COMPLETION_CALLBACKS_FIELD_NUMBER: builtins.int + LINKS_FIELD_NUMBER: builtins.int namespace: builtins.str identity: builtins.str """The identity of the client who initiated this request.""" @@ -12606,6 +13063,14 @@ class StartNexusOperationExecutionRequest(google.protobuf.message.Message): The default policy is NEXUS_OPERATION_ID_CONFLICT_POLICY_FAIL. """ @property + def on_conflict_options( + self, + ) -> temporalio.api.nexusoperation.v1.message_pb2.OnConflictOptions: + """Defines actions to be done to the existing running standalone Nexus when the conflict policy + NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING is used. If not set or set to a empty object + (all options with default value), it will not modify the running operation. + """ + @property def search_attributes( self, ) -> temporalio.api.common.v1.message_pb2.SearchAttributes: @@ -12624,6 +13089,22 @@ class StartNexusOperationExecutionRequest(google.protobuf.message.Message): @property def user_metadata(self) -> temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata: """Metadata for use by user interfaces to display the fixed as-of-start summary and details of the operation.""" + @property + def completion_callbacks( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.common.v1.message_pb2.Callback + ]: + """Completion callbacks to be invoked once the Nexus operation reaches a terminal state.""" + @property + def links( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.common.v1.message_pb2.Link + ]: + """Links to be associated with the Nexus operation. Callbacks may also have associated links; + links already included with a callback should not be duplicated here. + """ def __init__( self, *, @@ -12640,17 +13121,27 @@ class StartNexusOperationExecutionRequest(google.protobuf.message.Message): input: temporalio.api.common.v1.message_pb2.Payload | None = ..., id_reuse_policy: temporalio.api.enums.v1.nexus_pb2.NexusOperationIdReusePolicy.ValueType = ..., id_conflict_policy: temporalio.api.enums.v1.nexus_pb2.NexusOperationIdConflictPolicy.ValueType = ..., + on_conflict_options: temporalio.api.nexusoperation.v1.message_pb2.OnConflictOptions + | None = ..., search_attributes: temporalio.api.common.v1.message_pb2.SearchAttributes | None = ..., nexus_header: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., user_metadata: temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata | None = ..., + completion_callbacks: collections.abc.Iterable[ + temporalio.api.common.v1.message_pb2.Callback + ] + | None = ..., + links: collections.abc.Iterable[temporalio.api.common.v1.message_pb2.Link] + | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "input", b"input", + "on_conflict_options", + b"on_conflict_options", "schedule_to_close_timeout", b"schedule_to_close_timeout", "schedule_to_start_timeout", @@ -12666,6 +13157,8 @@ class StartNexusOperationExecutionRequest(google.protobuf.message.Message): def ClearField( self, field_name: typing_extensions.Literal[ + "completion_callbacks", + b"completion_callbacks", "endpoint", b"endpoint", "id_conflict_policy", @@ -12676,10 +13169,14 @@ class StartNexusOperationExecutionRequest(google.protobuf.message.Message): b"identity", "input", b"input", + "links", + b"links", "namespace", b"namespace", "nexus_header", b"nexus_header", + "on_conflict_options", + b"on_conflict_options", "operation", b"operation", "operation_id", @@ -12791,6 +13288,7 @@ class DescribeNexusOperationExecutionResponse(google.protobuf.message.Message): RESULT_FIELD_NUMBER: builtins.int FAILURE_FIELD_NUMBER: builtins.int LONG_POLL_TOKEN_FIELD_NUMBER: builtins.int + COMPLETION_CALLBACKS_FIELD_NUMBER: builtins.int run_id: builtins.str """The run ID of the operation, useful when run_id was not specified in the request.""" @property @@ -12809,6 +13307,15 @@ class DescribeNexusOperationExecutionResponse(google.protobuf.message.Message): """The failure if the operation completed unsuccessfully.""" long_poll_token: builtins.bytes """Token for follow-on long-poll requests. Absent only if the operation is complete.""" + @property + def completion_callbacks( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.nexusoperation.v1.message_pb2.CallbackInfo + ]: + """Completion callbacks to be invoked once the Nexus operation reaches a terminal state. + They will remain in the CALLBACK_STATE_STANDBY state until the Nexus operation is finished. + """ def __init__( self, *, @@ -12819,6 +13326,10 @@ class DescribeNexusOperationExecutionResponse(google.protobuf.message.Message): result: temporalio.api.common.v1.message_pb2.Payload | None = ..., failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., long_poll_token: builtins.bytes = ..., + completion_callbacks: collections.abc.Iterable[ + temporalio.api.nexusoperation.v1.message_pb2.CallbackInfo + ] + | None = ..., ) -> None: ... def HasField( self, @@ -12838,6 +13349,8 @@ class DescribeNexusOperationExecutionResponse(google.protobuf.message.Message): def ClearField( self, field_name: typing_extensions.Literal[ + "completion_callbacks", + b"completion_callbacks", "failure", b"failure", "info", diff --git a/temporalio/api/workflowservice/v1/service_pb2.py b/temporalio/api/workflowservice/v1/service_pb2.py index 92da5d6c2b..12492145e5 100644 --- a/temporalio/api/workflowservice/v1/service_pb2.py +++ b/temporalio/api/workflowservice/v1/service_pb2.py @@ -27,7 +27,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n-temporal/api/workflowservice/v1/service.proto\x12\x1ftemporal.api.workflowservice.v1\x1a\x1cgoogle/api/annotations.proto\x1a!nexusannotations/v1/options.proto\x1a+temporal/api/protometa/v1/annotations.proto\x1a\x36temporal/api/workflowservice/v1/request_response.proto2\xdc\xb2\x02\n\x0fWorkflowService\x12\xc3\x01\n\x11RegisterNamespace\x12\x39.temporal.api.workflowservice.v1.RegisterNamespaceRequest\x1a:.temporal.api.workflowservice.v1.RegisterNamespaceResponse"7\x82\xd3\xe4\x93\x02\x31"\x13/cluster/namespaces:\x01*Z\x17"\x12/api/v1/namespaces:\x01*\x12\xd5\x01\n\x11\x44\x65scribeNamespace\x12\x39.temporal.api.workflowservice.v1.DescribeNamespaceRequest\x1a:.temporal.api.workflowservice.v1.DescribeNamespaceResponse"I\x82\xd3\xe4\x93\x02\x43\x12\x1f/cluster/namespaces/{namespace}Z \x12\x1e/api/v1/namespaces/{namespace}\x12\xb4\x01\n\x0eListNamespaces\x12\x36.temporal.api.workflowservice.v1.ListNamespacesRequest\x1a\x37.temporal.api.workflowservice.v1.ListNamespacesResponse"1\x82\xd3\xe4\x93\x02+\x12\x13/cluster/namespacesZ\x14\x12\x12/api/v1/namespaces\x12\xe3\x01\n\x0fUpdateNamespace\x12\x37.temporal.api.workflowservice.v1.UpdateNamespaceRequest\x1a\x38.temporal.api.workflowservice.v1.UpdateNamespaceResponse"]\x82\xd3\xe4\x93\x02W"&/cluster/namespaces/{namespace}/update:\x01*Z*"%/api/v1/namespaces/{namespace}/update:\x01*\x12\x8f\x01\n\x12\x44\x65precateNamespace\x12:.temporal.api.workflowservice.v1.DeprecateNamespaceRequest\x1a;.temporal.api.workflowservice.v1.DeprecateNamespaceResponse"\x00\x12\xc6\x02\n\x16StartWorkflowExecution\x12>.temporal.api.workflowservice.v1.StartWorkflowExecutionRequest\x1a?.temporal.api.workflowservice.v1.StartWorkflowExecutionResponse"\xaa\x01\x82\xd3\xe4\x93\x02q"//namespaces/{namespace}/workflows/{workflow_id}:\x01*Z;"6/api/v1/namespaces/{namespace}/workflows/{workflow_id}:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16workflow:{workflow_id}\x12\xc2\x01\n\x15\x45xecuteMultiOperation\x12=.temporal.api.workflowservice.v1.ExecuteMultiOperationRequest\x1a>.temporal.api.workflowservice.v1.ExecuteMultiOperationResponse"*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xfe\x02\n\x1bGetWorkflowExecutionHistory\x12\x43.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryRequest\x1a\x44.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse"\xd3\x01\x82\xd3\xe4\x93\x02\x8f\x01\x12\x41/namespaces/{namespace}/workflows/{execution.workflow_id}/historyZJ\x12H/api/v1/namespaces/{namespace}/workflows/{execution.workflow_id}/history\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\xa3\x03\n"GetWorkflowExecutionHistoryReverse\x12J.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseRequest\x1aK.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseResponse"\xe3\x01\x82\xd3\xe4\x93\x02\x9f\x01\x12I/namespaces/{namespace}/workflows/{execution.workflow_id}/history-reverseZR\x12P/api/v1/namespaces/{namespace}/workflows/{execution.workflow_id}/history-reverse\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\xcd\x01\n\x15PollWorkflowTaskQueue\x12=.temporal.api.workflowservice.v1.PollWorkflowTaskQueueRequest\x1a>.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xd7\x01\n\x1cRespondWorkflowTaskCompleted\x12\x44.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest\x1a\x45.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedResponse"*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xce\x01\n\x19RespondWorkflowTaskFailed\x12\x41.temporal.api.workflowservice.v1.RespondWorkflowTaskFailedRequest\x1a\x42.temporal.api.workflowservice.v1.RespondWorkflowTaskFailedResponse"*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xcd\x01\n\x15PollActivityTaskQueue\x12=.temporal.api.workflowservice.v1.PollActivityTaskQueueRequest\x1a>.temporal.api.workflowservice.v1.PollActivityTaskQueueResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xc2\x02\n\x1bRecordActivityTaskHeartbeat\x12\x43.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatRequest\x1a\x44.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatResponse"\x97\x01\x82\xd3\xe4\x93\x02g"*/namespaces/{namespace}/activity-heartbeat:\x01*Z6"1/api/v1/namespaces/{namespace}/activity-heartbeat:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xa8\x04\n\x1fRecordActivityTaskHeartbeatById\x12G.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatByIdRequest\x1aH.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatByIdResponse"\xf1\x02\x82\xd3\xe4\x93\x02\xc0\x02":/namespaces/{namespace}/activities/{activity_id}/heartbeat:\x01*ZF"A/api/v1/namespaces/{namespace}/activities/{activity_id}/heartbeat:\x01*ZW"R/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/heartbeat:\x01*Z^"Y/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/heartbeat:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xc3\x02\n\x1cRespondActivityTaskCompleted\x12\x44.temporal.api.workflowservice.v1.RespondActivityTaskCompletedRequest\x1a\x45.temporal.api.workflowservice.v1.RespondActivityTaskCompletedResponse"\x95\x01\x82\xd3\xe4\x93\x02\x65")/namespaces/{namespace}/activity-complete:\x01*Z5"0/api/v1/namespaces/{namespace}/activity-complete:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xa7\x04\n RespondActivityTaskCompletedById\x12H.temporal.api.workflowservice.v1.RespondActivityTaskCompletedByIdRequest\x1aI.temporal.api.workflowservice.v1.RespondActivityTaskCompletedByIdResponse"\xed\x02\x82\xd3\xe4\x93\x02\xbc\x02"9/namespaces/{namespace}/activities/{activity_id}/complete:\x01*ZE"@/api/v1/namespaces/{namespace}/activities/{activity_id}/complete:\x01*ZV"Q/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/complete:\x01*Z]"X/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/complete:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xb2\x02\n\x19RespondActivityTaskFailed\x12\x41.temporal.api.workflowservice.v1.RespondActivityTaskFailedRequest\x1a\x42.temporal.api.workflowservice.v1.RespondActivityTaskFailedResponse"\x8d\x01\x82\xd3\xe4\x93\x02]"%/namespaces/{namespace}/activity-fail:\x01*Z1",/api/v1/namespaces/{namespace}/activity-fail:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\x8e\x04\n\x1dRespondActivityTaskFailedById\x12\x45.temporal.api.workflowservice.v1.RespondActivityTaskFailedByIdRequest\x1a\x46.temporal.api.workflowservice.v1.RespondActivityTaskFailedByIdResponse"\xdd\x02\x82\xd3\xe4\x93\x02\xac\x02"5/namespaces/{namespace}/activities/{activity_id}/fail:\x01*ZA".temporal.api.workflowservice.v1.ResetWorkflowExecutionRequest\x1a?.temporal.api.workflowservice.v1.ResetWorkflowExecutionResponse"\xf0\x01\x82\xd3\xe4\x93\x02\xa3\x01"H/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/reset:\x01*ZT"O/api/v1/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/reset:\x01*\x8a\x9d\xcc\x1b\x41\n\x14temporal-resource-id\x12)workflow:{workflow_execution.workflow_id}\x12\xa0\x03\n\x1aTerminateWorkflowExecution\x12\x42.temporal.api.workflowservice.v1.TerminateWorkflowExecutionRequest\x1a\x43.temporal.api.workflowservice.v1.TerminateWorkflowExecutionResponse"\xf8\x01\x82\xd3\xe4\x93\x02\xab\x01"L/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/terminate:\x01*ZX"S/api/v1/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/terminate:\x01*\x8a\x9d\xcc\x1b\x41\n\x14temporal-resource-id\x12)workflow:{workflow_execution.workflow_id}\x12\xe4\x01\n\x17\x44\x65leteWorkflowExecution\x12?.temporal.api.workflowservice.v1.DeleteWorkflowExecutionRequest\x1a@.temporal.api.workflowservice.v1.DeleteWorkflowExecutionResponse"F\x8a\x9d\xcc\x1b\x41\n\x14temporal-resource-id\x12)workflow:{workflow_execution.workflow_id}\x12\xa7\x01\n\x1aListOpenWorkflowExecutions\x12\x42.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsRequest\x1a\x43.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsResponse"\x00\x12\xad\x01\n\x1cListClosedWorkflowExecutions\x12\x44.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsRequest\x1a\x45.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsResponse"\x00\x12\xf0\x01\n\x16ListWorkflowExecutions\x12>.temporal.api.workflowservice.v1.ListWorkflowExecutionsRequest\x1a?.temporal.api.workflowservice.v1.ListWorkflowExecutionsResponse"U\x82\xd3\xe4\x93\x02O\x12!/namespaces/{namespace}/workflowsZ*\x12(/api/v1/namespaces/{namespace}/workflows\x12\x9a\x02\n\x1eListArchivedWorkflowExecutions\x12\x46.temporal.api.workflowservice.v1.ListArchivedWorkflowExecutionsRequest\x1aG.temporal.api.workflowservice.v1.ListArchivedWorkflowExecutionsResponse"g\x82\xd3\xe4\x93\x02\x61\x12*/namespaces/{namespace}/archived-workflowsZ3\x12\x31/api/v1/namespaces/{namespace}/archived-workflows\x12\x9b\x01\n\x16ScanWorkflowExecutions\x12>.temporal.api.workflowservice.v1.ScanWorkflowExecutionsRequest\x1a?.temporal.api.workflowservice.v1.ScanWorkflowExecutionsResponse"\x00\x12\xfd\x01\n\x17\x43ountWorkflowExecutions\x12?.temporal.api.workflowservice.v1.CountWorkflowExecutionsRequest\x1a@.temporal.api.workflowservice.v1.CountWorkflowExecutionsResponse"_\x82\xd3\xe4\x93\x02Y\x12&/namespaces/{namespace}/workflow-countZ/\x12-/api/v1/namespaces/{namespace}/workflow-count\x12\x92\x01\n\x13GetSearchAttributes\x12;.temporal.api.workflowservice.v1.GetSearchAttributesRequest\x1a<.temporal.api.workflowservice.v1.GetSearchAttributesResponse"\x00\x12\xd9\x01\n\x19RespondQueryTaskCompleted\x12\x41.temporal.api.workflowservice.v1.RespondQueryTaskCompletedRequest\x1a\x42.temporal.api.workflowservice.v1.RespondQueryTaskCompletedResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xd2\x01\n\x14ResetStickyTaskQueue\x12<.temporal.api.workflowservice.v1.ResetStickyTaskQueueRequest\x1a=.temporal.api.workflowservice.v1.ResetStickyTaskQueueResponse"=\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\x83\x01\n\x0eShutdownWorker\x12\x36.temporal.api.workflowservice.v1.ShutdownWorkerRequest\x1a\x37.temporal.api.workflowservice.v1.ShutdownWorkerResponse"\x00\x12\xfc\x02\n\rQueryWorkflow\x12\x35.temporal.api.workflowservice.v1.QueryWorkflowRequest\x1a\x36.temporal.api.workflowservice.v1.QueryWorkflowResponse"\xfb\x01\x82\xd3\xe4\x93\x02\xb7\x01"R/namespaces/{namespace}/workflows/{execution.workflow_id}/query/{query.query_type}:\x01*Z^"Y/api/v1/namespaces/{namespace}/workflows/{execution.workflow_id}/query/{query.query_type}:\x01*\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\xe7\x02\n\x19\x44\x65scribeWorkflowExecution\x12\x41.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest\x1a\x42.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse"\xc2\x01\x82\xd3\xe4\x93\x02\x7f\x12\x39/namespaces/{namespace}/workflows/{execution.workflow_id}ZB\x12@/api/v1/namespaces/{namespace}/workflows/{execution.workflow_id}\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\xc2\x02\n\x11\x44\x65scribeTaskQueue\x12\x39.temporal.api.workflowservice.v1.DescribeTaskQueueRequest\x1a:.temporal.api.workflowservice.v1.DescribeTaskQueueResponse"\xb5\x01\x82\xd3\xe4\x93\x02w\x12\x35/namespaces/{namespace}/task-queues/{task_queue.name}Z>\x12/namespaces/{namespace}/schedules/{schedule_id}/matching-timesZG\x12\x45/api/v1/namespaces/{namespace}/schedules/{schedule_id}/matching-times\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16schedule:{schedule_id}\x12\xa8\x02\n\x0e\x44\x65leteSchedule\x12\x36.temporal.api.workflowservice.v1.DeleteScheduleRequest\x1a\x37.temporal.api.workflowservice.v1.DeleteScheduleResponse"\xa4\x01\x82\xd3\xe4\x93\x02k*//namespaces/{namespace}/schedules/{schedule_id}Z8*6/api/v1/namespaces/{namespace}/schedules/{schedule_id}\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16schedule:{schedule_id}\x12\xd5\x01\n\rListSchedules\x12\x35.temporal.api.workflowservice.v1.ListSchedulesRequest\x1a\x36.temporal.api.workflowservice.v1.ListSchedulesResponse"U\x82\xd3\xe4\x93\x02O\x12!/namespaces/{namespace}/schedulesZ*\x12(/api/v1/namespaces/{namespace}/schedules\x12\xe2\x01\n\x0e\x43ountSchedules\x12\x36.temporal.api.workflowservice.v1.CountSchedulesRequest\x1a\x37.temporal.api.workflowservice.v1.CountSchedulesResponse"_\x82\xd3\xe4\x93\x02Y\x12&/namespaces/{namespace}/schedule-countZ/\x12-/api/v1/namespaces/{namespace}/schedule-count\x12\xb9\x01\n UpdateWorkerBuildIdCompatibility\x12H.temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest\x1aI.temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityResponse"\x00\x12\xe1\x02\n\x1dGetWorkerBuildIdCompatibility\x12\x45.temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityRequest\x1a\x46.temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityResponse"\xb0\x01\x82\xd3\xe4\x93\x02\xa9\x01\x12N/namespaces/{namespace}/task-queues/{task_queue}/worker-build-id-compatibilityZW\x12U/api/v1/namespaces/{namespace}/task-queues/{task_queue}/worker-build-id-compatibility\x12\xaa\x01\n\x1bUpdateWorkerVersioningRules\x12\x43.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest\x1a\x44.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesResponse"\x00\x12\xc6\x02\n\x18GetWorkerVersioningRules\x12@.temporal.api.workflowservice.v1.GetWorkerVersioningRulesRequest\x1a\x41.temporal.api.workflowservice.v1.GetWorkerVersioningRulesResponse"\xa4\x01\x82\xd3\xe4\x93\x02\x9d\x01\x12H/namespaces/{namespace}/task-queues/{task_queue}/worker-versioning-rulesZQ\x12O/api/v1/namespaces/{namespace}/task-queues/{task_queue}/worker-versioning-rules\x12\x97\x02\n\x19GetWorkerTaskReachability\x12\x41.temporal.api.workflowservice.v1.GetWorkerTaskReachabilityRequest\x1a\x42.temporal.api.workflowservice.v1.GetWorkerTaskReachabilityResponse"s\x82\xd3\xe4\x93\x02m\x12\x30/namespaces/{namespace}/worker-task-reachabilityZ9\x12\x37/api/v1/namespaces/{namespace}/worker-task-reachability\x12\xc8\x02\n\x12\x44\x65scribeDeployment\x12:.temporal.api.workflowservice.v1.DescribeDeploymentRequest\x1a;.temporal.api.workflowservice.v1.DescribeDeploymentResponse"\xb8\x01\x82\xd3\xe4\x93\x02\xb1\x01\x12R/namespaces/{namespace}/deployments/{deployment.series_name}/{deployment.build_id}Z[\x12Y/api/v1/namespaces/{namespace}/deployments/{deployment.series_name}/{deployment.build_id}\x12\x81\x04\n\x1f\x44\x65scribeWorkerDeploymentVersion\x12G.temporal.api.workflowservice.v1.DescribeWorkerDeploymentVersionRequest\x1aH.temporal.api.workflowservice.v1.DescribeWorkerDeploymentVersionResponse"\xca\x02\x82\xd3\xe4\x93\x02\xf7\x01\x12u/namespaces/{namespace}/worker-deployment-versions/{deployment_version.deployment_name}/{deployment_version.build_id}Z~\x12|/api/v1/namespaces/{namespace}/worker-deployment-versions/{deployment_version.deployment_name}/{deployment_version.build_id}\x8a\x9d\xcc\x1bG\n\x14temporal-resource-id\x12/deployment:{deployment_version.deployment_name}\x12\xdf\x01\n\x0fListDeployments\x12\x37.temporal.api.workflowservice.v1.ListDeploymentsRequest\x1a\x38.temporal.api.workflowservice.v1.ListDeploymentsResponse"Y\x82\xd3\xe4\x93\x02S\x12#/namespaces/{namespace}/deploymentsZ,\x12*/api/v1/namespaces/{namespace}/deployments\x12\xf7\x02\n\x19GetDeploymentReachability\x12\x41.temporal.api.workflowservice.v1.GetDeploymentReachabilityRequest\x1a\x42.temporal.api.workflowservice.v1.GetDeploymentReachabilityResponse"\xd2\x01\x82\xd3\xe4\x93\x02\xcb\x01\x12_/namespaces/{namespace}/deployments/{deployment.series_name}/{deployment.build_id}/reachabilityZh\x12\x66/api/v1/namespaces/{namespace}/deployments/{deployment.series_name}/{deployment.build_id}/reachability\x12\x99\x02\n\x14GetCurrentDeployment\x12<.temporal.api.workflowservice.v1.GetCurrentDeploymentRequest\x1a=.temporal.api.workflowservice.v1.GetCurrentDeploymentResponse"\x83\x01\x82\xd3\xe4\x93\x02}\x12\x38/namespaces/{namespace}/current-deployment/{series_name}ZA\x12?/api/v1/namespaces/{namespace}/current-deployment/{series_name}\x12\xb6\x02\n\x14SetCurrentDeployment\x12<.temporal.api.workflowservice.v1.SetCurrentDeploymentRequest\x1a=.temporal.api.workflowservice.v1.SetCurrentDeploymentResponse"\xa0\x01\x82\xd3\xe4\x93\x02\x99\x01"C/namespaces/{namespace}/current-deployment/{deployment.series_name}:\x01*ZO"J/api/v1/namespaces/{namespace}/current-deployment/{deployment.series_name}:\x01*\x12\xb0\x03\n!SetWorkerDeploymentCurrentVersion\x12I.temporal.api.workflowservice.v1.SetWorkerDeploymentCurrentVersionRequest\x1aJ.temporal.api.workflowservice.v1.SetWorkerDeploymentCurrentVersionResponse"\xf3\x01\x82\xd3\xe4\x93\x02\xb3\x01"P/namespaces/{namespace}/worker-deployments/{deployment_name}/set-current-version:\x01*Z\\"W/api/v1/namespaces/{namespace}/worker-deployments/{deployment_name}/set-current-version:\x01*\x8a\x9d\xcc\x1b\x34\n\x14temporal-resource-id\x12\x1c\x64\x65ployment:{deployment_name}\x12\xe7\x02\n\x18\x44\x65scribeWorkerDeployment\x12@.temporal.api.workflowservice.v1.DescribeWorkerDeploymentRequest\x1a\x41.temporal.api.workflowservice.v1.DescribeWorkerDeploymentResponse"\xc5\x01\x82\xd3\xe4\x93\x02\x85\x01\x12.temporal.api.workflowservice.v1.DeleteWorkerDeploymentRequest\x1a?.temporal.api.workflowservice.v1.DeleteWorkerDeploymentResponse"\xc5\x01\x82\xd3\xe4\x93\x02\x85\x01*.temporal.api.workflowservice.v1.ListWorkerDeploymentsResponse"g\x82\xd3\xe4\x93\x02\x61\x12*/namespaces/{namespace}/worker-deploymentsZ3\x12\x31/api/v1/namespaces/{namespace}/worker-deployments\x12\xae\x02\n\x16\x43reateWorkerDeployment\x12>.temporal.api.workflowservice.v1.CreateWorkerDeploymentRequest\x1a?.temporal.api.workflowservice.v1.CreateWorkerDeploymentResponse"\x92\x01\x82\xd3\xe4\x93\x02\x8b\x01".temporal.api.workflowservice.v1.DescribeBatchOperationRequest\x1a?.temporal.api.workflowservice.v1.DescribeBatchOperationResponse"\xa0\x01\x82\xd3\xe4\x93\x02o\x12\x31/namespaces/{namespace}/batch-operations/{job_id}Z:\x12\x38/api/v1/namespaces/{namespace}/batch-operations/{job_id}\x8a\x9d\xcc\x1b&\n\x14temporal-resource-id\x12\x0e\x62\x61tch:{job_id}\x12\xf5\x01\n\x13ListBatchOperations\x12;.temporal.api.workflowservice.v1.ListBatchOperationsRequest\x1a<.temporal.api.workflowservice.v1.ListBatchOperationsResponse"c\x82\xd3\xe4\x93\x02]\x12(/namespaces/{namespace}/batch-operationsZ1\x12//api/v1/namespaces/{namespace}/batch-operations\x12\xc4\x01\n\x12PollNexusTaskQueue\x12:.temporal.api.workflowservice.v1.PollNexusTaskQueueRequest\x1a;.temporal.api.workflowservice.v1.PollNexusTaskQueueResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xd9\x01\n\x19RespondNexusTaskCompleted\x12\x41.temporal.api.workflowservice.v1.RespondNexusTaskCompletedRequest\x1a\x42.temporal.api.workflowservice.v1.RespondNexusTaskCompletedResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xd0\x01\n\x16RespondNexusTaskFailed\x12>.temporal.api.workflowservice.v1.RespondNexusTaskFailedRequest\x1a?.temporal.api.workflowservice.v1.RespondNexusTaskFailedResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xe8\x02\n\x15UpdateActivityOptions\x12=.temporal.api.workflowservice.v1.UpdateActivityOptionsRequest\x1a>.temporal.api.workflowservice.v1.UpdateActivityOptionsResponse"\xcf\x01\x82\xd3\xe4\x93\x02\x8b\x01".temporal.api.workflowservice.v1.RecordWorkerHeartbeatResponse"\x95\x01\x82\xd3\xe4\x93\x02\x65")/namespaces/{namespace}/workers/heartbeat:\x01*Z5"0/api/v1/namespaces/{namespace}/workers/heartbeat:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xcb\x01\n\x0bListWorkers\x12\x33.temporal.api.workflowservice.v1.ListWorkersRequest\x1a\x34.temporal.api.workflowservice.v1.ListWorkersResponse"Q\x82\xd3\xe4\x93\x02K\x12\x1f/namespaces/{namespace}/workersZ(\x12&/api/v1/namespaces/{namespace}/workers\x12\xd8\x01\n\x0c\x43ountWorkers\x12\x34.temporal.api.workflowservice.v1.CountWorkersRequest\x1a\x35.temporal.api.workflowservice.v1.CountWorkersResponse"[\x82\xd3\xe4\x93\x02U\x12$/namespaces/{namespace}/worker-countZ-\x12+/api/v1/namespaces/{namespace}/worker-count\x12\xe2\x02\n\x15UpdateTaskQueueConfig\x12=.temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest\x1a>.temporal.api.workflowservice.v1.UpdateTaskQueueConfigResponse"\xc9\x01\x82\xd3\xe4\x93\x02\x8f\x01">/namespaces/{namespace}/task-queues/{task_queue}/update-config:\x01*ZJ"E/api/v1/namespaces/{namespace}/task-queues/{task_queue}/update-config:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16taskqueue:{task_queue}\x12\xa8\x02\n\x11\x46\x65tchWorkerConfig\x12\x39.temporal.api.workflowservice.v1.FetchWorkerConfigRequest\x1a:.temporal.api.workflowservice.v1.FetchWorkerConfigResponse"\x9b\x01\x82\xd3\xe4\x93\x02k",/namespaces/{namespace}/workers/fetch-config:\x01*Z8"3/api/v1/namespaces/{namespace}/workers/fetch-config:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xad\x02\n\x12UpdateWorkerConfig\x12:.temporal.api.workflowservice.v1.UpdateWorkerConfigRequest\x1a;.temporal.api.workflowservice.v1.UpdateWorkerConfigResponse"\x9d\x01\x82\xd3\xe4\x93\x02m"-/namespaces/{namespace}/workers/update-config:\x01*Z9"4/api/v1/namespaces/{namespace}/workers/update-config:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xcd\x02\n\x0e\x44\x65scribeWorker\x12\x36.temporal.api.workflowservice.v1.DescribeWorkerRequest\x1a\x37.temporal.api.workflowservice.v1.DescribeWorkerResponse"\xc9\x01\x82\xd3\xe4\x93\x02\x89\x01\x12>/namespaces/{namespace}/workers/describe/{worker_instance_key}ZG\x12\x45/api/v1/namespaces/{namespace}/workers/describe/{worker_instance_key}\x8a\x9d\xcc\x1b\x34\n\x14temporal-resource-id\x12\x1cworker:{worker_instance_key}\x12\xd2\x02\n\x16PauseWorkflowExecution\x12>.temporal.api.workflowservice.v1.PauseWorkflowExecutionRequest\x1a?.temporal.api.workflowservice.v1.PauseWorkflowExecutionResponse"\xb6\x01\x82\xd3\xe4\x93\x02}"5/namespaces/{namespace}/workflows/{workflow_id}/pause:\x01*ZA"/api/v1/namespaces/{namespace}/workflows/{workflow_id}/unpause:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16workflow:{workflow_id}\x12\xc8\x02\n\x16StartActivityExecution\x12>.temporal.api.workflowservice.v1.StartActivityExecutionRequest\x1a?.temporal.api.workflowservice.v1.StartActivityExecutionResponse"\xac\x01\x82\xd3\xe4\x93\x02s"0/namespaces/{namespace}/activities/{activity_id}:\x01*Z<"7/api/v1/namespaces/{namespace}/activities/{activity_id}:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\xb6\x02\n\x1cStartNexusOperationExecution\x12\x44.temporal.api.workflowservice.v1.StartNexusOperationExecutionRequest\x1a\x45.temporal.api.workflowservice.v1.StartNexusOperationExecutionResponse"\x88\x01\x82\xd3\xe4\x93\x02\x81\x01"7/namespaces/{namespace}/nexus-operations/{operation_id}:\x01*ZC">/api/v1/namespaces/{namespace}/nexus-operations/{operation_id}:\x01*\x12\xcb\x02\n\x19\x44\x65scribeActivityExecution\x12\x41.temporal.api.workflowservice.v1.DescribeActivityExecutionRequest\x1a\x42.temporal.api.workflowservice.v1.DescribeActivityExecutionResponse"\xa6\x01\x82\xd3\xe4\x93\x02m\x12\x30/namespaces/{namespace}/activities/{activity_id}Z9\x12\x37/api/v1/namespaces/{namespace}/activities/{activity_id}\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\xb8\x02\n\x1f\x44\x65scribeNexusOperationExecution\x12G.temporal.api.workflowservice.v1.DescribeNexusOperationExecutionRequest\x1aH.temporal.api.workflowservice.v1.DescribeNexusOperationExecutionResponse"\x81\x01\x82\xd3\xe4\x93\x02{\x12\x37/namespaces/{namespace}/nexus-operations/{operation_id}Z@\x12>/api/v1/namespaces/{namespace}/nexus-operations/{operation_id}\x12\xcf\x02\n\x15PollActivityExecution\x12=.temporal.api.workflowservice.v1.PollActivityExecutionRequest\x1a>.temporal.api.workflowservice.v1.PollActivityExecutionResponse"\xb6\x01\x82\xd3\xe4\x93\x02}\x12\x38/namespaces/{namespace}/activities/{activity_id}/outcomeZA\x12?/api/v1/namespaces/{namespace}/activities/{activity_id}/outcome\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\xb7\x02\n\x1bPollNexusOperationExecution\x12\x43.temporal.api.workflowservice.v1.PollNexusOperationExecutionRequest\x1a\x44.temporal.api.workflowservice.v1.PollNexusOperationExecutionResponse"\x8c\x01\x82\xd3\xe4\x93\x02\x85\x01\x12.temporal.api.workflowservice.v1.ListActivityExecutionsRequest\x1a?.temporal.api.workflowservice.v1.ListActivityExecutionsResponse"W\x82\xd3\xe4\x93\x02Q\x12"/namespaces/{namespace}/activitiesZ+\x12)/api/v1/namespaces/{namespace}/activities\x12\x90\x02\n\x1cListNexusOperationExecutions\x12\x44.temporal.api.workflowservice.v1.ListNexusOperationExecutionsRequest\x1a\x45.temporal.api.workflowservice.v1.ListNexusOperationExecutionsResponse"c\x82\xd3\xe4\x93\x02]\x12(/namespaces/{namespace}/nexus-operationsZ1\x12//api/v1/namespaces/{namespace}/nexus-operations\x12\xfd\x01\n\x17\x43ountActivityExecutions\x12?.temporal.api.workflowservice.v1.CountActivityExecutionsRequest\x1a@.temporal.api.workflowservice.v1.CountActivityExecutionsResponse"_\x82\xd3\xe4\x93\x02Y\x12&/namespaces/{namespace}/activity-countZ/\x12-/api/v1/namespaces/{namespace}/activity-count\x12\x9d\x02\n\x1d\x43ountNexusOperationExecutions\x12\x45.temporal.api.workflowservice.v1.CountNexusOperationExecutionsRequest\x1a\x46.temporal.api.workflowservice.v1.CountNexusOperationExecutionsResponse"m\x82\xd3\xe4\x93\x02g\x12-/namespaces/{namespace}/nexus-operation-countZ6\x12\x34/api/v1/namespaces/{namespace}/nexus-operation-count\x12\xef\x02\n\x1eRequestCancelActivityExecution\x12\x46.temporal.api.workflowservice.v1.RequestCancelActivityExecutionRequest\x1aG.temporal.api.workflowservice.v1.RequestCancelActivityExecutionResponse"\xbb\x01\x82\xd3\xe4\x93\x02\x81\x01"7/namespaces/{namespace}/activities/{activity_id}/cancel:\x01*ZC">/api/v1/namespaces/{namespace}/activities/{activity_id}/cancel:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\xdc\x02\n$RequestCancelNexusOperationExecution\x12L.temporal.api.workflowservice.v1.RequestCancelNexusOperationExecutionRequest\x1aM.temporal.api.workflowservice.v1.RequestCancelNexusOperationExecutionResponse"\x96\x01\x82\xd3\xe4\x93\x02\x8f\x01">/namespaces/{namespace}/nexus-operations/{operation_id}/cancel:\x01*ZJ"E/api/v1/namespaces/{namespace}/nexus-operations/{operation_id}/cancel:\x01*\x12\xe9\x02\n\x1aTerminateActivityExecution\x12\x42.temporal.api.workflowservice.v1.TerminateActivityExecutionRequest\x1a\x43.temporal.api.workflowservice.v1.TerminateActivityExecutionResponse"\xc1\x01\x82\xd3\xe4\x93\x02\x87\x01":/namespaces/{namespace}/activities/{activity_id}/terminate:\x01*ZF"A/api/v1/namespaces/{namespace}/activities/{activity_id}/terminate:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\x9e\x01\n\x17\x44\x65leteActivityExecution\x12?.temporal.api.workflowservice.v1.DeleteActivityExecutionRequest\x1a@.temporal.api.workflowservice.v1.DeleteActivityExecutionResponse"\x00\x12\xfd\x03\n\x16PauseActivityExecution\x12>.temporal.api.workflowservice.v1.PauseActivityExecutionRequest\x1a?.temporal.api.workflowservice.v1.PauseActivityExecutionResponse"\xe1\x02\x82\xd3\xe4\x93\x02\xb0\x02"6/namespaces/{namespace}/activities/{activity_id}/pause:\x01*ZB"=/api/v1/namespaces/{namespace}/activities/{activity_id}/pause:\x01*ZS"N/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/pause:\x01*ZZ"U/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/pause:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xfd\x03\n\x16ResetActivityExecution\x12>.temporal.api.workflowservice.v1.ResetActivityExecutionRequest\x1a?.temporal.api.workflowservice.v1.ResetActivityExecutionResponse"\xe1\x02\x82\xd3\xe4\x93\x02\xb0\x02"6/namespaces/{namespace}/activities/{activity_id}/reset:\x01*ZB"=/api/v1/namespaces/{namespace}/activities/{activity_id}/reset:\x01*ZS"N/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/reset:\x01*ZZ"U/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/reset:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\x8b\x04\n\x18UnpauseActivityExecution\x12@.temporal.api.workflowservice.v1.UnpauseActivityExecutionRequest\x1a\x41.temporal.api.workflowservice.v1.UnpauseActivityExecutionResponse"\xe9\x02\x82\xd3\xe4\x93\x02\xb8\x02"8/namespaces/{namespace}/activities/{activity_id}/unpause:\x01*ZD"?/api/v1/namespaces/{namespace}/activities/{activity_id}/unpause:\x01*ZU"P/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/unpause:\x01*Z\\"W/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/unpause:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xb9\x04\n\x1eUpdateActivityExecutionOptions\x12\x46.temporal.api.workflowservice.v1.UpdateActivityExecutionOptionsRequest\x1aG.temporal.api.workflowservice.v1.UpdateActivityExecutionOptionsResponse"\x85\x03\x82\xd3\xe4\x93\x02\xd4\x02"?/namespaces/{namespace}/activities/{activity_id}/update-options:\x01*ZK"F/api/v1/namespaces/{namespace}/activities/{activity_id}/update-options:\x01*Z\\"W/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/update-options:\x01*Zc"^/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/update-options:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xd6\x02\n TerminateNexusOperationExecution\x12H.temporal.api.workflowservice.v1.TerminateNexusOperationExecutionRequest\x1aI.temporal.api.workflowservice.v1.TerminateNexusOperationExecutionResponse"\x9c\x01\x82\xd3\xe4\x93\x02\x95\x01"A/namespaces/{namespace}/nexus-operations/{operation_id}/terminate:\x01*ZM"H/api/v1/namespaces/{namespace}/nexus-operations/{operation_id}/terminate:\x01*\x12\xb0\x01\n\x1d\x44\x65leteNexusOperationExecution\x12\x45.temporal.api.workflowservice.v1.DeleteNexusOperationExecutionRequest\x1a\x46.temporal.api.workflowservice.v1.DeleteNexusOperationExecutionResponse"\x00\x12\xc1\x03\n!PollWorkflowExecutionTimeSkipping\x12I.temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingRequest\x1aJ.temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponse"\x84\x02\x82\xd3\xe4\x93\x02\xb7\x01\x12U/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/time-skipping/pollZ^\x12\\/api/v1/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/time-skipping/poll\x8a\x9d\xcc\x1b\x41\n\x14temporal-resource-id\x12)workflow:{workflow_execution.workflow_id}B\xb6\x01\n"io.temporal.api.workflowservice.v1B\x0cServiceProtoP\x01Z5go.temporal.io/api/workflowservice/v1;workflowservice\xaa\x02!Temporalio.Api.WorkflowService.V1\xea\x02$Temporalio::Api::WorkflowService::V1b\x06proto3' + b'\n-temporal/api/workflowservice/v1/service.proto\x12\x1ftemporal.api.workflowservice.v1\x1a\x1cgoogle/api/annotations.proto\x1a!nexusannotations/v1/options.proto\x1a+temporal/api/protometa/v1/annotations.proto\x1a\x36temporal/api/workflowservice/v1/request_response.proto2\x86\xcd\x02\n\x0fWorkflowService\x12\xc3\x01\n\x11RegisterNamespace\x12\x39.temporal.api.workflowservice.v1.RegisterNamespaceRequest\x1a:.temporal.api.workflowservice.v1.RegisterNamespaceResponse"7\x82\xd3\xe4\x93\x02\x31"\x13/cluster/namespaces:\x01*Z\x17"\x12/api/v1/namespaces:\x01*\x12\xd5\x01\n\x11\x44\x65scribeNamespace\x12\x39.temporal.api.workflowservice.v1.DescribeNamespaceRequest\x1a:.temporal.api.workflowservice.v1.DescribeNamespaceResponse"I\x82\xd3\xe4\x93\x02\x43\x12\x1f/cluster/namespaces/{namespace}Z \x12\x1e/api/v1/namespaces/{namespace}\x12\xb4\x01\n\x0eListNamespaces\x12\x36.temporal.api.workflowservice.v1.ListNamespacesRequest\x1a\x37.temporal.api.workflowservice.v1.ListNamespacesResponse"1\x82\xd3\xe4\x93\x02+\x12\x13/cluster/namespacesZ\x14\x12\x12/api/v1/namespaces\x12\xe3\x01\n\x0fUpdateNamespace\x12\x37.temporal.api.workflowservice.v1.UpdateNamespaceRequest\x1a\x38.temporal.api.workflowservice.v1.UpdateNamespaceResponse"]\x82\xd3\xe4\x93\x02W"&/cluster/namespaces/{namespace}/update:\x01*Z*"%/api/v1/namespaces/{namespace}/update:\x01*\x12\x8f\x01\n\x12\x44\x65precateNamespace\x12:.temporal.api.workflowservice.v1.DeprecateNamespaceRequest\x1a;.temporal.api.workflowservice.v1.DeprecateNamespaceResponse"\x00\x12\xc6\x02\n\x16StartWorkflowExecution\x12>.temporal.api.workflowservice.v1.StartWorkflowExecutionRequest\x1a?.temporal.api.workflowservice.v1.StartWorkflowExecutionResponse"\xaa\x01\x82\xd3\xe4\x93\x02q"//namespaces/{namespace}/workflows/{workflow_id}:\x01*Z;"6/api/v1/namespaces/{namespace}/workflows/{workflow_id}:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16workflow:{workflow_id}\x12\xc2\x01\n\x15\x45xecuteMultiOperation\x12=.temporal.api.workflowservice.v1.ExecuteMultiOperationRequest\x1a>.temporal.api.workflowservice.v1.ExecuteMultiOperationResponse"*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xfe\x02\n\x1bGetWorkflowExecutionHistory\x12\x43.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryRequest\x1a\x44.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse"\xd3\x01\x82\xd3\xe4\x93\x02\x8f\x01\x12\x41/namespaces/{namespace}/workflows/{execution.workflow_id}/historyZJ\x12H/api/v1/namespaces/{namespace}/workflows/{execution.workflow_id}/history\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\xa3\x03\n"GetWorkflowExecutionHistoryReverse\x12J.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseRequest\x1aK.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseResponse"\xe3\x01\x82\xd3\xe4\x93\x02\x9f\x01\x12I/namespaces/{namespace}/workflows/{execution.workflow_id}/history-reverseZR\x12P/api/v1/namespaces/{namespace}/workflows/{execution.workflow_id}/history-reverse\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\xcd\x01\n\x15PollWorkflowTaskQueue\x12=.temporal.api.workflowservice.v1.PollWorkflowTaskQueueRequest\x1a>.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xd7\x01\n\x1cRespondWorkflowTaskCompleted\x12\x44.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest\x1a\x45.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedResponse"*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xce\x01\n\x19RespondWorkflowTaskFailed\x12\x41.temporal.api.workflowservice.v1.RespondWorkflowTaskFailedRequest\x1a\x42.temporal.api.workflowservice.v1.RespondWorkflowTaskFailedResponse"*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xcd\x01\n\x15PollActivityTaskQueue\x12=.temporal.api.workflowservice.v1.PollActivityTaskQueueRequest\x1a>.temporal.api.workflowservice.v1.PollActivityTaskQueueResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xc2\x02\n\x1bRecordActivityTaskHeartbeat\x12\x43.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatRequest\x1a\x44.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatResponse"\x97\x01\x82\xd3\xe4\x93\x02g"*/namespaces/{namespace}/activity-heartbeat:\x01*Z6"1/api/v1/namespaces/{namespace}/activity-heartbeat:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xa8\x04\n\x1fRecordActivityTaskHeartbeatById\x12G.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatByIdRequest\x1aH.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatByIdResponse"\xf1\x02\x82\xd3\xe4\x93\x02\xc0\x02":/namespaces/{namespace}/activities/{activity_id}/heartbeat:\x01*ZF"A/api/v1/namespaces/{namespace}/activities/{activity_id}/heartbeat:\x01*ZW"R/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/heartbeat:\x01*Z^"Y/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/heartbeat:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xc3\x02\n\x1cRespondActivityTaskCompleted\x12\x44.temporal.api.workflowservice.v1.RespondActivityTaskCompletedRequest\x1a\x45.temporal.api.workflowservice.v1.RespondActivityTaskCompletedResponse"\x95\x01\x82\xd3\xe4\x93\x02\x65")/namespaces/{namespace}/activity-complete:\x01*Z5"0/api/v1/namespaces/{namespace}/activity-complete:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xa7\x04\n RespondActivityTaskCompletedById\x12H.temporal.api.workflowservice.v1.RespondActivityTaskCompletedByIdRequest\x1aI.temporal.api.workflowservice.v1.RespondActivityTaskCompletedByIdResponse"\xed\x02\x82\xd3\xe4\x93\x02\xbc\x02"9/namespaces/{namespace}/activities/{activity_id}/complete:\x01*ZE"@/api/v1/namespaces/{namespace}/activities/{activity_id}/complete:\x01*ZV"Q/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/complete:\x01*Z]"X/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/complete:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xb2\x02\n\x19RespondActivityTaskFailed\x12\x41.temporal.api.workflowservice.v1.RespondActivityTaskFailedRequest\x1a\x42.temporal.api.workflowservice.v1.RespondActivityTaskFailedResponse"\x8d\x01\x82\xd3\xe4\x93\x02]"%/namespaces/{namespace}/activity-fail:\x01*Z1",/api/v1/namespaces/{namespace}/activity-fail:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\x8e\x04\n\x1dRespondActivityTaskFailedById\x12\x45.temporal.api.workflowservice.v1.RespondActivityTaskFailedByIdRequest\x1a\x46.temporal.api.workflowservice.v1.RespondActivityTaskFailedByIdResponse"\xdd\x02\x82\xd3\xe4\x93\x02\xac\x02"5/namespaces/{namespace}/activities/{activity_id}/fail:\x01*ZA"/namespaces/{namespace}/channels/{notification.channel}/notify:\x01*ZJ"E/api/v1/namespaces/{namespace}/channels/{notification.channel}/notify:\x01*Ze"`/namespaces/{namespace}/workflows/{execution.business_id}/channels/{notification.channel}/notify:\x01*Zl"g/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{notification.channel}/notify:\x01*Zf"a/namespaces/{namespace}/activities/{execution.business_id}/channels/{notification.channel}/notify:\x01*Zm"h/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{notification.channel}/notify:\x01*\x12\xa5\x05\n\x17RegisterChannelListener\x12?.temporal.api.workflowservice.v1.RegisterChannelListenerRequest\x1a@.temporal.api.workflowservice.v1.RegisterChannelListenerResponse"\x86\x04\x82\xd3\xe4\x93\x02\xff\x03"4/namespaces/{namespace}/channels/{channel}/listeners:\x01*Z@";/api/v1/namespaces/{namespace}/channels/{channel}/listeners:\x01*Z["V/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/listeners:\x01*Zb"]/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/listeners:\x01*Z\\"W/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/listeners:\x01*Zc"^/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/listeners:\x01*\x12\xed\x05\n\x19UnregisterChannelListener\x12\x41.temporal.api.workflowservice.v1.UnregisterChannelListenerRequest\x1a\x42.temporal.api.workflowservice.v1.UnregisterChannelListenerResponse"\xc8\x04\x82\xd3\xe4\x93\x02\xc1\x04*B/namespaces/{namespace}/channels/{channel}/listeners/{listener_id}ZK*I/api/v1/namespaces/{namespace}/channels/{channel}/listeners/{listener_id}Zf*d/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/listeners/{listener_id}Zm*k/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/listeners/{listener_id}Zg*e/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/listeners/{listener_id}Zn*l/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/listeners/{listener_id}\x12\x87\x05\n\x0bPollChannel\x12\x33.temporal.api.workflowservice.v1.PollChannelRequest\x1a\x34.temporal.api.workflowservice.v1.PollChannelResponse"\x8c\x04\x82\xd3\xe4\x93\x02\x85\x04\x12\x38/namespaces/{namespace}/channels/{channel}/notificationsZA\x12?/api/v1/namespaces/{namespace}/channels/{channel}/notificationsZ\\\x12Z/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/notificationsZc\x12\x61/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/notificationsZ]\x12[/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/notificationsZd\x12\x62/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/notifications\x12\xbf\x04\n\x0f\x44\x65scribeChannel\x12\x37.temporal.api.workflowservice.v1.DescribeChannelRequest\x1a\x38.temporal.api.workflowservice.v1.DescribeChannelResponse"\xb8\x03\x82\xd3\xe4\x93\x02\xb1\x03\x12*/namespaces/{namespace}/channels/{channel}Z3\x12\x31/api/v1/namespaces/{namespace}/channels/{channel}ZN\x12L/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}ZU\x12S/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}ZO\x12M/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}ZV\x12T/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}\x12\xb2\x03\n SignalWithStartWorkflowExecution\x12H.temporal.api.workflowservice.v1.SignalWithStartWorkflowExecutionRequest\x1aI.temporal.api.workflowservice.v1.SignalWithStartWorkflowExecutionResponse"\xf8\x01\xd2\x82\x04\t\x12\x07\x65xposed\x82\xd3\xe4\x93\x02\xb1\x01"O/namespaces/{namespace}/workflows/{workflow_id}/signal-with-start/{signal_name}:\x01*Z["V/api/v1/namespaces/{namespace}/workflows/{workflow_id}/signal-with-start/{signal_name}:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16workflow:{workflow_id}\x12\x8c\x03\n\x16ResetWorkflowExecution\x12>.temporal.api.workflowservice.v1.ResetWorkflowExecutionRequest\x1a?.temporal.api.workflowservice.v1.ResetWorkflowExecutionResponse"\xf0\x01\x82\xd3\xe4\x93\x02\xa3\x01"H/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/reset:\x01*ZT"O/api/v1/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/reset:\x01*\x8a\x9d\xcc\x1b\x41\n\x14temporal-resource-id\x12)workflow:{workflow_execution.workflow_id}\x12\xa0\x03\n\x1aTerminateWorkflowExecution\x12\x42.temporal.api.workflowservice.v1.TerminateWorkflowExecutionRequest\x1a\x43.temporal.api.workflowservice.v1.TerminateWorkflowExecutionResponse"\xf8\x01\x82\xd3\xe4\x93\x02\xab\x01"L/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/terminate:\x01*ZX"S/api/v1/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/terminate:\x01*\x8a\x9d\xcc\x1b\x41\n\x14temporal-resource-id\x12)workflow:{workflow_execution.workflow_id}\x12\xe4\x01\n\x17\x44\x65leteWorkflowExecution\x12?.temporal.api.workflowservice.v1.DeleteWorkflowExecutionRequest\x1a@.temporal.api.workflowservice.v1.DeleteWorkflowExecutionResponse"F\x8a\x9d\xcc\x1b\x41\n\x14temporal-resource-id\x12)workflow:{workflow_execution.workflow_id}\x12\xa7\x01\n\x1aListOpenWorkflowExecutions\x12\x42.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsRequest\x1a\x43.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsResponse"\x00\x12\xad\x01\n\x1cListClosedWorkflowExecutions\x12\x44.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsRequest\x1a\x45.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsResponse"\x00\x12\xf0\x01\n\x16ListWorkflowExecutions\x12>.temporal.api.workflowservice.v1.ListWorkflowExecutionsRequest\x1a?.temporal.api.workflowservice.v1.ListWorkflowExecutionsResponse"U\x82\xd3\xe4\x93\x02O\x12!/namespaces/{namespace}/workflowsZ*\x12(/api/v1/namespaces/{namespace}/workflows\x12\x9a\x02\n\x1eListArchivedWorkflowExecutions\x12\x46.temporal.api.workflowservice.v1.ListArchivedWorkflowExecutionsRequest\x1aG.temporal.api.workflowservice.v1.ListArchivedWorkflowExecutionsResponse"g\x82\xd3\xe4\x93\x02\x61\x12*/namespaces/{namespace}/archived-workflowsZ3\x12\x31/api/v1/namespaces/{namespace}/archived-workflows\x12\x9b\x01\n\x16ScanWorkflowExecutions\x12>.temporal.api.workflowservice.v1.ScanWorkflowExecutionsRequest\x1a?.temporal.api.workflowservice.v1.ScanWorkflowExecutionsResponse"\x00\x12\xfd\x01\n\x17\x43ountWorkflowExecutions\x12?.temporal.api.workflowservice.v1.CountWorkflowExecutionsRequest\x1a@.temporal.api.workflowservice.v1.CountWorkflowExecutionsResponse"_\x82\xd3\xe4\x93\x02Y\x12&/namespaces/{namespace}/workflow-countZ/\x12-/api/v1/namespaces/{namespace}/workflow-count\x12\x92\x01\n\x13GetSearchAttributes\x12;.temporal.api.workflowservice.v1.GetSearchAttributesRequest\x1a<.temporal.api.workflowservice.v1.GetSearchAttributesResponse"\x00\x12\xd9\x01\n\x19RespondQueryTaskCompleted\x12\x41.temporal.api.workflowservice.v1.RespondQueryTaskCompletedRequest\x1a\x42.temporal.api.workflowservice.v1.RespondQueryTaskCompletedResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xd2\x01\n\x14ResetStickyTaskQueue\x12<.temporal.api.workflowservice.v1.ResetStickyTaskQueueRequest\x1a=.temporal.api.workflowservice.v1.ResetStickyTaskQueueResponse"=\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\x83\x01\n\x0eShutdownWorker\x12\x36.temporal.api.workflowservice.v1.ShutdownWorkerRequest\x1a\x37.temporal.api.workflowservice.v1.ShutdownWorkerResponse"\x00\x12\xfc\x02\n\rQueryWorkflow\x12\x35.temporal.api.workflowservice.v1.QueryWorkflowRequest\x1a\x36.temporal.api.workflowservice.v1.QueryWorkflowResponse"\xfb\x01\x82\xd3\xe4\x93\x02\xb7\x01"R/namespaces/{namespace}/workflows/{execution.workflow_id}/query/{query.query_type}:\x01*Z^"Y/api/v1/namespaces/{namespace}/workflows/{execution.workflow_id}/query/{query.query_type}:\x01*\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\xe7\x02\n\x19\x44\x65scribeWorkflowExecution\x12\x41.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest\x1a\x42.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse"\xc2\x01\x82\xd3\xe4\x93\x02\x7f\x12\x39/namespaces/{namespace}/workflows/{execution.workflow_id}ZB\x12@/api/v1/namespaces/{namespace}/workflows/{execution.workflow_id}\x8a\x9d\xcc\x1b\x38\n\x14temporal-resource-id\x12 workflow:{execution.workflow_id}\x12\xc2\x02\n\x11\x44\x65scribeTaskQueue\x12\x39.temporal.api.workflowservice.v1.DescribeTaskQueueRequest\x1a:.temporal.api.workflowservice.v1.DescribeTaskQueueResponse"\xb5\x01\x82\xd3\xe4\x93\x02w\x12\x35/namespaces/{namespace}/task-queues/{task_queue.name}Z>\x12/namespaces/{namespace}/schedules/{schedule_id}/matching-timesZG\x12\x45/api/v1/namespaces/{namespace}/schedules/{schedule_id}/matching-times\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16schedule:{schedule_id}\x12\xa8\x02\n\x0e\x44\x65leteSchedule\x12\x36.temporal.api.workflowservice.v1.DeleteScheduleRequest\x1a\x37.temporal.api.workflowservice.v1.DeleteScheduleResponse"\xa4\x01\x82\xd3\xe4\x93\x02k*//namespaces/{namespace}/schedules/{schedule_id}Z8*6/api/v1/namespaces/{namespace}/schedules/{schedule_id}\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16schedule:{schedule_id}\x12\xd5\x01\n\rListSchedules\x12\x35.temporal.api.workflowservice.v1.ListSchedulesRequest\x1a\x36.temporal.api.workflowservice.v1.ListSchedulesResponse"U\x82\xd3\xe4\x93\x02O\x12!/namespaces/{namespace}/schedulesZ*\x12(/api/v1/namespaces/{namespace}/schedules\x12\xe2\x01\n\x0e\x43ountSchedules\x12\x36.temporal.api.workflowservice.v1.CountSchedulesRequest\x1a\x37.temporal.api.workflowservice.v1.CountSchedulesResponse"_\x82\xd3\xe4\x93\x02Y\x12&/namespaces/{namespace}/schedule-countZ/\x12-/api/v1/namespaces/{namespace}/schedule-count\x12\xb9\x01\n UpdateWorkerBuildIdCompatibility\x12H.temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest\x1aI.temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityResponse"\x00\x12\xe1\x02\n\x1dGetWorkerBuildIdCompatibility\x12\x45.temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityRequest\x1a\x46.temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityResponse"\xb0\x01\x82\xd3\xe4\x93\x02\xa9\x01\x12N/namespaces/{namespace}/task-queues/{task_queue}/worker-build-id-compatibilityZW\x12U/api/v1/namespaces/{namespace}/task-queues/{task_queue}/worker-build-id-compatibility\x12\xaa\x01\n\x1bUpdateWorkerVersioningRules\x12\x43.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest\x1a\x44.temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesResponse"\x00\x12\xc6\x02\n\x18GetWorkerVersioningRules\x12@.temporal.api.workflowservice.v1.GetWorkerVersioningRulesRequest\x1a\x41.temporal.api.workflowservice.v1.GetWorkerVersioningRulesResponse"\xa4\x01\x82\xd3\xe4\x93\x02\x9d\x01\x12H/namespaces/{namespace}/task-queues/{task_queue}/worker-versioning-rulesZQ\x12O/api/v1/namespaces/{namespace}/task-queues/{task_queue}/worker-versioning-rules\x12\x97\x02\n\x19GetWorkerTaskReachability\x12\x41.temporal.api.workflowservice.v1.GetWorkerTaskReachabilityRequest\x1a\x42.temporal.api.workflowservice.v1.GetWorkerTaskReachabilityResponse"s\x82\xd3\xe4\x93\x02m\x12\x30/namespaces/{namespace}/worker-task-reachabilityZ9\x12\x37/api/v1/namespaces/{namespace}/worker-task-reachability\x12\xc8\x02\n\x12\x44\x65scribeDeployment\x12:.temporal.api.workflowservice.v1.DescribeDeploymentRequest\x1a;.temporal.api.workflowservice.v1.DescribeDeploymentResponse"\xb8\x01\x82\xd3\xe4\x93\x02\xb1\x01\x12R/namespaces/{namespace}/deployments/{deployment.series_name}/{deployment.build_id}Z[\x12Y/api/v1/namespaces/{namespace}/deployments/{deployment.series_name}/{deployment.build_id}\x12\x81\x04\n\x1f\x44\x65scribeWorkerDeploymentVersion\x12G.temporal.api.workflowservice.v1.DescribeWorkerDeploymentVersionRequest\x1aH.temporal.api.workflowservice.v1.DescribeWorkerDeploymentVersionResponse"\xca\x02\x82\xd3\xe4\x93\x02\xf7\x01\x12u/namespaces/{namespace}/worker-deployment-versions/{deployment_version.deployment_name}/{deployment_version.build_id}Z~\x12|/api/v1/namespaces/{namespace}/worker-deployment-versions/{deployment_version.deployment_name}/{deployment_version.build_id}\x8a\x9d\xcc\x1bG\n\x14temporal-resource-id\x12/deployment:{deployment_version.deployment_name}\x12\xdf\x01\n\x0fListDeployments\x12\x37.temporal.api.workflowservice.v1.ListDeploymentsRequest\x1a\x38.temporal.api.workflowservice.v1.ListDeploymentsResponse"Y\x82\xd3\xe4\x93\x02S\x12#/namespaces/{namespace}/deploymentsZ,\x12*/api/v1/namespaces/{namespace}/deployments\x12\xf7\x02\n\x19GetDeploymentReachability\x12\x41.temporal.api.workflowservice.v1.GetDeploymentReachabilityRequest\x1a\x42.temporal.api.workflowservice.v1.GetDeploymentReachabilityResponse"\xd2\x01\x82\xd3\xe4\x93\x02\xcb\x01\x12_/namespaces/{namespace}/deployments/{deployment.series_name}/{deployment.build_id}/reachabilityZh\x12\x66/api/v1/namespaces/{namespace}/deployments/{deployment.series_name}/{deployment.build_id}/reachability\x12\x99\x02\n\x14GetCurrentDeployment\x12<.temporal.api.workflowservice.v1.GetCurrentDeploymentRequest\x1a=.temporal.api.workflowservice.v1.GetCurrentDeploymentResponse"\x83\x01\x82\xd3\xe4\x93\x02}\x12\x38/namespaces/{namespace}/current-deployment/{series_name}ZA\x12?/api/v1/namespaces/{namespace}/current-deployment/{series_name}\x12\xb6\x02\n\x14SetCurrentDeployment\x12<.temporal.api.workflowservice.v1.SetCurrentDeploymentRequest\x1a=.temporal.api.workflowservice.v1.SetCurrentDeploymentResponse"\xa0\x01\x82\xd3\xe4\x93\x02\x99\x01"C/namespaces/{namespace}/current-deployment/{deployment.series_name}:\x01*ZO"J/api/v1/namespaces/{namespace}/current-deployment/{deployment.series_name}:\x01*\x12\xb0\x03\n!SetWorkerDeploymentCurrentVersion\x12I.temporal.api.workflowservice.v1.SetWorkerDeploymentCurrentVersionRequest\x1aJ.temporal.api.workflowservice.v1.SetWorkerDeploymentCurrentVersionResponse"\xf3\x01\x82\xd3\xe4\x93\x02\xb3\x01"P/namespaces/{namespace}/worker-deployments/{deployment_name}/set-current-version:\x01*Z\\"W/api/v1/namespaces/{namespace}/worker-deployments/{deployment_name}/set-current-version:\x01*\x8a\x9d\xcc\x1b\x34\n\x14temporal-resource-id\x12\x1c\x64\x65ployment:{deployment_name}\x12\xe7\x02\n\x18\x44\x65scribeWorkerDeployment\x12@.temporal.api.workflowservice.v1.DescribeWorkerDeploymentRequest\x1a\x41.temporal.api.workflowservice.v1.DescribeWorkerDeploymentResponse"\xc5\x01\x82\xd3\xe4\x93\x02\x85\x01\x12.temporal.api.workflowservice.v1.DeleteWorkerDeploymentRequest\x1a?.temporal.api.workflowservice.v1.DeleteWorkerDeploymentResponse"\xc5\x01\x82\xd3\xe4\x93\x02\x85\x01*.temporal.api.workflowservice.v1.ListWorkerDeploymentsResponse"g\x82\xd3\xe4\x93\x02\x61\x12*/namespaces/{namespace}/worker-deploymentsZ3\x12\x31/api/v1/namespaces/{namespace}/worker-deployments\x12\xae\x02\n\x16\x43reateWorkerDeployment\x12>.temporal.api.workflowservice.v1.CreateWorkerDeploymentRequest\x1a?.temporal.api.workflowservice.v1.CreateWorkerDeploymentResponse"\x92\x01\x82\xd3\xe4\x93\x02\x8b\x01".temporal.api.workflowservice.v1.DescribeBatchOperationRequest\x1a?.temporal.api.workflowservice.v1.DescribeBatchOperationResponse"\xa0\x01\x82\xd3\xe4\x93\x02o\x12\x31/namespaces/{namespace}/batch-operations/{job_id}Z:\x12\x38/api/v1/namespaces/{namespace}/batch-operations/{job_id}\x8a\x9d\xcc\x1b&\n\x14temporal-resource-id\x12\x0e\x62\x61tch:{job_id}\x12\xf5\x01\n\x13ListBatchOperations\x12;.temporal.api.workflowservice.v1.ListBatchOperationsRequest\x1a<.temporal.api.workflowservice.v1.ListBatchOperationsResponse"c\x82\xd3\xe4\x93\x02]\x12(/namespaces/{namespace}/batch-operationsZ1\x12//api/v1/namespaces/{namespace}/batch-operations\x12\xc4\x01\n\x12PollNexusTaskQueue\x12:.temporal.api.workflowservice.v1.PollNexusTaskQueueRequest\x1a;.temporal.api.workflowservice.v1.PollNexusTaskQueueResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xd9\x01\n\x19RespondNexusTaskCompleted\x12\x41.temporal.api.workflowservice.v1.RespondNexusTaskCompletedRequest\x1a\x42.temporal.api.workflowservice.v1.RespondNexusTaskCompletedResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xd0\x01\n\x16RespondNexusTaskFailed\x12>.temporal.api.workflowservice.v1.RespondNexusTaskFailedRequest\x1a?.temporal.api.workflowservice.v1.RespondNexusTaskFailedResponse"5\x8a\x9d\xcc\x1b\x30\n\x14temporal-resource-id\x12\x18poller:{poller_group_id}\x12\xe8\x02\n\x15UpdateActivityOptions\x12=.temporal.api.workflowservice.v1.UpdateActivityOptionsRequest\x1a>.temporal.api.workflowservice.v1.UpdateActivityOptionsResponse"\xcf\x01\x82\xd3\xe4\x93\x02\x8b\x01".temporal.api.workflowservice.v1.RecordWorkerHeartbeatResponse"\x95\x01\x82\xd3\xe4\x93\x02\x65")/namespaces/{namespace}/workers/heartbeat:\x01*Z5"0/api/v1/namespaces/{namespace}/workers/heartbeat:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xcb\x01\n\x0bListWorkers\x12\x33.temporal.api.workflowservice.v1.ListWorkersRequest\x1a\x34.temporal.api.workflowservice.v1.ListWorkersResponse"Q\x82\xd3\xe4\x93\x02K\x12\x1f/namespaces/{namespace}/workersZ(\x12&/api/v1/namespaces/{namespace}/workers\x12\xd8\x01\n\x0c\x43ountWorkers\x12\x34.temporal.api.workflowservice.v1.CountWorkersRequest\x1a\x35.temporal.api.workflowservice.v1.CountWorkersResponse"[\x82\xd3\xe4\x93\x02U\x12$/namespaces/{namespace}/worker-countZ-\x12+/api/v1/namespaces/{namespace}/worker-count\x12\xe2\x02\n\x15UpdateTaskQueueConfig\x12=.temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest\x1a>.temporal.api.workflowservice.v1.UpdateTaskQueueConfigResponse"\xc9\x01\x82\xd3\xe4\x93\x02\x8f\x01">/namespaces/{namespace}/task-queues/{task_queue}/update-config:\x01*ZJ"E/api/v1/namespaces/{namespace}/task-queues/{task_queue}/update-config:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16taskqueue:{task_queue}\x12\xa8\x02\n\x11\x46\x65tchWorkerConfig\x12\x39.temporal.api.workflowservice.v1.FetchWorkerConfigRequest\x1a:.temporal.api.workflowservice.v1.FetchWorkerConfigResponse"\x9b\x01\x82\xd3\xe4\x93\x02k",/namespaces/{namespace}/workers/fetch-config:\x01*Z8"3/api/v1/namespaces/{namespace}/workers/fetch-config:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xad\x02\n\x12UpdateWorkerConfig\x12:.temporal.api.workflowservice.v1.UpdateWorkerConfigRequest\x1a;.temporal.api.workflowservice.v1.UpdateWorkerConfigResponse"\x9d\x01\x82\xd3\xe4\x93\x02m"-/namespaces/{namespace}/workers/update-config:\x01*Z9"4/api/v1/namespaces/{namespace}/workers/update-config:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xcd\x02\n\x0e\x44\x65scribeWorker\x12\x36.temporal.api.workflowservice.v1.DescribeWorkerRequest\x1a\x37.temporal.api.workflowservice.v1.DescribeWorkerResponse"\xc9\x01\x82\xd3\xe4\x93\x02\x89\x01\x12>/namespaces/{namespace}/workers/describe/{worker_instance_key}ZG\x12\x45/api/v1/namespaces/{namespace}/workers/describe/{worker_instance_key}\x8a\x9d\xcc\x1b\x34\n\x14temporal-resource-id\x12\x1cworker:{worker_instance_key}\x12\xd2\x02\n\x16PauseWorkflowExecution\x12>.temporal.api.workflowservice.v1.PauseWorkflowExecutionRequest\x1a?.temporal.api.workflowservice.v1.PauseWorkflowExecutionResponse"\xb6\x01\x82\xd3\xe4\x93\x02}"5/namespaces/{namespace}/workflows/{workflow_id}/pause:\x01*ZA"/api/v1/namespaces/{namespace}/workflows/{workflow_id}/unpause:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16workflow:{workflow_id}\x12\xc8\x02\n\x16StartActivityExecution\x12>.temporal.api.workflowservice.v1.StartActivityExecutionRequest\x1a?.temporal.api.workflowservice.v1.StartActivityExecutionResponse"\xac\x01\x82\xd3\xe4\x93\x02s"0/namespaces/{namespace}/activities/{activity_id}:\x01*Z<"7/api/v1/namespaces/{namespace}/activities/{activity_id}:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\xb6\x02\n\x1cStartNexusOperationExecution\x12\x44.temporal.api.workflowservice.v1.StartNexusOperationExecutionRequest\x1a\x45.temporal.api.workflowservice.v1.StartNexusOperationExecutionResponse"\x88\x01\x82\xd3\xe4\x93\x02\x81\x01"7/namespaces/{namespace}/nexus-operations/{operation_id}:\x01*ZC">/api/v1/namespaces/{namespace}/nexus-operations/{operation_id}:\x01*\x12\xcb\x02\n\x19\x44\x65scribeActivityExecution\x12\x41.temporal.api.workflowservice.v1.DescribeActivityExecutionRequest\x1a\x42.temporal.api.workflowservice.v1.DescribeActivityExecutionResponse"\xa6\x01\x82\xd3\xe4\x93\x02m\x12\x30/namespaces/{namespace}/activities/{activity_id}Z9\x12\x37/api/v1/namespaces/{namespace}/activities/{activity_id}\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\xb8\x02\n\x1f\x44\x65scribeNexusOperationExecution\x12G.temporal.api.workflowservice.v1.DescribeNexusOperationExecutionRequest\x1aH.temporal.api.workflowservice.v1.DescribeNexusOperationExecutionResponse"\x81\x01\x82\xd3\xe4\x93\x02{\x12\x37/namespaces/{namespace}/nexus-operations/{operation_id}Z@\x12>/api/v1/namespaces/{namespace}/nexus-operations/{operation_id}\x12\xcf\x02\n\x15PollActivityExecution\x12=.temporal.api.workflowservice.v1.PollActivityExecutionRequest\x1a>.temporal.api.workflowservice.v1.PollActivityExecutionResponse"\xb6\x01\x82\xd3\xe4\x93\x02}\x12\x38/namespaces/{namespace}/activities/{activity_id}/outcomeZA\x12?/api/v1/namespaces/{namespace}/activities/{activity_id}/outcome\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\xb7\x02\n\x1bPollNexusOperationExecution\x12\x43.temporal.api.workflowservice.v1.PollNexusOperationExecutionRequest\x1a\x44.temporal.api.workflowservice.v1.PollNexusOperationExecutionResponse"\x8c\x01\x82\xd3\xe4\x93\x02\x85\x01\x12.temporal.api.workflowservice.v1.ListActivityExecutionsRequest\x1a?.temporal.api.workflowservice.v1.ListActivityExecutionsResponse"W\x82\xd3\xe4\x93\x02Q\x12"/namespaces/{namespace}/activitiesZ+\x12)/api/v1/namespaces/{namespace}/activities\x12\x90\x02\n\x1cListNexusOperationExecutions\x12\x44.temporal.api.workflowservice.v1.ListNexusOperationExecutionsRequest\x1a\x45.temporal.api.workflowservice.v1.ListNexusOperationExecutionsResponse"c\x82\xd3\xe4\x93\x02]\x12(/namespaces/{namespace}/nexus-operationsZ1\x12//api/v1/namespaces/{namespace}/nexus-operations\x12\xfd\x01\n\x17\x43ountActivityExecutions\x12?.temporal.api.workflowservice.v1.CountActivityExecutionsRequest\x1a@.temporal.api.workflowservice.v1.CountActivityExecutionsResponse"_\x82\xd3\xe4\x93\x02Y\x12&/namespaces/{namespace}/activity-countZ/\x12-/api/v1/namespaces/{namespace}/activity-count\x12\x9d\x02\n\x1d\x43ountNexusOperationExecutions\x12\x45.temporal.api.workflowservice.v1.CountNexusOperationExecutionsRequest\x1a\x46.temporal.api.workflowservice.v1.CountNexusOperationExecutionsResponse"m\x82\xd3\xe4\x93\x02g\x12-/namespaces/{namespace}/nexus-operation-countZ6\x12\x34/api/v1/namespaces/{namespace}/nexus-operation-count\x12\xef\x02\n\x1eRequestCancelActivityExecution\x12\x46.temporal.api.workflowservice.v1.RequestCancelActivityExecutionRequest\x1aG.temporal.api.workflowservice.v1.RequestCancelActivityExecutionResponse"\xbb\x01\x82\xd3\xe4\x93\x02\x81\x01"7/namespaces/{namespace}/activities/{activity_id}/cancel:\x01*ZC">/api/v1/namespaces/{namespace}/activities/{activity_id}/cancel:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\xdc\x02\n$RequestCancelNexusOperationExecution\x12L.temporal.api.workflowservice.v1.RequestCancelNexusOperationExecutionRequest\x1aM.temporal.api.workflowservice.v1.RequestCancelNexusOperationExecutionResponse"\x96\x01\x82\xd3\xe4\x93\x02\x8f\x01">/namespaces/{namespace}/nexus-operations/{operation_id}/cancel:\x01*ZJ"E/api/v1/namespaces/{namespace}/nexus-operations/{operation_id}/cancel:\x01*\x12\xe9\x02\n\x1aTerminateActivityExecution\x12\x42.temporal.api.workflowservice.v1.TerminateActivityExecutionRequest\x1a\x43.temporal.api.workflowservice.v1.TerminateActivityExecutionResponse"\xc1\x01\x82\xd3\xe4\x93\x02\x87\x01":/namespaces/{namespace}/activities/{activity_id}/terminate:\x01*ZF"A/api/v1/namespaces/{namespace}/activities/{activity_id}/terminate:\x01*\x8a\x9d\xcc\x1b.\n\x14temporal-resource-id\x12\x16\x61\x63tivity:{activity_id}\x12\x9e\x01\n\x17\x44\x65leteActivityExecution\x12?.temporal.api.workflowservice.v1.DeleteActivityExecutionRequest\x1a@.temporal.api.workflowservice.v1.DeleteActivityExecutionResponse"\x00\x12\xfd\x03\n\x16PauseActivityExecution\x12>.temporal.api.workflowservice.v1.PauseActivityExecutionRequest\x1a?.temporal.api.workflowservice.v1.PauseActivityExecutionResponse"\xe1\x02\x82\xd3\xe4\x93\x02\xb0\x02"6/namespaces/{namespace}/activities/{activity_id}/pause:\x01*ZB"=/api/v1/namespaces/{namespace}/activities/{activity_id}/pause:\x01*ZS"N/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/pause:\x01*ZZ"U/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/pause:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xfd\x03\n\x16ResetActivityExecution\x12>.temporal.api.workflowservice.v1.ResetActivityExecutionRequest\x1a?.temporal.api.workflowservice.v1.ResetActivityExecutionResponse"\xe1\x02\x82\xd3\xe4\x93\x02\xb0\x02"6/namespaces/{namespace}/activities/{activity_id}/reset:\x01*ZB"=/api/v1/namespaces/{namespace}/activities/{activity_id}/reset:\x01*ZS"N/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/reset:\x01*ZZ"U/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/reset:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\x8b\x04\n\x18UnpauseActivityExecution\x12@.temporal.api.workflowservice.v1.UnpauseActivityExecutionRequest\x1a\x41.temporal.api.workflowservice.v1.UnpauseActivityExecutionResponse"\xe9\x02\x82\xd3\xe4\x93\x02\xb8\x02"8/namespaces/{namespace}/activities/{activity_id}/unpause:\x01*ZD"?/api/v1/namespaces/{namespace}/activities/{activity_id}/unpause:\x01*ZU"P/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/unpause:\x01*Z\\"W/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/unpause:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xb9\x04\n\x1eUpdateActivityExecutionOptions\x12\x46.temporal.api.workflowservice.v1.UpdateActivityExecutionOptionsRequest\x1aG.temporal.api.workflowservice.v1.UpdateActivityExecutionOptionsResponse"\x85\x03\x82\xd3\xe4\x93\x02\xd4\x02"?/namespaces/{namespace}/activities/{activity_id}/update-options:\x01*ZK"F/api/v1/namespaces/{namespace}/activities/{activity_id}/update-options:\x01*Z\\"W/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/update-options:\x01*Zc"^/api/v1/namespaces/{namespace}/workflows/{workflow_id}/activities/{activity_id}/update-options:\x01*\x8a\x9d\xcc\x1b%\n\x14temporal-resource-id\x12\r{resource_id}\x12\xd6\x02\n TerminateNexusOperationExecution\x12H.temporal.api.workflowservice.v1.TerminateNexusOperationExecutionRequest\x1aI.temporal.api.workflowservice.v1.TerminateNexusOperationExecutionResponse"\x9c\x01\x82\xd3\xe4\x93\x02\x95\x01"A/namespaces/{namespace}/nexus-operations/{operation_id}/terminate:\x01*ZM"H/api/v1/namespaces/{namespace}/nexus-operations/{operation_id}/terminate:\x01*\x12\xb0\x01\n\x1d\x44\x65leteNexusOperationExecution\x12\x45.temporal.api.workflowservice.v1.DeleteNexusOperationExecutionRequest\x1a\x46.temporal.api.workflowservice.v1.DeleteNexusOperationExecutionResponse"\x00\x12\xc1\x03\n!PollWorkflowExecutionTimeSkipping\x12I.temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingRequest\x1aJ.temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponse"\x84\x02\x82\xd3\xe4\x93\x02\xb7\x01\x12U/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/time-skipping/pollZ^\x12\\/api/v1/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/time-skipping/poll\x8a\x9d\xcc\x1b\x41\n\x14temporal-resource-id\x12)workflow:{workflow_execution.workflow_id}B\xb6\x01\n"io.temporal.api.workflowservice.v1B\x0cServiceProtoP\x01Z5go.temporal.io/api/workflowservice/v1;workflowservice\xaa\x02!Temporalio.Api.WorkflowService.V1\xea\x02$Temporalio::Api::WorkflowService::V1b\x06proto3' ) @@ -135,6 +135,26 @@ _WORKFLOWSERVICE.methods_by_name[ "SignalWorkflowExecution" ]._serialized_options = b'\202\323\344\223\002\301\001"W/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/signal/{signal_name}:\001*Zc"^/api/v1/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/signal/{signal_name}:\001*\212\235\314\033A\n\024temporal-resource-id\022)workflow:{workflow_execution.workflow_id}' + _WORKFLOWSERVICE.methods_by_name["NotifyChannel"]._options = None + _WORKFLOWSERVICE.methods_by_name[ + "NotifyChannel" + ]._serialized_options = b'\202\323\344\223\002\273\004">/namespaces/{namespace}/channels/{notification.channel}/notify:\001*ZJ"E/api/v1/namespaces/{namespace}/channels/{notification.channel}/notify:\001*Ze"`/namespaces/{namespace}/workflows/{execution.business_id}/channels/{notification.channel}/notify:\001*Zl"g/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{notification.channel}/notify:\001*Zf"a/namespaces/{namespace}/activities/{execution.business_id}/channels/{notification.channel}/notify:\001*Zm"h/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{notification.channel}/notify:\001*' + _WORKFLOWSERVICE.methods_by_name["RegisterChannelListener"]._options = None + _WORKFLOWSERVICE.methods_by_name[ + "RegisterChannelListener" + ]._serialized_options = b'\202\323\344\223\002\377\003"4/namespaces/{namespace}/channels/{channel}/listeners:\001*Z@";/api/v1/namespaces/{namespace}/channels/{channel}/listeners:\001*Z["V/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/listeners:\001*Zb"]/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/listeners:\001*Z\\"W/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/listeners:\001*Zc"^/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/listeners:\001*' + _WORKFLOWSERVICE.methods_by_name["UnregisterChannelListener"]._options = None + _WORKFLOWSERVICE.methods_by_name[ + "UnregisterChannelListener" + ]._serialized_options = b"\202\323\344\223\002\301\004*B/namespaces/{namespace}/channels/{channel}/listeners/{listener_id}ZK*I/api/v1/namespaces/{namespace}/channels/{channel}/listeners/{listener_id}Zf*d/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/listeners/{listener_id}Zm*k/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/listeners/{listener_id}Zg*e/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/listeners/{listener_id}Zn*l/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/listeners/{listener_id}" + _WORKFLOWSERVICE.methods_by_name["PollChannel"]._options = None + _WORKFLOWSERVICE.methods_by_name[ + "PollChannel" + ]._serialized_options = b"\202\323\344\223\002\205\004\0228/namespaces/{namespace}/channels/{channel}/notificationsZA\022?/api/v1/namespaces/{namespace}/channels/{channel}/notificationsZ\\\022Z/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/notificationsZc\022a/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}/notificationsZ]\022[/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/notificationsZd\022b/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}/notifications" + _WORKFLOWSERVICE.methods_by_name["DescribeChannel"]._options = None + _WORKFLOWSERVICE.methods_by_name[ + "DescribeChannel" + ]._serialized_options = b"\202\323\344\223\002\261\003\022*/namespaces/{namespace}/channels/{channel}Z3\0221/api/v1/namespaces/{namespace}/channels/{channel}ZN\022L/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}ZU\022S/api/v1/namespaces/{namespace}/workflows/{execution.business_id}/channels/{channel}ZO\022M/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}ZV\022T/api/v1/namespaces/{namespace}/activities/{execution.business_id}/channels/{channel}" _WORKFLOWSERVICE.methods_by_name["SignalWithStartWorkflowExecution"]._options = None _WORKFLOWSERVICE.methods_by_name[ "SignalWithStartWorkflowExecution" @@ -526,5 +546,5 @@ "PollWorkflowExecutionTimeSkipping" ]._serialized_options = b"\202\323\344\223\002\267\001\022U/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/time-skipping/pollZ^\022\\/api/v1/namespaces/{namespace}/workflows/{workflow_execution.workflow_id}/time-skipping/poll\212\235\314\033A\n\024temporal-resource-id\022)workflow:{workflow_execution.workflow_id}" _WORKFLOWSERVICE._serialized_start = 250 - _WORKFLOWSERVICE._serialized_end = 39510 + _WORKFLOWSERVICE._serialized_end = 42880 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/workflowservice/v1/service_pb2_grpc.py b/temporalio/api/workflowservice/v1/service_pb2_grpc.py index ac29720a33..9f639fa101 100644 --- a/temporalio/api/workflowservice/v1/service_pb2_grpc.py +++ b/temporalio/api/workflowservice/v1/service_pb2_grpc.py @@ -143,6 +143,31 @@ def __init__(self, channel): request_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.SignalWorkflowExecutionRequest.SerializeToString, response_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.SignalWorkflowExecutionResponse.FromString, ) + self.NotifyChannel = channel.unary_unary( + "/temporal.api.workflowservice.v1.WorkflowService/NotifyChannel", + request_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.NotifyChannelRequest.SerializeToString, + response_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.NotifyChannelResponse.FromString, + ) + self.RegisterChannelListener = channel.unary_unary( + "/temporal.api.workflowservice.v1.WorkflowService/RegisterChannelListener", + request_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.RegisterChannelListenerRequest.SerializeToString, + response_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.RegisterChannelListenerResponse.FromString, + ) + self.UnregisterChannelListener = channel.unary_unary( + "/temporal.api.workflowservice.v1.WorkflowService/UnregisterChannelListener", + request_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.UnregisterChannelListenerRequest.SerializeToString, + response_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.UnregisterChannelListenerResponse.FromString, + ) + self.PollChannel = channel.unary_unary( + "/temporal.api.workflowservice.v1.WorkflowService/PollChannel", + request_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.PollChannelRequest.SerializeToString, + response_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.PollChannelResponse.FromString, + ) + self.DescribeChannel = channel.unary_unary( + "/temporal.api.workflowservice.v1.WorkflowService/DescribeChannel", + request_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.DescribeChannelRequest.SerializeToString, + response_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.DescribeChannelResponse.FromString, + ) self.SignalWithStartWorkflowExecution = channel.unary_unary( "/temporal.api.workflowservice.v1.WorkflowService/SignalWithStartWorkflowExecution", request_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.SignalWithStartWorkflowExecutionRequest.SerializeToString, @@ -937,6 +962,53 @@ def SignalWorkflowExecution(self, request, context): context.set_details("Method not implemented!") raise NotImplementedError("Method not implemented!") + def NotifyChannel(self, request, context): + """NotifyChannel tells every listener of a channel that a source they consume + has moved. The writer names no addressee and never learns who listens. The + server wakes each listener: a Workflow with a Workflow Task, a callback by + invoking it. Nothing goes to History except the notifications a woken + Workflow Task carries on its scheduled event. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def RegisterChannelListener(self, request, context): + """RegisterChannelListener registers a callback as a listener of a channel. A + Workflow registers itself with the `SubscribeNotificationChannel` command + instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def UnregisterChannelListener(self, request, context): + """UnregisterChannelListener removes a listener from a channel. + + (-- api-linter: core::0136::http-method=disabled + aip.dev/not-precedent: Removing a listener is a delete of that listener. --) + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def PollChannel(self, request, context): + """PollChannel is a long poll for clients. It returns the retained + notifications of a channel with a counter above `after_counter`, waiting + up to `wait` for one when none is retained yet. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def DescribeChannel(self, request, context): + """DescribeChannel returns the listeners of a channel and its latest + notification. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + def SignalWithStartWorkflowExecution(self, request, context): """SignalWithStartWorkflowExecution is used to ensure a signal is sent to a workflow, even if it isn't yet started. @@ -2038,6 +2110,31 @@ def add_WorkflowServiceServicer_to_server(servicer, server): request_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.SignalWorkflowExecutionRequest.FromString, response_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.SignalWorkflowExecutionResponse.SerializeToString, ), + "NotifyChannel": grpc.unary_unary_rpc_method_handler( + servicer.NotifyChannel, + request_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.NotifyChannelRequest.FromString, + response_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.NotifyChannelResponse.SerializeToString, + ), + "RegisterChannelListener": grpc.unary_unary_rpc_method_handler( + servicer.RegisterChannelListener, + request_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.RegisterChannelListenerRequest.FromString, + response_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.RegisterChannelListenerResponse.SerializeToString, + ), + "UnregisterChannelListener": grpc.unary_unary_rpc_method_handler( + servicer.UnregisterChannelListener, + request_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.UnregisterChannelListenerRequest.FromString, + response_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.UnregisterChannelListenerResponse.SerializeToString, + ), + "PollChannel": grpc.unary_unary_rpc_method_handler( + servicer.PollChannel, + request_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.PollChannelRequest.FromString, + response_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.PollChannelResponse.SerializeToString, + ), + "DescribeChannel": grpc.unary_unary_rpc_method_handler( + servicer.DescribeChannel, + request_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.DescribeChannelRequest.FromString, + response_serializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.DescribeChannelResponse.SerializeToString, + ), "SignalWithStartWorkflowExecution": grpc.unary_unary_rpc_method_handler( servicer.SignalWithStartWorkflowExecution, request_deserializer=temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.SignalWithStartWorkflowExecutionRequest.FromString, @@ -3227,6 +3324,151 @@ def SignalWorkflowExecution( metadata, ) + @staticmethod + def NotifyChannel( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/temporal.api.workflowservice.v1.WorkflowService/NotifyChannel", + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.NotifyChannelRequest.SerializeToString, + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.NotifyChannelResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def RegisterChannelListener( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/temporal.api.workflowservice.v1.WorkflowService/RegisterChannelListener", + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.RegisterChannelListenerRequest.SerializeToString, + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.RegisterChannelListenerResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def UnregisterChannelListener( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/temporal.api.workflowservice.v1.WorkflowService/UnregisterChannelListener", + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.UnregisterChannelListenerRequest.SerializeToString, + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.UnregisterChannelListenerResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def PollChannel( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/temporal.api.workflowservice.v1.WorkflowService/PollChannel", + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.PollChannelRequest.SerializeToString, + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.PollChannelResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def DescribeChannel( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/temporal.api.workflowservice.v1.WorkflowService/DescribeChannel", + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.DescribeChannelRequest.SerializeToString, + temporal_dot_api_dot_workflowservice_dot_v1_dot_request__response__pb2.DescribeChannelResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + @staticmethod def SignalWithStartWorkflowExecution( request, diff --git a/temporalio/api/workflowservice/v1/service_pb2_grpc.pyi b/temporalio/api/workflowservice/v1/service_pb2_grpc.pyi index b743a159cc..38a22ebfff 100644 --- a/temporalio/api/workflowservice/v1/service_pb2_grpc.pyi +++ b/temporalio/api/workflowservice/v1/service_pb2_grpc.pyi @@ -279,6 +279,48 @@ class WorkflowServiceStub: This results in a `WORKFLOW_EXECUTION_SIGNALED` event recorded in the history and a workflow task being created for the execution. """ + NotifyChannel: grpc.UnaryUnaryMultiCallable[ + temporalio.api.workflowservice.v1.request_response_pb2.NotifyChannelRequest, + temporalio.api.workflowservice.v1.request_response_pb2.NotifyChannelResponse, + ] + """NotifyChannel tells every listener of a channel that a source they consume + has moved. The writer names no addressee and never learns who listens. The + server wakes each listener: a Workflow with a Workflow Task, a callback by + invoking it. Nothing goes to History except the notifications a woken + Workflow Task carries on its scheduled event. + """ + RegisterChannelListener: grpc.UnaryUnaryMultiCallable[ + temporalio.api.workflowservice.v1.request_response_pb2.RegisterChannelListenerRequest, + temporalio.api.workflowservice.v1.request_response_pb2.RegisterChannelListenerResponse, + ] + """RegisterChannelListener registers a callback as a listener of a channel. A + Workflow registers itself with the `SubscribeNotificationChannel` command + instead. + """ + UnregisterChannelListener: grpc.UnaryUnaryMultiCallable[ + temporalio.api.workflowservice.v1.request_response_pb2.UnregisterChannelListenerRequest, + temporalio.api.workflowservice.v1.request_response_pb2.UnregisterChannelListenerResponse, + ] + """UnregisterChannelListener removes a listener from a channel. + + (-- api-linter: core::0136::http-method=disabled + aip.dev/not-precedent: Removing a listener is a delete of that listener. --) + """ + PollChannel: grpc.UnaryUnaryMultiCallable[ + temporalio.api.workflowservice.v1.request_response_pb2.PollChannelRequest, + temporalio.api.workflowservice.v1.request_response_pb2.PollChannelResponse, + ] + """PollChannel is a long poll for clients. It returns the retained + notifications of a channel with a counter above `after_counter`, waiting + up to `wait` for one when none is retained yet. + """ + DescribeChannel: grpc.UnaryUnaryMultiCallable[ + temporalio.api.workflowservice.v1.request_response_pb2.DescribeChannelRequest, + temporalio.api.workflowservice.v1.request_response_pb2.DescribeChannelResponse, + ] + """DescribeChannel returns the listeners of a channel and its latest + notification. + """ SignalWithStartWorkflowExecution: grpc.UnaryUnaryMultiCallable[ temporalio.api.workflowservice.v1.request_response_pb2.SignalWithStartWorkflowExecutionRequest, temporalio.api.workflowservice.v1.request_response_pb2.SignalWithStartWorkflowExecutionResponse, @@ -1482,6 +1524,58 @@ class WorkflowServiceServicer(metaclass=abc.ABCMeta): task being created for the execution. """ @abc.abstractmethod + def NotifyChannel( + self, + request: temporalio.api.workflowservice.v1.request_response_pb2.NotifyChannelRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.workflowservice.v1.request_response_pb2.NotifyChannelResponse: + """NotifyChannel tells every listener of a channel that a source they consume + has moved. The writer names no addressee and never learns who listens. The + server wakes each listener: a Workflow with a Workflow Task, a callback by + invoking it. Nothing goes to History except the notifications a woken + Workflow Task carries on its scheduled event. + """ + @abc.abstractmethod + def RegisterChannelListener( + self, + request: temporalio.api.workflowservice.v1.request_response_pb2.RegisterChannelListenerRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.workflowservice.v1.request_response_pb2.RegisterChannelListenerResponse: + """RegisterChannelListener registers a callback as a listener of a channel. A + Workflow registers itself with the `SubscribeNotificationChannel` command + instead. + """ + @abc.abstractmethod + def UnregisterChannelListener( + self, + request: temporalio.api.workflowservice.v1.request_response_pb2.UnregisterChannelListenerRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.workflowservice.v1.request_response_pb2.UnregisterChannelListenerResponse: + """UnregisterChannelListener removes a listener from a channel. + + (-- api-linter: core::0136::http-method=disabled + aip.dev/not-precedent: Removing a listener is a delete of that listener. --) + """ + @abc.abstractmethod + def PollChannel( + self, + request: temporalio.api.workflowservice.v1.request_response_pb2.PollChannelRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.workflowservice.v1.request_response_pb2.PollChannelResponse: + """PollChannel is a long poll for clients. It returns the retained + notifications of a channel with a counter above `after_counter`, waiting + up to `wait` for one when none is retained yet. + """ + @abc.abstractmethod + def DescribeChannel( + self, + request: temporalio.api.workflowservice.v1.request_response_pb2.DescribeChannelRequest, + context: grpc.ServicerContext, + ) -> temporalio.api.workflowservice.v1.request_response_pb2.DescribeChannelResponse: + """DescribeChannel returns the listeners of a channel and its latest + notification. + """ + @abc.abstractmethod def SignalWithStartWorkflowExecution( self, request: temporalio.api.workflowservice.v1.request_response_pb2.SignalWithStartWorkflowExecutionRequest, diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 72f89d746a..d00649f067 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2588,7 +2588,7 @@ dependencies = [ [[package]] name = "temporalio-client" -version = "0.7.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", @@ -2620,7 +2620,7 @@ dependencies = [ [[package]] name = "temporalio-common" -version = "0.7.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", @@ -2661,7 +2661,7 @@ dependencies = [ [[package]] name = "temporalio-common-wasm" -version = "0.7.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", @@ -2686,7 +2686,7 @@ dependencies = [ [[package]] name = "temporalio-macros" -version = "0.7.0" +version = "1.0.0" dependencies = [ "proc-macro2", "quote", @@ -2695,7 +2695,7 @@ dependencies = [ [[package]] name = "temporalio-protos" -version = "0.7.0" +version = "0.9.0" dependencies = [ "anyhow", "base64 0.23.1", @@ -2704,7 +2704,6 @@ dependencies = [ "pbjson", "pbjson-build", "prost", - "prost-types", "prost-wkt-types", "serde", "serde_json", @@ -2716,7 +2715,7 @@ dependencies = [ [[package]] name = "temporalio-sdk-core" -version = "0.7.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index 3e4b3d37bf..44af176fdb 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -58,15 +58,16 @@ pythonize = "0.29" # default-features disabled so the rustls provider isn't pinned to `tls-ring`; # the bridge's tls-ring/tls-aws-lc features (above) select it. Re-add the # crate's non-TLS default (`envconfig`). -temporalio-client = { version = "0.7", path = "./sdk-core/crates/client", default-features = false, features = [ +temporalio-client = { version = "1.0", path = "./sdk-core/crates/client", default-features = false, features = [ "envconfig", + "experimental", ] } -temporalio-common = { version = "0.7", path = "./sdk-core/crates/common", features = [ +temporalio-common = { version = "1.0", path = "./sdk-core/crates/common", features = [ "envconfig", "otel" ]} # default-features disabled (drops the pinned `tls-ring`); re-add the non-TLS # defaults the bridge relied on (`envconfig`, `prometheus`) plus `ephemeral-server`. -temporalio-sdk-core = { version = "0.7", path = "./sdk-core/crates/sdk-core", default-features = false, features = [ +temporalio-sdk-core = { version = "0.9", path = "./sdk-core/crates/sdk-core", default-features = false, features = [ "ephemeral-server", "envconfig", "prometheus", diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index d8fa05edf3..92a37d7f45 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -308,6 +308,32 @@ async def _visit_coresdk_workflow_activation_ResolveNexusOperation( if o.HasField("result"): await self._visit_coresdk_nexus_NexusOperationResult(fs, o.result) + async def _visit_temporal_api_stream_v1_StreamRecord( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("body"): + await self._visit_temporal_api_common_v1_Payload(fs, o.body) + for v in o.metadata.values(): + await self._visit_temporal_api_common_v1_Payload(fs, v) + + async def _visit_coresdk_workflow_activation_DeliverStreamRecords( + self, fs: VisitorFunctions, o: Any + ): + for v in o.records: + await self._visit_temporal_api_stream_v1_StreamRecord(fs, v) + + async def _visit_temporal_api_notification_v1_Notification( + self, fs: VisitorFunctions, o: Any + ): + for v in o.metadata.values(): + await self._visit_temporal_api_common_v1_Payload(fs, v) + + async def _visit_coresdk_workflow_activation_NotificationsReceived( + self, fs: VisitorFunctions, o: Any + ): + for v in o.notifications: + await self._visit_temporal_api_notification_v1_Notification(fs, v) + async def _visit_coresdk_workflow_activation_WorkflowActivationJob( self, fs: VisitorFunctions, o: Any ): @@ -353,6 +379,14 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob( await self._visit_coresdk_workflow_activation_ResolveNexusOperation( fs, o.resolve_nexus_operation ) + elif o.HasField("deliver_stream_records"): + await self._visit_coresdk_workflow_activation_DeliverStreamRecords( + fs, o.deliver_stream_records + ) + elif o.HasField("notifications_received"): + await self._visit_coresdk_workflow_activation_NotificationsReceived( + fs, o.notifications_received + ) async def _visit_coresdk_workflow_activation_WorkflowActivation( self, fs: VisitorFunctions, o: Any @@ -492,6 +526,12 @@ async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( if o.HasField("input"): await self._visit_nexus_operation_input_payload(fs, o.input) + async def _visit_coresdk_workflow_commands_AppendStreamRecords( + self, fs: VisitorFunctions, o: Any + ): + for v in o.records: + await self._visit_temporal_api_stream_v1_StreamRecord(fs, v) + async def _visit_coresdk_workflow_commands_WorkflowCommand( self, fs: VisitorFunctions, o: Any ): @@ -551,6 +591,10 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand( await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( fs, o.schedule_nexus_operation ) + elif o.HasField("append_stream_records"): + await self._visit_coresdk_workflow_commands_AppendStreamRecords( + fs, o.append_stream_records + ) async def _visit_coresdk_workflow_completion_Success( self, fs: VisitorFunctions, o: Any diff --git a/temporalio/bridge/proto/core_interface_pb2.py b/temporalio/bridge/proto/core_interface_pb2.py index 19f330433a..e7343abbe6 100644 --- a/temporalio/bridge/proto/core_interface_pb2.py +++ b/temporalio/bridge/proto/core_interface_pb2.py @@ -33,6 +33,9 @@ from temporalio.bridge.proto.external_data import ( external_data_pb2 as temporal_dot_sdk_dot_core_dot_external__data_dot_external__data__pb2, ) +from temporalio.bridge.proto.external_stream import ( + external_stream_pb2 as temporal_dot_sdk_dot_core_dot_external__stream_dot_external__stream__pb2, +) from temporalio.bridge.proto.workflow_activation import ( workflow_activation_pb2 as temporal_dot_sdk_dot_core_dot_workflow__activation_dot_workflow__activation__pb2, ) @@ -44,7 +47,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n&temporal/sdk/core/core_interface.proto\x12\x07\x63oresdk\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x33temporal/sdk/core/activity_task/activity_task.proto\x1a%temporal/sdk/core/common/common.proto\x1a\x33temporal/sdk/core/external_data/external_data.proto\x1a?temporal/sdk/core/workflow_activation/workflow_activation.proto\x1a;temporal/sdk/core/workflow_commands/workflow_commands.proto\x1a?temporal/sdk/core/workflow_completion/workflow_completion.proto"Y\n\x11\x41\x63tivityHeartbeat\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x30\n\x07\x64\x65tails\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload"n\n\x16\x41\x63tivityTaskCompletion\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12@\n\x06result\x18\x02 \x01(\x0b\x32\x30.coresdk.activity_result.ActivityExecutionResult"<\n\x10WorkflowSlotInfo\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x11\n\tis_sticky\x18\x02 \x01(\x08")\n\x10\x41\x63tivitySlotInfo\x12\x15\n\ractivity_type\x18\x01 \x01(\t".\n\x15LocalActivitySlotInfo\x12\x15\n\ractivity_type\x18\x01 \x01(\t"3\n\rNexusSlotInfo\x12\x0f\n\x07service\x18\x01 \x01(\t\x12\x11\n\toperation\x18\x02 \x01(\t"\x86\x01\n\rNamespaceInfo\x12-\n\x06limits\x18\x01 \x01(\x0b\x32\x1d.coresdk.NamespaceInfo.Limits\x1a\x46\n\x06Limits\x12\x1d\n\x15\x62lob_size_limit_error\x18\x01 \x01(\x03\x12\x1d\n\x15memo_size_limit_error\x18\x02 \x01(\x03\x42\x33\xea\x02\x30Temporalio::Internal::Bridge::Api::CoreInterfaceb\x06proto3' + b'\n&temporal/sdk/core/core_interface.proto\x12\x07\x63oresdk\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x33temporal/sdk/core/activity_task/activity_task.proto\x1a%temporal/sdk/core/common/common.proto\x1a\x33temporal/sdk/core/external_data/external_data.proto\x1a\x37temporal/sdk/core/external_stream/external_stream.proto\x1a?temporal/sdk/core/workflow_activation/workflow_activation.proto\x1a;temporal/sdk/core/workflow_commands/workflow_commands.proto\x1a?temporal/sdk/core/workflow_completion/workflow_completion.proto"Y\n\x11\x41\x63tivityHeartbeat\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12\x30\n\x07\x64\x65tails\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload"n\n\x16\x41\x63tivityTaskCompletion\x12\x12\n\ntask_token\x18\x01 \x01(\x0c\x12@\n\x06result\x18\x02 \x01(\x0b\x32\x30.coresdk.activity_result.ActivityExecutionResult"<\n\x10WorkflowSlotInfo\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x11\n\tis_sticky\x18\x02 \x01(\x08")\n\x10\x41\x63tivitySlotInfo\x12\x15\n\ractivity_type\x18\x01 \x01(\t".\n\x15LocalActivitySlotInfo\x12\x15\n\ractivity_type\x18\x01 \x01(\t"3\n\rNexusSlotInfo\x12\x0f\n\x07service\x18\x01 \x01(\t\x12\x11\n\toperation\x18\x02 \x01(\t"\x86\x01\n\rNamespaceInfo\x12-\n\x06limits\x18\x01 \x01(\x0b\x32\x1d.coresdk.NamespaceInfo.Limits\x1a\x46\n\x06Limits\x12\x1d\n\x15\x62lob_size_limit_error\x18\x01 \x01(\x03\x12\x1d\n\x15memo_size_limit_error\x18\x02 \x01(\x03\x42\x33\xea\x02\x30Temporalio::Internal::Bridge::Api::CoreInterfaceb\x06proto3' ) @@ -148,20 +151,20 @@ DESCRIPTOR._serialized_options = ( b"\352\0020Temporalio::Internal::Bridge::Api::CoreInterface" ) - _ACTIVITYHEARTBEAT._serialized_start = 576 - _ACTIVITYHEARTBEAT._serialized_end = 665 - _ACTIVITYTASKCOMPLETION._serialized_start = 667 - _ACTIVITYTASKCOMPLETION._serialized_end = 777 - _WORKFLOWSLOTINFO._serialized_start = 779 - _WORKFLOWSLOTINFO._serialized_end = 839 - _ACTIVITYSLOTINFO._serialized_start = 841 - _ACTIVITYSLOTINFO._serialized_end = 882 - _LOCALACTIVITYSLOTINFO._serialized_start = 884 - _LOCALACTIVITYSLOTINFO._serialized_end = 930 - _NEXUSSLOTINFO._serialized_start = 932 - _NEXUSSLOTINFO._serialized_end = 983 - _NAMESPACEINFO._serialized_start = 986 - _NAMESPACEINFO._serialized_end = 1120 - _NAMESPACEINFO_LIMITS._serialized_start = 1050 - _NAMESPACEINFO_LIMITS._serialized_end = 1120 + _ACTIVITYHEARTBEAT._serialized_start = 633 + _ACTIVITYHEARTBEAT._serialized_end = 722 + _ACTIVITYTASKCOMPLETION._serialized_start = 724 + _ACTIVITYTASKCOMPLETION._serialized_end = 834 + _WORKFLOWSLOTINFO._serialized_start = 836 + _WORKFLOWSLOTINFO._serialized_end = 896 + _ACTIVITYSLOTINFO._serialized_start = 898 + _ACTIVITYSLOTINFO._serialized_end = 939 + _LOCALACTIVITYSLOTINFO._serialized_start = 941 + _LOCALACTIVITYSLOTINFO._serialized_end = 987 + _NEXUSSLOTINFO._serialized_start = 989 + _NEXUSSLOTINFO._serialized_end = 1040 + _NAMESPACEINFO._serialized_start = 1043 + _NAMESPACEINFO._serialized_end = 1177 + _NAMESPACEINFO_LIMITS._serialized_start = 1107 + _NAMESPACEINFO_LIMITS._serialized_end = 1177 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/external_data/__init__.py b/temporalio/bridge/proto/external_data/__init__.py index 836e97ef43..28f6b487a9 100644 --- a/temporalio/bridge/proto/external_data/__init__.py +++ b/temporalio/bridge/proto/external_data/__init__.py @@ -1,6 +1,21 @@ -from .external_data_pb2 import LocalActivityMarkerData, PatchedMarkerData +from .external_data_pb2 import ( + ExternalOutputSegmentManifest, + ExternalOutputStreamManifest, + ExternalOutputTopicManifest, + ExternalStreamMarkerData, + ExternalWaitMarker, + LocalActivityMarkerData, + ParkReason, + PatchedMarkerData, +) __all__ = [ + "ExternalOutputSegmentManifest", + "ExternalOutputStreamManifest", + "ExternalOutputTopicManifest", + "ExternalStreamMarkerData", + "ExternalWaitMarker", "LocalActivityMarkerData", + "ParkReason", "PatchedMarkerData", ] diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.py b/temporalio/bridge/proto/external_data/external_data_pb2.py index e8070efa79..07a8534cb6 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.py +++ b/temporalio/bridge/proto/external_data/external_data_pb2.py @@ -8,6 +8,7 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper # @@protoc_insertion_point(imports) @@ -18,12 +19,37 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\xfe\x01\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08\x42\x32\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' + b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\xfe\x01\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xa9\x02\n\x18\x45xternalStreamMarkerData\x12\x16\n\x0eschema_version\x18\x01 \x01(\r\x12\x1d\n\x15quiescence_generation\x18\x02 \x01(\x04\x12\x38\n\x05waits\x18\x03 \x03(\x0b\x32).coresdk.external_data.ExternalWaitMarker\x12\x19\n\x11replay_annotation\x18\x04 \x01(\x0c\x12<\n\x11terminal_boundary\x18\x05 \x01(\x0e\x32!.coresdk.external_data.ParkReason\x12\x43\n\x06output\x18\x06 \x01(\x0b\x32\x33.coresdk.external_data.ExternalOutputStreamManifest"9\n\x12\x45xternalWaitMarker\x12\x0f\n\x07wait_id\x18\x01 \x01(\r\x12\x12\n\ngeneration\x18\x02 \x01(\x04"\xda\x02\n\x1c\x45xternalOutputStreamManifest\x12\x16\n\x0eschema_version\x18\x01 \x01(\r\x12\x1b\n\x13\x66ingerprint_version\x18\x02 \x01(\r\x12\x13\n\x0bstage_token\x18\x03 \x01(\t\x12\x1e\n\x16history_floor_event_id\x18\x04 \x01(\x03\x12\x0e\n\x06run_id\x18\x05 \x01(\t\x12\x42\n\x06topics\x18\x06 \x03(\x0b\x32\x32.coresdk.external_data.ExternalOutputTopicManifest\x12\x46\n\x08segments\x18\x07 \x03(\x0b\x32\x34.coresdk.external_data.ExternalOutputSegmentManifest\x12\x13\n\x0bprovider_id\x18\x08 \x01(\t\x12\x1f\n\x17provider_format_version\x18\t \x01(\r"\x8d\x01\n\x1b\x45xternalOutputTopicManifest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x14\n\x0crecord_count\x18\x02 \x01(\r\x12\x1a\n\x12logical_byte_count\x18\x03 \x01(\x04\x12\x1b\n\x13logical_fingerprint\x18\x04 \x01(\x0c\x12\x10\n\x08\x66inished\x18\x05 \x01(\x08"?\n\x1d\x45xternalOutputSegmentManifest\x12\x1e\n\x16record_counts_by_topic\x18\x01 \x03(\r*\xde\x02\n\nParkReason\x12\x1b\n\x17PARK_REASON_UNSPECIFIED\x10\x00\x12\x14\n\x10PARK_REASON_IDLE\x10\x01\x12 \n\x1cPARK_REASON_ALL_WRITE_FENCED\x10\x02\x12\x18\n\x14PARK_REASON_SHUTDOWN\x10\x03\x12\x18\n\x14PARK_REASON_ROLLOVER\x10\x04\x12\x1f\n\x1bPARK_REASON_BUDGET_ROLLOVER\x10\x05\x12!\n\x1dPARK_REASON_COMMANDS_PRODUCED\x10\x06\x12"\n\x1ePARK_REASON_WORKFLOW_COMPLETED\x10\x07\x12\x1e\n\x1aPARK_REASON_TASK_COMPLETED\x10\x08\x12\x1e\n\x1aPARK_REASON_OUTPUT_LATENCY\x10\t\x12\x1f\n\x1bPARK_REASON_OUTPUT_CAPACITY\x10\nB2\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' ) +_PARKREASON = DESCRIPTOR.enum_types_by_name["ParkReason"] +ParkReason = enum_type_wrapper.EnumTypeWrapper(_PARKREASON) +PARK_REASON_UNSPECIFIED = 0 +PARK_REASON_IDLE = 1 +PARK_REASON_ALL_WRITE_FENCED = 2 +PARK_REASON_SHUTDOWN = 3 +PARK_REASON_ROLLOVER = 4 +PARK_REASON_BUDGET_ROLLOVER = 5 +PARK_REASON_COMMANDS_PRODUCED = 6 +PARK_REASON_WORKFLOW_COMPLETED = 7 +PARK_REASON_TASK_COMPLETED = 8 +PARK_REASON_OUTPUT_LATENCY = 9 +PARK_REASON_OUTPUT_CAPACITY = 10 + _LOCALACTIVITYMARKERDATA = DESCRIPTOR.message_types_by_name["LocalActivityMarkerData"] _PATCHEDMARKERDATA = DESCRIPTOR.message_types_by_name["PatchedMarkerData"] +_EXTERNALSTREAMMARKERDATA = DESCRIPTOR.message_types_by_name["ExternalStreamMarkerData"] +_EXTERNALWAITMARKER = DESCRIPTOR.message_types_by_name["ExternalWaitMarker"] +_EXTERNALOUTPUTSTREAMMANIFEST = DESCRIPTOR.message_types_by_name[ + "ExternalOutputStreamManifest" +] +_EXTERNALOUTPUTTOPICMANIFEST = DESCRIPTOR.message_types_by_name[ + "ExternalOutputTopicManifest" +] +_EXTERNALOUTPUTSEGMENTMANIFEST = DESCRIPTOR.message_types_by_name[ + "ExternalOutputSegmentManifest" +] LocalActivityMarkerData = _reflection.GeneratedProtocolMessageType( "LocalActivityMarkerData", (_message.Message,), @@ -46,13 +72,80 @@ ) _sym_db.RegisterMessage(PatchedMarkerData) +ExternalStreamMarkerData = _reflection.GeneratedProtocolMessageType( + "ExternalStreamMarkerData", + (_message.Message,), + { + "DESCRIPTOR": _EXTERNALSTREAMMARKERDATA, + "__module__": "temporal.sdk.core.external_data.external_data_pb2", + # @@protoc_insertion_point(class_scope:coresdk.external_data.ExternalStreamMarkerData) + }, +) +_sym_db.RegisterMessage(ExternalStreamMarkerData) + +ExternalWaitMarker = _reflection.GeneratedProtocolMessageType( + "ExternalWaitMarker", + (_message.Message,), + { + "DESCRIPTOR": _EXTERNALWAITMARKER, + "__module__": "temporal.sdk.core.external_data.external_data_pb2", + # @@protoc_insertion_point(class_scope:coresdk.external_data.ExternalWaitMarker) + }, +) +_sym_db.RegisterMessage(ExternalWaitMarker) + +ExternalOutputStreamManifest = _reflection.GeneratedProtocolMessageType( + "ExternalOutputStreamManifest", + (_message.Message,), + { + "DESCRIPTOR": _EXTERNALOUTPUTSTREAMMANIFEST, + "__module__": "temporal.sdk.core.external_data.external_data_pb2", + # @@protoc_insertion_point(class_scope:coresdk.external_data.ExternalOutputStreamManifest) + }, +) +_sym_db.RegisterMessage(ExternalOutputStreamManifest) + +ExternalOutputTopicManifest = _reflection.GeneratedProtocolMessageType( + "ExternalOutputTopicManifest", + (_message.Message,), + { + "DESCRIPTOR": _EXTERNALOUTPUTTOPICMANIFEST, + "__module__": "temporal.sdk.core.external_data.external_data_pb2", + # @@protoc_insertion_point(class_scope:coresdk.external_data.ExternalOutputTopicManifest) + }, +) +_sym_db.RegisterMessage(ExternalOutputTopicManifest) + +ExternalOutputSegmentManifest = _reflection.GeneratedProtocolMessageType( + "ExternalOutputSegmentManifest", + (_message.Message,), + { + "DESCRIPTOR": _EXTERNALOUTPUTSEGMENTMANIFEST, + "__module__": "temporal.sdk.core.external_data.external_data_pb2", + # @@protoc_insertion_point(class_scope:coresdk.external_data.ExternalOutputSegmentManifest) + }, +) +_sym_db.RegisterMessage(ExternalOutputSegmentManifest) + if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = ( b"\352\002/Temporalio::Internal::Bridge::Api::ExternalData" ) + _PARKREASON._serialized_start = 1371 + _PARKREASON._serialized_end = 1721 _LOCALACTIVITYMARKERDATA._serialized_start = 144 _LOCALACTIVITYMARKERDATA._serialized_end = 398 _PATCHEDMARKERDATA._serialized_start = 400 _PATCHEDMARKERDATA._serialized_end = 451 + _EXTERNALSTREAMMARKERDATA._serialized_start = 454 + _EXTERNALSTREAMMARKERDATA._serialized_end = 751 + _EXTERNALWAITMARKER._serialized_start = 753 + _EXTERNALWAITMARKER._serialized_end = 810 + _EXTERNALOUTPUTSTREAMMANIFEST._serialized_start = 813 + _EXTERNALOUTPUTSTREAMMANIFEST._serialized_end = 1159 + _EXTERNALOUTPUTTOPICMANIFEST._serialized_start = 1162 + _EXTERNALOUTPUTTOPICMANIFEST._serialized_end = 1303 + _EXTERNALOUTPUTSEGMENTMANIFEST._serialized_start = 1305 + _EXTERNALOUTPUTSEGMENTMANIFEST._serialized_end = 1368 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.pyi b/temporalio/bridge/proto/external_data/external_data_pb2.pyi index a1d29e39d8..07d07deaa3 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.pyi +++ b/temporalio/bridge/proto/external_data/external_data_pb2.pyi @@ -4,20 +4,109 @@ isort:skip_file """ import builtins +import collections.abc import sys +import typing import google.protobuf.descriptor import google.protobuf.duration_pb2 +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper import google.protobuf.message import google.protobuf.timestamp_pb2 -if sys.version_info >= (3, 8): +if sys.version_info >= (3, 10): import typing as typing_extensions else: import typing_extensions DESCRIPTOR: google.protobuf.descriptor.FileDescriptor +class _ParkReason: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _ParkReasonEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ParkReason.ValueType], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + PARK_REASON_UNSPECIFIED: _ParkReason.ValueType # 0 + PARK_REASON_IDLE: _ParkReason.ValueType # 1 + """The global quiescence timer expired with no record on any active wait.""" + PARK_REASON_ALL_WRITE_FENCED: _ParkReason.ValueType # 2 + """Every active wait in the quiescent snapshot reached a write fence with no later record + immediately available, so the idle delay was bypassed. + """ + PARK_REASON_SHUTDOWN: _ParkReason.ValueType # 3 + """The Worker is shutting down or the Run is being evicted while a Workflow Task is open.""" + PARK_REASON_ROLLOVER: _ParkReason.ValueType # 4 + """The Workflow Task rollover deadline expired. Core decided this one, so the terminal came + back through FinalizeExternalStreams. + """ + PARK_REASON_BUDGET_ROLLOVER: _ParkReason.ValueType # 5 + """The annotation reached its byte budget high-water mark and lang asked for a rollover. Unlike + PARK_REASON_ROLLOVER this needs no finalization round trip: the triggering + WorkflowStreamProgress already carried the terminal. + """ + PARK_REASON_COMMANDS_PRODUCED: _ParkReason.ValueType # 6 + """The completion carried server-bound commands (timer, activity, child workflow, signal), so + the Workflow Task completed normally with subscriptions left active and unparked. + """ + PARK_REASON_WORKFLOW_COMPLETED: _ParkReason.ValueType # 7 + """The completion carried a terminal command -- complete, fail, cancel, or continue-as-new.""" + PARK_REASON_TASK_COMPLETED: _ParkReason.ValueType # 8 + """The Workflow Task completed normally with nothing else to report: records were consumed and + no stream wait remained pending. Distinct from PARK_REASON_COMMANDS_PRODUCED so the marker + says which of the two it actually was rather than implying commands that never existed. + """ + PARK_REASON_OUTPUT_LATENCY: _ParkReason.ValueType # 9 + """The Workflow Task's output visibility deadline expired.""" + PARK_REASON_OUTPUT_CAPACITY: _ParkReason.ValueType # 10 + """The staged output reached its deterministic record or logical-byte capacity.""" + +class ParkReason(_ParkReason, metaclass=_ParkReasonEnumTypeWrapper): + """Why a Workflow Task holding external stream waits ended. Core knows the reason on every + completion path -- it either decided the boundary itself or was told -- which is why this + lives here, in the Core-readable marker envelope, and is deliberately *not* duplicated inside + the opaque replay annotation where a second copy could disagree with this one. + """ + +PARK_REASON_UNSPECIFIED: ParkReason.ValueType # 0 +PARK_REASON_IDLE: ParkReason.ValueType # 1 +"""The global quiescence timer expired with no record on any active wait.""" +PARK_REASON_ALL_WRITE_FENCED: ParkReason.ValueType # 2 +"""Every active wait in the quiescent snapshot reached a write fence with no later record +immediately available, so the idle delay was bypassed. +""" +PARK_REASON_SHUTDOWN: ParkReason.ValueType # 3 +"""The Worker is shutting down or the Run is being evicted while a Workflow Task is open.""" +PARK_REASON_ROLLOVER: ParkReason.ValueType # 4 +"""The Workflow Task rollover deadline expired. Core decided this one, so the terminal came +back through FinalizeExternalStreams. +""" +PARK_REASON_BUDGET_ROLLOVER: ParkReason.ValueType # 5 +"""The annotation reached its byte budget high-water mark and lang asked for a rollover. Unlike +PARK_REASON_ROLLOVER this needs no finalization round trip: the triggering +WorkflowStreamProgress already carried the terminal. +""" +PARK_REASON_COMMANDS_PRODUCED: ParkReason.ValueType # 6 +"""The completion carried server-bound commands (timer, activity, child workflow, signal), so +the Workflow Task completed normally with subscriptions left active and unparked. +""" +PARK_REASON_WORKFLOW_COMPLETED: ParkReason.ValueType # 7 +"""The completion carried a terminal command -- complete, fail, cancel, or continue-as-new.""" +PARK_REASON_TASK_COMPLETED: ParkReason.ValueType # 8 +"""The Workflow Task completed normally with nothing else to report: records were consumed and +no stream wait remained pending. Distinct from PARK_REASON_COMMANDS_PRODUCED so the marker +says which of the two it actually was rather than implying commands that never existed. +""" +PARK_REASON_OUTPUT_LATENCY: ParkReason.ValueType # 9 +"""The Workflow Task's output visibility deadline expired.""" +PARK_REASON_OUTPUT_CAPACITY: ParkReason.ValueType # 10 +"""The staged output reached its deterministic record or logical-byte capacity.""" +global___ParkReason = ParkReason + class LocalActivityMarkerData(google.protobuf.message.Message): """This file defines data that Core might write externally. The first motivating case being storing data in markers in event history. Defining such data as protos provides an easy way @@ -122,3 +211,246 @@ class PatchedMarkerData(google.protobuf.message.Message): ) -> None: ... global___PatchedMarkerData = PatchedMarkerData + +class ExternalStreamMarkerData(google.protobuf.message.Message): + """What Core writes into a marker for a Workflow Task that consumed external streams. Exactly one + of these is written per Workflow Task, however many progress reports that task carried. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SCHEMA_VERSION_FIELD_NUMBER: builtins.int + QUIESCENCE_GENERATION_FIELD_NUMBER: builtins.int + WAITS_FIELD_NUMBER: builtins.int + REPLAY_ANNOTATION_FIELD_NUMBER: builtins.int + TERMINAL_BOUNDARY_FIELD_NUMBER: builtins.int + OUTPUT_FIELD_NUMBER: builtins.int + schema_version: builtins.int + quiescence_generation: builtins.int + """Identifies the complete blocked snapshot this marker closes.""" + @property + def waits( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___ExternalWaitMarker + ]: ... + replay_annotation: builtins.bytes + """The accumulated replay annotation, opaque to Core. Core appends each observation delta to + this and never parses any of it. + """ + terminal_boundary: global___ParkReason.ValueType + @property + def output(self) -> global___ExternalOutputStreamManifest: + """The output staged for this Workflow Task, if any. Output shares this marker with input so + replay has one activation-segment schedule and never drains the Workflow event loop twice. + """ + def __init__( + self, + *, + schema_version: builtins.int = ..., + quiescence_generation: builtins.int = ..., + waits: collections.abc.Iterable[global___ExternalWaitMarker] | None = ..., + replay_annotation: builtins.bytes = ..., + terminal_boundary: global___ParkReason.ValueType = ..., + output: global___ExternalOutputStreamManifest | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["output", b"output"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "output", + b"output", + "quiescence_generation", + b"quiescence_generation", + "replay_annotation", + b"replay_annotation", + "schema_version", + b"schema_version", + "terminal_boundary", + b"terminal_boundary", + "waits", + b"waits", + ], + ) -> None: ... + +global___ExternalStreamMarkerData = ExternalStreamMarkerData + +class ExternalWaitMarker(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WAIT_ID_FIELD_NUMBER: builtins.int + GENERATION_FIELD_NUMBER: builtins.int + wait_id: builtins.int + generation: builtins.int + def __init__( + self, + *, + wait_id: builtins.int = ..., + generation: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "generation", b"generation", "wait_id", b"wait_id" + ], + ) -> None: ... + +global___ExternalWaitMarker = ExternalWaitMarker + +class ExternalOutputStreamManifest(google.protobuf.message.Message): + """Compact proof of the externally staged output belonging to one Workflow Task. + + This message contains no record payloads. The stage token identifies the immutable pending + batch in the provider, while the remaining fields let replay validate the deterministic logical + output and let a cold reader reconcile the pending batch against History. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SCHEMA_VERSION_FIELD_NUMBER: builtins.int + FINGERPRINT_VERSION_FIELD_NUMBER: builtins.int + STAGE_TOKEN_FIELD_NUMBER: builtins.int + HISTORY_FLOOR_EVENT_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + TOPICS_FIELD_NUMBER: builtins.int + SEGMENTS_FIELD_NUMBER: builtins.int + PROVIDER_ID_FIELD_NUMBER: builtins.int + PROVIDER_FORMAT_VERSION_FIELD_NUMBER: builtins.int + schema_version: builtins.int + fingerprint_version: builtins.int + stage_token: builtins.str + history_floor_event_id: builtins.int + """The event immediately preceding this Workflow Task's WorkflowTaskScheduled event in the + ordered History view used to build its activation. + """ + run_id: builtins.str + @property + def topics( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___ExternalOutputTopicManifest + ]: ... + @property + def segments( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___ExternalOutputSegmentManifest + ]: + """One entry per live activation segment, including segments which published no output.""" + provider_id: builtins.str + """Stable configured provider binding. Reconciliation refuses to interpret the batch through a + different provider or provider wire format. + """ + provider_format_version: builtins.int + def __init__( + self, + *, + schema_version: builtins.int = ..., + fingerprint_version: builtins.int = ..., + stage_token: builtins.str = ..., + history_floor_event_id: builtins.int = ..., + run_id: builtins.str = ..., + topics: collections.abc.Iterable[global___ExternalOutputTopicManifest] + | None = ..., + segments: collections.abc.Iterable[global___ExternalOutputSegmentManifest] + | None = ..., + provider_id: builtins.str = ..., + provider_format_version: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "fingerprint_version", + b"fingerprint_version", + "history_floor_event_id", + b"history_floor_event_id", + "provider_format_version", + b"provider_format_version", + "provider_id", + b"provider_id", + "run_id", + b"run_id", + "schema_version", + b"schema_version", + "segments", + b"segments", + "stage_token", + b"stage_token", + "topics", + b"topics", + ], + ) -> None: ... + +global___ExternalOutputStreamManifest = ExternalOutputStreamManifest + +class ExternalOutputTopicManifest(google.protobuf.message.Message): + """Logical, pre-codec identity of one topic sub-batch.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TOPIC_FIELD_NUMBER: builtins.int + RECORD_COUNT_FIELD_NUMBER: builtins.int + LOGICAL_BYTE_COUNT_FIELD_NUMBER: builtins.int + LOGICAL_FINGERPRINT_FIELD_NUMBER: builtins.int + FINISHED_FIELD_NUMBER: builtins.int + topic: builtins.str + record_count: builtins.int + logical_byte_count: builtins.int + logical_fingerprint: builtins.bytes + finished: builtins.bool + def __init__( + self, + *, + topic: builtins.str = ..., + record_count: builtins.int = ..., + logical_byte_count: builtins.int = ..., + logical_fingerprint: builtins.bytes = ..., + finished: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "finished", + b"finished", + "logical_byte_count", + b"logical_byte_count", + "logical_fingerprint", + b"logical_fingerprint", + "record_count", + b"record_count", + "topic", + b"topic", + ], + ) -> None: ... + +global___ExternalOutputTopicManifest = ExternalOutputTopicManifest + +class ExternalOutputSegmentManifest(google.protobuf.message.Message): + """Record counts for one activation segment. Entries correspond positionally to `topics` in the + enclosing ExternalOutputStreamManifest, retaining empty segments without repeating topic names. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RECORD_COUNTS_BY_TOPIC_FIELD_NUMBER: builtins.int + @property + def record_counts_by_topic( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[ + builtins.int + ]: ... + def __init__( + self, + *, + record_counts_by_topic: collections.abc.Iterable[builtins.int] | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "record_counts_by_topic", b"record_counts_by_topic" + ], + ) -> None: ... + +global___ExternalOutputSegmentManifest = ExternalOutputSegmentManifest diff --git a/temporalio/bridge/proto/external_stream/__init__.py b/temporalio/bridge/proto/external_stream/__init__.py new file mode 100644 index 0000000000..9f2eed766e --- /dev/null +++ b/temporalio/bridge/proto/external_stream/__init__.py @@ -0,0 +1,5 @@ +from .external_stream_pb2 import WakeSignal + +__all__ = [ + "WakeSignal", +] diff --git a/temporalio/bridge/proto/external_stream/external_stream_pb2.py b/temporalio/bridge/proto/external_stream/external_stream_pb2.py new file mode 100644 index 0000000000..b7051cba1d --- /dev/null +++ b/temporalio/bridge/proto/external_stream/external_stream_pb2.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporal/sdk/core/external_stream/external_stream.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n7temporal/sdk/core/external_stream/external_stream.proto\x12\x17\x63oresdk.external_stream"\xa2\x01\n\nWakeSignal\x12\x18\n\x10\x65nvelope_version\x18\x01 \x01(\r\x12\x13\n\x0bstream_name\x18\x02 \x01(\t\x12\x0f\n\x07wait_id\x18\x03 \x01(\r\x12\x17\n\x0fpark_generation\x18\x04 \x01(\x04\x12\x1e\n\x16\x66irst_execution_run_id\x18\x05 \x01(\t\x12\x1b\n\x13producer_session_id\x18\x06 \x01(\tB4\xea\x02\x31Temporalio::Internal::Bridge::Api::ExternalStreamb\x06proto3' +) + + +_WAKESIGNAL = DESCRIPTOR.message_types_by_name["WakeSignal"] +WakeSignal = _reflection.GeneratedProtocolMessageType( + "WakeSignal", + (_message.Message,), + { + "DESCRIPTOR": _WAKESIGNAL, + "__module__": "temporal.sdk.core.external_stream.external_stream_pb2", + # @@protoc_insertion_point(class_scope:coresdk.external_stream.WakeSignal) + }, +) +_sym_db.RegisterMessage(WakeSignal) + +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = ( + b"\352\0021Temporalio::Internal::Bridge::Api::ExternalStream" + ) + _WAKESIGNAL._serialized_start = 85 + _WAKESIGNAL._serialized_end = 247 +# @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/external_stream/external_stream_pb2.pyi b/temporalio/bridge/proto/external_stream/external_stream_pb2.pyi new file mode 100644 index 0000000000..4702ac5541 --- /dev/null +++ b/temporalio/bridge/proto/external_stream/external_stream_pb2.pyi @@ -0,0 +1,95 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Wire formats for External Workflow Streams that are neither commands nor activation jobs. + +The one member today is the reserved wake Signal's envelope. It lives here, at the protocol +level, rather than being a user payload: Core must read its fields, and Core has no access to +the user's DataConverter or codec. A user codec that encrypts payloads would otherwise make the +envelope unreadable to the component that must read it. +""" + +import builtins +import sys + +import google.protobuf.descriptor +import google.protobuf.message + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class WakeSignal(google.protobuf.message.Message): + """The payload of the reserved `__temporal_external_stream_wake` Signal. + + Sent as a single argument whose Payload uses metadata + `encoding = "binary/protobuf"` and + `messageType = "coresdk.external_stream.WakeSignal"`, serialized with the protocol's own + serialization rather than through the user's DataConverter. + + The Signal carries no user data, so bypassing the codec leaks nothing. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ENVELOPE_VERSION_FIELD_NUMBER: builtins.int + STREAM_NAME_FIELD_NUMBER: builtins.int + WAIT_ID_FIELD_NUMBER: builtins.int + PARK_GENERATION_FIELD_NUMBER: builtins.int + FIRST_EXECUTION_RUN_ID_FIELD_NUMBER: builtins.int + PRODUCER_SESSION_ID_FIELD_NUMBER: builtins.int + envelope_version: builtins.int + """Starts at 1. Core ignores an unknown version harmlessly rather than failing -- an old Core + must not break because a newer producer learned a new field. + """ + stream_name: builtins.str + """The stream this wake was prompted by. Only a hint: on wakeup the runtime rechecks every + active subscription, not just this one. + """ + wait_id: builtins.int + park_generation: builtins.int + """0 is reserved and means "no park generation" -- an unparked wake, which Core accepts as a + recheck request after validating chain identity. Park generations are quiescence + generations, which start at 1, so a *non-zero* generation the current Run does not + recognize is a claim that turned out to be wrong and is ignored as stale. + """ + first_execution_run_id: builtins.str + """Chain identity, not the current Run. The Signal is addressed to the Workflow ID without a + Run ID so it always lands on the current Run of the chain; this field is what lets Core + tell "same chain, stale generation" from "different chain". + """ + producer_session_id: builtins.str + """Diagnostics only. Never used for validation or deduplication.""" + def __init__( + self, + *, + envelope_version: builtins.int = ..., + stream_name: builtins.str = ..., + wait_id: builtins.int = ..., + park_generation: builtins.int = ..., + first_execution_run_id: builtins.str = ..., + producer_session_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "envelope_version", + b"envelope_version", + "first_execution_run_id", + b"first_execution_run_id", + "park_generation", + b"park_generation", + "producer_session_id", + b"producer_session_id", + "stream_name", + b"stream_name", + "wait_id", + b"wait_id", + ], + ) -> None: ... + +global___WakeSignal = WakeSignal diff --git a/temporalio/bridge/proto/workflow_activation/__init__.py b/temporalio/bridge/proto/workflow_activation/__init__.py index 5c178e2185..9e085c84d1 100644 --- a/temporalio/bridge/proto/workflow_activation/__init__.py +++ b/temporalio/bridge/proto/workflow_activation/__init__.py @@ -1,17 +1,23 @@ from .workflow_activation_pb2 import ( CancelWorkflow, + DeliverStreamRecords, DoUpdate, + FinalizeExternalStreams, FireTimer, InitializeWorkflow, + NotificationsReceived, NotifyHasPatch, + PrepareExternalStreamPark, QueryWorkflow, RemoveFromCache, + ReplayExternalStreams, ResolveActivity, ResolveChildWorkflowExecution, ResolveChildWorkflowExecutionStart, ResolveChildWorkflowExecutionStartCancelled, ResolveChildWorkflowExecutionStartFailure, ResolveChildWorkflowExecutionStartSuccess, + ResolveExternalStreamWaits, ResolveNexusOperation, ResolveNexusOperationStart, ResolveRequestCancelExternalWorkflow, @@ -24,18 +30,24 @@ __all__ = [ "CancelWorkflow", + "DeliverStreamRecords", "DoUpdate", + "FinalizeExternalStreams", "FireTimer", "InitializeWorkflow", + "NotificationsReceived", "NotifyHasPatch", + "PrepareExternalStreamPark", "QueryWorkflow", "RemoveFromCache", + "ReplayExternalStreams", "ResolveActivity", "ResolveChildWorkflowExecution", "ResolveChildWorkflowExecutionStart", "ResolveChildWorkflowExecutionStartCancelled", "ResolveChildWorkflowExecutionStartFailure", "ResolveChildWorkflowExecutionStartSuccess", + "ResolveExternalStreamWaits", "ResolveNexusOperation", "ResolveNexusOperationStart", "ResolveRequestCancelExternalWorkflow", diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py index 0caca7d48a..08dfd94b4c 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py @@ -21,12 +21,21 @@ from temporalio.api.common.v1 import ( message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, ) +from temporalio.api.enums.v1 import ( + failed_cause_pb2 as temporal_dot_api_dot_enums_dot_v1_dot_failed__cause__pb2, +) from temporalio.api.enums.v1 import ( workflow_pb2 as temporal_dot_api_dot_enums_dot_v1_dot_workflow__pb2, ) from temporalio.api.failure.v1 import ( message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, ) +from temporalio.api.notification.v1 import ( + message_pb2 as temporal_dot_api_dot_notification_dot_v1_dot_message__pb2, +) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.update.v1 import ( message_pb2 as temporal_dot_api_dot_update_dot_v1_dot_message__pb2, ) @@ -39,17 +48,33 @@ from temporalio.bridge.proto.common import ( common_pb2 as temporal_dot_sdk_dot_core_dot_common_dot_common__pb2, ) +from temporalio.bridge.proto.external_data import ( + external_data_pb2 as temporal_dot_sdk_dot_core_dot_external__data_dot_external__data__pb2, +) from temporalio.bridge.proto.nexus import ( nexus_pb2 as temporal_dot_sdk_dot_core_dot_nexus_dot_nexus__pb2, ) +from temporalio.bridge.proto.workflow_commands import ( + workflow_commands_pb2 as temporal_dot_sdk_dot_core_dot_workflow__commands_dot_workflow__commands__pb2, +) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xe0\n\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xfc\n\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"_\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"f\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xcb\x02\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\nB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' + b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a*temporal/api/notification/v1/message.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a\x33temporal/sdk/core/external_data/external_data.proto\x1a;temporal/sdk/core/workflow_commands/workflow_commands.proto"\xc2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08\x12\x1e\n\x16history_floor_event_id\x18\r \x01(\x03"\xff\x0e\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12`\n\x1dresolve_external_stream_waits\x18\x11 \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveExternalStreamWaitsH\x00\x12^\n\x1cprepare_external_stream_park\x18\x12 \x01(\x0b\x32\x36.coresdk.workflow_activation.PrepareExternalStreamParkH\x00\x12U\n\x17replay_external_streams\x18\x13 \x01(\x0b\x32\x32.coresdk.workflow_activation.ReplayExternalStreamsH\x00\x12Y\n\x19\x66inalize_external_streams\x18\x14 \x01(\x0b\x32\x34.coresdk.workflow_activation.FinalizeExternalStreamsH\x00\x12S\n\x16\x64\x65liver_stream_records\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_activation.DeliverStreamRecordsH\x00\x12T\n\x16notifications_received\x18\x16 \x01(\x0b\x32\x32.coresdk.workflow_activation.NotificationsReceivedH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"Z\n\x15NotificationsReceived\x12\x41\n\rnotifications\x18\x01 \x03(\x0b\x32*.temporal.api.notification.v1.Notification"\x7f\n\x1aResolveExternalStreamWaits\x12\x1d\n\x15quiescence_generation\x18\x01 \x01(\x04\x12\x42\n\x0bready_hints\x18\x02 \x03(\x0b\x32-.coresdk.workflow_commands.ExternalStreamWait"\xab\x01\n\x19PrepareExternalStreamPark\x12\x1d\n\x15quiescence_generation\x18\x01 \x01(\x04\x12<\n\x05waits\x18\x02 \x03(\x0b\x32-.coresdk.workflow_commands.ExternalStreamWait\x12\x31\n\x06reason\x18\x03 \x01(\x0e\x32!.coresdk.external_data.ParkReason"\x92\x02\n\x15ReplayExternalStreams\x12\x1d\n\x15quiescence_generation\x18\x01 \x01(\x04\x12<\n\x05waits\x18\x02 \x03(\x0b\x32-.coresdk.workflow_commands.ExternalStreamWait\x12\x19\n\x11replay_annotation\x18\x03 \x01(\x0c\x12<\n\x11terminal_boundary\x18\x04 \x01(\x0e\x32!.coresdk.external_data.ParkReason\x12\x43\n\x06output\x18\x05 \x01(\x0b\x32\x33.coresdk.external_data.ExternalOutputStreamManifest"\xa9\x01\n\x17\x46inalizeExternalStreams\x12\x1d\n\x15quiescence_generation\x18\x01 \x01(\x04\x12<\n\x05waits\x18\x02 \x03(\x0b\x32-.coresdk.workflow_commands.ExternalStreamWait\x12\x31\n\x06reason\x18\x03 \x01(\x0e\x32!.coresdk.external_data.ParkReason"\x88\x01\n\x14\x44\x65liverStreamRecords\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x02 \x01(\x03\x12\x11\n\tto_offset\x18\x03 \x01(\x03\x12\x35\n\x07records\x18\x04 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord"\xfc\n\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xcb\x02\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\nB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' ) _WORKFLOWACTIVATION = DESCRIPTOR.message_types_by_name["WorkflowActivation"] _WORKFLOWACTIVATIONJOB = DESCRIPTOR.message_types_by_name["WorkflowActivationJob"] +_NOTIFICATIONSRECEIVED = DESCRIPTOR.message_types_by_name["NotificationsReceived"] +_RESOLVEEXTERNALSTREAMWAITS = DESCRIPTOR.message_types_by_name[ + "ResolveExternalStreamWaits" +] +_PREPAREEXTERNALSTREAMPARK = DESCRIPTOR.message_types_by_name[ + "PrepareExternalStreamPark" +] +_REPLAYEXTERNALSTREAMS = DESCRIPTOR.message_types_by_name["ReplayExternalStreams"] +_FINALIZEEXTERNALSTREAMS = DESCRIPTOR.message_types_by_name["FinalizeExternalStreams"] +_DELIVERSTREAMRECORDS = DESCRIPTOR.message_types_by_name["DeliverStreamRecords"] _INITIALIZEWORKFLOW = DESCRIPTOR.message_types_by_name["InitializeWorkflow"] _INITIALIZEWORKFLOW_HEADERSENTRY = _INITIALIZEWORKFLOW.nested_types_by_name[ "HeadersEntry" @@ -114,6 +139,72 @@ ) _sym_db.RegisterMessage(WorkflowActivationJob) +NotificationsReceived = _reflection.GeneratedProtocolMessageType( + "NotificationsReceived", + (_message.Message,), + { + "DESCRIPTOR": _NOTIFICATIONSRECEIVED, + "__module__": "temporal.sdk.core.workflow_activation.workflow_activation_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_activation.NotificationsReceived) + }, +) +_sym_db.RegisterMessage(NotificationsReceived) + +ResolveExternalStreamWaits = _reflection.GeneratedProtocolMessageType( + "ResolveExternalStreamWaits", + (_message.Message,), + { + "DESCRIPTOR": _RESOLVEEXTERNALSTREAMWAITS, + "__module__": "temporal.sdk.core.workflow_activation.workflow_activation_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_activation.ResolveExternalStreamWaits) + }, +) +_sym_db.RegisterMessage(ResolveExternalStreamWaits) + +PrepareExternalStreamPark = _reflection.GeneratedProtocolMessageType( + "PrepareExternalStreamPark", + (_message.Message,), + { + "DESCRIPTOR": _PREPAREEXTERNALSTREAMPARK, + "__module__": "temporal.sdk.core.workflow_activation.workflow_activation_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_activation.PrepareExternalStreamPark) + }, +) +_sym_db.RegisterMessage(PrepareExternalStreamPark) + +ReplayExternalStreams = _reflection.GeneratedProtocolMessageType( + "ReplayExternalStreams", + (_message.Message,), + { + "DESCRIPTOR": _REPLAYEXTERNALSTREAMS, + "__module__": "temporal.sdk.core.workflow_activation.workflow_activation_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_activation.ReplayExternalStreams) + }, +) +_sym_db.RegisterMessage(ReplayExternalStreams) + +FinalizeExternalStreams = _reflection.GeneratedProtocolMessageType( + "FinalizeExternalStreams", + (_message.Message,), + { + "DESCRIPTOR": _FINALIZEEXTERNALSTREAMS, + "__module__": "temporal.sdk.core.workflow_activation.workflow_activation_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_activation.FinalizeExternalStreams) + }, +) +_sym_db.RegisterMessage(FinalizeExternalStreams) + +DeliverStreamRecords = _reflection.GeneratedProtocolMessageType( + "DeliverStreamRecords", + (_message.Message,), + { + "DESCRIPTOR": _DELIVERSTREAMRECORDS, + "__module__": "temporal.sdk.core.workflow_activation.workflow_activation_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_activation.DeliverStreamRecords) + }, +) +_sym_db.RegisterMessage(DeliverStreamRecords) + InitializeWorkflow = _reflection.GeneratedProtocolMessageType( "InitializeWorkflow", (_message.Message,), @@ -376,56 +467,68 @@ _SIGNALWORKFLOW_HEADERSENTRY._serialized_options = b"8\001" _DOUPDATE_HEADERSENTRY._options = None _DOUPDATE_HEADERSENTRY._serialized_options = b"8\001" - _WORKFLOWACTIVATION._serialized_start = 532 - _WORKFLOWACTIVATION._serialized_end = 1078 - _WORKFLOWACTIVATIONJOB._serialized_start = 1081 - _WORKFLOWACTIVATIONJOB._serialized_end = 2457 - _INITIALIZEWORKFLOW._serialized_start = 2460 - _INITIALIZEWORKFLOW._serialized_end = 3864 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 3785 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 3864 - _FIRETIMER._serialized_start = 3866 - _FIRETIMER._serialized_end = 3890 - _RESOLVEACTIVITY._serialized_start = 3892 - _RESOLVEACTIVITY._serialized_end = 4001 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4004 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4341 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4343 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4402 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4405 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 4571 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 4573 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 4669 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 4671 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 4776 - _UPDATERANDOMSEED._serialized_start = 4778 - _UPDATERANDOMSEED._serialized_end = 4821 - _QUERYWORKFLOW._serialized_start = 4824 - _QUERYWORKFLOW._serialized_end = 5084 - _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 3785 - _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 3864 - _CANCELWORKFLOW._serialized_start = 5086 - _CANCELWORKFLOW._serialized_end = 5118 - _SIGNALWORKFLOW._serialized_start = 5121 - _SIGNALWORKFLOW._serialized_end = 5410 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 3785 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 3864 - _NOTIFYHASPATCH._serialized_start = 5412 - _NOTIFYHASPATCH._serialized_end = 5446 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5448 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 5543 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 5545 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 5647 - _DOUPDATE._serialized_start = 5650 - _DOUPDATE._serialized_end = 5981 - _DOUPDATE_HEADERSENTRY._serialized_start = 3785 - _DOUPDATE_HEADERSENTRY._serialized_end = 3864 - _RESOLVENEXUSOPERATIONSTART._serialized_start = 5984 - _RESOLVENEXUSOPERATIONSTART._serialized_end = 6138 - _RESOLVENEXUSOPERATION._serialized_start = 6140 - _RESOLVENEXUSOPERATION._serialized_end = 6229 - _REMOVEFROMCACHE._serialized_start = 6232 - _REMOVEFROMCACHE._serialized_end = 6584 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 6346 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 6584 + _WORKFLOWACTIVATION._serialized_start = 770 + _WORKFLOWACTIVATION._serialized_end = 1348 + _WORKFLOWACTIVATIONJOB._serialized_start = 1351 + _WORKFLOWACTIVATIONJOB._serialized_end = 3270 + _NOTIFICATIONSRECEIVED._serialized_start = 3272 + _NOTIFICATIONSRECEIVED._serialized_end = 3362 + _RESOLVEEXTERNALSTREAMWAITS._serialized_start = 3364 + _RESOLVEEXTERNALSTREAMWAITS._serialized_end = 3491 + _PREPAREEXTERNALSTREAMPARK._serialized_start = 3494 + _PREPAREEXTERNALSTREAMPARK._serialized_end = 3665 + _REPLAYEXTERNALSTREAMS._serialized_start = 3668 + _REPLAYEXTERNALSTREAMS._serialized_end = 3942 + _FINALIZEEXTERNALSTREAMS._serialized_start = 3945 + _FINALIZEEXTERNALSTREAMS._serialized_end = 4114 + _DELIVERSTREAMRECORDS._serialized_start = 4117 + _DELIVERSTREAMRECORDS._serialized_end = 4253 + _INITIALIZEWORKFLOW._serialized_start = 4256 + _INITIALIZEWORKFLOW._serialized_end = 5660 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 5581 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 5660 + _FIRETIMER._serialized_start = 5662 + _FIRETIMER._serialized_end = 5686 + _RESOLVEACTIVITY._serialized_start = 5688 + _RESOLVEACTIVITY._serialized_end = 5797 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 5800 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 6137 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 6139 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 6198 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 6201 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 6367 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 6369 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 6465 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 6467 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 6572 + _UPDATERANDOMSEED._serialized_start = 6574 + _UPDATERANDOMSEED._serialized_end = 6617 + _QUERYWORKFLOW._serialized_start = 6620 + _QUERYWORKFLOW._serialized_end = 6880 + _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 5581 + _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 5660 + _CANCELWORKFLOW._serialized_start = 6882 + _CANCELWORKFLOW._serialized_end = 6914 + _SIGNALWORKFLOW._serialized_start = 6917 + _SIGNALWORKFLOW._serialized_end = 7206 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 5581 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 5660 + _NOTIFYHASPATCH._serialized_start = 7208 + _NOTIFYHASPATCH._serialized_end = 7242 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 7245 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 7422 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 7425 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 7609 + _DOUPDATE._serialized_start = 7612 + _DOUPDATE._serialized_end = 7943 + _DOUPDATE_HEADERSENTRY._serialized_start = 5581 + _DOUPDATE_HEADERSENTRY._serialized_end = 5660 + _RESOLVENEXUSOPERATIONSTART._serialized_start = 7946 + _RESOLVENEXUSOPERATIONSTART._serialized_end = 8100 + _RESOLVENEXUSOPERATION._serialized_start = 8102 + _RESOLVENEXUSOPERATION._serialized_end = 8191 + _REMOVEFROMCACHE._serialized_start = 8194 + _REMOVEFROMCACHE._serialized_end = 8546 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 8308 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 8546 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi index afc7eb4abd..bff7a735d1 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi @@ -19,13 +19,18 @@ import google.protobuf.message import google.protobuf.timestamp_pb2 import temporalio.api.common.v1.message_pb2 +import temporalio.api.enums.v1.failed_cause_pb2 import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 +import temporalio.api.notification.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.update.v1.message_pb2 import temporalio.bridge.proto.activity_result.activity_result_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 import temporalio.bridge.proto.common.common_pb2 +import temporalio.bridge.proto.external_data.external_data_pb2 import temporalio.bridge.proto.nexus.nexus_pb2 +import temporalio.bridge.proto.workflow_commands.workflow_commands_pb2 if sys.version_info >= (3, 10): import typing as typing_extensions @@ -44,7 +49,7 @@ class WorkflowActivation(google.protobuf.message.Message): 1. init workflow 2. patches 3. random-seed-updates - 4. signals/updates + 4. signals/updates/channel notifications 5. all others 6. local activity resolutions 7. queries @@ -95,6 +100,7 @@ class WorkflowActivation(google.protobuf.message.Message): LAST_SDK_VERSION_FIELD_NUMBER: builtins.int SUGGEST_CONTINUE_AS_NEW_REASONS_FIELD_NUMBER: builtins.int TARGET_WORKER_DEPLOYMENT_VERSION_CHANGED_FIELD_NUMBER: builtins.int + HISTORY_FLOOR_EVENT_ID_FIELD_NUMBER: builtins.int run_id: builtins.str """The id of the currently active run of the workflow. Also used as a cache key. There may only ever be one active workflow task (and hence activation) of a run at one time. @@ -156,6 +162,11 @@ class WorkflowActivation(google.protobuf.message.Message): the workflow is Pinned. Experimental. """ + history_floor_event_id: builtins.int + """The event immediately preceding this Workflow Task's WorkflowTaskScheduled event in the + ordered History view Core used to build the activation. Zero means Core could not identify + that exact predecessor, in which case output must not be staged for this task. + """ def __init__( self, *, @@ -175,6 +186,7 @@ class WorkflowActivation(google.protobuf.message.Message): ] | None = ..., target_worker_deployment_version_changed: builtins.bool = ..., + history_floor_event_id: builtins.int = ..., ) -> None: ... def HasField( self, @@ -194,6 +206,8 @@ class WorkflowActivation(google.protobuf.message.Message): b"continue_as_new_suggested", "deployment_version_for_current_task", b"deployment_version_for_current_task", + "history_floor_event_id", + b"history_floor_event_id", "history_length", b"history_length", "history_size_bytes", @@ -235,6 +249,12 @@ class WorkflowActivationJob(google.protobuf.message.Message): DO_UPDATE_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_START_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + RESOLVE_EXTERNAL_STREAM_WAITS_FIELD_NUMBER: builtins.int + PREPARE_EXTERNAL_STREAM_PARK_FIELD_NUMBER: builtins.int + REPLAY_EXTERNAL_STREAMS_FIELD_NUMBER: builtins.int + FINALIZE_EXTERNAL_STREAMS_FIELD_NUMBER: builtins.int + DELIVER_STREAM_RECORDS_FIELD_NUMBER: builtins.int + NOTIFICATIONS_RECEIVED_FIELD_NUMBER: builtins.int REMOVE_FROM_CACHE_FIELD_NUMBER: builtins.int @property def initialize_workflow(self) -> global___InitializeWorkflow: @@ -295,6 +315,30 @@ class WorkflowActivationJob(google.protobuf.message.Message): def resolve_nexus_operation(self) -> global___ResolveNexusOperation: """A nexus operation resolved.""" @property + def resolve_external_stream_waits(self) -> global___ResolveExternalStreamWaits: + """One or more external stream waits may have data buffered. Runs user workflow code.""" + @property + def prepare_external_stream_park(self) -> global___PrepareExternalStreamPark: + """Runtime-internal: install park intents and recheck every stream. Runs no user code.""" + @property + def replay_external_streams(self) -> global___ReplayExternalStreams: + """Replay: deliver the recorded external stream observations from a marker.""" + @property + def finalize_external_streams(self) -> global___FinalizeExternalStreams: + """Runtime-internal: encode the terminal boundary for a marker Core is about to write. + Runs no user code. + """ + @property + def deliver_stream_records(self) -> global___DeliverStreamRecords: + """The number below is shared with the native stream tree, which leaves 17 + to 20 to the jobs above, and must not be reused. + + A range of a stream the workflow subscribed to. + """ + @property + def notifications_received(self) -> global___NotificationsReceived: + """Notifications from the channels the workflow subscribed to.""" + @property def remove_from_cache(self) -> global___RemoveFromCache: """Remove the workflow identified by the [WorkflowActivation] containing this job from the cache after performing the activation. It is guaranteed that this will be the only job @@ -322,6 +366,12 @@ class WorkflowActivationJob(google.protobuf.message.Message): do_update: global___DoUpdate | None = ..., resolve_nexus_operation_start: global___ResolveNexusOperationStart | None = ..., resolve_nexus_operation: global___ResolveNexusOperation | None = ..., + resolve_external_stream_waits: global___ResolveExternalStreamWaits | None = ..., + prepare_external_stream_park: global___PrepareExternalStreamPark | None = ..., + replay_external_streams: global___ReplayExternalStreams | None = ..., + finalize_external_streams: global___FinalizeExternalStreams | None = ..., + deliver_stream_records: global___DeliverStreamRecords | None = ..., + notifications_received: global___NotificationsReceived | None = ..., remove_from_cache: global___RemoveFromCache | None = ..., ) -> None: ... def HasField( @@ -329,24 +379,36 @@ class WorkflowActivationJob(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "cancel_workflow", b"cancel_workflow", + "deliver_stream_records", + b"deliver_stream_records", "do_update", b"do_update", + "finalize_external_streams", + b"finalize_external_streams", "fire_timer", b"fire_timer", "initialize_workflow", b"initialize_workflow", + "notifications_received", + b"notifications_received", "notify_has_patch", b"notify_has_patch", + "prepare_external_stream_park", + b"prepare_external_stream_park", "query_workflow", b"query_workflow", "remove_from_cache", b"remove_from_cache", + "replay_external_streams", + b"replay_external_streams", "resolve_activity", b"resolve_activity", "resolve_child_workflow_execution", b"resolve_child_workflow_execution", "resolve_child_workflow_execution_start", b"resolve_child_workflow_execution_start", + "resolve_external_stream_waits", + b"resolve_external_stream_waits", "resolve_nexus_operation", b"resolve_nexus_operation", "resolve_nexus_operation_start", @@ -368,24 +430,36 @@ class WorkflowActivationJob(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "cancel_workflow", b"cancel_workflow", + "deliver_stream_records", + b"deliver_stream_records", "do_update", b"do_update", + "finalize_external_streams", + b"finalize_external_streams", "fire_timer", b"fire_timer", "initialize_workflow", b"initialize_workflow", + "notifications_received", + b"notifications_received", "notify_has_patch", b"notify_has_patch", + "prepare_external_stream_park", + b"prepare_external_stream_park", "query_workflow", b"query_workflow", "remove_from_cache", b"remove_from_cache", + "replay_external_streams", + b"replay_external_streams", "resolve_activity", b"resolve_activity", "resolve_child_workflow_execution", b"resolve_child_workflow_execution", "resolve_child_workflow_execution_start", b"resolve_child_workflow_execution_start", + "resolve_external_stream_waits", + b"resolve_external_stream_waits", "resolve_nexus_operation", b"resolve_nexus_operation", "resolve_nexus_operation_start", @@ -421,6 +495,12 @@ class WorkflowActivationJob(google.protobuf.message.Message): "do_update", "resolve_nexus_operation_start", "resolve_nexus_operation", + "resolve_external_stream_waits", + "prepare_external_stream_park", + "replay_external_streams", + "finalize_external_streams", + "deliver_stream_records", + "notifications_received", "remove_from_cache", ] | None @@ -428,6 +508,300 @@ class WorkflowActivationJob(google.protobuf.message.Message): global___WorkflowActivationJob = WorkflowActivationJob +class NotificationsReceived(google.protobuf.message.Message): + """Hand a workflow the notifications the server folded for its channels. + + They come from the scheduled event of the Workflow Task this activation + belongs to. History is the record, so a replay yields the same job with the + same notifications at the same point. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NOTIFICATIONS_FIELD_NUMBER: builtins.int + @property + def notifications( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.notification.v1.message_pb2.Notification + ]: ... + def __init__( + self, + *, + notifications: collections.abc.Iterable[ + temporalio.api.notification.v1.message_pb2.Notification + ] + | None = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["notifications", b"notifications"] + ) -> None: ... + +global___NotificationsReceived = NotificationsReceived + +class ResolveExternalStreamWaits(google.protobuf.message.Message): + """Tells lang that one or more external stream waits may now have data. + + This job contains no records. The listed waits are readiness *hints*, not an exhaustive + availability claim: on receipt lang probes every active wait, drains all currently available + inputs, and only then resumes Workflow futures. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + QUIESCENCE_GENERATION_FIELD_NUMBER: builtins.int + READY_HINTS_FIELD_NUMBER: builtins.int + quiescence_generation: builtins.int + @property + def ready_hints( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.ExternalStreamWait + ]: ... + def __init__( + self, + *, + quiescence_generation: builtins.int = ..., + ready_hints: collections.abc.Iterable[ + temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.ExternalStreamWait + ] + | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "quiescence_generation", + b"quiescence_generation", + "ready_hints", + b"ready_hints", + ], + ) -> None: ... + +global___ResolveExternalStreamWaits = ResolveExternalStreamWaits + +class PrepareExternalStreamPark(google.protobuf.message.Message): + """Asks lang to park the complete active wait set through the backend handshake. + + Runtime-internal: no user Workflow code runs. Lang installs a park intent per subscription, + rechecks every stream, and answers with ExternalStreamParkResult. A recheck that finds records + answers `became_ready` instead, and Core issues a normal resolve activation next. + + Requires backend I/O, so lang must not handle this on the synchronous Workflow thread. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + QUIESCENCE_GENERATION_FIELD_NUMBER: builtins.int + WAITS_FIELD_NUMBER: builtins.int + REASON_FIELD_NUMBER: builtins.int + quiescence_generation: builtins.int + @property + def waits( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.ExternalStreamWait + ]: + """Always the complete active wait set.""" + reason: temporalio.bridge.proto.external_data.external_data_pb2.ParkReason.ValueType + """One of PARK_REASON_IDLE, PARK_REASON_ALL_WRITE_FENCED, or PARK_REASON_SHUTDOWN.""" + def __init__( + self, + *, + quiescence_generation: builtins.int = ..., + waits: collections.abc.Iterable[ + temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.ExternalStreamWait + ] + | None = ..., + reason: temporalio.bridge.proto.external_data.external_data_pb2.ParkReason.ValueType = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "quiescence_generation", + b"quiescence_generation", + "reason", + b"reason", + "waits", + b"waits", + ], + ) -> None: ... + +global___PrepareExternalStreamPark = PrepareExternalStreamPark + +class ReplayExternalStreams(google.protobuf.message.Message): + """Carries a recorded marker's annotation back to lang during replay. + + Core issues exactly one of these per marker. Lang's replay driver walks the annotation's + segments in order, performing one event-loop drain per segment, so the live run's k + activations become k drains inside this one activation and condition evaluation matches. + + Requires backend I/O (the recorded range reads and their integrity validation), so lang must + prepare it off the Workflow thread and deliver from memory once it is on it. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + QUIESCENCE_GENERATION_FIELD_NUMBER: builtins.int + WAITS_FIELD_NUMBER: builtins.int + REPLAY_ANNOTATION_FIELD_NUMBER: builtins.int + TERMINAL_BOUNDARY_FIELD_NUMBER: builtins.int + OUTPUT_FIELD_NUMBER: builtins.int + quiescence_generation: builtins.int + @property + def waits( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.ExternalStreamWait + ]: ... + replay_annotation: builtins.bytes + """Opaque to Core.""" + terminal_boundary: ( + temporalio.bridge.proto.external_data.external_data_pb2.ParkReason.ValueType + ) + @property + def output( + self, + ) -> temporalio.bridge.proto.external_data.external_data_pb2.ExternalOutputStreamManifest: + """The output manifest recorded in the same marker, if this Workflow Task published output.""" + def __init__( + self, + *, + quiescence_generation: builtins.int = ..., + waits: collections.abc.Iterable[ + temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.ExternalStreamWait + ] + | None = ..., + replay_annotation: builtins.bytes = ..., + terminal_boundary: temporalio.bridge.proto.external_data.external_data_pb2.ParkReason.ValueType = ..., + output: temporalio.bridge.proto.external_data.external_data_pb2.ExternalOutputStreamManifest + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["output", b"output"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "output", + b"output", + "quiescence_generation", + b"quiescence_generation", + "replay_annotation", + b"replay_annotation", + "terminal_boundary", + b"terminal_boundary", + "waits", + b"waits", + ], + ) -> None: ... + +global___ReplayExternalStreams = ReplayExternalStreams + +class FinalizeExternalStreams(google.protobuf.message.Message): + """Issued before Core writes a marker for a boundary Core itself decided and no park handshake + will run: rollover-deadline expiry, and shutdown or eviction with a Workflow Task open. + + Runs no user Workflow code, cannot resolve futures, and its only legal responses are + ExternalStreamFinalized or an activation failure. It requires *no* backend I/O: lang reads the + blocked cursor snapshot from its own in-memory manager state. If lang cannot answer, Core + writes no marker and the Workflow Task fails for retry -- there is no best-effort path. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + QUIESCENCE_GENERATION_FIELD_NUMBER: builtins.int + WAITS_FIELD_NUMBER: builtins.int + REASON_FIELD_NUMBER: builtins.int + quiescence_generation: builtins.int + @property + def waits( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.ExternalStreamWait + ]: ... + reason: temporalio.bridge.proto.external_data.external_data_pb2.ParkReason.ValueType + """One of PARK_REASON_ROLLOVER or PARK_REASON_SHUTDOWN.""" + def __init__( + self, + *, + quiescence_generation: builtins.int = ..., + waits: collections.abc.Iterable[ + temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.ExternalStreamWait + ] + | None = ..., + reason: temporalio.bridge.proto.external_data.external_data_pb2.ParkReason.ValueType = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "quiescence_generation", + b"quiescence_generation", + "reason", + b"reason", + "waits", + b"waits", + ], + ) -> None: ... + +global___FinalizeExternalStreams = FinalizeExternalStreams + +class DeliverStreamRecords(google.protobuf.message.Message): + """Hand a workflow the next range of a stream it subscribed to. + + The range is delivered once, on the task the server decided it belongs to, + and the offsets it covered are recorded in History rather than the payloads. + On replay the server re-supplies the same range by reading the stream again, + so this job appears at the same point with the same contents both times. + + An empty range is still delivered: a task where the subscription saw nothing + is a fact replay has to reproduce, not an absence of one. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_ID_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + TO_OFFSET_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + stream_id: builtins.str + """Id of the stream this range came from.""" + from_offset: builtins.int + """Inclusive.""" + to_offset: builtins.int + """Exclusive. Equal to from_offset when the subscription saw nothing.""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: ... + def __init__( + self, + *, + stream_id: builtins.str = ..., + from_offset: builtins.int = ..., + to_offset: builtins.int = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "from_offset", + b"from_offset", + "records", + b"records", + "stream_id", + b"stream_id", + "to_offset", + b"to_offset", + ], + ) -> None: ... + +global___DeliverStreamRecords = DeliverStreamRecords + class InitializeWorkflow(google.protobuf.message.Message): """Initialize a new workflow""" @@ -1182,6 +1556,7 @@ class ResolveSignalExternalWorkflow(google.protobuf.message.Message): SEQ_FIELD_NUMBER: builtins.int FAILURE_FIELD_NUMBER: builtins.int + CAUSE_FIELD_NUMBER: builtins.int seq: builtins.int """Sequence number as provided by lang in the corresponding SignalExternalWorkflowExecution command @@ -1191,18 +1566,25 @@ class ResolveSignalExternalWorkflow(google.protobuf.message.Message): """If populated, this signal either failed to be sent or was cancelled depending on failure type / info. """ + cause: temporalio.api.enums.v1.failed_cause_pb2.SignalExternalWorkflowExecutionFailedCause.ValueType + """The server-reported cause when the signal failed. Unspecified when the signal succeeded or + was cancelled before being sent. + """ def __init__( self, *, seq: builtins.int = ..., failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + cause: temporalio.api.enums.v1.failed_cause_pb2.SignalExternalWorkflowExecutionFailedCause.ValueType = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["failure", b"failure"] ) -> builtins.bool: ... def ClearField( self, - field_name: typing_extensions.Literal["failure", b"failure", "seq", b"seq"], + field_name: typing_extensions.Literal[ + "cause", b"cause", "failure", b"failure", "seq", b"seq" + ], ) -> None: ... global___ResolveSignalExternalWorkflow = ResolveSignalExternalWorkflow @@ -1212,27 +1594,31 @@ class ResolveRequestCancelExternalWorkflow(google.protobuf.message.Message): SEQ_FIELD_NUMBER: builtins.int FAILURE_FIELD_NUMBER: builtins.int + CAUSE_FIELD_NUMBER: builtins.int seq: builtins.int """Sequence number as provided by lang in the corresponding RequestCancelExternalWorkflowExecution command """ @property def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: - """If populated, this signal either failed to be sent or was cancelled depending on failure - type / info. - """ + """If populated, the cancellation request failed.""" + cause: temporalio.api.enums.v1.failed_cause_pb2.CancelExternalWorkflowExecutionFailedCause.ValueType + """The server-reported cause when the cancellation request failed.""" def __init__( self, *, seq: builtins.int = ..., failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + cause: temporalio.api.enums.v1.failed_cause_pb2.CancelExternalWorkflowExecutionFailedCause.ValueType = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["failure", b"failure"] ) -> builtins.bool: ... def ClearField( self, - field_name: typing_extensions.Literal["failure", b"failure", "seq", b"seq"], + field_name: typing_extensions.Literal[ + "cause", b"cause", "failure", b"failure", "seq", b"seq" + ], ) -> None: ... global___ResolveRequestCancelExternalWorkflow = ResolveRequestCancelExternalWorkflow diff --git a/temporalio/bridge/proto/workflow_commands/__init__.py b/temporalio/bridge/proto/workflow_commands/__init__.py index f599917afa..039d7a3634 100644 --- a/temporalio/bridge/proto/workflow_commands/__init__.py +++ b/temporalio/bridge/proto/workflow_commands/__init__.py @@ -1,13 +1,18 @@ from .workflow_commands_pb2 import ( ActivityCancellationType, + AppendStreamRecords, CancelChildWorkflowExecution, CancelSignalWorkflow, CancelTimer, CancelWorkflowExecution, CompleteWorkflowExecution, ContinueAsNewWorkflowExecution, + ExternalStreamFinalized, + ExternalStreamParkResult, + ExternalStreamWait, FailWorkflowExecution, ModifyWorkflowProperties, + ParkSetConfirmed, QueryResult, QuerySuccess, RequestCancelActivity, @@ -21,21 +26,35 @@ SignalExternalWorkflowExecution, StartChildWorkflowExecution, StartTimer, + StreamSetBecameReady, + SubscribeNotificationChannel, + SubscribeStream, + UnsubscribeNotificationChannel, UpdateResponse, UpsertWorkflowSearchAttributes, WorkflowCommand, + WorkflowOutputStreamBuffered, + WorkflowOutputStreamCommit, + WorkflowStreamChannels, + WorkflowStreamProgress, + WorkflowStreamQuiescent, ) __all__ = [ "ActivityCancellationType", + "AppendStreamRecords", "CancelChildWorkflowExecution", "CancelSignalWorkflow", "CancelTimer", "CancelWorkflowExecution", "CompleteWorkflowExecution", "ContinueAsNewWorkflowExecution", + "ExternalStreamFinalized", + "ExternalStreamParkResult", + "ExternalStreamWait", "FailWorkflowExecution", "ModifyWorkflowProperties", + "ParkSetConfirmed", "QueryResult", "QuerySuccess", "RequestCancelActivity", @@ -49,7 +68,16 @@ "SignalExternalWorkflowExecution", "StartChildWorkflowExecution", "StartTimer", + "StreamSetBecameReady", + "SubscribeNotificationChannel", + "SubscribeStream", + "UnsubscribeNotificationChannel", "UpdateResponse", "UpsertWorkflowSearchAttributes", "WorkflowCommand", + "WorkflowOutputStreamBuffered", + "WorkflowOutputStreamCommit", + "WorkflowStreamChannels", + "WorkflowStreamProgress", + "WorkflowStreamQuiescent", ] diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py index 0cdc02c6ba..6b26131200 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py @@ -34,18 +34,24 @@ from temporalio.api.sdk.v1 import ( user_metadata_pb2 as temporal_dot_api_dot_sdk_dot_v1_dot_user__metadata__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.bridge.proto.child_workflow import ( child_workflow_pb2 as temporal_dot_sdk_dot_core_dot_child__workflow_dot_child__workflow__pb2, ) from temporalio.bridge.proto.common import ( common_pb2 as temporal_dot_sdk_dot_core_dot_common_dot_common__pb2, ) +from temporalio.bridge.proto.external_data import ( + external_data_pb2 as temporal_dot_sdk_dot_core_dot_external__data_dot_external__data__pb2, +) from temporalio.bridge.proto.nexus import ( nexus_pb2 as temporal_dot_sdk_dot_core_dot_nexus_dot_nexus__pb2, ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xa9\x10\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x95\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12%\n\x1dinclude_arguments_into_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\x96\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' + b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a$temporal/api/stream/v1/message.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto\x1a\x33temporal/sdk/core/external_data/external_data.proto"\x8c\x18\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x12U\n\x18workflow_stream_progress\x18\x17 \x01(\x0b\x32\x31.coresdk.workflow_commands.WorkflowStreamProgressH\x00\x12W\n\x19workflow_stream_quiescent\x18\x18 \x01(\x0b\x32\x32.coresdk.workflow_commands.WorkflowStreamQuiescentH\x00\x12Z\n\x1b\x65xternal_stream_park_result\x18\x19 \x01(\x0b\x32\x33.coresdk.workflow_commands.ExternalStreamParkResultH\x00\x12W\n\x19\x65xternal_stream_finalized\x18\x1a \x01(\x0b\x32\x32.coresdk.workflow_commands.ExternalStreamFinalizedH\x00\x12^\n\x1dworkflow_output_stream_commit\x18\x1b \x01(\x0b\x32\x35.coresdk.workflow_commands.WorkflowOutputStreamCommitH\x00\x12\x62\n\x1fworkflow_output_stream_buffered\x18\x1c \x01(\x0b\x32\x37.coresdk.workflow_commands.WorkflowOutputStreamBufferedH\x00\x12\x46\n\x10subscribe_stream\x18\x1d \x01(\x0b\x32*.coresdk.workflow_commands.SubscribeStreamH\x00\x12O\n\x15\x61ppend_stream_records\x18\x1e \x01(\x0b\x32..coresdk.workflow_commands.AppendStreamRecordsH\x00\x12\x61\n\x1esubscribe_notification_channel\x18\x1f \x01(\x0b\x32\x37.coresdk.workflow_commands.SubscribeNotificationChannelH\x00\x12\x65\n unsubscribe_notification_channel\x18 \x01(\x0b\x32\x39.coresdk.workflow_commands.UnsubscribeNotificationChannelH\x00\x12U\n\x18workflow_stream_channels\x18! \x01(\x0b\x32\x31.coresdk.workflow_commands.WorkflowStreamChannelsH\x00\x42\t\n\x07variant"1\n\x1eUnsubscribeNotificationChannel\x12\x0f\n\x07\x63hannel\x18\x01 \x01(\t"*\n\x16WorkflowStreamChannels\x12\x10\n\x08\x63hannels\x18\x01 \x03(\t"/\n\x1cSubscribeNotificationChannel\x12\x0f\n\x07\x63hannel\x18\x01 \x01(\t"M\n\x16WorkflowStreamProgress\x12\x19\n\x11observation_delta\x18\x01 \x01(\x0c\x12\x18\n\x10request_rollover\x18\x02 \x01(\x08"\xa7\x01\n\x17WorkflowStreamQuiescent\x12\x1d\n\x15quiescence_generation\x18\x01 \x01(\x04\x12<\n\x05waits\x18\x02 \x03(\x0b\x32-.coresdk.workflow_commands.ExternalStreamWait\x12/\n\x0cidle_timeout\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration"W\n\x12\x45xternalStreamWait\x12\x0f\n\x07wait_id\x18\x01 \x01(\r\x12\x12\n\ngeneration\x18\x02 \x01(\x04\x12\x1c\n\x14immediately_parkable\x18\x03 \x01(\x08"\xf0\x01\n\x18\x45xternalStreamParkResult\x12\x1d\n\x15quiescence_generation\x18\x01 \x01(\x04\x12@\n\tconfirmed\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ParkSetConfirmedH\x00\x12G\n\x0c\x62\x65\x63\x61me_ready\x18\x03 \x01(\x0b\x32/.coresdk.workflow_commands.StreamSetBecameReadyH\x00\x12\x1f\n\x17\x66inal_observation_delta\x18\x04 \x01(\x0c\x42\t\n\x07outcome"\x12\n\x10ParkSetConfirmed"\x16\n\x14StreamSetBecameReady"Y\n\x17\x45xternalStreamFinalized\x12\x1d\n\x15quiescence_generation\x18\x01 \x01(\x04\x12\x1f\n\x17\x66inal_observation_delta\x18\x02 \x01(\x0c"}\n\x1aWorkflowOutputStreamCommit\x12\x45\n\x08manifest\x18\x01 \x01(\x0b\x32\x33.coresdk.external_data.ExternalOutputStreamManifest\x12\x18\n\x10request_rollover\x18\x02 \x01(\x08"V\n\x1cWorkflowOutputStreamBuffered\x12\x36\n\x13max_publish_latency\x18\x01 \x01(\x0b\x32\x19.google.protobuf.Duration"a\n\x13\x41ppendStreamRecords\x12\x13\n\x0bstream_name\x18\x01 \x01(\t\x12\x35\n\x07records\x18\x02 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord"\x87\x01\n\x0fSubscribeStream\x12\x19\n\x11stream_name_or_id\x18\x01 \x01(\t\x12\x14\n\x0cstart_offset\x18\x02 \x01(\x03\x12\x43\n\x0estart_position\x18\x03 \x01(\x0b\x32+.temporal.api.stream.v1.StreamStartPosition"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\x96\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' ) _ACTIVITYCANCELLATIONTYPE = DESCRIPTOR.enum_types_by_name["ActivityCancellationType"] @@ -56,6 +62,28 @@ _WORKFLOWCOMMAND = DESCRIPTOR.message_types_by_name["WorkflowCommand"] +_UNSUBSCRIBENOTIFICATIONCHANNEL = DESCRIPTOR.message_types_by_name[ + "UnsubscribeNotificationChannel" +] +_WORKFLOWSTREAMCHANNELS = DESCRIPTOR.message_types_by_name["WorkflowStreamChannels"] +_SUBSCRIBENOTIFICATIONCHANNEL = DESCRIPTOR.message_types_by_name[ + "SubscribeNotificationChannel" +] +_WORKFLOWSTREAMPROGRESS = DESCRIPTOR.message_types_by_name["WorkflowStreamProgress"] +_WORKFLOWSTREAMQUIESCENT = DESCRIPTOR.message_types_by_name["WorkflowStreamQuiescent"] +_EXTERNALSTREAMWAIT = DESCRIPTOR.message_types_by_name["ExternalStreamWait"] +_EXTERNALSTREAMPARKRESULT = DESCRIPTOR.message_types_by_name["ExternalStreamParkResult"] +_PARKSETCONFIRMED = DESCRIPTOR.message_types_by_name["ParkSetConfirmed"] +_STREAMSETBECAMEREADY = DESCRIPTOR.message_types_by_name["StreamSetBecameReady"] +_EXTERNALSTREAMFINALIZED = DESCRIPTOR.message_types_by_name["ExternalStreamFinalized"] +_WORKFLOWOUTPUTSTREAMCOMMIT = DESCRIPTOR.message_types_by_name[ + "WorkflowOutputStreamCommit" +] +_WORKFLOWOUTPUTSTREAMBUFFERED = DESCRIPTOR.message_types_by_name[ + "WorkflowOutputStreamBuffered" +] +_APPENDSTREAMRECORDS = DESCRIPTOR.message_types_by_name["AppendStreamRecords"] +_SUBSCRIBESTREAM = DESCRIPTOR.message_types_by_name["SubscribeStream"] _STARTTIMER = DESCRIPTOR.message_types_by_name["StartTimer"] _CANCELTIMER = DESCRIPTOR.message_types_by_name["CancelTimer"] _SCHEDULEACTIVITY = DESCRIPTOR.message_types_by_name["ScheduleActivity"] @@ -130,6 +158,160 @@ ) _sym_db.RegisterMessage(WorkflowCommand) +UnsubscribeNotificationChannel = _reflection.GeneratedProtocolMessageType( + "UnsubscribeNotificationChannel", + (_message.Message,), + { + "DESCRIPTOR": _UNSUBSCRIBENOTIFICATIONCHANNEL, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.UnsubscribeNotificationChannel) + }, +) +_sym_db.RegisterMessage(UnsubscribeNotificationChannel) + +WorkflowStreamChannels = _reflection.GeneratedProtocolMessageType( + "WorkflowStreamChannels", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWSTREAMCHANNELS, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.WorkflowStreamChannels) + }, +) +_sym_db.RegisterMessage(WorkflowStreamChannels) + +SubscribeNotificationChannel = _reflection.GeneratedProtocolMessageType( + "SubscribeNotificationChannel", + (_message.Message,), + { + "DESCRIPTOR": _SUBSCRIBENOTIFICATIONCHANNEL, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.SubscribeNotificationChannel) + }, +) +_sym_db.RegisterMessage(SubscribeNotificationChannel) + +WorkflowStreamProgress = _reflection.GeneratedProtocolMessageType( + "WorkflowStreamProgress", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWSTREAMPROGRESS, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.WorkflowStreamProgress) + }, +) +_sym_db.RegisterMessage(WorkflowStreamProgress) + +WorkflowStreamQuiescent = _reflection.GeneratedProtocolMessageType( + "WorkflowStreamQuiescent", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWSTREAMQUIESCENT, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.WorkflowStreamQuiescent) + }, +) +_sym_db.RegisterMessage(WorkflowStreamQuiescent) + +ExternalStreamWait = _reflection.GeneratedProtocolMessageType( + "ExternalStreamWait", + (_message.Message,), + { + "DESCRIPTOR": _EXTERNALSTREAMWAIT, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.ExternalStreamWait) + }, +) +_sym_db.RegisterMessage(ExternalStreamWait) + +ExternalStreamParkResult = _reflection.GeneratedProtocolMessageType( + "ExternalStreamParkResult", + (_message.Message,), + { + "DESCRIPTOR": _EXTERNALSTREAMPARKRESULT, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.ExternalStreamParkResult) + }, +) +_sym_db.RegisterMessage(ExternalStreamParkResult) + +ParkSetConfirmed = _reflection.GeneratedProtocolMessageType( + "ParkSetConfirmed", + (_message.Message,), + { + "DESCRIPTOR": _PARKSETCONFIRMED, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.ParkSetConfirmed) + }, +) +_sym_db.RegisterMessage(ParkSetConfirmed) + +StreamSetBecameReady = _reflection.GeneratedProtocolMessageType( + "StreamSetBecameReady", + (_message.Message,), + { + "DESCRIPTOR": _STREAMSETBECAMEREADY, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.StreamSetBecameReady) + }, +) +_sym_db.RegisterMessage(StreamSetBecameReady) + +ExternalStreamFinalized = _reflection.GeneratedProtocolMessageType( + "ExternalStreamFinalized", + (_message.Message,), + { + "DESCRIPTOR": _EXTERNALSTREAMFINALIZED, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.ExternalStreamFinalized) + }, +) +_sym_db.RegisterMessage(ExternalStreamFinalized) + +WorkflowOutputStreamCommit = _reflection.GeneratedProtocolMessageType( + "WorkflowOutputStreamCommit", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWOUTPUTSTREAMCOMMIT, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.WorkflowOutputStreamCommit) + }, +) +_sym_db.RegisterMessage(WorkflowOutputStreamCommit) + +WorkflowOutputStreamBuffered = _reflection.GeneratedProtocolMessageType( + "WorkflowOutputStreamBuffered", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWOUTPUTSTREAMBUFFERED, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.WorkflowOutputStreamBuffered) + }, +) +_sym_db.RegisterMessage(WorkflowOutputStreamBuffered) + +AppendStreamRecords = _reflection.GeneratedProtocolMessageType( + "AppendStreamRecords", + (_message.Message,), + { + "DESCRIPTOR": _APPENDSTREAMRECORDS, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.AppendStreamRecords) + }, +) +_sym_db.RegisterMessage(AppendStreamRecords) + +SubscribeStream = _reflection.GeneratedProtocolMessageType( + "SubscribeStream", + (_message.Message,), + { + "DESCRIPTOR": _SUBSCRIBESTREAM, + "__module__": "temporal.sdk.core.workflow_commands.workflow_commands_pb2", + # @@protoc_insertion_point(class_scope:coresdk.workflow_commands.SubscribeStream) + }, +) +_sym_db.RegisterMessage(SubscribeStream) + StartTimer = _reflection.GeneratedProtocolMessageType( "StartTimer", (_message.Message,), @@ -484,70 +666,98 @@ _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_options = b"8\001" _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._options = None _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_options = b"8\001" - _ACTIVITYCANCELLATIONTYPE._serialized_start = 8672 - _ACTIVITYCANCELLATIONTYPE._serialized_end = 8760 - _WORKFLOWCOMMAND._serialized_start = 518 - _WORKFLOWCOMMAND._serialized_end = 2607 - _STARTTIMER._serialized_start = 2609 - _STARTTIMER._serialized_end = 2692 - _CANCELTIMER._serialized_start = 2694 - _CANCELTIMER._serialized_end = 2720 - _SCHEDULEACTIVITY._serialized_start = 2723 - _SCHEDULEACTIVITY._serialized_end = 3547 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3468 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3547 - _SCHEDULELOCALACTIVITY._serialized_start = 3550 - _SCHEDULELOCALACTIVITY._serialized_end = 4339 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3468 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3547 - _REQUESTCANCELACTIVITY._serialized_start = 4341 - _REQUESTCANCELACTIVITY._serialized_end = 4377 - _REQUESTCANCELLOCALACTIVITY._serialized_start = 4379 - _REQUESTCANCELLOCALACTIVITY._serialized_end = 4420 - _QUERYRESULT._serialized_start = 4423 - _QUERYRESULT._serialized_end = 4579 - _QUERYSUCCESS._serialized_start = 4581 - _QUERYSUCCESS._serialized_end = 4646 - _COMPLETEWORKFLOWEXECUTION._serialized_start = 4648 - _COMPLETEWORKFLOWEXECUTION._serialized_end = 4724 - _FAILWORKFLOWEXECUTION._serialized_start = 4726 - _FAILWORKFLOWEXECUTION._serialized_end = 4800 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 4803 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 5717 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5560 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5636 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3468 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3547 - _CANCELWORKFLOWEXECUTION._serialized_start = 5719 - _CANCELWORKFLOWEXECUTION._serialized_end = 5795 - _SETPATCHMARKER._serialized_start = 5797 - _SETPATCHMARKER._serialized_end = 5851 - _STARTCHILDWORKFLOWEXECUTION._serialized_start = 5854 - _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7028 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3468 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3547 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5560 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5636 - _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7030 - _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7104 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7107 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7249 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7252 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 7651 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3468 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3547 - _CANCELSIGNALWORKFLOW._serialized_start = 7653 - _CANCELSIGNALWORKFLOW._serialized_end = 7688 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 7690 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 7791 - _MODIFYWORKFLOWPROPERTIES._serialized_start = 7793 - _MODIFYWORKFLOWPROPERTIES._serialized_end = 7872 - _UPDATERESPONSE._serialized_start = 7875 - _UPDATERESPONSE._serialized_end = 8085 - _SCHEDULENEXUSOPERATION._serialized_start = 8088 - _SCHEDULENEXUSOPERATION._serialized_end = 8626 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 8576 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 8626 - _REQUESTCANCELNEXUSOPERATION._serialized_start = 8628 - _REQUESTCANCELNEXUSOPERATION._serialized_end = 8670 + _ACTIVITYCANCELLATIONTYPE._serialized_start = 11068 + _ACTIVITYCANCELLATIONTYPE._serialized_end = 11156 + _WORKFLOWCOMMAND._serialized_start = 609 + _WORKFLOWCOMMAND._serialized_end = 3693 + _UNSUBSCRIBENOTIFICATIONCHANNEL._serialized_start = 3695 + _UNSUBSCRIBENOTIFICATIONCHANNEL._serialized_end = 3744 + _WORKFLOWSTREAMCHANNELS._serialized_start = 3746 + _WORKFLOWSTREAMCHANNELS._serialized_end = 3788 + _SUBSCRIBENOTIFICATIONCHANNEL._serialized_start = 3790 + _SUBSCRIBENOTIFICATIONCHANNEL._serialized_end = 3837 + _WORKFLOWSTREAMPROGRESS._serialized_start = 3839 + _WORKFLOWSTREAMPROGRESS._serialized_end = 3916 + _WORKFLOWSTREAMQUIESCENT._serialized_start = 3919 + _WORKFLOWSTREAMQUIESCENT._serialized_end = 4086 + _EXTERNALSTREAMWAIT._serialized_start = 4088 + _EXTERNALSTREAMWAIT._serialized_end = 4175 + _EXTERNALSTREAMPARKRESULT._serialized_start = 4178 + _EXTERNALSTREAMPARKRESULT._serialized_end = 4418 + _PARKSETCONFIRMED._serialized_start = 4420 + _PARKSETCONFIRMED._serialized_end = 4438 + _STREAMSETBECAMEREADY._serialized_start = 4440 + _STREAMSETBECAMEREADY._serialized_end = 4462 + _EXTERNALSTREAMFINALIZED._serialized_start = 4464 + _EXTERNALSTREAMFINALIZED._serialized_end = 4553 + _WORKFLOWOUTPUTSTREAMCOMMIT._serialized_start = 4555 + _WORKFLOWOUTPUTSTREAMCOMMIT._serialized_end = 4680 + _WORKFLOWOUTPUTSTREAMBUFFERED._serialized_start = 4682 + _WORKFLOWOUTPUTSTREAMBUFFERED._serialized_end = 4768 + _APPENDSTREAMRECORDS._serialized_start = 4770 + _APPENDSTREAMRECORDS._serialized_end = 4867 + _SUBSCRIBESTREAM._serialized_start = 4870 + _SUBSCRIBESTREAM._serialized_end = 5005 + _STARTTIMER._serialized_start = 5007 + _STARTTIMER._serialized_end = 5090 + _CANCELTIMER._serialized_start = 5092 + _CANCELTIMER._serialized_end = 5118 + _SCHEDULEACTIVITY._serialized_start = 5121 + _SCHEDULEACTIVITY._serialized_end = 5945 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 5866 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 5945 + _SCHEDULELOCALACTIVITY._serialized_start = 5948 + _SCHEDULELOCALACTIVITY._serialized_end = 6735 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 5866 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 5945 + _REQUESTCANCELACTIVITY._serialized_start = 6737 + _REQUESTCANCELACTIVITY._serialized_end = 6773 + _REQUESTCANCELLOCALACTIVITY._serialized_start = 6775 + _REQUESTCANCELLOCALACTIVITY._serialized_end = 6816 + _QUERYRESULT._serialized_start = 6819 + _QUERYRESULT._serialized_end = 6975 + _QUERYSUCCESS._serialized_start = 6977 + _QUERYSUCCESS._serialized_end = 7042 + _COMPLETEWORKFLOWEXECUTION._serialized_start = 7044 + _COMPLETEWORKFLOWEXECUTION._serialized_end = 7120 + _FAILWORKFLOWEXECUTION._serialized_start = 7122 + _FAILWORKFLOWEXECUTION._serialized_end = 7196 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 7199 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 8113 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 7956 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 8032 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 5866 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 5945 + _CANCELWORKFLOWEXECUTION._serialized_start = 8115 + _CANCELWORKFLOWEXECUTION._serialized_end = 8191 + _SETPATCHMARKER._serialized_start = 8193 + _SETPATCHMARKER._serialized_end = 8247 + _STARTCHILDWORKFLOWEXECUTION._serialized_start = 8250 + _STARTCHILDWORKFLOWEXECUTION._serialized_end = 9424 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 5866 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 5945 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 7956 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 8032 + _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 9426 + _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 9500 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 9503 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 9645 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 9648 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 10047 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 5866 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 5945 + _CANCELSIGNALWORKFLOW._serialized_start = 10049 + _CANCELSIGNALWORKFLOW._serialized_end = 10084 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 10086 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 10187 + _MODIFYWORKFLOWPROPERTIES._serialized_start = 10189 + _MODIFYWORKFLOWPROPERTIES._serialized_end = 10268 + _UPDATERESPONSE._serialized_start = 10271 + _UPDATERESPONSE._serialized_end = 10481 + _SCHEDULENEXUSOPERATION._serialized_start = 10484 + _SCHEDULENEXUSOPERATION._serialized_end = 11022 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 10972 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 11022 + _REQUESTCANCELNEXUSOPERATION._serialized_start = 11024 + _REQUESTCANCELNEXUSOPERATION._serialized_end = 11066 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi index ad4a5f487f..9199b814e2 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi @@ -25,8 +25,10 @@ import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.sdk.v1.event_group_marker_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 import temporalio.bridge.proto.common.common_pb2 +import temporalio.bridge.proto.external_data.external_data_pb2 import temporalio.bridge.proto.nexus.nexus_pb2 if sys.version_info >= (3, 10): @@ -103,6 +105,17 @@ class WorkflowCommand(google.protobuf.message.Message): UPDATE_RESPONSE_FIELD_NUMBER: builtins.int SCHEDULE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int REQUEST_CANCEL_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + WORKFLOW_STREAM_PROGRESS_FIELD_NUMBER: builtins.int + WORKFLOW_STREAM_QUIESCENT_FIELD_NUMBER: builtins.int + EXTERNAL_STREAM_PARK_RESULT_FIELD_NUMBER: builtins.int + EXTERNAL_STREAM_FINALIZED_FIELD_NUMBER: builtins.int + WORKFLOW_OUTPUT_STREAM_COMMIT_FIELD_NUMBER: builtins.int + WORKFLOW_OUTPUT_STREAM_BUFFERED_FIELD_NUMBER: builtins.int + SUBSCRIBE_STREAM_FIELD_NUMBER: builtins.int + APPEND_STREAM_RECORDS_FIELD_NUMBER: builtins.int + SUBSCRIBE_NOTIFICATION_CHANNEL_FIELD_NUMBER: builtins.int + UNSUBSCRIBE_NOTIFICATION_CHANNEL_FIELD_NUMBER: builtins.int + WORKFLOW_STREAM_CHANNELS_FIELD_NUMBER: builtins.int @property def user_metadata(self) -> temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata: """User metadata that may or may not be persisted into history depending on the command type. @@ -177,6 +190,37 @@ class WorkflowCommand(google.protobuf.message.Message): def request_cancel_nexus_operation( self, ) -> global___RequestCancelNexusOperation: ... + @property + def workflow_stream_progress(self) -> global___WorkflowStreamProgress: ... + @property + def workflow_stream_quiescent(self) -> global___WorkflowStreamQuiescent: ... + @property + def external_stream_park_result(self) -> global___ExternalStreamParkResult: ... + @property + def external_stream_finalized(self) -> global___ExternalStreamFinalized: ... + @property + def workflow_output_stream_commit(self) -> global___WorkflowOutputStreamCommit: ... + @property + def workflow_output_stream_buffered( + self, + ) -> global___WorkflowOutputStreamBuffered: ... + @property + def subscribe_stream(self) -> global___SubscribeStream: + """The two numbers below are shared with the native stream tree, which + leaves 23 to 28 to the commands above, and must not be reused. + """ + @property + def append_stream_records(self) -> global___AppendStreamRecords: ... + @property + def subscribe_notification_channel( + self, + ) -> global___SubscribeNotificationChannel: ... + @property + def unsubscribe_notification_channel( + self, + ) -> global___UnsubscribeNotificationChannel: ... + @property + def workflow_stream_channels(self) -> global___WorkflowStreamChannels: ... def __init__( self, *, @@ -215,10 +259,26 @@ class WorkflowCommand(google.protobuf.message.Message): schedule_nexus_operation: global___ScheduleNexusOperation | None = ..., request_cancel_nexus_operation: global___RequestCancelNexusOperation | None = ..., + workflow_stream_progress: global___WorkflowStreamProgress | None = ..., + workflow_stream_quiescent: global___WorkflowStreamQuiescent | None = ..., + external_stream_park_result: global___ExternalStreamParkResult | None = ..., + external_stream_finalized: global___ExternalStreamFinalized | None = ..., + workflow_output_stream_commit: global___WorkflowOutputStreamCommit | None = ..., + workflow_output_stream_buffered: global___WorkflowOutputStreamBuffered + | None = ..., + subscribe_stream: global___SubscribeStream | None = ..., + append_stream_records: global___AppendStreamRecords | None = ..., + subscribe_notification_channel: global___SubscribeNotificationChannel + | None = ..., + unsubscribe_notification_channel: global___UnsubscribeNotificationChannel + | None = ..., + workflow_stream_channels: global___WorkflowStreamChannels | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ + "append_stream_records", + b"append_stream_records", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -231,6 +291,10 @@ class WorkflowCommand(google.protobuf.message.Message): b"complete_workflow_execution", "continue_as_new_workflow_execution", b"continue_as_new_workflow_execution", + "external_stream_finalized", + b"external_stream_finalized", + "external_stream_park_result", + b"external_stream_park_result", "fail_workflow_execution", b"fail_workflow_execution", "modify_workflow_properties", @@ -259,6 +323,12 @@ class WorkflowCommand(google.protobuf.message.Message): b"start_child_workflow_execution", "start_timer", b"start_timer", + "subscribe_notification_channel", + b"subscribe_notification_channel", + "subscribe_stream", + b"subscribe_stream", + "unsubscribe_notification_channel", + b"unsubscribe_notification_channel", "update_response", b"update_response", "upsert_workflow_search_attributes", @@ -267,11 +337,23 @@ class WorkflowCommand(google.protobuf.message.Message): b"user_metadata", "variant", b"variant", + "workflow_output_stream_buffered", + b"workflow_output_stream_buffered", + "workflow_output_stream_commit", + b"workflow_output_stream_commit", + "workflow_stream_channels", + b"workflow_stream_channels", + "workflow_stream_progress", + b"workflow_stream_progress", + "workflow_stream_quiescent", + b"workflow_stream_quiescent", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ + "append_stream_records", + b"append_stream_records", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -286,6 +368,10 @@ class WorkflowCommand(google.protobuf.message.Message): b"continue_as_new_workflow_execution", "event_group_markers", b"event_group_markers", + "external_stream_finalized", + b"external_stream_finalized", + "external_stream_park_result", + b"external_stream_park_result", "fail_workflow_execution", b"fail_workflow_execution", "modify_workflow_properties", @@ -314,6 +400,12 @@ class WorkflowCommand(google.protobuf.message.Message): b"start_child_workflow_execution", "start_timer", b"start_timer", + "subscribe_notification_channel", + b"subscribe_notification_channel", + "subscribe_stream", + b"subscribe_stream", + "unsubscribe_notification_channel", + b"unsubscribe_notification_channel", "update_response", b"update_response", "upsert_workflow_search_attributes", @@ -322,6 +414,16 @@ class WorkflowCommand(google.protobuf.message.Message): b"user_metadata", "variant", b"variant", + "workflow_output_stream_buffered", + b"workflow_output_stream_buffered", + "workflow_output_stream_commit", + b"workflow_output_stream_commit", + "workflow_stream_channels", + b"workflow_stream_channels", + "workflow_stream_progress", + b"workflow_stream_progress", + "workflow_stream_quiescent", + b"workflow_stream_quiescent", ], ) -> None: ... def WhichOneof( @@ -350,12 +452,516 @@ class WorkflowCommand(google.protobuf.message.Message): "update_response", "schedule_nexus_operation", "request_cancel_nexus_operation", + "workflow_stream_progress", + "workflow_stream_quiescent", + "external_stream_park_result", + "external_stream_finalized", + "workflow_output_stream_commit", + "workflow_output_stream_buffered", + "subscribe_stream", + "append_stream_records", + "subscribe_notification_channel", + "unsubscribe_notification_channel", + "workflow_stream_channels", ] | None ): ... global___WorkflowCommand = WorkflowCommand +class UnsubscribeNotificationChannel(google.protobuf.message.Message): + """End this workflow's subscription to a notification channel. Notifications + already recorded on a scheduled event still reach that Workflow Task. + + The server records the event whether or not the run held a subscription, so + every command has an event to match on replay. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CHANNEL_FIELD_NUMBER: builtins.int + channel: builtins.str + """Name of the channel, scoped to the namespace.""" + def __init__( + self, + *, + channel: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["channel", b"channel"] + ) -> None: ... + +global___UnsubscribeNotificationChannel = UnsubscribeNotificationChannel + +class WorkflowStreamChannels(google.protobuf.message.Message): + """The complete set of notification channels the run listens on, one per open external stream + reader, in a stable order replay reproduces. + + Core subscribes the run to a channel it has not yet subscribed on the completion that ends the + Workflow Task, after the progress marker and never on a run-ending completion. Issued from the + activation that opened the reader, the server-bound subscribe command would end a task that + was meant to stay retained. Reported on every completion once a reader has been opened, the + empty set included: an absent command leaves Core's view unchanged, which is what the park and + finalization answers rely on. Carries no retention request of its own. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CHANNELS_FIELD_NUMBER: builtins.int + @property + def channels( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[ + builtins.str + ]: ... + def __init__( + self, + *, + channels: collections.abc.Iterable[builtins.str] | None = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["channels", b"channels"] + ) -> None: ... + +global___WorkflowStreamChannels = WorkflowStreamChannels + +class SubscribeNotificationChannel(google.protobuf.message.Message): + """Subscribe this workflow to a notification channel, so the scheduled event of + each later Workflow Task carries the notifications folded for it. + + The notifications live in History rather than arriving by a side channel, so + a replay reads the same ones the live run saw. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CHANNEL_FIELD_NUMBER: builtins.int + channel: builtins.str + """Name of the channel, scoped to the namespace.""" + def __init__( + self, + *, + channel: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["channel", b"channel"] + ) -> None: ... + +global___SubscribeNotificationChannel = SubscribeNotificationChannel + +class WorkflowStreamProgress(google.protobuf.message.Message): + """Commits an observation delta for external streams. + + Emitted on *every* completion path where replay-visible stream state changed -- which includes + an activation that observed no records at all -- and independent of whether the Workflow Task + is retained. Consuming a record and committing that consumption are separate steps, and the + second is not conditional on why the Workflow Task ended. + + Ordering is normative: this command precedes every command whose value could depend on the + consumed data. On replay that guarantees integrity validation for a record runs before the + command derived from it is matched. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + OBSERVATION_DELTA_FIELD_NUMBER: builtins.int + REQUEST_ROLLOVER_FIELD_NUMBER: builtins.int + observation_delta: builtins.bytes + """The segments produced since the previous progress report for this Workflow Task. Opaque to + Core, which appends it to the accumulated annotation and never parses it. + """ + request_rollover: builtins.bool + """Set when the encoder is approaching the annotation byte budget and the runtime wants the + Workflow Task rolled over rather than the marker grown further. Core takes the rollover + path minus the finalization round trip, because this command already carries the terminal. + """ + def __init__( + self, + *, + observation_delta: builtins.bytes = ..., + request_rollover: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "observation_delta", + b"observation_delta", + "request_rollover", + b"request_rollover", + ], + ) -> None: ... + +global___WorkflowStreamProgress = WorkflowStreamProgress + +class WorkflowStreamQuiescent(google.protobuf.message.Message): + """Asks Core to retain the open Workflow Task. Carries no annotation data -- retention and + progress are separate questions, and a completion may carry either command, both, or neither. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + QUIESCENCE_GENERATION_FIELD_NUMBER: builtins.int + WAITS_FIELD_NUMBER: builtins.int + IDLE_TIMEOUT_FIELD_NUMBER: builtins.int + quiescence_generation: builtins.int + """Identifies this complete blocked snapshot.""" + @property + def waits( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___ExternalStreamWait + ]: + """The *complete* set of waits the Workflow is blocked on. A partial set would let one idle + stream park a Workflow Task another stream is still driving. + """ + @property + def idle_timeout(self) -> google.protobuf.duration_pb2.Duration: + """Already reduced by `min` over the quiescent set in wait_id order. Core rejects a + non-positive value as a malformed completion rather than coercing it. + """ + def __init__( + self, + *, + quiescence_generation: builtins.int = ..., + waits: collections.abc.Iterable[global___ExternalStreamWait] | None = ..., + idle_timeout: google.protobuf.duration_pb2.Duration | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["idle_timeout", b"idle_timeout"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "idle_timeout", + b"idle_timeout", + "quiescence_generation", + b"quiescence_generation", + "waits", + b"waits", + ], + ) -> None: ... + +global___WorkflowStreamQuiescent = WorkflowStreamQuiescent + +class ExternalStreamWait(google.protobuf.message.Message): + """One external stream wait. Shared by the commands above and by the activation jobs, so it takes + no command tag of its own. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + WAIT_ID_FIELD_NUMBER: builtins.int + GENERATION_FIELD_NUMBER: builtins.int + IMMEDIATELY_PARKABLE_FIELD_NUMBER: builtins.int + wait_id: builtins.int + generation: builtins.int + """Increments each time this subscription re-enters the blocked state. Never leaves the + Core/lang boundary. + """ + immediately_parkable: builtins.bool + """Normally set after a write fence. A single fenced stream does not park the Workflow Task; + Core bypasses the idle delay only when *every* active wait in the snapshot is set. + """ + def __init__( + self, + *, + wait_id: builtins.int = ..., + generation: builtins.int = ..., + immediately_parkable: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "generation", + b"generation", + "immediately_parkable", + b"immediately_parkable", + "wait_id", + b"wait_id", + ], + ) -> None: ... + +global___ExternalStreamWait = ExternalStreamWait + +class ExternalStreamParkResult(google.protobuf.message.Message): + """Lang's answer to PrepareExternalStreamPark.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + QUIESCENCE_GENERATION_FIELD_NUMBER: builtins.int + CONFIRMED_FIELD_NUMBER: builtins.int + BECAME_READY_FIELD_NUMBER: builtins.int + FINAL_OBSERVATION_DELTA_FIELD_NUMBER: builtins.int + quiescence_generation: builtins.int + @property + def confirmed(self) -> global___ParkSetConfirmed: ... + @property + def became_ready(self) -> global___StreamSetBecameReady: ... + final_observation_delta: builtins.bytes + """Terminal observation delta for the marker Core is about to write. Present on `confirmed`: + only lang can encode the boundary, and Core must not write a marker whose annotation has + no terminal. + """ + def __init__( + self, + *, + quiescence_generation: builtins.int = ..., + confirmed: global___ParkSetConfirmed | None = ..., + became_ready: global___StreamSetBecameReady | None = ..., + final_observation_delta: builtins.bytes = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "became_ready", + b"became_ready", + "confirmed", + b"confirmed", + "outcome", + b"outcome", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "became_ready", + b"became_ready", + "confirmed", + b"confirmed", + "final_observation_delta", + b"final_observation_delta", + "outcome", + b"outcome", + "quiescence_generation", + b"quiescence_generation", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["outcome", b"outcome"] + ) -> typing_extensions.Literal["confirmed", "became_ready"] | None: ... + +global___ExternalStreamParkResult = ExternalStreamParkResult + +class ParkSetConfirmed(google.protobuf.message.Message): + """Every stream was still empty after intents were installed and every stream was rechecked.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___ParkSetConfirmed = ParkSetConfirmed + +class StreamSetBecameReady(google.protobuf.message.Message): + """A recheck found records, so this parking generation is aborted and Core issues a normal + resolve activation rather than running user code from inside the park path. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___StreamSetBecameReady = StreamSetBecameReady + +class ExternalStreamFinalized(google.protobuf.message.Message): + """Lang's answer to FinalizeExternalStreams, for the paths where Core decides the boundary and no + park handshake runs. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + QUIESCENCE_GENERATION_FIELD_NUMBER: builtins.int + FINAL_OBSERVATION_DELTA_FIELD_NUMBER: builtins.int + quiescence_generation: builtins.int + final_observation_delta: builtins.bytes + def __init__( + self, + *, + quiescence_generation: builtins.int = ..., + final_observation_delta: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "final_observation_delta", + b"final_observation_delta", + "quiescence_generation", + b"quiescence_generation", + ], + ) -> None: ... + +global___ExternalStreamFinalized = ExternalStreamFinalized + +class WorkflowOutputStreamCommit(google.protobuf.message.Message): + """Reports that the output batch for this Workflow Task has been durably staged outside History. + Core records only this compact manifest in the shared external-stream marker. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + MANIFEST_FIELD_NUMBER: builtins.int + REQUEST_ROLLOVER_FIELD_NUMBER: builtins.int + @property + def manifest( + self, + ) -> temporalio.bridge.proto.external_data.external_data_pb2.ExternalOutputStreamManifest: ... + request_rollover: builtins.bool + """The deterministic output capacity was reached and this task must request a replacement.""" + def __init__( + self, + *, + manifest: temporalio.bridge.proto.external_data.external_data_pb2.ExternalOutputStreamManifest + | None = ..., + request_rollover: builtins.bool = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["manifest", b"manifest"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "manifest", b"manifest", "request_rollover", b"request_rollover" + ], + ) -> None: ... + +global___WorkflowOutputStreamCommit = WorkflowOutputStreamCommit + +class WorkflowOutputStreamBuffered(google.protobuf.message.Message): + """Reports that external output is buffered in lang but has not yet been staged. Core retains the + open Workflow Task until the earliest reported deadline and then asks lang to finalize it. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + MAX_PUBLISH_LATENCY_FIELD_NUMBER: builtins.int + @property + def max_publish_latency(self) -> google.protobuf.duration_pb2.Duration: ... + def __init__( + self, + *, + max_publish_latency: google.protobuf.duration_pb2.Duration | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "max_publish_latency", b"max_publish_latency" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "max_publish_latency", b"max_publish_latency" + ], + ) -> None: ... + +global___WorkflowOutputStreamBuffered = WorkflowOutputStreamBuffered + +class AppendStreamRecords(google.protobuf.message.Message): + """Append a batch of records to a stream this workflow owns. + + The bodies go to the stream's own log rather than into History, which gets + one fixed-size event naming the offset range. That is what makes the batch + size free: a thousand records cost the same in History as one. The server + stores each record with an empty producer id, because the workflow is the + producer here. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_NAME_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + stream_name: builtins.str + """Name of a stream this workflow owns, scoped to the workflow. Created on + first use. Empty means the workflow's default output stream. A workflow + cannot append to a stream in another execution, so this is never the id + of a standalone stream. + """ + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: ... + def __init__( + self, + *, + stream_name: builtins.str = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "records", b"records", "stream_name", b"stream_name" + ], + ) -> None: ... + +global___AppendStreamRecords = AppendStreamRecords + +class SubscribeStream(google.protobuf.message.Message): + """Subscribe this workflow to a stream, so later Workflow Tasks carry the + ranges it has not consumed yet. + + The stream's addressing is resolved by the server. A workflow cannot look it + up without doing I/O, and a value it carried would be a reading rather than a + fact, so it could differ on replay. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_NAME_OR_ID_FIELD_NUMBER: builtins.int + START_OFFSET_FIELD_NUMBER: builtins.int + START_POSITION_FIELD_NUMBER: builtins.int + stream_name_or_id: builtins.str + """Stream to consume, named either way round: a stream this workflow owns + by the name it appends under, a stream in another execution by its id. + The server tries them in that order, so a workflow that owns a stream + under this name cannot reach a standalone stream with the same id. When + neither exists the workflow gets a stream of its own by that name, which + is how a reader subscribes before the first record is written. + """ + start_offset: builtins.int + """Where to start, as an absolute offset. Read only when start_position is + unset. The server refuses a negative value. + """ + @property + def start_position( + self, + ) -> temporalio.api.stream.v1.message_pb2.StreamStartPosition: + """Where to start: an offset, the earliest record held, the tail, or the + last N records. The server resolves it and records the resulting offset, + so replay does not resolve it again and Core does not keep it. + """ + def __init__( + self, + *, + stream_name_or_id: builtins.str = ..., + start_offset: builtins.int = ..., + start_position: temporalio.api.stream.v1.message_pb2.StreamStartPosition + | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["start_position", b"start_position"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "start_offset", + b"start_offset", + "start_position", + b"start_position", + "stream_name_or_id", + b"stream_name_or_id", + ], + ) -> None: ... + +global___SubscribeStream = SubscribeStream + class StartTimer(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -620,7 +1226,7 @@ class ScheduleLocalActivity(google.protobuf.message.Message): RETRY_POLICY_FIELD_NUMBER: builtins.int LOCAL_RETRY_THRESHOLD_FIELD_NUMBER: builtins.int CANCELLATION_TYPE_FIELD_NUMBER: builtins.int - INCLUDE_ARGUMENTS_INTO_MARKER_FIELD_NUMBER: builtins.int + INCLUDE_ARGUMENTS_IN_MARKER_FIELD_NUMBER: builtins.int seq: builtins.int """Lang's incremental sequence number, used as the operation identifier""" activity_id: builtins.str @@ -685,7 +1291,7 @@ class ScheduleLocalActivity(google.protobuf.message.Message): confirmed. Lang should default this to `WAIT_CANCELLATION_COMPLETED`, even though proto will default to `TRY_CANCEL` automatically. """ - include_arguments_into_marker: builtins.bool + include_arguments_in_marker: builtins.bool """If set, the local activity arguments will be included in the resulting marker under the `input` key. This is disabled by default to avoid increasing history size unless the lang SDK explicitly chooses to expose it. @@ -712,7 +1318,7 @@ class ScheduleLocalActivity(google.protobuf.message.Message): retry_policy: temporalio.api.common.v1.message_pb2.RetryPolicy | None = ..., local_retry_threshold: google.protobuf.duration_pb2.Duration | None = ..., cancellation_type: global___ActivityCancellationType.ValueType = ..., - include_arguments_into_marker: builtins.bool = ..., + include_arguments_in_marker: builtins.bool = ..., ) -> None: ... def HasField( self, @@ -746,8 +1352,8 @@ class ScheduleLocalActivity(google.protobuf.message.Message): b"cancellation_type", "headers", b"headers", - "include_arguments_into_marker", - b"include_arguments_into_marker", + "include_arguments_in_marker", + b"include_arguments_in_marker", "local_retry_threshold", b"local_retry_threshold", "original_schedule_time", diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 85b71d7ecd..08716165cb 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 85b71d7ecd4f2bf677fa1cee17f3fbc1ab10f1b9 +Subproject commit 08716165cb663e914373f0c65b3eba435f496356 diff --git a/temporalio/bridge/services_generated.py b/temporalio/bridge/services_generated.py index c9960fcb5f..b8298fde79 100644 --- a/temporalio/bridge/services_generated.py +++ b/temporalio/bridge/services_generated.py @@ -369,6 +369,24 @@ async def describe_batch_operation( timeout=timeout, ) + async def describe_channel( + self, + req: temporalio.api.workflowservice.v1.DescribeChannelRequest, + retry: bool = False, + metadata: Mapping[str, str | bytes] = {}, + timeout: timedelta | None = None, + ) -> temporalio.api.workflowservice.v1.DescribeChannelResponse: + """Invokes the WorkflowService.describe_channel rpc method.""" + return await self._client._rpc_call( + rpc="describe_channel", + req=req, + service=self._service, + resp_type=temporalio.api.workflowservice.v1.DescribeChannelResponse, + retry=retry, + metadata=metadata, + timeout=timeout, + ) + async def describe_deployment( self, req: temporalio.api.workflowservice.v1.DescribeDeploymentRequest, @@ -1035,6 +1053,24 @@ async def list_workflow_rules( timeout=timeout, ) + async def notify_channel( + self, + req: temporalio.api.workflowservice.v1.NotifyChannelRequest, + retry: bool = False, + metadata: Mapping[str, str | bytes] = {}, + timeout: timedelta | None = None, + ) -> temporalio.api.workflowservice.v1.NotifyChannelResponse: + """Invokes the WorkflowService.notify_channel rpc method.""" + return await self._client._rpc_call( + rpc="notify_channel", + req=req, + service=self._service, + resp_type=temporalio.api.workflowservice.v1.NotifyChannelResponse, + retry=retry, + metadata=metadata, + timeout=timeout, + ) + async def patch_schedule( self, req: temporalio.api.workflowservice.v1.PatchScheduleRequest, @@ -1143,6 +1179,24 @@ async def poll_activity_task_queue( timeout=timeout, ) + async def poll_channel( + self, + req: temporalio.api.workflowservice.v1.PollChannelRequest, + retry: bool = False, + metadata: Mapping[str, str | bytes] = {}, + timeout: timedelta | None = None, + ) -> temporalio.api.workflowservice.v1.PollChannelResponse: + """Invokes the WorkflowService.poll_channel rpc method.""" + return await self._client._rpc_call( + rpc="poll_channel", + req=req, + service=self._service, + resp_type=temporalio.api.workflowservice.v1.PollChannelResponse, + retry=retry, + metadata=metadata, + timeout=timeout, + ) + async def poll_nexus_operation_execution( self, req: temporalio.api.workflowservice.v1.PollNexusOperationExecutionRequest, @@ -1305,6 +1359,24 @@ async def record_worker_heartbeat( timeout=timeout, ) + async def register_channel_listener( + self, + req: temporalio.api.workflowservice.v1.RegisterChannelListenerRequest, + retry: bool = False, + metadata: Mapping[str, str | bytes] = {}, + timeout: timedelta | None = None, + ) -> temporalio.api.workflowservice.v1.RegisterChannelListenerResponse: + """Invokes the WorkflowService.register_channel_listener rpc method.""" + return await self._client._rpc_call( + rpc="register_channel_listener", + req=req, + service=self._service, + resp_type=temporalio.api.workflowservice.v1.RegisterChannelListenerResponse, + retry=retry, + metadata=metadata, + timeout=timeout, + ) + async def register_namespace( self, req: temporalio.api.workflowservice.v1.RegisterNamespaceRequest, @@ -2007,6 +2079,24 @@ async def unpause_workflow_execution( timeout=timeout, ) + async def unregister_channel_listener( + self, + req: temporalio.api.workflowservice.v1.UnregisterChannelListenerRequest, + retry: bool = False, + metadata: Mapping[str, str | bytes] = {}, + timeout: timedelta | None = None, + ) -> temporalio.api.workflowservice.v1.UnregisterChannelListenerResponse: + """Invokes the WorkflowService.unregister_channel_listener rpc method.""" + return await self._client._rpc_call( + rpc="unregister_channel_listener", + req=req, + service=self._service, + resp_type=temporalio.api.workflowservice.v1.UnregisterChannelListenerResponse, + retry=retry, + metadata=metadata, + timeout=timeout, + ) + async def update_activity_execution_options( self, req: temporalio.api.workflowservice.v1.UpdateActivityExecutionOptionsRequest, diff --git a/temporalio/bridge/src/client_rpc_generated.rs b/temporalio/bridge/src/client_rpc_generated.rs index ea00d4fd51..3d3c980183 100644 --- a/temporalio/bridge/src/client_rpc_generated.rs +++ b/temporalio/bridge/src/client_rpc_generated.rs @@ -191,6 +191,15 @@ impl ClientRef { describe_batch_operation ) } + "describe_channel" => { + rpc_call!( + connection, + call, + WorkflowService, + workflow_service, + describe_channel + ) + } "describe_deployment" => { rpc_call!( connection, @@ -524,6 +533,15 @@ impl ClientRef { list_workflow_rules ) } + "notify_channel" => { + rpc_call!( + connection, + call, + WorkflowService, + workflow_service, + notify_channel + ) + } "patch_schedule" => { rpc_call!( connection, @@ -578,6 +596,15 @@ impl ClientRef { poll_activity_task_queue ) } + "poll_channel" => { + rpc_call!( + connection, + call, + WorkflowService, + workflow_service, + poll_channel + ) + } "poll_nexus_operation_execution" => { rpc_call!( connection, @@ -659,6 +686,15 @@ impl ClientRef { record_worker_heartbeat ) } + "register_channel_listener" => { + rpc_call!( + connection, + call, + WorkflowService, + workflow_service, + register_channel_listener + ) + } "register_namespace" => { rpc_call!( connection, @@ -1010,6 +1046,15 @@ impl ClientRef { unpause_workflow_execution ) } + "unregister_channel_listener" => { + rpc_call!( + connection, + call, + WorkflowService, + workflow_service, + unregister_channel_listener + ) + } "update_activity_execution_options" => { rpc_call!( connection, diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index e74698e944..84d8b81996 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -14,12 +14,13 @@ use temporalio_common::protos::coresdk::{ nexus::NexusTaskCompletion, ActivityHeartbeat, ActivityTaskCompletion, }; use temporalio_common::protos::temporal::api::history::v1::History; +use temporalio_common::protos::temporal::api::stream::v1::StreamSlice; use temporalio_common::protos::temporal::api::worker::v1::{PluginInfo, StorageDriverInfo}; use temporalio_sdk_core::replay::{HistoryForReplay, ReplayWorkerInput}; use temporalio_sdk_core::{ - PollError, SlotInfo, SlotInfoTrait, SlotKind, SlotKindType, SlotMarkUsedContext, - SlotReleaseContext, SlotReservationContext, SlotSupplier as SlotSupplierTrait, - SlotSupplierPermit, WorkflowErrorType, + ExternalStreamReadyResult, ExternalStreamRunStatus, PollError, SlotInfo, SlotInfoTrait, + SlotKind, SlotKindType, SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext, + SlotSupplier as SlotSupplierTrait, SlotSupplierPermit, WorkflowErrorType, }; use tokio::sync::mpsc::{channel, Sender}; use tokio_stream::wrappers::ReceiverStream; @@ -671,6 +672,63 @@ impl WorkerRef { Ok(()) } + /// Tell Core a record is buffered for one external stream wait. + /// + /// **Synchronous and blocking**, like `record_activity_heartbeat`, but unlike it + /// **acknowledged**: the caller is a watcher whose next action -- keep waiting, re-probe, send + /// a wake Signal, or tear itself down -- depends entirely on which of the five results comes + /// back, so a fire-and-forget send would leave it unable to tell "Core will activate" from + /// "nothing here will". + /// + /// Synchronous rather than a Python awaitable on purpose. There is one watcher per + /// subscription and they do not share an event loop, and a future produced by + /// `future_into_py` is bound to the loop that created it -- so an awaitable here would work + /// from the worker's own loop and hang from anywhere else. Blocking with the GIL released is + /// safe: the answer comes off Core's serialized local-input lane, which does no I/O. + /// + /// Returns one of `Accepted`, `Stale`, `Parked`, `NoOpenWorkflowTask`, `RunNotFound`. + fn notify_external_stream_ready( + &self, + py: Python<'_>, + run_id: String, + wait_id: u32, + wait_generation: u64, + ) -> PyResult<&'static str> { + let worker = self.worker.as_ref().unwrap().clone(); + // Deliberately *not* `enter_sync!`: entering the tokio handle would put this thread in an + // async context, and `Handle::block_on` panics there. + let handle = self.runtime.core.tokio_handle(); + let result = py.detach(move || { + handle.block_on(worker.notify_external_stream_ready(&run_id, wait_id, wait_generation)) + }); + Ok(match result { + ExternalStreamReadyResult::Accepted => "Accepted", + ExternalStreamReadyResult::Stale => "Stale", + ExternalStreamReadyResult::Parked => "Parked", + ExternalStreamReadyResult::NoOpenWorkflowTask => "NoOpenWorkflowTask", + ExternalStreamReadyResult::RunNotFound => "RunNotFound", + }) + } + + /// Ask what state a run's external stream wait set is in, changing nothing. + /// + /// Deliberately not the readiness call: readiness means "a record is buffered", so probing + /// with it would assert something false and manufacture a spurious activation on the way out + /// of a shutting-down worker. + /// + /// Returns one of `WftOpen`, `Parked`, `NoOpenWorkflowTask`, `RunNotFound`. + fn external_stream_run_status(&self, py: Python<'_>, run_id: String) -> PyResult<&'static str> { + let worker = self.worker.as_ref().unwrap().clone(); + let handle = self.runtime.core.tokio_handle(); + let status = py.detach(move || handle.block_on(worker.external_stream_run_status(&run_id))); + Ok(match status { + ExternalStreamRunStatus::WftOpen => "WftOpen", + ExternalStreamRunStatus::Parked => "Parked", + ExternalStreamRunStatus::NoOpenWorkflowTask => "NoOpenWorkflowTask", + ExternalStreamRunStatus::RunNotFound => "RunNotFound", + }) + } + fn request_workflow_eviction(&self, run_id: &str) -> PyResult<()> { enter_sync!(self.runtime); self.worker @@ -950,14 +1008,24 @@ impl HistoryPusher { #[pymethods] impl HistoryPusher { + /// Feed one history to the replay worker. `stream_slices` are serialized + /// `temporal.api.stream.v1.StreamSlice` messages carrying the records the + /// history's completed tasks consumed, which History itself never holds. + #[pyo3(signature = (workflow_id, history_proto, stream_slices = Vec::new()))] fn push_history<'p>( &self, py: Python<'p>, workflow_id: &str, history_proto: &Bound<'_, PyBytes>, + stream_slices: Vec>, ) -> PyResult> { let history = History::decode(history_proto.as_bytes()) .map_err(|err| PyValueError::new_err(format!("Invalid proto: {err}")))?; + let slices = stream_slices + .iter() + .map(|slice| StreamSlice::decode(slice.as_bytes())) + .collect::, _>>() + .map_err(|err| PyValueError::new_err(format!("Invalid stream slice proto: {err}")))?; let wfid = workflow_id.to_string(); let tx = if let Some(tx) = self.tx.as_ref() { tx.clone() @@ -968,7 +1036,7 @@ impl HistoryPusher { }; // We accept this doesn't have logging/tracing self.runtime.future_into_py(py, async move { - tx.send(HistoryForReplay::new(history, wfid)) + tx.send(HistoryForReplay::new(history, wfid).with_stream_slices(slices)) .await .map_err(|_| { PyRuntimeError::new_err( diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index 4b7f55d092..bff886a4f8 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -5,6 +5,8 @@ from __future__ import annotations +import asyncio +import enum from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass from typing import ( @@ -32,6 +34,62 @@ from temporalio.worker._command_aware_visitor import CommandAwarePayloadVisitor +class ExternalStreamReadyResult(enum.Enum): + """What Core did with an external stream readiness notification. + + Five values, not two, because a watcher does something different with each + and an operator should conclude something different from each. + + ``NO_OPEN_WORKFLOW_TASK`` and ``RUN_NOT_FOUND`` are separate on purpose. A + Run cached between Workflow Tasks is the healthy post-completion state; + reporting it as a missing Run would both corrupt the metric and tell the + watcher to tear itself down while it is still needed. + """ + + ACCEPTED = "Accepted" + """Serialized into an open Workflow Task. Core will activate; do nothing.""" + + STALE = "Stale" + """The wait moved on. Re-probe; do **not** send a Signal.""" + + PARKED = "Parked" + """A confirmed park generation exists. Send the reserved wake Signal.""" + + NO_OPEN_WORKFLOW_TASK = "NoOpenWorkflowTask" + """Cached with waits registered, no task open. Signal, and **keep** the watcher.""" + + RUN_NOT_FOUND = "RunNotFound" + """Absent from this Worker's cache. Signal, then tear the watcher down.""" + + @property + def needs_wake_signal(self) -> bool: + """Whether local readiness could not be delivered and a Signal is owed.""" + return self in ( + ExternalStreamReadyResult.PARKED, + ExternalStreamReadyResult.NO_OPEN_WORKFLOW_TASK, + ExternalStreamReadyResult.RUN_NOT_FOUND, + ) + + +class ExternalStreamRunStatus(enum.Enum): + """What state a Run's external stream wait set is in. + + The answer to a **read-only** probe. Asking changes nothing. + """ + + WFT_OPEN = "WftOpen" + """A Workflow Task is open, retained by the wait set.""" + + PARKED = "Parked" + """The complete wait set is parked.""" + + NO_OPEN_WORKFLOW_TASK = "NoOpenWorkflowTask" + """Cached with active waits, but no Workflow Task open.""" + + RUN_NOT_FOUND = "RunNotFound" + """Absent from this Worker's cache.""" + + @dataclass class WorkerConfig: """Python representation of the Rust struct for configuring a worker.""" @@ -259,6 +317,64 @@ def record_activity_heartbeat( """Record an activity heartbeat.""" self._ref.record_activity_heartbeat(comp.SerializeToString()) # type: ignore[reportOptionalMemberAccess] + def notify_external_stream_ready_sync( + self, run_id: str, wait_id: int, wait_generation: int + ) -> ExternalStreamReadyResult: + """Tell Core a record is **buffered** for one external stream wait. + + Buffered, not merely available: a notification for a record the caller + has not yet read into its buffer would produce an activation whose + drain must block, which is the deadlock hazard the out-of-thread buffer + exists to remove. + + Acknowledged rather than fire-and-forget, because what the caller does + next depends entirely on the answer -- see + :py:class:`ExternalStreamReadyResult`. + + Blocks while Core answers on its serialized local-input lane. Safe from + any thread, and safe to block in -- the lane does no I/O. + + .. warning:: + Do **not** call this from the event loop running the Worker. That + loop is what drives the poll this answer comes from, so blocking it + here waits for something only it can produce. Use + :py:meth:`notify_external_stream_ready` from a loop. + """ + return ExternalStreamReadyResult( + self._ref.notify_external_stream_ready( # type: ignore[reportOptionalMemberAccess] + run_id, wait_id, wait_generation + ) + ) + + async def notify_external_stream_ready( + self, run_id: str, wait_id: int, wait_generation: int + ) -> ExternalStreamReadyResult: + """:py:meth:`notify_external_stream_ready_sync`, off the calling loop. + + There is one watcher per subscription and they do not share a loop, so + this runs the blocking call in a thread rather than binding a future to + whichever loop happened to create it. + """ + return await asyncio.to_thread( + self.notify_external_stream_ready_sync, run_id, wait_id, wait_generation + ) + + def external_stream_run_status_sync(self, run_id: str) -> ExternalStreamRunStatus: + """Ask what state a Run's external stream wait set is in, changing nothing. + + A separate call from :py:meth:`notify_external_stream_ready` on purpose: + readiness means "a record is buffered", so probing with it would assert + something false and manufacture a spurious Workflow Task on the way out + of a shutting-down Worker. + """ + return ExternalStreamRunStatus( + self._ref.external_stream_run_status(run_id) # type: ignore[reportOptionalMemberAccess] + ) + + async def external_stream_run_status(self, run_id: str) -> ExternalStreamRunStatus: + """:py:meth:`external_stream_run_status_sync`, off the calling loop.""" + return await asyncio.to_thread(self.external_stream_run_status_sync, run_id) + def request_workflow_eviction(self, run_id: str) -> None: """Request a workflow be evicted.""" self._ref.request_workflow_eviction(run_id) # type: ignore[reportOptionalMemberAccess] diff --git a/temporalio/client/__init__.py b/temporalio/client/__init__.py index 2eef41a39d..7ed7187be6 100644 --- a/temporalio/client/__init__.py +++ b/temporalio/client/__init__.py @@ -64,6 +64,14 @@ from ._callback import ( Callback, ) +from ._channel import ( + ChannelAddress, + ChannelDescription, + ChannelKind, + ChannelListener, + ChannelSubscriptionInfo, + stream_channel, +) from ._client import ( Client, ClientConfig, @@ -106,6 +114,7 @@ CreateScheduleInput, DeleteScheduleInput, DescribeActivityInput, + DescribeChannelInput, DescribeNexusOperationInput, DescribeScheduleInput, DescribeWorkflowInput, @@ -120,10 +129,13 @@ ListNexusOperationsInput, ListSchedulesInput, ListWorkflowsInput, + NotifyChannelInput, OutboundInterceptor, PauseActivityInput, PauseScheduleInput, + PollChannelInput, QueryWorkflowInput, + RegisterChannelListenerInput, ReportCancellationAsyncActivityInput, SignalWorkflowInput, StartActivityInput, @@ -137,6 +149,7 @@ TriggerScheduleInput, UnpauseActivityInput, UnpauseScheduleInput, + UnregisterChannelListenerInput, UpdateActivityOptionsInput, UpdateScheduleInput, UpdateWithStartStartWorkflowInput, @@ -315,6 +328,11 @@ "TerminateNexusOperationInput", "ListNexusOperationsInput", "CountNexusOperationsInput", + "DescribeChannelInput", + "NotifyChannelInput", + "PollChannelInput", + "RegisterChannelListenerInput", + "UnregisterChannelListenerInput", "StartWorkflowUpdateInput", "UpdateWithStartUpdateWorkflowInput", "UpdateWithStartStartWorkflowInput", @@ -351,6 +369,12 @@ "CloudOperationsClient", "Plugin", "Callback", + "ChannelDescription", + "ChannelKind", + "ChannelListener", + "ChannelAddress", + "ChannelSubscriptionInfo", + "stream_channel", "_ClientImpl", "_apply_headers", "_decode_user_metadata", diff --git a/temporalio/client/_channel.py b/temporalio/client/_channel.py new file mode 100644 index 0000000000..3bdfa3ef22 --- /dev/null +++ b/temporalio/client/_channel.py @@ -0,0 +1,272 @@ +"""Notification channel descriptions as the client reports them.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import IntEnum + +import temporalio.api.notification.v1 +import temporalio.api.workflow.v1 +import temporalio.common +from temporalio.streams._ref import StreamRef +from temporalio.workflow import Notification + +from ._callback import Callback + +__all__ = [ + "ChannelAddress", + "ChannelDescription", + "ChannelKind", + "ChannelListener", + "ChannelSubscriptionInfo", + "stream_channel", +] + +STREAM_CHANNEL_PREFIX = "stream/" +"""The first segment of the channel a native stream notifies.""" + + +class ChannelKind(IntEnum): + """Where a channel lives, which decides how a call addresses it. + + .. warning:: + This API is experimental and unstable. + """ + + UNSPECIFIED = int( + temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_UNSPECIFIED + ) + """The server did not say; an older server answers this.""" + + INDEPENDENT = int( + temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_INDEPENDENT + ) + """Its own execution, keyed by namespace and channel name. + + Any number of workflows subscribe to it and callbacks register on it. + """ + + LINKED = int(temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_LINKED) + """Kept in one execution's state, keyed by namespace, execution and name. + + The owning execution, a workflow or a standalone activity, is its listener + by construction. A call reaches it with the ``execution`` argument, or + with ``workflow_id`` when the owner is a workflow. + """ + + +@dataclass(frozen=True) +class ChannelListener: + """One listener of a channel: a workflow or a callback. + + .. warning:: + This API is experimental and unstable. + """ + + listener_id: str + """Assigned by the server when the listener registered.""" + + workflow_id: str | None + """The subscribed workflow, when the listener is one.""" + + run_id: str | None + """The run that subscribed. Delivery follows the chain's current run.""" + + callback: Callback | None + """The callback the server invokes, when the listener is one.""" + + registered_time: datetime | None + """When the listener registered.""" + + @staticmethod + def _from_proto( + proto: temporalio.api.notification.v1.ChannelListener, + ) -> ChannelListener: + callback: Callback | None = None + if proto.HasField("callback") and proto.callback.HasField("nexus"): + callback = Callback( + url=proto.callback.nexus.url, headers=dict(proto.callback.nexus.header) + ) + workflow = proto.workflow if proto.HasField("workflow") else None + return ChannelListener( + listener_id=proto.listener_id, + workflow_id=workflow.workflow_id if workflow else None, + run_id=workflow.run_id if workflow else None, + callback=callback, + registered_time=( + proto.registered_time.ToDatetime(tzinfo=timezone.utc) + if proto.HasField("registered_time") + else None + ), + ) + + +@dataclass(frozen=True) +class ChannelDescription: + """What the server knows about a channel. + + .. warning:: + This API is experimental and unstable. + """ + + listeners: Sequence[ChannelListener] + """Who is listening, workflows and callbacks alike.""" + + latest: Notification | None + """The notification with the highest counter the channel retains.""" + + retained_count: int + """How many notifications the channel keeps for pollers.""" + + kind: ChannelKind = ChannelKind.UNSPECIFIED + """Which kind of channel this is. + + A linked channel of a running execution exists by construction, so a + describe with ``execution`` answers :attr:`ChannelKind.LINKED` with no + listeners and nothing retained for a name nobody has notified yet. + """ + + linked_to: temporalio.common.Execution | None = None + """The owner of a linked channel and the run that holds it. + + ``None`` for an independent channel. + """ + + +@dataclass(frozen=True) +class ChannelSubscriptionInfo: + """A workflow's standing on one channel, as its description reports it. + + An independent channel is listed from the subscribe event until the run + unsubscribes or closes. A linked channel is listed once it holds state. A + closed run keeps listing what it stood on, and a continue-as-new + successor starts with nothing. + + .. warning:: + This API is experimental and unstable. + """ + + channel: str + """Channel name.""" + + kind: ChannelKind + """:attr:`ChannelKind.INDEPENDENT` for a subscription the workflow made by + command, :attr:`ChannelKind.LINKED` for a channel linked to it.""" + + subscribed_event_id: int + """Id of the event that recorded the subscription. Zero for the linked kind.""" + + last_counter: int + """Highest counter the workflow has accepted from the channel. Zero when + none has arrived.""" + + pending_notification: Notification | None + """The notification held for the workflow's next Workflow Task, when one + is pending.""" + + scheduled_counter: int + """Counter carried by the scheduled event of a Workflow Task that has not + started yet. Zero otherwise.""" + + listener_count: int + """Linked kind: callback listeners registered on the channel.""" + + retained_count: int + """Linked kind: notifications retained for pollers.""" + + accepted_count: int + """Linked kind: notifications the channel has accepted over its life.""" + + @staticmethod + def _from_proto( + proto: temporalio.api.workflow.v1.ChannelSubscriptionInfo, + ) -> ChannelSubscriptionInfo: + return ChannelSubscriptionInfo( + channel=proto.channel, + kind=ChannelKind(proto.kind), + subscribed_event_id=proto.subscribed_event_id, + last_counter=proto.last_counter, + pending_notification=( + Notification._from_proto(proto.pending_notification) + if proto.HasField("pending_notification") + else None + ), + scheduled_counter=proto.scheduled_counter, + listener_count=proto.listener_count, + retained_count=proto.retained_count, + accepted_count=proto.accepted_count, + ) + + +@dataclass(frozen=True) +class ChannelAddress: + """Where a channel call reaches a channel: its name and, when linked, its owner. + + .. warning:: + This API is experimental and unstable. + """ + + channel: str + """The channel name.""" + + execution: temporalio.common.Execution | None + """The execution the channel is linked to, or ``None`` for an independent one. + + Pass both to :py:meth:`temporalio.client.Client.poll_channel` and the + other channel calls as ``channel`` and ``execution``. + """ + + @property + def workflow_id(self) -> str | None: + """The owning workflow's id, when the owner is a workflow. + + ``None`` for an independent channel and for one a standalone activity + owns, which only ``execution`` reaches. + """ + if ( + self.execution is not None + and self.execution.type == temporalio.common.ExecutionType.WORKFLOW + ): + return self.execution.business_id + return None + + +def stream_channel(ref: StreamRef) -> ChannelAddress: + """The channel a native stream notifies on every append and on its close. + + The server derives the name from the stream's identity, and this helper + derives the same one, so a client polls or registers a callback without + asking. A stream a workflow owns notifies ``stream/`` linked to the + owning workflow. A stream an activity owns notifies + ``stream//`` linked to the workflow that scheduled + the activity, or ``stream/`` linked to the activity execution + itself when the activity is a standalone one. A standalone stream notifies + the independent channel ``stream/``, whatever the topic, since + its topics share one stream on the server. The address names the owner + without a run, so it reaches the owner's current run. + + Each change arrives as one notification: the stream's change sequence as + the counter, the head after the change as the position, and ``closed`` + set in the metadata on the close. + """ + if ref.kind == "workflow": + assert ref.workflow_id is not None + return ChannelAddress( + STREAM_CHANNEL_PREFIX + ref.topic, + temporalio.common.Execution.workflow(ref.workflow_id), + ) + if ref.kind == "activity": + assert ref.activity_id is not None + if not ref.workflow_id: + return ChannelAddress( + STREAM_CHANNEL_PREFIX + ref.topic, + temporalio.common.Execution.activity(ref.activity_id), + ) + return ChannelAddress( + f"{STREAM_CHANNEL_PREFIX}{ref.activity_id}/{ref.topic}", + temporalio.common.Execution.workflow(ref.workflow_id), + ) + assert ref.stream_id is not None + return ChannelAddress(STREAM_CHANNEL_PREFIX + ref.stream_id, None) diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index b5db0c4b7d..5330bf7afe 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -27,6 +27,7 @@ import temporalio.converter import temporalio.runtime import temporalio.service +import temporalio.streams import temporalio.workflow from temporalio.service import ( ConnectConfig, @@ -39,6 +40,7 @@ ServiceClient, TLSConfig, ) +from temporalio.streams._ref import open_ref from ..common import HeaderCodecBehavior from ..types import ( @@ -62,22 +64,29 @@ AsyncActivityHandle, AsyncActivityIDReference, ) +from ._callback import Callback +from ._channel import ChannelDescription from ._impl import _ClientImpl from ._interceptor import ( CountActivitiesInput, CountNexusOperationsInput, CountWorkflowsInput, CreateScheduleInput, + DescribeChannelInput, GetWorkerBuildIdCompatibilityInput, GetWorkerTaskReachabilityInput, ListActivitiesInput, ListNexusOperationsInput, ListSchedulesInput, ListWorkflowsInput, + NotifyChannelInput, OutboundInterceptor, + PollChannelInput, + RegisterChannelListenerInput, StartActivityInput, StartWorkflowInput, StartWorkflowUpdateWithStartInput, + UnregisterChannelListenerInput, UpdateWithStartUpdateWorkflowInput, UpdateWorkerBuildIdCompatibilityInput, ) @@ -113,6 +122,9 @@ from ._interceptor import Interceptor from ._plugin import Plugin +DEFAULT_CHANNEL_POLL_WAIT = timedelta(seconds=30) +"""How long :py:meth:`Client.poll_channel` waits for a notification by default.""" + class Client: """Client for accessing Temporal. @@ -155,6 +167,7 @@ async def connect( grpc_compression: GrpcCompression = GrpcCompression.GZIP, payload_limits: PayloadLimitsConfig = PayloadLimitsConfig(), header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + stream_provider: temporalio.streams.StreamProvider | None = None, ) -> Self: """Connect to a Temporal server. @@ -220,6 +233,11 @@ async def connect( payload_limits: Warning thresholds for outbound payload/memo sizes. Over-threshold fields are logged but still sent. Set a threshold to 0 to disable it. header_codec_behavior: Encoding behavior for headers sent by the client. + stream_provider: Experimental. The stream provider + :py:meth:`get_stream_handle` opens handles from, see + :py:mod:`temporalio.streams`. A provider that is also a + :py:class:`Plugin` sets this itself when passed in ``plugins``, + and workers built from this client inherit it. """ connect_config = temporalio.service.ConnectConfig( target_host=target_host, @@ -256,6 +274,7 @@ def make_lambda( default_workflow_query_reject_condition=default_workflow_query_reject_condition, header_codec_behavior=header_codec_behavior, plugins=plugins, + stream_provider=stream_provider, ) def __init__( @@ -269,6 +288,7 @@ def __init__( default_workflow_query_reject_condition: None | (temporalio.common.QueryRejectCondition) = None, header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + stream_provider: temporalio.streams.StreamProvider | None = None, ): """Create a Temporal client from a service client. @@ -283,6 +303,7 @@ def __init__( interceptors=interceptors, default_workflow_query_reject_condition=default_workflow_query_reject_condition, header_codec_behavior=header_codec_behavior, + stream_provider=stream_provider, ) self._initial_config = config.copy() @@ -886,6 +907,139 @@ def get_workflow_handle( result_type=result_type, ) + def get_stream_handle( + self, + workflow_id: str | temporalio.streams.StreamRef | None = None, + *, + run_id: str | None = None, + activity_id: str | None = None, + stream_id: str | None = None, + ) -> temporalio.streams.StreamHandle: + """Get a handle on a stream from the provider registered on this client. + + Mirrors :py:meth:`get_workflow_handle`: without ``run_id`` the handle + follows the workflow's execution chain across continue-as-new, with + one it is pinned to that run. With ``activity_id`` the handle is on + the streams that activity owns: a standalone activity's when + ``workflow_id`` is left out, and ``run_id`` then pins the activity's + run, or an activity that ``workflow_id`` scheduled. With + ``stream_id`` it is on a standalone stream, one with an id of its own + and no owner, which :py:meth:`create_stream` made; it takes no other + argument. A :py:class:`temporalio.streams.StreamRef` in place of + ``workflow_id`` opens the stream the ref names, whatever owns it, and + takes no other argument either: the handle's calls that name no topic + then address the ref's topic. The provider is the one registered + with ``plugins=[provider]`` at :py:meth:`connect`, or passed as + ``stream_provider``. The handle's ``read``, ``latest`` and + ``producer`` take a topic, and without one address the owner's + default topic, :py:data:`temporalio.streams.DEFAULT_TOPIC`. A + ``read`` starts at :py:data:`temporalio.streams.BEGINNING`, at + :py:data:`temporalio.streams.END` or at the last ``N`` records with + ``last=N``, and resumes only after a cursor it was handed. See + :py:mod:`temporalio.streams`. + + Args: + workflow_id: Workflow ID whose stream to get a handle to, the + workflow that scheduled ``activity_id``, or a + :py:class:`temporalio.streams.StreamRef` naming the stream. + run_id: Run ID to pin the handle to. + activity_id: Activity ID whose own streams to get a handle to. + stream_id: ID of the standalone stream to get a handle to. + + Returns: + The stream handle. + + Raises: + ValueError: No owner was named, or a ref or ``stream_id`` was + given together with another argument. + temporalio.streams.StreamUnsupportedError: No stream provider is + registered on this client, or it cannot hold a stream an + activity owns or a stream without an owner. + """ + provider = self._config.get("stream_provider") + if provider is None: + raise temporalio.streams.StreamUnsupportedError( + "no stream provider is registered on this client; connect with " + "plugins=[provider]" + ) + if isinstance(workflow_id, temporalio.streams.StreamRef): + if run_id is not None or activity_id is not None or stream_id is not None: + raise ValueError( + "a StreamRef names the stream in full, so it takes no run_id, " + "activity_id or stream_id" + ) + return open_ref(provider, self, workflow_id) + if stream_id is not None: + if workflow_id is not None or run_id is not None or activity_id is not None: + raise ValueError( + "stream_id names a standalone stream, which has no workflow_id, " + "run_id or activity_id" + ) + return provider.get_standalone_stream_handle(self, stream_id) + if activity_id is not None: + return provider.get_activity_stream_handle( + self, activity_id, workflow_id=workflow_id, run_id=run_id + ) + if workflow_id is None: + raise ValueError( + "name the workflow_id, the activity_id or the stream_id to address" + ) + return provider.get_stream_handle(self, workflow_id, run_id=run_id) + + async def create_stream( + self, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> temporalio.streams.StreamHandle: + """Create a standalone stream and get a handle on it. + + A standalone stream has an id of its own and no owner, so it is + created here on purpose rather than by its first write, and it is + sealed on purpose with the handle's ``close()``, after which appends + are refused and the retained records stay readable. The three policy + arguments bound what it retains: records older than ``retention``, + beyond the newest ``max_records`` or past ``max_bytes`` of stored + records are dropped, and ``None`` leaves a bound to the provider's + default. Creating a stream that exists with the same policy returns a + handle on it, so a retried create is harmless. Another process + reaches the stream with ``get_stream_handle(stream_id=...)`` or with + the handle's ``ref()``. + + Args: + stream_id: ID of the stream to create. + retention: How long a record is kept. + max_records: How many of the newest records are kept. + max_bytes: How many bytes of records are kept. + + Returns: + A handle on the new or existing stream. + + Raises: + ValueError: ``stream_id`` is empty, a bound is not positive, or + the stream exists with a different policy. + temporalio.streams.StreamUnsupportedError: No stream provider is + registered on this client, or it cannot hold a stream without + an owner. + """ + provider = self._config.get("stream_provider") + if provider is None: + raise temporalio.streams.StreamUnsupportedError( + "no stream provider is registered on this client; connect with " + "plugins=[provider]" + ) + if not stream_id: + raise ValueError("stream_id must not be empty") + return await provider.create_standalone_stream( + self, + stream_id, + retention=retention, + max_records=max_records, + max_bytes=max_bytes, + ) + def get_workflow_handle_for( self, workflow: ( @@ -2842,6 +2996,270 @@ async def get_worker_task_reachability( ) ) + async def notify_channel( + self, + channel: str, + *, + position: bytes = b"", + counter: int = 0, + metadata: Mapping[str, Any] | None = None, + execution: temporalio.common.Execution | None = None, + workflow_id: str | None = None, + run_id: str | None = None, + rpc_metadata: Mapping[str, str | bytes] = {}, + rpc_timeout: timedelta | None = None, + ) -> int: + """Notify the listeners of ``channel`` that a source they consume has moved. + + A workflow listening with :py:func:`temporalio.workflow.subscribe_channel` + or :py:func:`temporalio.workflow.linked_channel` runs a Workflow Task + that carries the notification. The server folds notifications per + listener while one is pending, keeping the one with the highest + ``counter``, so a burst of writes costs a listener one task. + + .. warning:: + This API is experimental and unstable. + + Args: + channel: Name of the channel. Scoped to the namespace, or to the + execution when one is given. + position: Where the source stands after the write, in the writer's + own terms. Opaque to the server. + counter: Orders notifications from this channel's writers. Derive it + from ``position``, since only the source can order its positions. + metadata: Details for the listener, such as which topic moved. Each + value is encoded with the client's data converter. + execution: Address the channel linked to this execution, a workflow + or a standalone activity, instead of the independent channel of + that name. Without a run id the call reaches the current run of + a workflow chain, as a Signal does. + workflow_id: Shorthand for ``execution`` naming a workflow. Not + with ``execution``. + run_id: With ``workflow_id``, the run of its chain to address. + rpc_metadata: Headers used on the RPC call. Keys here override + client-level RPC metadata keys. + rpc_timeout: Optional RPC deadline to set for the RPC call. + + Returns: + How many listeners the channel had when the notification arrived. + + Raises: + ValueError: Both ``execution`` and ``workflow_id`` were given, or + ``run_id`` without ``workflow_id``. + """ + return await self._impl.notify_channel( + NotifyChannelInput( + channel=channel, + position=position, + counter=counter, + metadata=metadata, + rpc_metadata=rpc_metadata, + rpc_timeout=rpc_timeout, + execution=_channel_execution(execution, workflow_id, run_id), + ) + ) + + async def poll_channel( + self, + channel: str, + *, + after_counter: int = 0, + wait: bool | timedelta = True, + max_notifications: int = 100, + execution: temporalio.common.Execution | None = None, + workflow_id: str | None = None, + run_id: str | None = None, + rpc_metadata: Mapping[str, str | bytes] = {}, + rpc_timeout: timedelta | None = None, + ) -> list[temporalio.workflow.Notification]: + """Read the notifications ``channel`` retains above a counter. + + .. warning:: + This API is experimental and unstable. + + Args: + channel: Name of the channel. Scoped to the namespace, or to the + execution when one is given. + after_counter: Only notifications with a counter above this one are + returned. Pass the highest counter seen so far to page. + wait: How long the server holds the call when nothing is retained + above ``after_counter``. ``True`` waits up to + :py:data:`DEFAULT_CHANNEL_POLL_WAIT`, ``False`` returns at once. + max_notifications: Upper bound on the notifications returned. + execution: Address the channel linked to this execution, a workflow + or a standalone activity, instead of the independent channel of + that name. Without a run id the call reaches the current run of + a workflow chain. + workflow_id: Shorthand for ``execution`` naming a workflow. Not + with ``execution``. + run_id: With ``workflow_id``, the run of its chain to address. + rpc_metadata: Headers used on the RPC call. Keys here override + client-level RPC metadata keys. + rpc_timeout: Optional RPC deadline to set for the RPC call. + + Returns: + The notifications, oldest first. Empty when the wait ran out. + + Raises: + ValueError: Both ``execution`` and ``workflow_id`` were given, or + ``run_id`` without ``workflow_id``. + """ + if wait is True: + wait_for: timedelta | None = DEFAULT_CHANNEL_POLL_WAIT + elif wait is False: + wait_for = None + else: + wait_for = wait + return await self._impl.poll_channel( + PollChannelInput( + channel=channel, + after_counter=after_counter, + wait=wait_for, + max_notifications=max_notifications, + rpc_metadata=rpc_metadata, + rpc_timeout=rpc_timeout, + execution=_channel_execution(execution, workflow_id, run_id), + ) + ) + + async def describe_channel( + self, + channel: str, + *, + execution: temporalio.common.Execution | None = None, + workflow_id: str | None = None, + run_id: str | None = None, + rpc_metadata: Mapping[str, str | bytes] = {}, + rpc_timeout: timedelta | None = None, + ) -> ChannelDescription: + """Describe ``channel``: its kind, its listeners and what it retains. + + .. warning:: + This API is experimental and unstable. + + Args: + channel: Name of the channel. Scoped to the namespace, or to the + execution when one is given. + execution: Describe the channel linked to this execution, a + workflow or a standalone activity, instead of the independent + channel of that name. A linked channel of a running execution + exists by construction, so the answer for a name nobody has + notified yet is a linked channel with no listeners and nothing + retained, not a not-found error. + workflow_id: Shorthand for ``execution`` naming a workflow. Not + with ``execution``. + run_id: With ``workflow_id``, the run of its chain to address. + rpc_metadata: Headers used on the RPC call. Keys here override + client-level RPC metadata keys. + rpc_timeout: Optional RPC deadline to set for the RPC call. + + Raises: + ValueError: Both ``execution`` and ``workflow_id`` were given, or + ``run_id`` without ``workflow_id``. + """ + return await self._impl.describe_channel( + DescribeChannelInput( + channel=channel, + rpc_metadata=rpc_metadata, + rpc_timeout=rpc_timeout, + execution=_channel_execution(execution, workflow_id, run_id), + ) + ) + + async def register_channel_listener( + self, + channel: str, + callback: Callback, + *, + execution: temporalio.common.Execution | None = None, + workflow_id: str | None = None, + run_id: str | None = None, + rpc_metadata: Mapping[str, str | bytes] = {}, + rpc_timeout: timedelta | None = None, + ) -> str: + """Register ``callback`` as a listener of ``channel``. + + The server invokes the callback with each notification on the channel + until :py:meth:`unregister_channel_listener` removes it. + + .. warning:: + This API is experimental and unstable. + + Args: + channel: Name of the channel. Scoped to the namespace, or to the + execution when one is given. + callback: The callback to invoke. + execution: Listen on the channel linked to this execution, a + workflow or a standalone activity, instead of the independent + channel of that name. The listener lives in that execution's + state and ends with its run. + workflow_id: Shorthand for ``execution`` naming a workflow. Not + with ``execution``. + run_id: With ``workflow_id``, the run of its chain to address. + rpc_metadata: Headers used on the RPC call. Keys here override + client-level RPC metadata keys. + rpc_timeout: Optional RPC deadline to set for the RPC call. + + Returns: + The listener id the server assigned. + + Raises: + ValueError: Both ``execution`` and ``workflow_id`` were given, or + ``run_id`` without ``workflow_id``. + """ + return await self._impl.register_channel_listener( + RegisterChannelListenerInput( + channel=channel, + callback=callback, + rpc_metadata=rpc_metadata, + rpc_timeout=rpc_timeout, + execution=_channel_execution(execution, workflow_id, run_id), + ) + ) + + async def unregister_channel_listener( + self, + channel: str, + listener_id: str, + *, + execution: temporalio.common.Execution | None = None, + workflow_id: str | None = None, + run_id: str | None = None, + rpc_metadata: Mapping[str, str | bytes] = {}, + rpc_timeout: timedelta | None = None, + ) -> None: + """Remove a listener from ``channel``. + + .. warning:: + This API is experimental and unstable. + + Args: + channel: Name of the channel. Scoped to the namespace, or to the + execution when one is given. + listener_id: The id :py:meth:`register_channel_listener` returned. + execution: The execution whose linked channel the listener is on, + when it was registered with one. + workflow_id: Shorthand for ``execution`` naming a workflow. Not + with ``execution``. + run_id: With ``workflow_id``, the run of its chain to address. + rpc_metadata: Headers used on the RPC call. Keys here override + client-level RPC metadata keys. + rpc_timeout: Optional RPC deadline to set for the RPC call. + + Raises: + ValueError: Both ``execution`` and ``workflow_id`` were given, or + ``run_id`` without ``workflow_id``. + """ + await self._impl.unregister_channel_listener( + UnregisterChannelListenerInput( + channel=channel, + listener_id=listener_id, + rpc_metadata=rpc_metadata, + rpc_timeout=rpc_timeout, + execution=_channel_execution(execution, workflow_id, run_id), + ) + ) + def create_nexus_client( self, service: type[NexusServiceType] | str, @@ -3018,6 +3436,7 @@ class ClientConnectConfig(TypedDict, total=False): grpc_compression: GrpcCompression payload_limits: PayloadLimitsConfig header_codec_behavior: HeaderCodecBehavior + stream_provider: temporalio.streams.StreamProvider | None class ClientConfig(TypedDict, total=False): @@ -3032,3 +3451,25 @@ class ClientConfig(TypedDict, total=False): temporalio.common.QueryRejectCondition | None ] header_codec_behavior: Required[HeaderCodecBehavior] + stream_provider: temporalio.streams.StreamProvider | None + + +def _channel_execution( + execution: temporalio.common.Execution | None, + workflow_id: str | None, + run_id: str | None, +) -> temporalio.common.Execution | None: + """The owner a channel call names, by ``execution`` or by the workflow id shorthand. + + The two spellings do not combine, and a run id needs its workflow id. + ``None`` names the independent channel. + """ + if execution is not None: + if workflow_id is not None or run_id is not None: + raise ValueError("pass execution or workflow_id, not both") + return execution + if workflow_id is None: + if run_id is not None: + raise ValueError("run_id needs workflow_id") + return None + return temporalio.common.Execution.workflow(workflow_id, run_id) diff --git a/temporalio/client/_impl.py b/temporalio/client/_impl.py index abe3f8c091..4be55aa90f 100644 --- a/temporalio/client/_impl.py +++ b/temporalio/client/_impl.py @@ -23,6 +23,7 @@ import temporalio.api.enums.v1 import temporalio.api.errordetails.v1 import temporalio.api.failure.v1 +import temporalio.api.notification.v1 import temporalio.api.schedule.v1 import temporalio.api.taskqueue.v1 import temporalio.api.update.v1 @@ -32,6 +33,7 @@ import temporalio.exceptions import temporalio.nexus import temporalio.nexus._operation_context +import temporalio.workflow from temporalio.activity import ActivityCancellationDetails from temporalio.converter import ( ActivitySerializationContext, @@ -54,6 +56,7 @@ ActivityHandle, AsyncActivityIDReference, ) +from ._channel import ChannelDescription, ChannelKind, ChannelListener from ._exceptions import ( AsyncActivityCancelledError, ScheduleAlreadyRunningError, @@ -74,6 +77,7 @@ CreateScheduleInput, DeleteScheduleInput, DescribeActivityInput, + DescribeChannelInput, DescribeNexusOperationInput, DescribeScheduleInput, DescribeWorkflowInput, @@ -87,10 +91,13 @@ ListNexusOperationsInput, ListSchedulesInput, ListWorkflowsInput, + NotifyChannelInput, OutboundInterceptor, PauseActivityInput, PauseScheduleInput, + PollChannelInput, QueryWorkflowInput, + RegisterChannelListenerInput, ReportCancellationAsyncActivityInput, SignalWorkflowInput, StartActivityInput, @@ -104,6 +111,7 @@ TriggerScheduleInput, UnpauseActivityInput, UnpauseScheduleInput, + UnregisterChannelListenerInput, UpdateActivityOptionsInput, UpdateScheduleInput, UpdateWithStartStartWorkflowInput, @@ -1764,6 +1772,137 @@ async def count_nexus_operations( ) ) + ### Notification channel calls + + async def notify_channel(self, input: NotifyChannelInput) -> int: + notification = temporalio.api.notification.v1.Notification( + channel=input.channel, position=input.position, counter=input.counter + ) + for key, value in (input.metadata or {}).items(): + [payload] = await self._client.data_converter.encode([value]) + notification.metadata[key].CopyFrom(payload) + resp = await self._client.workflow_service.notify_channel( + temporalio.api.workflowservice.v1.NotifyChannelRequest( + namespace=self._client.namespace, + notification=notification, + identity=self._client.identity, + request_id=str(uuid.uuid4()), + execution=_channel_owner(input.execution), + ), + retry=True, + metadata=input.rpc_metadata, + timeout=input.rpc_timeout, + ) + return resp.listener_count + + async def poll_channel( + self, input: PollChannelInput + ) -> list[temporalio.workflow.Notification]: + req = temporalio.api.workflowservice.v1.PollChannelRequest( + namespace=self._client.namespace, + channel=input.channel, + after_counter=input.after_counter, + max_notifications=input.max_notifications, + execution=_channel_owner(input.execution), + ) + if input.wait is not None: + req.wait.FromTimedelta(input.wait) + resp = await self._client.workflow_service.poll_channel( + req, retry=True, metadata=input.rpc_metadata, timeout=input.rpc_timeout + ) + return [await self._notification_from_proto(n) for n in resp.notifications] + + async def describe_channel(self, input: DescribeChannelInput) -> ChannelDescription: + resp = await self._client.workflow_service.describe_channel( + temporalio.api.workflowservice.v1.DescribeChannelRequest( + namespace=self._client.namespace, + channel=input.channel, + execution=_channel_owner(input.execution), + ), + retry=True, + metadata=input.rpc_metadata, + timeout=input.rpc_timeout, + ) + return ChannelDescription( + listeners=[ + ChannelListener._from_proto(listener) for listener in resp.listeners + ], + latest=( + await self._notification_from_proto(resp.latest) + if resp.HasField("latest") + else None + ), + retained_count=resp.retained_count, + kind=ChannelKind(resp.kind), + linked_to=( + temporalio.common.Execution.from_proto(resp.linked_to) + if resp.HasField("linked_to") + else None + ), + ) + + async def register_channel_listener( + self, input: RegisterChannelListenerInput + ) -> str: + resp = await self._client.workflow_service.register_channel_listener( + temporalio.api.workflowservice.v1.RegisterChannelListenerRequest( + namespace=self._client.namespace, + channel=input.channel, + callback=temporalio.api.common.v1.Callback( + nexus=temporalio.api.common.v1.Callback.Nexus( + url=input.callback.url, header=input.callback.headers + ) + ), + request_id=str(uuid.uuid4()), + identity=self._client.identity, + execution=_channel_owner(input.execution), + ), + retry=True, + metadata=input.rpc_metadata, + timeout=input.rpc_timeout, + ) + return resp.listener_id + + async def unregister_channel_listener( + self, input: UnregisterChannelListenerInput + ) -> None: + await self._client.workflow_service.unregister_channel_listener( + temporalio.api.workflowservice.v1.UnregisterChannelListenerRequest( + namespace=self._client.namespace, + channel=input.channel, + listener_id=input.listener_id, + identity=self._client.identity, + execution=_channel_owner(input.execution), + ), + retry=True, + metadata=input.rpc_metadata, + timeout=input.rpc_timeout, + ) + + async def _notification_from_proto( + self, proto: temporalio.api.notification.v1.Notification + ) -> temporalio.workflow.Notification: + # The worker runs the codec over a workflow's notifications before + # they reach workflow code. The client does the same here, so both + # sides hand out payloads a converter can read. + metadata = dict(proto.metadata.items()) + codec = self._client.data_converter.payload_codec + if codec and metadata: + keys = list(metadata) + decoded = await codec.decode([metadata[k] for k in keys]) + metadata = dict(zip(keys, decoded)) + return temporalio.workflow.Notification( + channel=proto.channel, + position=proto.position, + counter=proto.counter, + metadata=metadata, + linked_to=( + temporalio.common.Execution.from_proto(proto.linked_to) + if proto.HasField("linked_to") + else None + ), + ) + async def _apply_headers( self, source: Mapping[str, temporalio.api.common.v1.Payload] | None, @@ -1776,3 +1915,10 @@ async def _apply_headers( == HeaderCodecBehavior.CODEC, self._client.data_converter, ) + + +def _channel_owner( + execution: temporalio.common.Execution | None, +) -> temporalio.api.common.v1.Execution | None: + """The owner on the wire, or ``None`` for an independent channel.""" + return None if execution is None else execution.to_proto() diff --git a/temporalio/client/_interceptor.py b/temporalio/client/_interceptor.py index 68077ebc93..c35a1281fb 100644 --- a/temporalio/client/_interceptor.py +++ b/temporalio/client/_interceptor.py @@ -21,6 +21,8 @@ from temporalio.converter import DataConverter if TYPE_CHECKING: + import temporalio.workflow + from ._activity import ( ActivityExecutionAsyncIterator, ActivityExecutionCount, @@ -30,6 +32,8 @@ ActivityOptionsUpdate, AsyncActivityIDReference, ) + from ._callback import Callback + from ._channel import ChannelDescription from ._nexus import ( NexusOperationExecutionAsyncIterator, NexusOperationExecutionCount, @@ -698,6 +702,84 @@ class CountNexusOperationsInput: rpc_timeout: timedelta | None +@dataclass +class NotifyChannelInput: + """Input for :py:meth:`OutboundInterceptor.notify_channel`. + + .. warning:: + This API is experimental and unstable. + """ + + channel: str + position: bytes + counter: int + metadata: Mapping[str, Any] | None + rpc_metadata: Mapping[str, str | bytes] + rpc_timeout: timedelta | None + execution: temporalio.common.Execution | None = None + + +@dataclass +class PollChannelInput: + """Input for :py:meth:`OutboundInterceptor.poll_channel`. + + .. warning:: + This API is experimental and unstable. + """ + + channel: str + after_counter: int + wait: timedelta | None + max_notifications: int + rpc_metadata: Mapping[str, str | bytes] + rpc_timeout: timedelta | None + execution: temporalio.common.Execution | None = None + + +@dataclass +class DescribeChannelInput: + """Input for :py:meth:`OutboundInterceptor.describe_channel`. + + .. warning:: + This API is experimental and unstable. + """ + + channel: str + rpc_metadata: Mapping[str, str | bytes] + rpc_timeout: timedelta | None + execution: temporalio.common.Execution | None = None + + +@dataclass +class RegisterChannelListenerInput: + """Input for :py:meth:`OutboundInterceptor.register_channel_listener`. + + .. warning:: + This API is experimental and unstable. + """ + + channel: str + callback: Callback + rpc_metadata: Mapping[str, str | bytes] + rpc_timeout: timedelta | None + execution: temporalio.common.Execution | None = None + + +@dataclass +class UnregisterChannelListenerInput: + """Input for :py:meth:`OutboundInterceptor.unregister_channel_listener`. + + .. warning:: + This API is experimental and unstable. + """ + + channel: str + listener_id: str + rpc_metadata: Mapping[str, str | bytes] + rpc_timeout: timedelta | None + execution: temporalio.common.Execution | None = None + + @dataclass class Interceptor: """Interceptor for clients. @@ -1001,3 +1083,51 @@ async def count_nexus_operations( This API is experimental and unstable. """ return await self.next.count_nexus_operations(input) + + ### Notification channel calls + + async def notify_channel(self, input: NotifyChannelInput) -> int: + """Called for every :py:meth:`Client.notify_channel` call. + + .. warning:: + This API is experimental and unstable. + """ + return await self.next.notify_channel(input) + + async def poll_channel( + self, input: PollChannelInput + ) -> list[temporalio.workflow.Notification]: + """Called for every :py:meth:`Client.poll_channel` call. + + .. warning:: + This API is experimental and unstable. + """ + return await self.next.poll_channel(input) + + async def describe_channel(self, input: DescribeChannelInput) -> ChannelDescription: + """Called for every :py:meth:`Client.describe_channel` call. + + .. warning:: + This API is experimental and unstable. + """ + return await self.next.describe_channel(input) + + async def register_channel_listener( + self, input: RegisterChannelListenerInput + ) -> str: + """Called for every :py:meth:`Client.register_channel_listener` call. + + .. warning:: + This API is experimental and unstable. + """ + return await self.next.register_channel_listener(input) + + async def unregister_channel_listener( + self, input: UnregisterChannelListenerInput + ) -> None: + """Called for every :py:meth:`Client.unregister_channel_listener` call. + + .. warning:: + This API is experimental and unstable. + """ + await self.next.unregister_channel_listener(input) diff --git a/temporalio/client/_workflow.py b/temporalio/client/_workflow.py index 0607ade0ab..c2e57cc92b 100644 --- a/temporalio/client/_workflow.py +++ b/temporalio/client/_workflow.py @@ -4,6 +4,7 @@ import asyncio import functools +import json import warnings from asyncio import Future from collections.abc import ( @@ -31,6 +32,7 @@ import temporalio.api.common.v1 import temporalio.api.enums.v1 import temporalio.api.history.v1 +import temporalio.api.stream.v1 import temporalio.api.update.v1 import temporalio.api.workflow.v1 import temporalio.api.workflowservice.v1 @@ -59,6 +61,7 @@ ReturnType, SelfType, ) +from ._channel import ChannelSubscriptionInfo from ._exceptions import ( WorkflowContinuedAsNewError, WorkflowFailureError, @@ -1418,6 +1421,18 @@ class WorkflowExecutionDescription(WorkflowExecution): raw_description: temporalio.api.workflowservice.v1.DescribeWorkflowExecutionResponse """Underlying protobuf description.""" + channel_subscriptions: Sequence[ChannelSubscriptionInfo] = () + """The notification channels this run stands on. + + The independent channels it subscribed to and the channels linked to it + that hold any state, sorted by name with the independent kind first. + Empty when there are none. See + :py:class:`temporalio.client.ChannelSubscriptionInfo`. + + .. warning:: + This API is experimental and unstable. + """ + _static_summary: str | None = None _static_details: str | None = None _metadata_decoded: bool = False @@ -1456,6 +1471,10 @@ async def _from_raw_description( namespace=namespace, converter=converter, raw_description=description, + channel_subscriptions=tuple( + ChannelSubscriptionInfo._from_proto(info) + for info in description.channel_subscriptions + ), ) @@ -1681,6 +1700,20 @@ class WorkflowHistory: events: Sequence[temporalio.api.history.v1.HistoryEvent] """History events for the workflow.""" + stream_slices: Sequence[temporalio.api.stream.v1.StreamSlice] = () + """The stream records the workflow's completed tasks consumed, if captured. + + History records only the offsets each Workflow Task consumed from a + server-side stream; the records live in the stream. A history whose tasks + consumed records and that carries none here cannot be replayed without the + server that still holds the stream. One slice per recorded range, tagged + with the ``WorkflowTaskCompleted`` event that recorded it, in the shape the + server puts on a poll response. + :py:meth:`temporalio.worker.Replayer.fetch_stream_slices` fills it while + the stream is retained, and :py:meth:`to_json` and :py:meth:`from_json` + carry it, so an exported history file is the whole replay input. + """ + @property def run_id(self) -> str: """Run ID extracted from the first event.""" @@ -1702,31 +1735,62 @@ def from_json(workflow_id: str, history: str | dict[str, Any]) -> WorkflowHistor Args: workflow_id: The workflow's ID history: A string or parsed-to-dict representation of workflow - history + history. A ``streamSlices`` list beside ``events``, as + :py:meth:`to_json` writes one, is read into + :py:attr:`stream_slices`. Returns: Workflow history """ parsed = _history_from_json(history) - return WorkflowHistory(workflow_id, parsed.events) + raw: Any = [] + if isinstance(history, dict): + raw = history.get("streamSlices") or history.get("stream_slices") or [] + elif '"streamSlices"' in history or '"stream_slices"' in history: + # Read again only when the text mentions them. Almost every + # history has none, and handing the parsed dict to + # _history_from_json instead would make it deep-copy an export + # that can be very large. + decoded = json.loads(history) + if isinstance(decoded, dict): + raw = decoded.get("streamSlices") or decoded.get("stream_slices") or [] + slices: list[temporalio.api.stream.v1.StreamSlice] = [ + google.protobuf.json_format.ParseDict( + entry, + temporalio.api.stream.v1.StreamSlice(), + ignore_unknown_fields=True, + ) + for entry in raw + ] + return WorkflowHistory(workflow_id, parsed.events, slices) def to_json(self) -> str: """Convert this history to JSON. - Note, this does not include the workflow ID. + Note, this does not include the workflow ID. The stream slices, when + there are any, are written as a ``streamSlices`` list beside the + events; without them the output is the history proto's JSON alone. """ - return google.protobuf.json_format.MessageToJson( - temporalio.api.history.v1.History(events=self.events) - ) + if not self.stream_slices: + return google.protobuf.json_format.MessageToJson( + temporalio.api.history.v1.History(events=self.events) + ) + return json.dumps(self.to_json_dict(), indent=2) def to_json_dict(self) -> dict[str, Any]: """Convert this history to JSON-compatible dict. - Note, this does not include the workflow ID. + Note, this does not include the workflow ID. See :py:meth:`to_json` + for how the stream slices are carried. """ - return google.protobuf.json_format.MessageToDict( + out = google.protobuf.json_format.MessageToDict( temporalio.api.history.v1.History(events=self.events) ) + if self.stream_slices: + out["streamSlices"] = [ + google.protobuf.json_format.MessageToDict(s) for s in self.stream_slices + ] + return out @dataclass diff --git a/temporalio/client_stream.py b/temporalio/client_stream.py new file mode 100644 index 0000000000..a7b5f8d32b --- /dev/null +++ b/temporalio/client_stream.py @@ -0,0 +1,1218 @@ +"""Client for Temporal server-side streams. + +A stream is a durable, offset-addressed append-only sequence that lives beside +Workflow History rather than inside it. Appending schedules no Workflow Task, +and each reader holds its own cursor, so adding a reader costs nothing on the +write side. The record is ``temporal.api.stream.v1.StreamRecord`` on the wire +and in the store, so a reader in any language decodes the same bytes. + +Prototype support for AI-198. Four things about it are temporary and will +change before this is a real feature: + +- **Requires the ``grpc`` extra.** The rest of this SDK reaches the server + through sdk-core, which does not know about this service yet, so the client + here opens its own channel with ``grpcio``. +- **The protos are vendored** under ``temporalio.api.streamservice.v1`` instead of + coming from the api submodule, because the service is still defined in the + server. That is why the wire names read as server-internal. +- **This client is for use outside a Workflow.** Workflow code publishes and + consumes with ``workflow.append_stream_records`` and + ``workflow.read_stream_records`` instead. +- **The channel mirrors the client's connection rather than sharing it.** + :class:`Connection` reads a :class:`temporalio.service.ConnectConfig` and + opens a ``grpcio`` channel with the same target, TLS material, API key, + headers and keep-alive, so a client connected to Temporal Cloud reaches the + stream service the same way. What it cannot mirror is noted on that class. + +A failed call raises :class:`temporalio.streams.StreamNotFoundError` when the +server answers ``NOT_FOUND``, +:class:`temporalio.streams.StreamProducerError` when it refuses a producer +sequence it already holds, :class:`temporalio.streams.StreamCursorError` when +it refuses a read below the retention floor, +:class:`temporalio.streams.StreamClosedError` when it refuses an append to a +sealed stream, and :class:`temporalio.service.RPCError` otherwise, never the +transport's own exception type. :func:`translate_error` is the one place that +decides. + +A failure sdk-core would retry is retried here, on the same codes and with the +same default :class:`temporalio.service.RetryConfig`, because this channel is +not Core's and gets none of its retrying. ``RESOURCE_EXHAUSTED`` backs off +longer than the rest, as in Core, so a caller the server is throttling does not +add to the load. A call the server cannot tell from its own repeat, an append +without a producer id or a create, is retried only on ``RESOURCE_EXHAUSTED``, +which the server sends before it does anything. The budget is bounded; a caller +that wants a shorter one cancels, as with ``asyncio.timeout``, and the +cancellation lands whether an attempt or a wait between attempts is in progress. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +import time +import weakref +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import TYPE_CHECKING, Any, TypeVar + +import grpc +import grpc.aio +from google.protobuf.message import Message + +import temporalio.api.streamservice.v1 as stream +from temporalio.api.common.v1 import GrpcStatus +from temporalio.api.enums.v1 import ResourceExhaustedCause +from temporalio.api.errordetails.v1 import ResourceExhaustedFailure +from temporalio.api.stream.v1 import StreamRecord, StreamStartPosition +from temporalio.api.streamservice.v1 import service_pb2_grpc +from temporalio.service import ( + ConnectConfig, + RetryConfig, + RPCError, + RPCStatusCode, + TLSConfig, + __version__, +) +from temporalio.streams import ( + StreamClosedError, + StreamCursorError, + StreamNotFoundError, + StreamProducerError, +) + +if TYPE_CHECKING: + from temporalio.client import Client + +__all__ = [ + "Appended", + "Connection", + "Page", + "StreamClient", + "StreamEntry", + "StreamHandle", + "WorkflowStreamHandle", + "close_shared_clients", + "shared_client", + "shared_key", + "translate_error", +] + +_T = TypeVar("_T") + +logger = logging.getLogger(__name__) + +# A refusal the caller has to act on is a FAILED_PRECONDITION whose message +# begins with a reason token and ": ", since the service carries no typed +# detail for these yet. A repeat with different content and one behind the +# sequence the server accepted last are both a producer error; a read below +# the retention floor is a cursor error. +_REASON_SEPARATOR = ": " +_REASONS: dict[str, type[Exception]] = { + "STREAM_PRODUCER_CONFLICT": StreamProducerError, + "STREAM_PRODUCER_STALE_SEQUENCE": StreamProducerError, + "STREAM_CURSOR_BELOW_FLOOR": StreamCursorError, + "STREAM_CLOSED": StreamClosedError, + # A create of an id that exists with another policy is the caller's + # mistake, which the interface contract spells as ValueError. + "STREAM_POLICY_MISMATCH": ValueError, +} +# The phrases a server built before the tokens existed sends for the same +# refusals, so a reader of either server gets the typed error. The sealed +# stream's message is matched whole. +_PRODUCER_PHRASE = "producer sequence" +_CURSOR_PHRASE = "below the stream's floor" +_CLOSED_PHRASE = "stream is closed" + +# The codes sdk-core retries. +_RETRYABLE = frozenset( + { + grpc.StatusCode.DATA_LOSS, + grpc.StatusCode.INTERNAL, + grpc.StatusCode.UNKNOWN, + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.ABORTED, + grpc.StatusCode.OUT_OF_RANGE, + grpc.StatusCode.UNAVAILABLE, + } +) +# What the server sends before it does anything, so a call that cannot be told +# from its own repeat is still safe to make again on it. +_REFUSED = frozenset({grpc.StatusCode.RESOURCE_EXHAUSTED}) +# A message over the channel's limit comes back as RESOURCE_EXHAUSTED and is +# the same size every time. +_TOO_LARGE = ( + "grpc: received message larger than max", + "grpc: message after decompression larger than max", + "grpc: received message after decompression larger than max", +) +# The floor under a throttled call's wait, sdk-core's own. +_THROTTLE = RetryConfig( + initial_interval_millis=1000, + multiplier=2.0, + max_interval_millis=10000, + max_elapsed_time_millis=None, + max_retries=0, +) + + +class _Backoff: + """Exponential backoff over a :class:`RetryConfig`, with sdk-core's arithmetic.""" + + def __init__(self, config: RetryConfig) -> None: + self._config = config + self._started = time.monotonic() + self._interval = config.initial_interval_millis / 1000 + self._failures = 0 + + def next(self) -> float | None: + """Seconds to wait before the next attempt, or ``None`` once the budget is spent.""" + config = self._config + self._failures += 1 + if config.max_retries and self._failures >= config.max_retries: + return None + base = self._interval + self._interval = min( + base * config.multiplier, config.max_interval_millis / 1000 + ) + spread = base * config.randomization_factor + delay = max(base + random.uniform(-spread, spread), 0.0) + if config.max_elapsed_time_millis is not None and ( + time.monotonic() - self._started + delay + > config.max_elapsed_time_millis / 1000 + ): + return None + return delay + + +def _raw_status(error: grpc.aio.AioRpcError) -> bytes: + # The aio metadata iterates as (key, value) pairs at runtime, whatever + # shape the stubs give its items. + trailing: Any = error.trailing_metadata() + for item in trailing or (): + key, value = item[0], item[1] + if key == "grpc-status-details-bin" and isinstance(value, bytes): + return value + return b"" + + +def _exhausted_cause(error: grpc.aio.AioRpcError) -> int | None: + """The cause the server attached to a ``RESOURCE_EXHAUSTED``, when it attached one.""" + raw = _raw_status(error) + if not raw: + return None + status = GrpcStatus() + status.ParseFromString(raw) + for detail in status.details: + if detail.Is(ResourceExhaustedFailure.DESCRIPTOR): + failure = ResourceExhaustedFailure() + detail.Unpack(failure) + return failure.cause + return None + + +def _worth_waiting_out(error: grpc.aio.AioRpcError) -> bool: + """Whether a ``RESOURCE_EXHAUSTED`` is load the server will shed, rather than a limit.""" + if (error.details() or "").startswith(_TOO_LARGE): + return False + # A stream's budget refuses an append for as long as the stream is that + # full, which no wait changes. + return _exhausted_cause(error) != ( + ResourceExhaustedCause.RESOURCE_EXHAUSTED_CAUSE_PERSISTENCE_STORAGE_LIMIT + ) + + +def _retry_after( + error: grpc.aio.AioRpcError, + backoff: _Backoff, + throttle: _Backoff, + *, + idempotent: bool, +) -> float | None: + """Seconds to wait before making the call again, or ``None`` to raise it.""" + code = error.code() + if code not in _RETRYABLE or (not idempotent and code not in _REFUSED): + return None + throttled = code is grpc.StatusCode.RESOURCE_EXHAUSTED + if throttled and not _worth_waiting_out(error): + return None + delay = backoff.next() + if delay is None: + return None + if throttled: + delay = max(delay, throttle.next() or 0.0) + return delay + + +class _Headers(grpc.aio.UnaryUnaryClientInterceptor): + """Attaches the connection's headers to every call, as Core's interceptor does.""" + + def __init__(self, headers: Sequence[tuple[str, str | bytes]]) -> None: + self._headers = headers + + async def intercept_unary_unary( # type: ignore[override] + self, + continuation: Callable[[grpc.aio.ClientCallDetails, Any], Awaitable[Any]], + client_call_details: grpc.aio.ClientCallDetails, + request: Any, + ) -> Any: + # The aio metadata iterates as (key, value) pairs at runtime, whatever + # shape the stubs give its items. + given: Any = client_call_details.metadata + metadata = grpc.aio.Metadata(*(given or ())) + for key, value in self._headers: + # A header the caller set on the call wins over the connection's. + if key not in metadata: + metadata.add(key, value) + details = client_call_details._replace(metadata=metadata) # type: ignore[attr-defined] + return await continuation(details, request) + + +@dataclass(frozen=True) +class Connection: + """How a stream channel reaches a frontend, taken from a client's connection. + + :meth:`from_config` reads what ``Client.connect`` was given and this opens + a ``grpc.aio`` channel that behaves the same way: the target, TLS with the + same root CA, client certificate and key, the API key as a bearer + ``authorization`` header, the client's default headers and keep-alive. + Two clients with the same settings yield equal connections, which is what + lets them share one channel per namespace. + + Two things ``grpcio`` cannot express the way sdk-core does. It has one + override for both the TLS server name it sends and the name it verifies, + so ``verification_server_name`` takes that override when set and + ``domain`` otherwise, while ``domain`` alone still sets the HTTP/2 + authority. And it reads the settings once, when the channel is opened, so + an API key or header updated on the client afterwards reaches the stream + channel only through a new connection. + """ + + target: str + secure: bool + server_root_ca_cert: bytes | None = None + client_cert: bytes | None = None + client_private_key: bytes | None = None + server_name: str | None = None + authority: str | None = None + headers: tuple[tuple[str, str | bytes], ...] = () + keep_alive: tuple[int, int] | None = None + http_proxy: str | None = None + + @staticmethod + def from_config(config: ConnectConfig) -> Connection: + """Read a :class:`temporalio.service.ConnectConfig` the way the bridge does.""" + target = config.target_host + tls: TLSConfig | None = None + if "://" in target: + # The bridge still accepts a URL with a scheme; the scheme decides. + scheme, _, target = target.partition("://") + secure = scheme == "https" + if isinstance(config.tls, TLSConfig): + tls = config.tls + elif isinstance(config.tls, TLSConfig): + secure, tls = True, config.tls + elif config.tls: + secure = True + else: + # TLS is on by default when an API key is given and tls was left unset. + secure = config.tls is None and config.api_key is not None + + headers: list[tuple[str, str | bytes]] = [ + ("client-name", "temporal-python"), + ("client-version", __version__), + ] + given = {key.lower() for key in config.rpc_metadata} + if config.api_key is not None and "authorization" not in given: + headers.append(("authorization", f"Bearer {config.api_key}")) + headers.extend(config.rpc_metadata.items()) + + proxy = config.http_connect_proxy_config + http_proxy: str | None = None + if proxy is not None: + auth = ( + f"{proxy.basic_auth[0]}:{proxy.basic_auth[1]}@" + if proxy.basic_auth + else "" + ) + http_proxy = f"http://{auth}{proxy.target_host}" + + keep_alive = config.keep_alive_config + return Connection( + target=target, + secure=secure, + server_root_ca_cert=tls.server_root_ca_cert if tls else None, + client_cert=tls.client_cert if tls else None, + client_private_key=tls.client_private_key if tls else None, + server_name=((tls.verification_server_name or tls.domain) if tls else None), + authority=tls.domain if tls else None, + headers=tuple(headers), + keep_alive=( + (keep_alive.interval_millis, keep_alive.timeout_millis) + if keep_alive + else None + ), + http_proxy=http_proxy, + ) + + def channel(self) -> grpc.aio.Channel: + """Open a channel with these settings. Nothing is sent until the first call.""" + options: list[tuple[str, Any]] = [] + if self.keep_alive is not None: + options.append(("grpc.keepalive_time_ms", self.keep_alive[0])) + options.append(("grpc.keepalive_timeout_ms", self.keep_alive[1])) + if self.server_name: + options.append(("grpc.ssl_target_name_override", self.server_name)) + if self.authority: + options.append(("grpc.default_authority", self.authority)) + if self.http_proxy: + options.append(("grpc.http_proxy", self.http_proxy)) + # The stubs do not know the aio interceptor base as a ClientInterceptor. + interceptors: Any = [_Headers(self.headers)] if self.headers else None + if not self.secure: + return grpc.aio.insecure_channel( + self.target, options=options, interceptors=interceptors + ) + credentials = grpc.ssl_channel_credentials( + root_certificates=self.server_root_ca_cert, + private_key=self.client_private_key, + certificate_chain=self.client_cert, + ) + return grpc.aio.secure_channel( + self.target, credentials, options=options, interceptors=interceptors + ) + + +@dataclass(frozen=True) +class Appended: + """Where one append landed. + + ``deduplicated`` says the server already held this producer's batch at + this sequence and wrote nothing; the offsets are then the original ones. + """ + + first_offset: int + next_offset: int + count: int + deduplicated: bool = False + + +@dataclass(frozen=True) +class StreamEntry: + """One record read from a stream, with where it sits. + + Offsets are assigned over the unfiltered stream, so a topic-filtered read + hands back entries whose offsets are not contiguous. A reader that resumes + between records takes ``offset`` rather than counting what it received. + """ + + record: StreamRecord + offset: int = 0 + + +@dataclass(frozen=True) +class Page: + """One read of a stream. + + ``closed`` with ``next_offset >= head_offset`` is the end: nothing more can + be added and this reader has everything. ``run_id`` names the execution + holding the stream, which is how a reader of a workflow's stream learns + which run it is on when it did not pin one. + """ + + entries: list[StreamEntry] + next_offset: int + head_offset: int + closed: bool + run_id: str = "" + + +def _copy_by_name(source: Message, target: Message, *, skip: frozenset[str]) -> None: + """Copy every field ``source`` has set onto the field of ``target`` with that name. + + By descriptor rather than field by field, so a field added to + ``StreamRecord`` crosses in both directions without anybody remembering to + add a line here. A field the target does not have raises, which is the + answer a reader wants: better a loud failure than a body that arrives + without the thing that described it. + """ + fields = target.DESCRIPTOR.fields_by_name + for descriptor, value in source.ListFields(): + if descriptor.name in skip: + continue + if descriptor.name not in fields: + raise ValueError( + f"{source.DESCRIPTOR.full_name}.{descriptor.name} has no counterpart " + f"on {target.DESCRIPTOR.full_name}" + ) + field = getattr(target, descriptor.name) + if descriptor.message_type is not None and ( + descriptor.message_type.GetOptions().map_entry + ): + holds_message = ( + descriptor.message_type.fields_by_name["value"].message_type is not None + ) + for key, item in value.items(): + if holds_message: + field[key].CopyFrom(item) + else: + field[key] = item + elif hasattr(field, "extend"): + # A plain repeated field; maps answered above and everything else + # takes an assignment or a CopyFrom. + field.extend(value) + elif descriptor.type == descriptor.TYPE_MESSAGE: + field.CopyFrom(value) + else: + setattr(target, descriptor.name, value) + + +def _to_service(record: StreamRecord) -> stream.StreamRecord: + # The stored shape is the public record plus the offset a read assigns. + # An unset body stays unset, so a FINISH record reads back as one. + out = stream.StreamRecord() + _copy_by_name(record, out, skip=frozenset()) + return out + + +def _to_public(record: stream.StreamRecord) -> StreamEntry: + out = StreamRecord() + # The offset is the store's, not the record's; it rides on the entry. + _copy_by_name(record, out, skip=frozenset({"offset"})) + return StreamEntry(record=out, offset=record.offset) + + +def translate_error( + code: grpc.StatusCode, details: str, raw_status: bytes = b"" +) -> Exception: + """The SDK error for one failed call, from its status code and message. + + ``NOT_FOUND`` is :class:`temporalio.streams.StreamNotFoundError`. A + ``FAILED_PRECONDITION`` whose message begins with a reason token is the + error the token names: a producer refusal, a repeat with different content + or one behind the sequence the server accepted last, is + :class:`temporalio.streams.StreamProducerError`, because the caller asked + to be deduplicated and could not be; a read below the retention floor is + :class:`temporalio.streams.StreamCursorError`. Everything else is + :class:`temporalio.service.RPCError` with the code and the raw status. + """ + details = details or code.name + if code is grpc.StatusCode.NOT_FOUND: + return StreamNotFoundError(details) + if code is grpc.StatusCode.FAILED_PRECONDITION: + token, separator, _ = details.partition(_REASON_SEPARATOR) + typed = _REASONS.get(token) if separator else None + if typed is not None: + return typed(details) + if _CURSOR_PHRASE in details: + return StreamCursorError(details) + if details == _CLOSED_PHRASE: + return StreamClosedError(details) + if code is grpc.StatusCode.INVALID_ARGUMENT and _PRODUCER_PHRASE in details: + return StreamProducerError(details) + return RPCError(details, RPCStatusCode(code.value[0]), raw_status) + + +def _translate(error: grpc.aio.AioRpcError) -> Exception: + return translate_error(error.code(), error.details() or "", _raw_status(error)) + + +async def _call( + method: Callable[[Any], Awaitable[_T]], + request: Any, + *, + retry_config: RetryConfig | None = None, + idempotent: bool = True, +) -> _T: + """Make one stub call, retried as sdk-core would, translating the failure to the SDK's. + + ``idempotent`` is false for a call the server cannot tell from its own + repeat, which is then retried only on a refusal the server sent before it + did anything. The request is the same object on every attempt, so a + numbered append is deduplicated by the server whichever attempt landed. + """ + config = retry_config or RetryConfig() + backoff = _Backoff(config) + throttle = _Backoff(_THROTTLE) + attempts = 0 + while True: + attempts += 1 + try: + return await method(request) + except grpc.aio.AioRpcError as error: + delay = _retry_after(error, backoff, throttle, idempotent=idempotent) + if delay is None: + raise _translate(error) from error + _log_retry(error, attempts, config) + await asyncio.sleep(delay) + + +def _log_retry(error: grpc.aio.AioRpcError, attempts: int, config: RetryConfig) -> None: + # Quiet at first and louder once half the budget is gone, as sdk-core does, + # so a single throttled call is not a warning but a struggling one is. + level = logging.DEBUG + if config.max_retries and attempts * 2 >= config.max_retries: + level = logging.WARNING + logger.log( + level, + "stream call failed with %s on attempt %d, retrying: %s", + error.code().name, + attempts, + error.details(), + ) + + +class StreamClient: + """Creates and opens streams on a namespace.""" + + def __init__( + self, + channel: Any, + namespace: str, + *, + retry_config: RetryConfig | None = None, + ) -> None: + """Wrap an existing ``grpc.aio`` channel. Prefer :meth:`connect`. + + ``retry_config`` is the policy every call made through this client + retries under; ``None`` is the SDK's default. + """ + self._channel = channel + self._namespace = namespace + self._retry_config = retry_config + # The generated stub is typed for a synchronous channel. This client + # drives it over ``grpc.aio``, where every call is awaited. + self._stub: Any = service_pb2_grpc.StreamServiceStub(channel) + + @staticmethod + def connect( + target_host: str, + namespace: str = "default", + *, + retry_config: RetryConfig | None = None, + ) -> StreamClient: + """Open a plaintext channel to a frontend, for a local server. + + Separate from ``Client.connect`` because this does not share the + connection the rest of the SDK uses; :meth:`for_connection` opens + one with a client's settings. + """ + return StreamClient( + grpc.aio.insecure_channel(target_host), + namespace, + retry_config=retry_config, + ) + + @staticmethod + def for_connection( + connection: Connection, + namespace: str = "default", + *, + retry_config: RetryConfig | None = None, + ) -> StreamClient: + """Open a channel the way ``connection`` describes. + + ``retry_config`` left ``None`` is the SDK's default; pass the client's + own to retry as its other calls do. + """ + return StreamClient(connection.channel(), namespace, retry_config=retry_config) + + async def close(self) -> None: + """Close the underlying channel.""" + await self._channel.close() + + async def create( + self, + stream_id: str, + *, + retention: float | timedelta | None = None, + max_items: int | None = None, + max_bytes: int | None = None, + ) -> StreamHandle: + """Create a stream and return a handle to it. + + ``retention`` is the age past which an open stream's records are + reclaimed and how long a closed stream stays readable, in seconds or + as a ``timedelta``. ``max_items`` caps how many records remain + readable, dropping the oldest, which bounds storage for a stream + nobody truncates. ``max_bytes`` caps the bytes held; an append that + would cross it is refused rather than reclaiming anything. + """ + lifecycle = stream.StreamLifecycle() + if retention is not None: + if not isinstance(retention, timedelta): + retention = timedelta(seconds=retention) + lifecycle.retention.FromTimedelta(retention) + if max_items is not None: + lifecycle.max_items = max_items + if max_bytes is not None: + lifecycle.max_bytes = max_bytes + + # A create that landed but was not answered would be refused as a + # repeat, so it goes again only on a refusal. + response = await _call( + self._stub.CreateStream, + stream.CreateStreamRequest( + frontend_request=stream.CreateStreamInput( + namespace=self._namespace, + stream_id=stream_id, + lifecycle=lifecycle, + ) + ), + retry_config=self._retry_config, + idempotent=False, + ) + return StreamHandle( + self._stub, + self._namespace, + stream_id, + run_id=response.frontend_response.run_id, + retry_config=self._retry_config, + ) + + def get(self, stream_id: str) -> StreamHandle: + """Open an existing stream without a round trip.""" + return StreamHandle( + self._stub, self._namespace, stream_id, retry_config=self._retry_config + ) + + def workflow_stream( + self, workflow_id: str, name: str = "", *, owner_run_id: str = "" + ) -> WorkflowStreamHandle: + """Open a stream a workflow owns. + + A stream a workflow owns lives inside that workflow's execution and has + no id of its own, so it is named by its owner and its name. An empty + name is the workflow's default output stream, which is what + ``workflow.append_stream_records`` writes to when it is given none. + + The stream is created by the workflow's first publish or subscription, + or by the first append from outside. Reading one that does not exist + yet is not an error: it reads as empty and a + :meth:`WorkflowStreamHandle.follow` parked on it wakes when something + is written. + + ``owner_run_id`` pins the handle to one run of the workflow; see + :meth:`WorkflowStreamHandle.pin` for why a follower wants that. + """ + return WorkflowStreamHandle( + self._stub, + self._namespace, + workflow_id, + name, + owner_run_id, + retry_config=self._retry_config, + ) + + def activity_stream( + self, + activity_id: str, + name: str = "", + *, + workflow_id: str = "", + run_id: str = "", + ) -> WorkflowStreamHandle: + """Open a stream an activity owns. + + Without ``workflow_id`` the activity is a standalone one, an execution + of its own, and ``run_id`` pins one run of it. With ``workflow_id`` it + is an activity that workflow scheduled, reached through the workflow, + and ``run_id`` pins the workflow's run. Either way the stream is apart + from the workflow's streams, one per activity execution rather than + per attempt, and it reads as closed once the activity reaches a + terminal status. An empty name is the activity's default stream. + """ + return WorkflowStreamHandle( + self._stub, + self._namespace, + workflow_id, + name, + run_id, + activity_id=activity_id, + retry_config=self._retry_config, + ) + + +class StreamHandle: + """A handle to one standalone stream.""" + + def __init__( + self, + stub: Any, + namespace: str, + stream_id: str, + run_id: str = "", + *, + retry_config: RetryConfig | None = None, + ) -> None: + """Prefer :meth:`StreamClient.get` or :meth:`StreamClient.create`.""" + self._stub = stub + self._namespace = namespace + self._id = stream_id + # Passing this back saves the server resolving the current run on every + # call, which is otherwise a persistence lookup per request. + self._run_id = run_id + self._retry_config = retry_config + + @property + def id(self) -> str: + """Id of the stream this handle points at.""" + return self._id + + async def append( + self, + *records: StreamRecord, + producer_id: str = "", + sequence: int = 0, + ) -> Appended: + """Append records and return where they landed. + + Supplying ``producer_id`` and ``sequence`` makes the append idempotent: + a retry with the same pair returns the original offsets rather than + appending twice, and says so. That is also what lets a failed append + be made again here on every code sdk-core retries. Without them the + append is at-least-once, which is only the right trade when duplicates + are harmless, and it is made again only on a refusal the server sent + before it did anything. + """ + response = await _call( + self._stub.AddMessages, + stream.AddMessagesRequest( + frontend_request=stream.AddMessagesInput( + namespace=self._namespace, + stream_id=self._id, + run_id=self._run_id, + records=[_to_service(record) for record in records], + producer_id=producer_id, + sequence=sequence, + ) + ), + retry_config=self._retry_config, + idempotent=bool(producer_id), + ) + return _appended(response.frontend_response) + + async def read( + self, + *, + from_offset: int = 0, + start: StreamStartPosition | None = None, + max_records: int = 0, + topics: Sequence[str] = (), + wait: bool = False, + ) -> tuple[list[StreamEntry], int]: + """Read once from ``from_offset``, returning the entries and the + offset to read from next. + + A reader with no offset yet passes ``start`` instead: the oldest + record held, the tail, or the last N records. The server resolves it + in the same read, so it cannot race with truncation, and the offset + returned is where to continue. Passing both is refused. + + With ``wait`` set, blocks until something arrives, the stream closes, + or the server's long-poll window elapses. A window that elapses returns + an empty list rather than raising, so the caller just reads again. + """ + page = await self.poll( + from_offset=from_offset, + start=start, + max_records=max_records, + topics=topics, + wait=wait, + ) + return page.entries, page.next_offset + + async def follow( + self, + *, + from_offset: int = 0, + start: StreamStartPosition | None = None, + topics: Sequence[str] = (), + ) -> AsyncIterator[StreamEntry]: + """Yield entries as they arrive, starting at ``from_offset`` or ``start``. + + Ends once the stream is closed and this reader has drained it. A closed + stream stays readable until its retention expires, so a reader that + starts late still sees everything instead of having to coordinate a + shutdown with the producer. + """ + offset = from_offset + while True: + page = await self.poll(from_offset=offset, start=start, topics=topics) + # The first page carries where the start resolved to, and every + # later poll continues from the offset it handed back. + start = None + for entry in page.entries: + yield entry + offset = page.next_offset + if page.closed and offset >= page.head_offset: + return + + async def poll( + self, + *, + from_offset: int = 0, + start: StreamStartPosition | None = None, + topics: Sequence[str] = (), + max_records: int = 0, + wait: bool = True, + ) -> Page: + """One read, with everything the caller needs to decide what to do next.""" + response = await _call( + self._stub.PollMessages, + stream.PollMessagesRequest( + frontend_request=stream.PollMessagesInput( + namespace=self._namespace, + stream_id=self._id, + run_id=self._run_id, + from_offset=from_offset, + start_position=start, + max_messages=max_records, + topics=list(topics), + wait_new_messages=wait, + ) + ), + retry_config=self._retry_config, + ) + return _page(response.frontend_response) + + async def truncate(self, new_base_offset: int) -> None: + """Drop every record below ``new_base_offset``. + + A reader that asks for an offset below it is refused. One that has no + offset yet and wants the oldest record left asks for + ``StreamStartPosition(earliest=True)`` rather than offset zero. + """ + await _call( + self._stub.TruncateStream, + stream.TruncateStreamRequest( + frontend_request=stream.TruncateStreamInput( + namespace=self._namespace, + stream_id=self._id, + new_base_offset=new_base_offset, + ) + ), + retry_config=self._retry_config, + ) + + async def finish_writing(self, producer_id: str) -> None: + """Declare one producer done without ending the stream for others.""" + await _call( + self._stub.FinishWriting, + stream.FinishWritingRequest( + frontend_request=stream.FinishWritingInput( + namespace=self._namespace, + stream_id=self._id, + producer_id=producer_id, + ) + ), + retry_config=self._retry_config, + ) + + async def close(self) -> None: + """Seal the stream. Readers can still drain what is already there.""" + await _call( + self._stub.CloseStream, + stream.CloseStreamRequest( + frontend_request=stream.CloseStreamInput( + namespace=self._namespace, stream_id=self._id + ) + ), + retry_config=self._retry_config, + ) + + async def describe(self) -> stream.StreamState: + """Read the stream's current frontier, floor and closed state.""" + response = await _call( + self._stub.DescribeStream, + stream.DescribeStreamRequest( + frontend_request=stream.DescribeStreamInput( + namespace=self._namespace, stream_id=self._id + ) + ), + retry_config=self._retry_config, + ) + return response.frontend_response.state + + +class WorkflowStreamHandle: + """A handle to a stream a workflow or an activity owns. + + A workflow writes to its own from inside its Workflow Task, which costs it + no transition of its own. Anything else writes through :meth:`append`, + which costs one transition on the owning execution per batch. Both land in + the same log in the order the server accepted them. An activity has only + the second path. + """ + + def __init__( + self, + stub: Any, + namespace: str, + workflow_id: str, + name: str = "", + owner_run_id: str = "", + *, + activity_id: str = "", + retry_config: RetryConfig | None = None, + ) -> None: + """Prefer :meth:`StreamClient.workflow_stream` or :meth:`StreamClient.activity_stream`.""" + self._stub = stub + self._namespace = namespace + self._workflow_id = workflow_id + self._name = name + self._owner_run_id = owner_run_id + self._activity_id = activity_id + self._retry_config = retry_config + + @property + def workflow_id(self) -> str: + """Id of the workflow that owns this stream, or that scheduled the activity that does.""" + return self._workflow_id + + @property + def activity_id(self) -> str: + """Id of the activity that owns this stream, empty when a workflow does.""" + return self._activity_id + + def _owner(self) -> dict[str, Any]: + # A workflow owner goes out in the workflow fields, which a server + # without owner support still routes on; only an activity needs the + # owner reference. + if not self._activity_id: + return { + "workflow_id": self._workflow_id, + "owner_run_id": self._owner_run_id, + } + if self._workflow_id: + owner = stream.StreamOwner( + kind=stream.STREAM_OWNER_KIND_WORKFLOW_ACTIVITY, + id=self._workflow_id, + run_id=self._owner_run_id, + activity_id=self._activity_id, + ) + else: + owner = stream.StreamOwner( + kind=stream.STREAM_OWNER_KIND_ACTIVITY, + id=self._activity_id, + run_id=self._owner_run_id, + ) + return {"owner": owner} + + @property + def name(self) -> str: + """Name the owner publishes under, empty for its default stream.""" + return self._name + + @property + def owner_run_id(self) -> str: + """The run this handle is pinned to, empty while it follows the current run.""" + return self._owner_run_id + + def pin(self, run_id: str) -> None: + """Address one run of the workflow from now on. + + Unpinned, every call resolves to whichever run is current. A follower + holding an offset from one run would then be redirected when the + workflow continues as new: the successor's stream starts empty at + offset zero, so the server reads the follower's offset as "caught up", + parks it until the successor has published that many records, and + then skips exactly that many. Pinned, the run's end shows up as + ``closed`` on the page, and the caller decides whether to follow the + successor from the beginning. + """ + self._owner_run_id = run_id + + async def append( + self, + *records: StreamRecord, + producer_id: str = "", + sequence: int = 0, + ) -> Appended: + """Append from outside the workflow, as :meth:`StreamHandle.append`. + + Batch where you can. The append costs one transition on the owning + execution whatever its size, so a batch of a hundred costs what a batch + of one does. + """ + response = await _call( + self._stub.AddWorkflowMessages, + stream.AddWorkflowMessagesRequest( + frontend_request=stream.AddWorkflowMessagesInput( + namespace=self._namespace, + **self._owner(), + stream_name=self._name, + records=[_to_service(record) for record in records], + producer_id=producer_id, + sequence=sequence, + ) + ), + retry_config=self._retry_config, + idempotent=bool(producer_id), + ) + return _appended(response.frontend_response) + + async def read( + self, + *, + from_offset: int = 0, + start: StreamStartPosition | None = None, + max_records: int = 0, + topics: Sequence[str] = (), + wait: bool = False, + ) -> tuple[list[StreamEntry], int]: + """Read once from ``from_offset``, as :meth:`StreamHandle.read`.""" + page = await self.poll( + from_offset=from_offset, + start=start, + max_records=max_records, + topics=topics, + wait=wait, + ) + return page.entries, page.next_offset + + async def follow( + self, + *, + from_offset: int = 0, + start: StreamStartPosition | None = None, + topics: Sequence[str] = (), + ) -> AsyncIterator[StreamEntry]: + """Yield entries as they arrive, as :meth:`StreamHandle.follow`.""" + offset = from_offset + while True: + page = await self.poll(from_offset=offset, start=start, topics=topics) + # The first page carries where the start resolved to, and every + # later poll continues from the offset it handed back. + start = None + for entry in page.entries: + yield entry + offset = page.next_offset + if page.closed and offset >= page.head_offset: + return + + async def poll( + self, + *, + from_offset: int = 0, + start: StreamStartPosition | None = None, + topics: Sequence[str] = (), + max_records: int = 0, + wait: bool = True, + ) -> Page: + """One read, with everything the caller needs to decide what to do next. + + :meth:`read` is the same call without the frontier and the closed flag. + A caller that has to tell "nothing yet" from "nothing ever" needs both. + On a pinned handle ``closed`` also says the run has ended. + """ + response = await _call( + self._stub.PollWorkflowMessages, + stream.PollWorkflowMessagesRequest( + frontend_request=stream.PollWorkflowMessagesInput( + namespace=self._namespace, + **self._owner(), + stream_name=self._name, + from_offset=from_offset, + start_position=start, + max_messages=max_records, + topics=list(topics), + wait_new_messages=wait, + ) + ), + retry_config=self._retry_config, + ) + return _page(response.frontend_response) + + async def describe(self) -> stream.StreamState: + """Read the stream's current frontier, floor and closed state. + + A reader that wants only what comes next starts from the head this + reports rather than from zero. + """ + response = await _call( + self._stub.DescribeWorkflowStream, + stream.DescribeWorkflowStreamRequest( + frontend_request=stream.DescribeWorkflowStreamInput( + namespace=self._namespace, + **self._owner(), + stream_name=self._name, + ) + ), + retry_config=self._retry_config, + ) + return response.frontend_response.state + + +def _appended(out: stream.AddMessagesOutput) -> Appended: + return Appended( + first_offset=out.first_offset, + next_offset=out.next_offset, + count=out.count, + deduplicated=out.deduplicated, + ) + + +def _page(out: stream.PollMessagesOutput) -> Page: + return Page( + entries=[_to_public(record) for record in out.records], + next_offset=out.next_offset, + head_offset=out.head_offset, + closed=out.closed, + run_id=out.run_id, + ) + + +# One channel per loop, connection and namespace, shared by every handle in the +# process. A channel is multiplexed and long lived, and callers open a handle +# per subscription, which would otherwise be a connection per subscription. The +# loop is the key because a grpc.aio channel belongs to the loop that made it, +# and it is held weakly so a loop that is gone cannot lend its channel to a +# successor that happens to reuse its id. +SharedKey = tuple[Connection, str] + +_shared: weakref.WeakKeyDictionary[ + asyncio.AbstractEventLoop, dict[SharedKey, StreamClient] +] = weakref.WeakKeyDictionary() + + +def shared_key(client: Client) -> SharedKey: + """What names the shared channel ``client`` reaches the stream service through.""" + return Connection.from_config(client.service_client.config), client.namespace + + +def shared_client(client: Client) -> StreamClient: + """The process-wide stream client on this loop for ``client``'s connection and namespace. + + Opened with the client's connection settings and its ``retry_config``, so + a call on it authenticates and retries as the client's other calls do. + """ + per_loop = _shared.setdefault(asyncio.get_running_loop(), {}) + key = shared_key(client) + existing = per_loop.get(key) + if existing is None: + existing = per_loop[key] = StreamClient.for_connection( + key[0], key[1], retry_config=client.service_client.config.retry_config + ) + return existing + + +async def close_shared_clients(*keys: SharedKey) -> None: + """Close the shared clients this loop opened for ``keys``, or all of them. + + A provider closes the ones it opened, named by :func:`shared_key`: + another provider on the same loop may still be reading through a channel + of its own, and taking that out from under it is not this one's to do. + With no keys it closes every one, which is what a process finished with + streams wants, and what a test that opened a loop of its own wants. + """ + loop = asyncio.get_running_loop() + if not keys: + per_loop = _shared.pop(loop, {}) + closing = list(per_loop.values()) + else: + per_loop = _shared.get(loop, {}) + closing = [per_loop.pop(key) for key in keys if key in per_loop] + for client in closing: + await client.close() diff --git a/temporalio/common.py b/temporalio/common.py index 04081bf19c..cf126ff95e 100644 --- a/temporalio/common.py +++ b/temporalio/common.py @@ -18,6 +18,7 @@ Generic, TypeAlias, TypeVar, + cast, get_origin, get_type_hints, overload, @@ -108,6 +109,63 @@ def _validate(self) -> None: raise ValueError("Maximum attempts cannot be negative") +class ExecutionType(IntEnum): + """What kind of execution an :class:`Execution` names. + + .. warning:: + This API is experimental and unstable. + """ + + UNSPECIFIED = int(temporalio.api.enums.v1.ExecutionType.EXECUTION_TYPE_UNSPECIFIED) + WORKFLOW = int(temporalio.api.enums.v1.ExecutionType.EXECUTION_TYPE_WORKFLOW) + ACTIVITY = int(temporalio.api.enums.v1.ExecutionType.EXECUTION_TYPE_ACTIVITY) + """A standalone activity, one started by a client rather than a workflow.""" + + +@dataclass(frozen=True) +class Execution: + """One execution in a namespace: a workflow or a standalone activity. + + ``business_id`` is the id the caller chose, the workflow id or the + activity id. ``run_id`` pins one run of it. Unset, a call reaches the + current run of a workflow chain, as a Signal does. + + .. warning:: + This API is experimental and unstable. + """ + + type: ExecutionType + business_id: str + run_id: str | None = None + + @classmethod + def workflow(cls, workflow_id: str, run_id: str | None = None) -> Execution: + """A workflow execution.""" + return cls(ExecutionType.WORKFLOW, workflow_id, run_id) + + @classmethod + def activity(cls, activity_id: str, run_id: str | None = None) -> Execution: + """A standalone activity execution.""" + return cls(ExecutionType.ACTIVITY, activity_id, run_id) + + def to_proto(self) -> temporalio.api.common.v1.Execution: + """This execution as the API names it.""" + return temporalio.api.common.v1.Execution( + type=cast( + "temporalio.api.enums.v1.ExecutionType.ValueType", int(self.type) + ), + business_id=self.business_id, + run_id=self.run_id or "", + ) + + @staticmethod + def from_proto(proto: temporalio.api.common.v1.Execution) -> Execution: + """From the API's form. An empty run id reads as unset.""" + return Execution( + ExecutionType(proto.type), proto.business_id, proto.run_id or None + ) + + class WorkflowIDReusePolicy(IntEnum): """How already-in-use workflow IDs are handled on start. diff --git a/temporalio/contrib/external_workflow_streams/__init__.py b/temporalio/contrib/external_workflow_streams/__init__.py new file mode 100644 index 0000000000..a04387f214 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/__init__.py @@ -0,0 +1,179 @@ +"""External Workflow Streams for Temporal Workflows. + +.. warning:: + This package is experimental and may change in future versions. + +High-volume stream payloads live in a pluggable external backend such as Redis +Streams and never enter Temporal History. Deterministic replay is preserved +with compact markers recording consumed offset ranges and the +availability/blocking boundaries the original execution observed. + +Workflow code creates topics with :data:`external_stream` and subscribes to +them. A Worker receives a :class:`StreamBackend` through +``Worker(external_stream_backend=...)``. External processes publish with an +:class:`ExternalStreamProducer` bound to the Workflow's +:class:`WorkflowChainKey`. + +The complementary output direction uses :data:`external_output_stream` inside +Workflow code, :class:`ExternalOutputStreamProducer` in Activities and external +processes, and :class:`ExternalOutputStreamClient` for resumable external +consumers. Output payloads are staged behind a pending provider barrier until a +compact marker in Temporal History proves their producing Workflow Task was +accepted. + +This is a **mirror image** of the shipped +:py:mod:`temporalio.contrib.workflow_streams`, not a replacement for it, and +the two coexist (ADR-001). No name here may begin with +``__temporal_workflow_stream``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from temporalio.contrib.external_workflow_streams._api import ( + ExternalStreamOptions, + ExternalStreamSubscription, + ExternalStreamTopic, + external_stream, + merge, +) +from temporalio.contrib.external_workflow_streams._backend import ( + AppendConflictError, + ParkIntent, + ParkIntentRemoval, + StreamBackend, + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._errors import ( + ConcurrentStreamConsumerError, + ExternalStreamCapacityError, + StreamDecodeError, + StreamError, + StreamIntegrityError, + StreamStorageError, +) +from temporalio.contrib.external_workflow_streams._output_api import ( + ExternalOutputStreamOptions, + ExternalOutputStreamTopic, + external_output_stream, +) +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputReadResult, + OutputStage, + OutputStageConflictError, + OutputStageManifest, + OutputStageNotFoundError, + OutputStageResolutionError, + OutputStageStatus, + OutputStreamBackend, + OutputStreamRecord, + PendingOutputBarrier, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._output_client import ( + ExternalOutputStreamClient, + ExternalOutputStreamClientTopic, + ExternalOutputStreamItem, +) +from temporalio.contrib.external_workflow_streams._output_producer import ( + ExternalOutputStreamProducer, + ExternalOutputStreamProducerTopic, + OutputAppendNotAcknowledgedError, +) +from temporalio.contrib.external_workflow_streams._producer import ( + AppendNotAcknowledgedError, + ChainKeyMismatchError, + ExternalStreamProducer, + ExternalStreamProducerTopic, + PrecedingWriteFailedError, + WakeNotAcknowledgedError, + WorkflowChainKey, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + IdempotencyKey, + Offset, + OffsetComparator, + RecordKind, + StartAtTail, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._wake import WakeRequest + +if TYPE_CHECKING: + from temporalio.contrib.external_workflow_streams._redis import RedisStreamBackend + +__all__ = [ + "AFTER", + "BEGINNING", + "AppendConflictError", + "AppendNotAcknowledgedError", + "ChainKeyMismatchError", + "ConcurrentStreamConsumerError", + "Cursor", + "ExternalStreamCapacityError", + "ExternalOutputStreamClient", + "ExternalOutputStreamClientTopic", + "ExternalOutputStreamItem", + "ExternalOutputStreamOptions", + "ExternalOutputStreamProducer", + "ExternalOutputStreamProducerTopic", + "ExternalOutputStreamTopic", + "ExternalStreamOptions", + "ExternalStreamProducer", + "ExternalStreamProducerTopic", + "ExternalStreamSubscription", + "ExternalStreamTopic", + "StartAtTail", + "IdempotencyKey", + "Offset", + "OffsetComparator", + "OutputAppendNotAcknowledgedError", + "OutputReadResult", + "OutputStage", + "OutputStageConflictError", + "OutputStageManifest", + "OutputStageNotFoundError", + "OutputStageResolutionError", + "OutputStageStatus", + "OutputStreamBackend", + "OutputStreamRecord", + "ParkIntent", + "ParkIntentRemoval", + "PrecedingWriteFailedError", + "RecordKind", + "RedisStreamBackend", + "StreamBackend", + "StreamDecodeError", + "StreamError", + "StreamIntegrityError", + "StreamDirection", + "StreamKey", + "StreamPayloadCodec", + "StreamRecord", + "StreamStorageError", + "StagedOutputRecord", + "PendingOutputBarrier", + "WakeNotAcknowledgedError", + "WakeRequest", + "WorkflowChainKey", + "external_stream", + "external_output_stream", + "merge", +] + + +def __getattr__(name: str) -> Any: + """Lazily expose the Redis provider without importing it in Workflows.""" + if name == "RedisStreamBackend": + from temporalio.contrib.external_workflow_streams._redis import ( + RedisStreamBackend, + ) + + return RedisStreamBackend + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/temporalio/contrib/external_workflow_streams/_annotation.py b/temporalio/contrib/external_workflow_streams/_annotation.py new file mode 100644 index 0000000000..a55cfb0f81 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_annotation.py @@ -0,0 +1,726 @@ +"""The replay annotation codec (P5). + +The opaque bytes Python encodes, Core stores in a marker, and Python decodes on +replay. **Core never parses any of it** -- it appends observation deltas to a +byte buffer and hands the result back. + +That last fact drives the encoding's shape. Core accumulates by byte +concatenation, so an annotation is a sequence of self-delimiting **frames**: + +.. code-block:: text + + annotation := schema_version, header_frame + , (bindings_frame | segment_frame)*, terminal_frame + + header := streams[] // wait_id -> binding + bindings := streams[] // the same, for waits added later + binding := (stream_key, start_cursor, provider_id + , provider_format_version) + segment := run*, segment_end_reason + run := (wait_id, first_offset, last_offset, count, control_positions) + terminal := blocked_snapshot[] // wait_id -> BEGINNING | AFTER(offset) + +The provider identity is carried with each wait so every recorded cursor and +range can be checked against the backend configured on the replaying Worker +before that backend interprets it. + +A subscription may be created at **any** activation of a retained Workflow +Task, which is later than the header frame that already went to Core. Core +appends bytes and never rewrites them, so a header cannot be extended in place; +the binding rides its own frame instead, emitted with the delta of the +activation that registered the wait and before the segment that first records a +run for it. Decoding merges every bindings frame into ``header.streams``, so +what replay reads is one complete ``wait_id -> binding`` table however late a +wait joined. Without this a wait registered after the first delta reaches the +marker as runs and a terminal entry with no stream key, no backend, and no start +cursor -- and replay of *unchanged* code fails as "the Workflow did not create" +that wait. + +Concatenating the deltas of one Workflow Task therefore *is* the annotation, +with no reassembly step that could disagree with Core's. + +Two properties the encoding is built around: + +- **No field is per-record.** A run costs two offsets, a count, and a sparse + control list whether it covers ten records or a hundred thousand. This is + what makes marker size scale with cross-stream schedule transitions rather + than with item count. +- **Both run endpoints are recorded, not a start plus a count.** Backend + offsets are ordered but not dense, so ``(first, count)`` does not determine + where a run ends -- and a deletion inside the range would be undetectable. +""" + +from __future__ import annotations + +import enum +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Final + +import temporalio.exceptions +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + Offset, +) + +__all__ = [ + "MAX_ANNOTATION_BYTES", + "Annotation", + "AnnotationAccumulator", + "AnnotationDecodeError", + "AnnotationHeader", + "Run", + "Segment", + "SegmentEndReason", + "StreamBinding", + "decode_annotation", + "encode_annotation", + "encode_bindings", + "encoded_run_size", + "encoded_segment_size", +] + +SCHEMA_VERSION: Final = 1 +"""Leads the encoding, so a marker written by an older SDK stays readable. + +It is also the extension point that keeps ADR-003's accepted risk bounded: a +per-record content-hash mode can be added later without a format break. + +There are no legacy decoders. The feature is private and unreleased, so the +format has one current schema and rejects everything else. +""" + +MAX_ANNOTATION_BYTES: Final = 64 * 1024 +"""Hard cap on one marker's annotation, well below the server's event limit. + +A constant rather than a guideline: it is enforced while encoding, so an +annotation can never exceed it. The runtime rolls the Workflow Task over +instead of growing the marker. +""" + +ROLLOVER_HIGH_WATER: Final = 0.75 +"""Fraction of the budget at which the encoder asks for a rollover. + +The margin exists because the request has to travel to Core and come back: +a rollover asked for at 100% would arrive too late to prevent the overflow it +was meant to avoid. +""" + +_FRAME_HEADER: Final = 0x01 +_FRAME_SEGMENT: Final = 0x02 +_FRAME_TERMINAL: Final = 0x03 +_FRAME_BINDINGS: Final = 0x04 + +_CURSOR_BEGINNING: Final = 0x00 +_CURSOR_AFTER: Final = 0x01 + + +@enum.unique +class SegmentEndReason(enum.IntEnum): + """Why one activation's segment ended. + + Recorded because replay must reproduce not only *what* was delivered but + *where the runtime returned control with nothing further available* -- + those boundaries are what make ``wait_condition`` predicates fire the same + number of times. + """ + + NO_DATA_AVAILABLE = 1 + BATCH_LIMIT = 2 + FENCE_REACHED = 3 + BUDGET_ROLLOVER = 4 + """This batch continues in the following marker.""" + + +class AnnotationDecodeError(Exception): + """The annotation bytes are not a well-formed annotation.""" + + +class AnnotationBudgetExceeded(temporalio.exceptions.ApplicationError): + """Encoding would push the annotation past :data:`MAX_ANNOTATION_BYTES`. + + Unreachable if the three preventions hold, and each closes a way it used to + be reachable: + + - the closing frames are **reserved** rather than checked, so the terminal + and any late bindings frame always fit (see + :attr:`AnnotationAccumulator.reserved`); + - the runtime **stops delivering** rather than growing a segment it could not + then record, and asks Core to roll the Workflow Task over; + - ``subscribe()`` **refuses** a subscription set whose own header and + terminal could not fit an empty annotation, at the point the Workflow makes + it. + + An :py:class:`temporalio.exceptions.ApplicationError` marked + non-retryable, and that is the substance of this class rather than a detail. + A plain exception here fails the *Workflow Task*, and the server retries + Workflow Task failures forever: the encoding that overflowed overflows again + on every retry, so the Workflow is stuck permanently with no marker, no + terminal, and no rollover ever requested. ADR-007 rejects exactly that + check-and-fail behaviour. Failing the Workflow instead is still bad news, but + it is bounded, visible, and reported once. + """ + + def __init__(self, message: str) -> None: + super().__init__(message, type="AnnotationBudgetExceeded", non_retryable=True) + + +# --- the decoded shape ------------------------------------------------------ + + +@dataclass(frozen=True) +class StreamBinding: + """What a ``wait_id`` was subscribed to, where it started, and through what. + + ``start_cursor`` is explicit rather than re-derived: it is what makes an + annotation with no segments at all -- a subscription to an empty stream -- + a complete replay instruction. + + ``stream_key`` is chosen by Workflow code, so a mismatch is nondeterminism. + ``provider_id`` and ``provider_format_version`` describe the backend + configured on the Worker, so a mismatch is a deployment problem: the + Workflow is unchanged and the store need not be damaged. + """ + + stream_key: StreamKey + start_cursor: Cursor + provider_id: str + """The configured backend's provider when this wait was recorded.""" + provider_format_version: int + """That provider's on-the-wire format version when this wait was recorded. + + Checked on replay rather than merely stored: a backend implementation that + keeps its provider id while changing how it lays records out would otherwise + be read as though nothing had happened. + """ + + +@dataclass(frozen=True) +class Run: + """A maximal set of consecutive deliveries from **one** stream. + + Alternating streams produce one run per delivery; a single-stream batch of + 100,000 records produces one run. That difference is the whole reason the + schedule is encoded as runs. + """ + + wait_id: int + first_offset: Offset + last_offset: Offset + count: int + control_positions: tuple[int, ...] = () + """Sparse relative indices within the run at which a control record sat. + + Sparse, not one kind tag per record: fences are rare by construction, so + this keeps a run's control encoding proportional to the number of fences + rather than to the number of records. + """ + + def __post_init__(self) -> None: + """Validate the run length and sparse control positions.""" + if self.count < 1: + raise ValueError(f"a run covers at least one record, got {self.count}") + if any(not 0 <= p < self.count for p in self.control_positions): + raise ValueError( + f"control positions {self.control_positions} fall outside a run " + f"of {self.count} record(s)" + ) + if list(self.control_positions) != sorted(set(self.control_positions)): + raise ValueError( + f"control positions must be strictly increasing, got " + f"{self.control_positions}" + ) + + +@dataclass(frozen=True) +class Segment: + """One original activation's worth of deliveries. + + ``runs`` may be empty and that is meaningful: an activation that drained + and found nothing still ran one event-loop drain, and replay must reproduce + it or conditions fire a different number of times. + """ + + runs: tuple[Run, ...] + end_reason: SegmentEndReason + + +@dataclass(frozen=True) +class AnnotationHeader: + """The bindings, and nothing else. + + Holds **every** wait's binding once decoded, including the ones that arrived + in a later bindings frame because their ``subscribe()`` call ran after the + header had already gone to Core. Where a binding was carried is an encoding + detail; the decoded table is flat, so replay never has to ask when a wait + joined. + + There is deliberately no annotation-wide provider here. One existed through + schema version 1 and was taken from whichever subscription happened to be + registered first, which made it wrong for every other wait in a + multi-backend annotation. + """ + + streams: dict[int, StreamBinding] = field(default_factory=dict) + schema_version: int = SCHEMA_VERSION + + +@dataclass(frozen=True) +class Annotation: + """A complete marker annotation. + + ``terminal`` is ``None`` only for an annotation still being accumulated. + Core refuses to write a marker for one without a terminal, so a decoded + annotation always has one. + """ + + header: AnnotationHeader + segments: tuple[Segment, ...] = () + terminal: dict[int, Cursor] | None = None + + +# --- primitives ------------------------------------------------------------- + + +def _put_uvarint(out: bytearray, value: int) -> None: + if value < 0: + raise ValueError(f"cannot encode a negative value: {value}") + while True: + byte = value & 0x7F + value >>= 7 + out.append(byte | (0x80 if value else 0)) + if not value: + return + + +def _put_str(out: bytearray, value: str) -> None: + encoded = value.encode() + _put_uvarint(out, len(encoded)) + out += encoded + + +def _put_cursor(out: bytearray, cursor: Cursor) -> None: + """Cursors and offsets are distinct types in the encoding, on purpose. + + A boundary and a record identity are not interchangeable, and an encoding + that spelled them the same way would let one be read as the other. + """ + if cursor.is_beginning: + out.append(_CURSOR_BEGINNING) + return + assert cursor.offset is not None + out.append(_CURSOR_AFTER) + _put_str(out, cursor.offset.serialize()) + + +def _put_stream_key(out: bytearray, key: StreamKey) -> None: + _put_str(out, key.namespace) + _put_str(out, key.workflow_id) + _put_str(out, key.first_execution_run_id) + _put_str(out, key.stream_name) + + +class _Reader: + def __init__(self, data: bytes) -> None: + self._data = data + self._at = 0 + + @property + def exhausted(self) -> bool: + return self._at >= len(self._data) + + def _take(self, n: int) -> bytes: + if self._at + n > len(self._data): + raise AnnotationDecodeError( + f"annotation truncated: wanted {n} byte(s) at offset {self._at}, " + f"only {len(self._data) - self._at} remain" + ) + chunk = self._data[self._at : self._at + n] + self._at += n + return chunk + + def byte(self) -> int: + return self._take(1)[0] + + def uvarint(self) -> int: + value = 0 + shift = 0 + while True: + byte = self.byte() + value |= (byte & 0x7F) << shift + if not byte & 0x80: + return value + shift += 7 + if shift > 63: + raise AnnotationDecodeError("varint is too long to be valid") + + def string(self) -> str: + return self._take(self.uvarint()).decode() + + def cursor(self) -> Cursor: + kind = self.byte() + if kind == _CURSOR_BEGINNING: + return BEGINNING + if kind == _CURSOR_AFTER: + return AFTER(Offset.deserialize(self.string())) + raise AnnotationDecodeError(f"unknown cursor kind {kind:#x}") + + def stream_key(self) -> StreamKey: + return StreamKey(self.string(), self.string(), self.string(), self.string()) + + def bindings(self) -> dict[int, StreamBinding]: + streams: dict[int, StreamBinding] = {} + for _ in range(self.uvarint()): + # Read into locals rather than nesting the calls in the constructor: + # argument evaluation order is not the thing that should decide which + # field a byte lands in. + wait_id = self.uvarint() + stream_key = self.stream_key() + start_cursor = self.cursor() + provider_id = self.string() + provider_format_version = self.uvarint() + streams[wait_id] = StreamBinding( + stream_key, + start_cursor, + provider_id, + provider_format_version, + ) + return streams + + +# --- frame encoding --------------------------------------------------------- + + +def _put_bindings(out: bytearray, streams: Mapping[int, StreamBinding]) -> None: + _put_uvarint(out, len(streams)) + for wait_id in sorted(streams): + binding = streams[wait_id] + _put_uvarint(out, wait_id) + _put_stream_key(out, binding.stream_key) + _put_cursor(out, binding.start_cursor) + _put_str(out, binding.provider_id) + _put_uvarint(out, binding.provider_format_version) + + +def encode_header(header: AnnotationHeader) -> bytes: + """The schema version and header frame, which lead every annotation.""" + out = bytearray() + _put_uvarint(out, header.schema_version) + out.append(_FRAME_HEADER) + _put_bindings(out, header.streams) + return bytes(out) + + +def encode_bindings(streams: Mapping[int, StreamBinding]) -> bytes: + """Bindings for waits that were registered after the header was emitted. + + The same body as the header frame under a different tag. A second *header* + frame would have been ambiguous -- it also carries the schema version, and + a decoder would have to decide whether the later one replaced the earlier -- + whereas a bindings frame says exactly one thing: these waits exist too. + """ + out = bytearray() + out.append(_FRAME_BINDINGS) + _put_bindings(out, streams) + return bytes(out) + + +def _put_run(out: bytearray, run: Run) -> None: + _put_uvarint(out, run.wait_id) + _put_str(out, run.first_offset.serialize()) + _put_str(out, run.last_offset.serialize()) + _put_uvarint(out, run.count) + _put_uvarint(out, len(run.control_positions)) + previous = 0 + for position in run.control_positions: + # Delta-encoded: control positions ascend, so the gaps are smaller + # numbers than the absolute indices and cost fewer varint bytes. + _put_uvarint(out, position - previous) + previous = position + + +def encode_segment(segment: Segment) -> bytes: + out = bytearray() + out.append(_FRAME_SEGMENT) + _put_uvarint(out, len(segment.runs)) + for run in segment.runs: + _put_run(out, run) + out.append(int(segment.end_reason)) + return bytes(out) + + +def encoded_run_size(run: Run) -> int: + """What one run costs inside a segment frame, in bytes. + + Exposed because the runtime has to know what an *open* segment costs before + it is closed: a segment frame that no longer fits the annotation's remaining + budget cannot be deferred to the next annotation -- its deliveries happened + in this Workflow Task and the marker for that task is where replay must find + them -- so the only place to act is before the records that would grow it are + handed over. Measured rather than estimated per record, because a run's cost + is dominated by two provider-supplied offset strings whose length this side + does not choose. + """ + out = bytearray() + _put_run(out, run) + return len(out) + + +def encoded_segment_size(run_sizes: Sequence[int]) -> int: + """What a segment frame costs, given what each of its runs costs. + + Takes the per-run sizes rather than the runs so that a caller extending one + run at a time re-measures only the run it changed. + """ + out = bytearray() + _put_uvarint(out, len(run_sizes)) + # Frame tag, the run count, the runs themselves, and the end-reason byte. + return 1 + len(out) + sum(run_sizes) + 1 + + +def encode_terminal(blocked: dict[int, Cursor]) -> bytes: + out = bytearray() + out.append(_FRAME_TERMINAL) + _put_uvarint(out, len(blocked)) + for wait_id in sorted(blocked): + _put_uvarint(out, wait_id) + _put_cursor(out, blocked[wait_id]) + return bytes(out) + + +def encode_annotation(annotation: Annotation) -> bytes: + """A whole annotation, as Core would have accumulated it.""" + parts = [encode_header(annotation.header)] + parts.extend(encode_segment(s) for s in annotation.segments) + if annotation.terminal is not None: + parts.append(encode_terminal(annotation.terminal)) + return b"".join(parts) + + +def decode_annotation(data: bytes) -> Annotation: + """Parses a complete annotation. + + Accepts an annotation with no terminal so a partially accumulated one can + be inspected; Core is what refuses to *write* a marker for one. + """ + reader = _Reader(data) + schema_version = reader.uvarint() + if schema_version != SCHEMA_VERSION: + raise AnnotationDecodeError( + f"annotation schema version {schema_version} is not supported by this " + f"SDK, which writes and reads version {SCHEMA_VERSION}" + ) + + if reader.byte() != _FRAME_HEADER: + raise AnnotationDecodeError("an annotation must begin with its header frame") + streams = reader.bindings() + + segments: list[Segment] = [] + terminal: dict[int, Cursor] | None = None + while not reader.exhausted: + frame = reader.byte() + if frame == _FRAME_BINDINGS: + if terminal is not None: + raise AnnotationDecodeError("a bindings frame follows the terminal") + for wait_id, binding in reader.bindings().items(): + if wait_id in streams: + # A wait is bound once. A second binding for the same id + # would leave replay choosing between two stream keys, and + # whichever it chose could be the one the records were not + # written to. + raise AnnotationDecodeError( + f"external stream wait {wait_id} is bound twice in one " + "annotation" + ) + streams[wait_id] = binding + elif frame == _FRAME_SEGMENT: + if terminal is not None: + raise AnnotationDecodeError("a segment frame follows the terminal") + runs = [] + for _ in range(reader.uvarint()): + wait_id = reader.uvarint() + first = Offset.deserialize(reader.string()) + last = Offset.deserialize(reader.string()) + count = reader.uvarint() + positions: list[int] = [] + previous = 0 + for _ in range(reader.uvarint()): + previous += reader.uvarint() + positions.append(previous) + runs.append(Run(wait_id, first, last, count, tuple(positions))) + segments.append(Segment(tuple(runs), SegmentEndReason(reader.byte()))) + elif frame == _FRAME_TERMINAL: + if terminal is not None: + raise AnnotationDecodeError("an annotation has one terminal, not two") + terminal = {} + for _ in range(reader.uvarint()): + # Read into locals rather than `terminal[reader.uvarint()] = + # reader.cursor()`: Python evaluates the assigned value before + # the subscript, which would read the two fields backwards. + wait_id = reader.uvarint() + terminal[wait_id] = reader.cursor() + else: + raise AnnotationDecodeError(f"unknown frame tag {frame:#x}") + + return Annotation( + AnnotationHeader(streams, schema_version), tuple(segments), terminal + ) + + +# --- accumulation and the byte budget --------------------------------------- + + +class AnnotationAccumulator: + """Builds one Workflow Task's annotation, one observation delta at a time. + + Mirrors what Core holds: every delta this emits is appended verbatim to + ``ExternalWaitSet.replay_annotation``, so the concatenation of everything + emitted here *is* the annotation the marker carries. There is no separate + assembly step that could disagree. + + The budget is enforced here, while encoding, rather than checked afterwards + -- by the time an oversized annotation exists it is too late to do anything + but discard work. + """ + + def __init__( + self, + header: AnnotationHeader, + *, + max_bytes: int = MAX_ANNOTATION_BYTES, + high_water: float = ROLLOVER_HIGH_WATER, + ) -> None: + """Start an accumulator with its immutable header and byte limits.""" + self._max_bytes = max_bytes + self._high_water_bytes = int(max_bytes * high_water) + self._emitted: list[bytes] = [] + self._size = 0 + self._closing = 0 + self._spill = 0 + self._terminated = False + self._emit(encode_header(header), closing=True) + + @property + def size(self) -> int: + """Bytes accumulated so far, which is what the marker will carry.""" + return self._size + + @property + def reserved(self) -> int: + """Bytes held back from the ordinary run of segments. + + Two parts, and they are held back against different things: + + - **closing** -- the terminal, plus a bindings frame for any wait + registered since the header went out. Neither may ever be refused: both + record something that has already happened, and an annotation whose + terminal does not fit is one Core writes *without* a terminal -- + durable, and undecodable past the frame after it. Only they may spend + this. + - **spill** -- a margin a segment frame may overrun into. The runtime + stops delivering before a segment it could not record, but it prices a + record it has not seen yet, and a provider chooses how long its offsets + are. The margin is what turns a misprice into a rollover instead of a + refusal; an annotation that dips into it has already asked Core to end + the Workflow Task. + """ + return self._closing + self._spill + + def reserve(self, closing: int, *, spill: int = 0) -> None: + """Sets what closing costs, and how much a segment may overrun.""" + self._closing = closing + self._spill = spill + + @property + def headroom(self) -> int: + """Bytes available to a segment before it starts spending the margin. + + What the runtime measures affordability against. Zero does not mean the + annotation is full; it means the next segment overruns into the spill and + the Workflow Task has to end. + """ + return max(0, self._max_bytes - self._closing - self._spill - self._size) + + def fits(self, frame_bytes: int) -> bool: + """Whether a segment of this size fits without spending the margin.""" + return frame_bytes <= self.headroom + + @property + def request_rollover(self) -> bool: + """Whether the next progress report should ask Core to roll over. + + Set once the high-water mark is passed. Core then rolls the task over + *without* a finalization round trip, because the progress report + carrying this flag already carried the terminal. + + The high-water mark alone is not the whole condition -- it is a fraction + of the budget, and a frame can be larger than the fraction that is left. + The runtime adds the other half by refusing to grow a segment it could not + then record; see ``WorkflowStreamRuntime.request_rollover``. + """ + return self._size >= self._high_water_bytes + + @property + def terminated(self) -> bool: + """Whether a terminal frame has already been emitted.""" + return self._terminated + + def accumulated(self) -> bytes: + """Everything emitted so far, concatenated -- what Core now holds.""" + return b"".join(self._emitted) + + def add_bindings(self, streams: Mapping[int, StreamBinding]) -> bytes: + """Encodes bindings for waits registered after the header went out. + + Its own frame rather than an amended header: Core appends the deltas it + is given and never rewrites what it already holds, so the only way a + binding decided later can reach the marker is to be appended after the + bytes that preceded it. + """ + if self._terminated: + raise ValueError("cannot bind a wait after the terminal") + if not streams: + raise ValueError("a bindings frame binds at least one wait") + # A closing frame for budget purposes: the wait it binds was admitted by + # `register`, which charged this frame into the reserve at the time, and + # refusing it now would leave the wait in the terminal with no stream key, + # no backend, and no start cursor -- which replay reports as a wait the + # Workflow never created. + return self._emit(encode_bindings(streams), closing=True) + + def add_segment(self, segment: Segment) -> bytes: + """Encodes one activation's segment and returns it as a delta.""" + if self._terminated: + raise ValueError("cannot add a segment after the terminal") + return self._emit(encode_segment(segment)) + + def add_terminal(self, blocked: dict[int, Cursor]) -> bytes: + """Encodes the blocked snapshot that closes this annotation.""" + if self._terminated: + raise ValueError("an annotation has one terminal, not two") + delta = self._emit(encode_terminal(blocked), closing=True) + self._terminated = True + return delta + + def _emit(self, frame: bytes, *, closing: bool = False) -> bytes: + # A closing frame -- the header, a bindings frame, the terminal -- may + # spend everything, including the margin, because it is what makes the + # annotation readable at all. A segment may spend the margin but not the + # closing reserve: overrunning into the margin is a rollover, overrunning + # past it is the failure this is the last line against. + cap = self._max_bytes if closing else self._max_bytes - self._closing + if self._size + len(frame) > cap: + raise AnnotationBudgetExceeded( + f"encoding {len(frame)} more byte(s) would take the annotation to " + f"{self._size + len(frame)}, past the {cap}-byte limit " + f"({self._max_bytes}-byte budget less {self._closing} reserved for " + "the frames that close it); the runtime should have stopped " + "delivering and asked Core to roll the Workflow Task over before " + "this" + ) + self._emitted.append(frame) + self._size += len(frame) + return frame diff --git a/temporalio/contrib/external_workflow_streams/_api.py b/temporalio/contrib/external_workflow_streams/_api.py new file mode 100644 index 0000000000..90bfa4d857 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_api.py @@ -0,0 +1,957 @@ +"""The Workflow-facing API (P9). + +.. code-block:: python + + streams = external_stream.with_options(idle_timeout=timedelta(seconds=1)) + tokens = streams.topic("tokens", type=str) + + async for token in tokens.subscribe(): + process(token) + +Workflow code never constructs or imports the backend. Its provider instance +holds connections and credentials, lives on the Worker outside the sandbox, +and is reached only through an opaque handle. + +Everything here is a mirror image of the shipped +:py:mod:`temporalio.contrib.workflow_streams`, not a second implementation of +it, so no name may collide -- and in particular no name here may begin with +``__temporal_workflow_stream``. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncGenerator, AsyncIterator, Sequence +from dataclasses import dataclass, field, replace +from datetime import timedelta +from typing import Any, Generic, Protocol + +import temporalio.workflow +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._errors import ( + ConcurrentStreamConsumerError, + StreamError, + classify_read_failure, +) +from temporalio.contrib.external_workflow_streams._record import ( + Cursor, + Offset, + StartAtTail, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._wake import channel_for +from temporalio.types import AnyType + +__all__ = [ + "merge", + "ExternalStreamOptions", + "ExternalStreamSubscription", + "ExternalStreamTopic", + "external_stream", +] + +DEFAULT_IDLE_TIMEOUT = timedelta(seconds=1) +"""How long the complete blocked set waits before parking. + +A property of the **set**, not of one subscription: one idle stream must not +park a Workflow Task another stream is still driving. +""" + +MAX_RECORDS_PER_ACTIVATION = 256 +"""How many records one activation may hand to Workflow code. + +Without a cap, a producer that keeps the buffer non-empty makes one +``activate()`` call never return: the iterator re-fills before every record and +always finds one. Activations run on a thread-pool executor under a **2-second +deadlock timeout**, so the Workflow Task fails -- and every retry hits the same +producer, so the Workflow is stuck permanently rather than merely slowed. Driving +the real iterator against a never-empty buffer consumed 316,086 records in two +seconds and blocked zero times. + +A **record count, never a duration**. A time-based cap would be nondeterministic, +and because the boundary each activation stopped at is recorded in the +annotation, a replay of the same records would cut the segments somewhere else +and diverge from the live run. +""" + +#: Where per-Run subscription state hangs off the SDK's Run object. Reserved, +#: and deliberately not in the ``__temporal_workflow_stream*`` namespace the +#: shipped contrib feature already owns. +_RUN_STATE_ATTR = "__temporal_external_stream_state" + + +class ExternalStreamRuntime(Protocol): + """What Workflow code needs from the Worker, and nothing more. + + An opaque handle across the sandbox boundary. Workflow code never sees the + provider instance configured on the Worker. + """ + + def stream_key(self, stream_name: str) -> StreamKey: + """The full stream identity for a name, from the running Run's chain.""" + ... + + def register( + self, + *, + wait_id: int, + stream_key: StreamKey, + idle_timeout: timedelta, + start_cursor: Cursor | None = None, + start_at_tail: StartAtTail | None = None, + ) -> None: + """Registers a wait with the Worker's subscription manager. + + ``idle_timeout`` is the *configured* value for this one subscription. + The runtime reduces the quiescent set's values to one; this side has no + business doing that reduction, because it can only see one member. + """ + ... + + def drain(self, wait_id: int, max_records: int | None = None) -> list[StreamRecord]: + """Pops buffered records. Performs no I/O.""" + ... + + def delivery_budget_remaining(self) -> int: + """How many more records this activation may hand to Workflow code. + + Lives on the runtime rather than on a subscription because the budget is + an *activation* budget: :py:func:`merge` consumes from several + subscriptions in one activation, and a per-subscription counter would let + *n* streams run *n* times as long. + """ + ... + + def codec_for( + self, value_type: type | None, wait_id: int + ) -> StreamPayloadCodec[Any]: + """The Workflow's DataConverter, bound to a topic's declared type. + + ``wait_id`` selects the *context* the converter carries. It is this + Run's own for a live record, and the one the marker recorded for a wait + a replayed record arrives on -- which an offline ``Replayer``, running + under its own namespace, is not otherwise able to reproduce. + """ + ... + + def new_readiness_future(self) -> asyncio.Future[None]: + """A future the readiness activation handler will resolve. + + Created by the runtime rather than here because it must belong to the + Workflow's own deterministic event loop, which this module has no + business reaching into. + """ + ... + + def record_delivery(self, wait_id: int, record: StreamRecord) -> None: + """Notes one record reaching the Workflow, in observed global order. + + Called for control records too. They are never yielded, but they occupy + offsets inside a run, so the annotation has to know about them. + """ + ... + + def record_consumption(self, wait_id: int, record: StreamRecord) -> None: + """Notes one delivered record leaving the subscription's ready list.""" + ... + + def note_blocked(self, wait_id: int, blocked: bool) -> None: + """Records whether Workflow code is currently waiting on this wait.""" + ... + + def unsubscribe(self, wait_id: int) -> None: + """Ends a wait and stops the Worker serving it. + + Keeps the wait's recorded state -- replay and a Continue-As-New + successor both still need it -- and only makes it unblockable. + """ + ... + + def register_pending(self, wait_id: int, future: asyncio.Future[None]) -> None: + """Registers the future the readiness activation will resolve.""" + ... + + def discard_pending(self, wait_id: int) -> None: + """Forgets a future that is no longer being awaited.""" + ... + + +@dataclass +class _RunState: + """Per-Run subscription bookkeeping. + + Lives on the SDK's object for the Run rather than in a module global, so it + shares the Run's lifetime exactly -- a module global would outlive an evicted + Run and hand its wait ids to the next one. Not on the user's Workflow object + either: that object does not exist while its ``@workflow.init`` constructor + runs, and a constructor may publish. + """ + + runtime: ExternalStreamRuntime | None = None + next_wait_id: int = 1 + #: ``wait_id -> Future``, resolved by the readiness activation handler. + pending: dict[int, Any] = field(default_factory=dict) + + +def _run_holder() -> Any: + """The object per-Run state hangs off: the SDK's runtime for the current Run.""" + return temporalio.workflow._Runtime.current() + + +def _run_state() -> _RunState: + holder = _run_holder() + state = getattr(holder, _RUN_STATE_ATTR, None) + if state is None: + state = _RunState() + setattr(holder, _RUN_STATE_ATTR, state) + return state + + +def _install_runtime( # pyright: ignore[reportUnusedFunction] + holder: Any, runtime: ExternalStreamRuntime +) -> None: + """Gives a Run its handle to the Worker's manager. + + Called by the Worker with its object for the Run, before the Workflow's + constructor runs, so a constructor can subscribe and publish. + """ + state = getattr(holder, _RUN_STATE_ATTR, None) + if state is None: + state = _RunState() + setattr(holder, _RUN_STATE_ATTR, state) + state.runtime = runtime + + +@dataclass(frozen=True) +class ExternalStreamOptions: + """The entry point, and the options every topic under it inherits.""" + + idle_timeout: timedelta = DEFAULT_IDLE_TIMEOUT + + def with_options( + self, *, idle_timeout: timedelta | None = None + ) -> ExternalStreamOptions: + """A copy with the given options replaced. + + Args: + idle_timeout: How long the complete blocked set waits with no record + on any active subscription before parking. + + Raises: + ValueError: The timeout is not positive. Rejected rather than + coerced -- zero would park instantly and a negative value means + nothing at all, so neither can be what a caller intended. + """ + if idle_timeout is not None and idle_timeout <= timedelta(0): + raise ValueError( + f"idle_timeout must be positive, got {idle_timeout}. A " + "non-positive timeout is a configuration error rather than a " + "request to park immediately." + ) + return replace( + self, + idle_timeout=self.idle_timeout if idle_timeout is None else idle_timeout, + ) + + def topic( + self, name: str, *, type: type[AnyType] | None = None + ) -> ExternalStreamTopic[Any]: + """A handle for one stream. + + Args: + name: The stream name. The only place it appears. + type: The value type, used as the decode hint. + """ + if not name: + raise ValueError("a topic needs a non-empty name") + return ExternalStreamTopic(name=name, value_type=type, options=self) + + +@dataclass(frozen=True) +class ExternalStreamTopic(Generic[AnyType]): + """One stream, from the Workflow's side. + + Has ``subscribe`` and no ``publish``: the consumer and producer handles are + distinct types, not one object passed across a process boundary. + """ + + name: str + value_type: type[AnyType] | None + options: ExternalStreamOptions + + def subscribe( + self, + *, + start_cursor: Cursor | None = None, + start_at_tail: StartAtTail | None = None, + ) -> ExternalStreamSubscription[AnyType]: + """Starts a new subscription and returns its async iterator. + + Each call is an **independent** subscription with its own ``wait_id``, + cursor, and park intent -- even two calls naming the same stream. + Delivery is broadcast, so each sees every record from its own cursor. + + ``wait_id`` comes from a per-Run counter in call order, which puts it in + the same hazard class as timers and activities: inserting, removing, or + reordering a ``subscribe()`` call renumbers every later wait in the Run + and must be gated behind ``workflow.patched()``. + + Args: + start_cursor: The boundary the subscription begins after. ``None`` + resumes where the predecessor Run committed this wait, or at + ``BEGINNING`` on a first execution. A boundary the Workflow + names is recorded in the marker's header like the restored one, + so it must be derived deterministically: replay names it again. + start_at_tail: Start at the stream's tail instead, or at its newest + ``last`` records. The Worker resolves the boundary against the + store after this Workflow Task and records it with the + subscription, so replay starts where the live run did rather + than asking the store again. Exclusive with ``start_cursor``. + """ + state = _run_state() + if state.runtime is None: + raise RuntimeError( + "external streams are not configured on this Worker; pass " + "external_stream_backend=... to the Worker" + ) + wait_id = state.next_wait_id + state.next_wait_id += 1 + + stream_key = state.runtime.stream_key(self.name) + # Only a named boundary travels; the runtime derives the default itself + # from the predecessor Run's continuation. + start: dict[str, Any] = ( + {} if start_cursor is None else {"start_cursor": start_cursor} + ) + if start_at_tail is not None: + start["start_at_tail"] = start_at_tail + state.runtime.register( + wait_id=wait_id, + stream_key=stream_key, + # Passed on registration rather than read back from the subscription + # later, because the runtime is what holds the quiescent set and + # reduces it with `min`. Leaving it out is not a smaller default -- + # it silently substitutes DEFAULT_IDLE_TIMEOUT for whatever + # `with_options` was given, so no configured value can ever reach + # the reduction and every set parks after one second. + idle_timeout=self.options.idle_timeout, + **start, + ) + # The channel the stream's writers notify. Asked of the SDK's object for + # the Run rather than of the stream runtime, because the answer is part + # of the Run's completion and completions are the Run's to emit; it + # also decides, from the server it talks to, whether the Run is the + # channel's listener already or has to be subscribed. + subscribe = getattr(_run_holder(), "subscribe_stream_channel", None) + if subscribe is not None: + subscribe(channel_for(stream_key)) + # Registering a wait is not blocking on one. The quiescent snapshot is a + # request to Core to retain the Workflow Task and, once the idle timer + # expires, to park it; a subscription Workflow code has not begun + # iterating has no coroutine waiting on it, so including it would ask + # Core to hold a Workflow Task open -- and eventually park it -- for a + # wait nothing will ever resolve. The first `_await_readiness` is what + # enters the blocked state, and that transition is what the wait + # generation counts. + state.runtime.note_blocked(wait_id, False) + return ExternalStreamSubscription( + topic=self, wait_id=wait_id, stream_key=stream_key, state=state + ) + + +async def merge( + *subscriptions: ExternalStreamSubscription[Any], +) -> AsyncGenerator[tuple[ExternalStreamSubscription[Any], Any], None]: + """Iterates several subscriptions as one wait, in delivery order. + + Yields ``(subscription, value)`` rather than bare values: the streams may + carry different types, and a merged value whose origin had to be guessed + from its shape would be unusable for anything but logging. + + The subscriptions are already one wait set -- every subscription a Run holds + is -- so this adds no coordination of its own. What it adds is the ability to + *wait on all of them at once*: iterating them one at a time would block on + the first while records piled up on the second, and the idle timer covering + the set would then fire against a Workflow that was not actually idle. + + Each pass takes **at most one record from each subscription**, in ``wait_id`` + order, resuming after the subscription that last took one. All three of + those are load-bearing: + + - *In ``wait_id`` order*, which is what makes the interleaving reproduce. + Records that arrived in one batch across two streams have no inherent + order between them, so an order that depended on dict iteration, on + arrival time, or on which watcher happened to run first would replay + differently than it ran. ``wait_id`` comes from a per-Run counter in + ``subscribe()`` call order, so replay reconstructs the same total order + from the Workflow code itself, and the pass then depends only on which + waits have a record ready -- which replay reconstructs from the recorded + segments. + - *At most one record*, which is what makes it a merge rather than a + priority order. Draining one subscription's whole ready list first lets + the lowest ``wait_id`` spend the entire + :py:data:`MAX_RECORDS_PER_ACTIVATION` budget by itself, and the next + activation starts the same pass in the same order, so a continuously + backlogged first stream starves every later one forever. + - *Resuming after the last take*, which is what makes "at most one each" add + up to fairness across activations rather than only within a pass. The + budget covers the merged set, so a pass can be cut anywhere inside it, and + a pass that always restarted at the lowest wait id would be cut in the + same place every activation and re-privilege the same prefix forever. With + 257 ready subscriptions the 257th is never asked at all -- ``_fill`` + returns on the spent budget before it reaches the manager, so that wait is + not merely served nothing, it is never enquired after. With 100 the first + 56 take one record per activation more than the rest, and the gap grows + without bound. Rotating the start is what makes the skew between any two + continuously ready streams what it is claimed to be: a single record. + + A control record spends the subscription's turn: it is consumed, because it + occupies an offset inside a run, and the pass moves on. Filling one record + at a time is also what keeps the budget exact -- a fill that took the whole + remaining budget into one subscription's ready list would let the rest of + that list be consumed after the budget was spent, and would strand it there, + since the completion path re-arms readiness from the *manager's* buffers and + knows nothing about records already popped out of them. + + The recorded delivery schedule follows: alternating records across two + streams encode as one run per delivery, because a run is a maximal + consecutive stretch from a single wait. + + The delivery budget covers the whole set rather than each member, so a merge + over *n* never-empty streams still returns after + :py:data:`MAX_RECORDS_PER_ACTIVATION` records. A per-subscription budget + would let the activation run *n* times as long, which is the same deadlock + with a larger constant in front of it. + + **Ending a merge does not end the subscriptions it merged.** Stopping at a + value -- breaking out of the ``async for``, or ``aclose()`` on the generator + -- leaves every subscription open, at its own cursor, and consumable again + either singly or in another merge. Closing them is + :meth:`ExternalStreamSubscription.close`, and that is a separate decision, + because it is the one that drops what the buffer still holds without + consuming it. Nothing is left blocked in the meantime: the merged wait + unblocks every member when it ends, so a Workflow that stops consuming and + goes on to wait for something else is not holding a Workflow Task open for a + stream nobody is reading. + """ + ordered = sorted(subscriptions, key=lambda s: s.wait_id) + if not ordered: + raise ValueError("merge() needs at least one subscription") + if len({s.wait_id for s in ordered}) != len(ordered): + raise ValueError( + "merge() was given the same subscription twice; each one is a " + "separate wait with its own cursor, and merging one with itself " + "would deliver every record to it twice" + ) + + # The wait that last took a record, so the next pass resumes after it rather + # than restarting at the lowest wait id. + # + # Local to this generator, recorded nowhere, and that is what makes it + # replay-safe rather than merely convenient. Replay serves a drain from the + # **front** of the recorded segment and only while that front belongs to the + # asking wait, so a wait asked out of turn gets nothing and the record stays + # for whoever asks next. Every active wait is still asked exactly once per + # pass, so the yielded sequence is the recorded global order whatever + # position the pass starts at. Under replay the budget is unbounded, so + # passes are not cut where they were cut live and this cursor generally ends + # up somewhere else than it did -- which steers nothing but *later live* + # fairness, and no live schedule was ever recorded for History to contradict. + resume_after = 0 + while True: + # A closed subscription leaves the set rather than blocking it: it has + # no coroutine behind it, so including it in the wait would ask Core to + # retain the Workflow Task for a wait nothing can resolve. + active = [s for s in ordered if not s._finished] + if not active: + return + # The first wait past the last take, wrapping to the front when there is + # none -- which is also what happens when the wait the cursor named has + # since closed and left the set. + start = next((i for i, s in enumerate(active) if s.wait_id > resume_after), 0) + delivered_any = False + for step in range(len(active)): + subscription = active[(start + step) % len(active)] + subscription._fill(1) + record = subscription._peek() + if record is None: + continue + delivered_any = True + # Advanced only where a record was actually taken, control records + # included, because those spend the turn and the budget too. A wait + # that had nothing has not had its turn, and moving the cursor past + # it would cost it the turn it never got -- which is the starvation + # this exists to end, reintroduced from the other side. + resume_after = subscription.wait_id + if record.is_control: + subscription._commit(record) + continue + # Decoded before consumption is committed, for the same reason as in + # `_iterate`: a decode that raises must leave the record where a + # later pass can still find it. + value = subscription._decode(record) + subscription._commit(record) + yield subscription, value + if not delivered_any: + # Nothing anywhere: block on all of them at once. Whichever wait + # Core resolves first wakes this, and the next pass picks it up. + if await _await_any_readiness(active): + continue + + +async def _await_any_readiness( + subscriptions: Sequence[ExternalStreamSubscription[Any]], +) -> bool: + """Blocks until any one of the waits is resolved. Returns whether it skipped. + + **One scoped operation over the whole group**, and both halves of that follow + from it. + + Every wait is marked blocked while it runs, because the quiescent snapshot + must name the **complete** set the Workflow is waiting on: a set missing one + member would let Core park the Workflow Task while that member was still + live. + + And every wait leaves the blocked set when it ends, however it ends -- a + resolved future, the double-check below returning early, or cancellation. + Only one member can have a record in hand afterwards, and :meth:`_peek` clears + the flag only for that one, so a member left blocked here is one no coroutine + is awaiting: the quiescent snapshot names it, and Core retains and eventually + parks the Workflow Task on behalf of a group wait that is already over. The + next genuine group wait marks every member blocked again and advances its + wait generation, which is a new blocking epoch rather than a continuation of + this one. + """ + runtime = subscriptions[0]._state.runtime + assert runtime is not None + # Every member is checked before any is registered. A merge that refused + # half-way would leave the earlier waits registered and blocked with no + # coroutine behind them, which is the state that asks Core to retain a + # Workflow Task for nobody. + for subscription in subscriptions: + subscription._refuse_a_second_waiter() + futures: list[asyncio.Future[None]] = [] + try: + # Inside the `try`, so that registering the group is covered by the same + # cleanup that ends it. A raise part-way through would otherwise leave + # the members already registered blocked with their futures unreachable + # -- the very state the refusal loop above is careful not to create. + for subscription in subscriptions: + runtime.note_blocked(subscription.wait_id, True) + future = runtime.new_readiness_future() + runtime.register_pending(subscription.wait_id, future) + # Also on the subscription, so `close()` can resume a merge that is + # sitting on this wait: the runtime's map is keyed for the side that + # resolves readiness, and closing is neither that side nor this one. + subscription._pending_future = future + futures.append(future) + # The same last look the single-subscription path takes, for the same + # reason: a record buffered before these waits were registered had its + # readiness reported to nobody. And for the same reason as there, it + # goes through `_fill` and so finds nothing once the budget is spent -- + # otherwise a merge over a busy stream would resume immediately and the + # activation would never end. + # + # One record, matching the pass above: a fill that took the whole + # remaining budget here would put records into a ready list the budget + # can no longer pay for, and the completion path re-arms readiness from + # the manager's buffers, which no longer hold them. + for subscription in subscriptions: + subscription._fill(1) + if subscription._ready: + return True + await asyncio.wait(futures, return_when=asyncio.FIRST_COMPLETED) + return False + finally: + for subscription in subscriptions: + # Unblocking belongs here, beside the removal of the futures it is + # the counterpart of: a wait whose readiness future has just been + # discarded is a wait nothing can be waiting on. See this function's + # docstring for what leaving one blocked would ask of Core -- and + # note that a member the loop above never reached is already + # unblocked, so this is a no-op for it rather than a claim about it. + runtime.note_blocked(subscription.wait_id, False) + subscription._pending_future = None + runtime.discard_pending(subscription.wait_id) + for future in futures: + if not future.done(): + future.cancel() + + +class ExternalStreamSubscription(Generic[AnyType]): + """One subscription's async iterator over decoded values. + + **One consumer.** The cursor, the readiness future, and the blocked flag are + all the subscription's rather than an iterator's, so two coroutines waiting on + it at once is refused with + :class:`temporalio.contrib.external_workflow_streams.ConcurrentStreamConsumerError` + rather than served -- see :meth:`_refuse_a_second_waiter` for what sharing + them would do. Two consumers of the same *stream* is a supported shape and + the way to ask for it is a second ``subscribe()``: delivery is a broadcast + (ADR-021), so each wait gets every record and keeps its own cursor. + + Iterating the same subscription again after the previous consumer has stopped + is fine, and resumes where it left off. + """ + + def __init__( + self, + *, + topic: ExternalStreamTopic[AnyType], + wait_id: int, + stream_key: StreamKey, + state: _RunState, + ) -> None: + """Bind one wait ID and stream key to its per-Run state.""" + self._topic = topic + self._wait_id = wait_id + self._stream_key = stream_key + self._state = state + self._ready: list[StreamRecord] = [] + #: Set by :meth:`close`, and the only thing that ends iteration. + self._finished = False + #: The readiness future currently being awaited, if any. Held here as + #: well as on the runtime because :meth:`close` has to be able to resume + #: the coroutine sitting on it, and the runtime's map is keyed by wait + #: id for the *resolving* side rather than for this one. + self._pending_future: asyncio.Future[None] | None = None + + @property + def wait_id(self) -> int: + """The Run-local identity Core uses for this subscription.""" + return self._wait_id + + @property + def stream_key(self) -> StreamKey: + """The durable stream identity this subscription reads.""" + return self._stream_key + + @property + def idle_timeout(self) -> timedelta: + """This subscription's configured contribution to the wait-set timeout.""" + return self._topic.options.idle_timeout + + def __aiter__(self) -> AsyncIterator[AnyType]: + """A fresh generator over the *same* cursor, not an independent view. + + Which is why a second one running concurrently is refused rather than + interleaved: see :meth:`_refuse_a_second_waiter`. + """ + return self._iterate() + + async def _iterate(self) -> AsyncIterator[AnyType]: + async for _offset, value in self.records(): + yield value + + async def records(self) -> AsyncIterator[tuple[Offset, AnyType]]: + """Each value with the provider offset it was read from. + + Same delivery, same consumption, same commit order as iterating values. + A reader that has to name where it got to, or hand a position to + something outside the Workflow, cannot do it from the values alone. + """ + while not self._finished: + # Re-filled before *every* record rather than once per batch. A + # record buffered while Workflow code was doing something else -- a + # timer, an activity, another stream -- has already had its readiness + # reported and consumed, so nothing will report it again. Blocking + # without looking would wait forever on a record that is already + # here. Filling is a buffer pop and costs nothing. + self._fill() + record = self._peek() + if record is not None: + if record.is_control: + self._commit(record) + continue + # Decoded first, committed second. Consumption is a claim that + # Workflow code *received* this record, and the claim is only + # true once a value exists: a decode that raises out of a + # mismatched converter never yields anything. Committing first + # makes the claim false in exactly that case -- the record + # leaves the ready list, the buffer it came from is already + # empty, and the consumption cursor a Continue-As-New successor + # resumes from has stepped over a record nothing ever saw. Left + # uncommitted it stays at the head of the ready list and the + # next `__anext__` retries it. + # + # Synchronous, so there is no longer a point *inside* decoding + # at which cancellation can land at all: the record either + # becomes a value or raises, and neither outcome can leave the + # ready list half-consumed. + value = self._decode(record) + offset = record.offset + self._commit(record) + assert offset is not None + yield offset, value + continue + await self._await_readiness() + + def _fill(self, limit: int | None = None) -> None: + """Moves whatever is buffered into this subscription's ready list. + + Draining is a buffer pop and never touches the backend -- the record is + already here or it is not, and if it is not, only Core can say when to + look again. + + Bounded by the activation's remaining delivery budget, **and charged + against it here**. A drain that took the whole buffer would be handed + straight back by a producer that keeps refilling it, and this activation + would never return. Charging where the records move rather than where they + are later consumed is what makes the bound a reservation: a drain that + checked the budget and charged nothing left the same room visible to the + next subscription's drain, so two subscriptions consumed in two + independent coroutines took the whole budget each. + + ``limit`` bounds it further, for a caller that will consume fewer records + than the budget allows. :py:func:`merge` passes 1: it takes one record + per subscription per pass, and anything it pulled out of the manager's + buffer and did not take would be stranded there -- the completion path + re-arms readiness from the manager's buffers, so a record already popped + out of one has nothing left to announce it. + """ + if self._ready: + # Nothing to check the budget for: these records are already charged + # -- by the drain that took them, and again by the `begin_activation` + # of any later activation that inherits them, since each activation + # pays for what it may hand over. A check here would refuse a record + # this activation has already been billed for. + return + assert self._state.runtime is not None + budget = self._state.runtime.delivery_budget_remaining() + if limit is not None: + budget = min(budget, limit) + if budget <= 0: + # Nothing is taken even though records are sitting right here, so the + # caller blocks and the activation ends. The records are not lost: + # the completion path re-reports readiness for every buffer that is + # still non-empty, which is what brings the next activation in. + return + drained = self._state.runtime.drain(self._wait_id, budget) + for record in drained: + # Recorded for *every* record, control ones included: they occupy + # offsets inside a run, so a run's count includes them and their + # positions go in `control_positions`. Omitting them would make + # replay's range read find more records than the marker claims. + self._state.runtime.record_delivery(self._wait_id, record) + # Control records stay in the list rather than being filtered out here, + # so that consumption advances past them in order. A filtered control + # record would be neither consumed nor left behind, and the continuation + # cursor would step over it. + self._ready = list(drained) + + def _peek(self) -> StreamRecord | None: + """The next ready record, or ``None``. Commits nothing. + + Leaving this wait's blocked state is right here rather than at commit + time: a record is in hand, so no coroutine is waiting on this wait, and a + quiescent snapshot taken while the value is being decoded must not name + it. + """ + if not self._ready: + return None + assert self._state.runtime is not None + self._state.runtime.note_blocked(self._wait_id, False) + return self._ready[0] + + def _commit(self, record: StreamRecord) -> None: + """Pops the record and records that Workflow code now has it. + + Consumption is not delivery. A batch is delivered whole, but a Workflow + that stops iterating part-way through has consumed only its prefix, and + a successor Run resuming from the delivery cursor would step over the + rest. + + Called only once the record has actually become a value -- or, for a + control record, once it has been skipped, which is the whole of what + receiving one means. + + Spends no delivery budget: the drain that put this record here already + did. What it does tell the runtime is that the record has left the ready + list, which is what stops the next activation being charged for it. + """ + assert self._state.runtime is not None + assert self._ready and self._ready[0] is record + self._ready.pop(0) + self._state.runtime.record_consumption(self._wait_id, record) + + def _refuse_a_second_waiter(self) -> None: + """Refuses a second coroutine blocking on this subscription. + + One subscription, one consumer. Everything a blocked wait is found + through is single-slot -- :attr:`_pending_future` here, and the runtime's + pending map keyed by wait id -- so a second waiter does not queue behind + the first, it *replaces* it: readiness resolves only the newer future, its + ``finally`` removes the map entry, and the older one is left unreachable by + the readiness activation and by :meth:`close` alike. That coroutine is + stranded for the life of the Run, and the shared blocked flag can even say + the wait is no longer blocked while it is still sitting there. + + Refused here, at the point a *second waiter appears*, rather than at + ``__aiter__``. Two coroutines inside ``_iterate`` at once is only possible + while one of them is suspended, and the only suspension point in there is + this wait -- a generator suspended at its ``yield`` is not inside + ``_iterate`` at all, it is between ``__anext__`` calls. So this catches + every genuinely concurrent consumer, while a guard on the iterator would + also refuse the sequential shape that works today: taking some records, + breaking out of the loop, and coming back to the same subscription later. + A broken-out-of ``async for`` leaves its generator suspended rather than + closed, so the iterator-level guard cannot tell that shape from this one + (ADR-037). + """ + if self._pending_future is None: + return + raise ConcurrentStreamConsumerError( + f"external stream wait {self._wait_id} already has a coroutine " + "blocked on it. A subscription is a single consumer: it has one " + "cursor and one readiness future, so a second waiter would replace " + "the first and strand it permanently. Iterate one subscription from " + "one coroutine, or subscribe again -- a second subscription to the " + "same stream is its own wait with its own cursor, and delivery is a " + "broadcast." + ) + + async def _await_readiness(self) -> None: + """Blocks until the readiness activation resolves this wait. + + The future is resolved from the ``ResolveExternalStreamWaits`` branch of + the activation dispatch, which is the only thing that knows a record + arrived. + """ + assert self._state.runtime is not None + self._refuse_a_second_waiter() + # Entering the blocked state is what the wait generation counts, and is + # what later makes a readiness notification for *this* block + # distinguishable from one for a block already resolved. + self._state.runtime.note_blocked(self._wait_id, True) + future = self._state.runtime.new_readiness_future() + self._state.runtime.register_pending(self._wait_id, future) + self._pending_future = future + try: + # Look once more, now that this wait is registered. A record buffered + # between the last drain and this registration had its readiness + # reported while nothing was waiting for it, and no second + # notification is coming -- the watcher has already moved its + # prefetch cursor past it. Checking after registering is what closes + # that window: anything earlier is found here, anything later + # resolves the future. + # + # It goes through `_fill`, so it observes the delivery budget like + # every other drain. A refill that ignored the budget would hand this + # wait a record the instant the budget stopped it, and the activation + # would go right back to never returning -- the double-check would + # undo the cap rather than merely coexist with it. + self._fill() + if self._ready: + return + await future + except BaseException: + # Abandoned rather than resolved: cancellation, most often because + # Workflow code raced this stream against a timer and cancelled the + # loser. Nothing is awaiting this wait any more, so it must leave the + # blocked set -- a wait left in it is named by the quiescent + # snapshot, and Core would retain and eventually park the Workflow + # Task for a coroutine that no longer exists. + self._state.runtime.note_blocked(self._wait_id, False) + raise + finally: + self._pending_future = None + self._state.runtime.discard_pending(self._wait_id) + + def _decode(self, record: StreamRecord) -> AnyType: + """This record as a value of the topic's declared type. + + **Synchronous, and that is the point.** The Workflow thread runs one + half of ``DataConverter.decode`` -- ``from_payloads``, which needs the + topic's type and performs no I/O. The other half, external-payload + retrieval and the user's ``PayloadCodec``, already ran on the Worker's + loop before this record was buffered, exactly as it does for every other + payload an activation carries. Awaiting a codec here would perform real + I/O inside a deterministic event loop, synthesize Workflow commands out + of the codec's internals, and put arbitrary user work under the + 2-second deadlock timeout. + + Failures of *either* half surface from here, because here is where the + record would have become a value. A preparation error carried over from + the Worker's loop is raised at the delivery it belongs to rather than + where it happened: the watcher has no Workflow to tell, and a record + that is never delivered must not fail anything. + """ + assert self._state.runtime is not None + # By wait, not by Run: the asynchronous half ran under the stream key + # the record's own wait was bound to, and on replay that key comes from + # the marker rather than from this Run's identity. Passing the wait is + # what lets the two halves agree. + codec = self._state.runtime.codec_for(self._topic.value_type, self._wait_id) + prepared = getattr(record, "prepared_payload", None) + failed = getattr(record, "prepare_error", None) + try: + if failed is not None: + raise failed + if prepared is None: + prepared = codec.parse_unprepared(record.payload) + return codec.convert(prepared) + except StreamError: + # Already classified -- a storage failure reaching the payload store + # stays row one rather than being relabelled as the consumer's + # converter mismatch. + raise + except Exception as err: + # `range_validated=True` for both delivery paths, and for the same + # reason. Replay validated the recorded range before this record was + # prepared; live delivery read the record out of the backend and the + # provider guarantees a record's bytes cannot change once written + # (ADR-003). Either way the bytes are the bytes that were written, + # so what failed is the consumer's converter -- row three -- and not + # the stream (ADR-015). + raise classify_read_failure(range_validated=True, cause=err) from err + + def close(self) -> None: + """Ends this subscription: iteration stops and the wait goes away. + + Synchronous and idempotent, because the ordinary shape is a ``finally`` + that cannot know whether the iterator already ended. + + Three things happen, and each answers a distinct way an abandoned + subscription is visible: + + - the wait leaves the **blocked set**, so no quiescent snapshot names it + and Core is not asked to retain -- and eventually park -- the Workflow + Task for it; + - any coroutine sitting on this wait's readiness future is **resumed** + rather than cancelled, so ``_iterate`` reaches its loop condition and + the iteration ends with ``StopAsyncIteration`` instead of raising + ``CancelledError`` into Workflow code that merely closed a stream; + - records already drained but never handed over are **dropped without + being consumed**, so the consumption cursor stops short of them and a + Continue-As-New successor receives them. + + The Worker-side half -- stopping this wait's watcher and removing its + park intent -- is the manager's, and is reached through the runtime + rather than from here, because it has to be scheduled onto the Worker's + loop: this runs on the Workflow thread, where creating a task is + silently ineffective. + + What closing does **not** do is forget the subscription. Its recorded + binding is what replay needs to know what this wait was reading, and its + cursor is what stops a Continue-As-New successor restarting the stream + from the beginning. + """ + if self._finished: + return + self._finished = True + runtime = self._state.runtime + if runtime is None: + return + runtime.note_blocked(self._wait_id, False) + runtime.discard_pending(self._wait_id) + self._ready.clear() + future = self._pending_future + self._pending_future = None + if future is not None and not future.done(): + future.set_result(None) + runtime.unsubscribe(self._wait_id) + + +#: The entry point Workflow code uses. +external_stream = ExternalStreamOptions() diff --git a/temporalio/contrib/external_workflow_streams/_backend.py b/temporalio/contrib/external_workflow_streams/_backend.py new file mode 100644 index 0000000000..6c05adaa5b --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_backend.py @@ -0,0 +1,515 @@ +"""The stream backend contract (P2). + +What a provider must implement to be registrable. The contract is small on +purpose -- every operation here exists because replay or parking cannot be made +correct without it. + +The obligations that are easy to get subtly wrong, and that the conformance +suite exists to catch: + +- **The range read is inclusive of both endpoints; the watch is exclusive.** + These are two different operations, not one with a flag. Replay issues an + inclusive read for a range the marker already names, and never asks the + backend what comes next. Implementing the inclusive read with exclusive + semantics is invisible until the first replay drops a record. +- **Offsets are compared by the provider's rule, not lexically.** Redis stream + ids compare as numeric ``(ms, seq)`` tuples, and string comparison is wrong + the moment the millisecond component changes width. +- **A cursor is a boundary, never the identity of a future record.** No + operation may require naming a record that does not exist yet, or a consumer + parked at the tail could not resume. +- **Append is idempotent on identity, not on key alone.** Reusing a + ``(session_id, sequence)`` with byte-identical content is a no-op returning + the original offset; reusing it with *different* bytes is an error, because + silently accepting it would let a retried producer overwrite history. +- **Records are immutable.** Mandatory, and checked at registration rather than + compensated for at runtime. +""" + +from __future__ import annotations + +import abc +import enum +import time +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import timedelta +from typing import ClassVar, Final + +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + Cursor, + IdempotencyKey, + Offset, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._wake import WakeTransport + +__all__ = [ + "AppendConflictError", + "ParkIntent", + "ParkIntentRemoval", + "StreamBackend", + "StreamDirection", + "StreamKey", +] + + +@dataclass(frozen=True) +class ParkIntent: + """One subscription's declaration that it is about to park. + + The key is ``(stream key, wait_id)``; everything here is the value. + """ + + wait_id: int + cursor: Cursor + """Where this subscription stopped consuming. What ``recheck`` reads past.""" + park_generation: int + """The quiescence generation being parked. A wake Signal names this.""" + run_id: str + """The Run that installed it -- part of the value so a new Run's intent + replaces its predecessor's for the same key rather than accumulating.""" + + def __post_init__(self) -> None: + """Reject the generation reserved for unparked wakes.""" + if self.park_generation < 1: + # 0 is the reserved unparked-wake sentinel, so a real park + # generation can never take it. + raise ValueError( + f"a park generation is a quiescence generation and starts at 1, " + f"got {self.park_generation}" + ) + + +@enum.unique +class StreamDirection(enum.Enum): + """Which endpoint role a physical stream serves. + + Direction is part of provider identity rather than a user-controlled prefix + on ``stream_name``. This permits an input and output topic to share a name + without sharing records, idempotency state, or coordination metadata. + """ + + INPUT = "input" + """Records produced externally and consumed by a Workflow.""" + + OUTPUT = "output" + """Records produced by a Workflow or Activity for an external client.""" + + +@dataclass(frozen=True) +class StreamKey: + """What identifies a stream, for the whole life of a Continue-As-New chain. + + ``first_execution_run_id`` rather than the current Run ID: it prevents + collisions after Workflow ID reuse while staying stable across the chain, + so a new Run continues the same stream rather than starting a fresh one. + """ + + namespace: str + workflow_id: str + first_execution_run_id: str + stream_name: str + direction: StreamDirection = StreamDirection.INPUT + """The endpoint direction, defaulting to the existing input protocol. + + The default preserves the meaning of every four-argument ``StreamKey`` + constructor used by existing input providers and applications. + """ + + def __post_init__(self) -> None: + """Require every component of the durable stream identity.""" + for field_name in ( + "namespace", + "workflow_id", + "first_execution_run_id", + "stream_name", + ): + if not getattr(self, field_name): + raise ValueError(f"a stream key needs a non-empty {field_name}") + if not isinstance(self.direction, StreamDirection): + raise TypeError( + "a stream key direction must be a StreamDirection, got " + f"{type(self.direction).__name__}" + ) + + def __str__(self) -> str: + """Return a slash-delimited diagnostic representation.""" + return ( + f"{self.namespace}/{self.workflow_id}/" + f"{self.first_execution_run_id}/{self.direction.value}/{self.stream_name}" + ) + + +class AppendConflictError(Exception): + """An idempotency key was reused with different bytes. + + Distinct from a successful idempotent retry, which returns the original + offset and appends nothing. Accepting this quietly would let a retried + producer attempt with changed input rewrite what a consumer may already + have delivered. + """ + + def __init__(self, key: IdempotencyKey) -> None: + """Create a conflict for the key reused with different content.""" + super().__init__( + f"idempotency key {key} was already used with different content; " + "an append is idempotent on identity, not on the key alone" + ) + self.key = key + + +#: How long a watch blocks before returning empty, when no timeout is given. +DEFAULT_WATCH_BLOCK: Final = timedelta(seconds=5) + + +class ParkIntentRemoval(enum.Enum): + """What a conditional park-intent removal found at the key. + + Three outcomes rather than a Boolean, because two of them mean the intent + the consumer named is gone and one means it never got out of the way, and + the consumer acts differently on each: a removal that happened ends a wake + suppression that may owe a record an announcement, while a live intent that + replaced it does not. + """ + + REMOVED = 1 + """The intent matched the named Run and park generation, and is gone.""" + + ABSENT = 2 + """Nothing was installed at the key. + + Reported for a key that is already clear -- including one this provider + itself cleared on a call whose reply never arrived. It is *not* the same as + a key some other intent holds. + """ + + MISMATCH = 3 + """A different Run or park generation is installed, and was left alone.""" + + +class StreamBackend(abc.ABC): + """A stream provider. + + Instances live **outside** the Workflow sandbox, on the Worker. Workflow + code never imports or holds the configured instance. + """ + + guarantees_immutability: ClassVar[bool | None] = None + """Whether this provider guarantees a record's bytes cannot change. + + ``None`` means *undeclared*, which is what an implementer who never + considered the question leaves it as. Registration requires ``True``, so + both "forgot" and "cannot" are rejected at Worker construction -- loudly, + before any Workflow can name the backend -- rather than at replay, quietly, + after data has already been consumed. + + The guarantee is what makes the four cheap range checks sufficient: given + it, the only damage replay has to detect is a record that is no longer + there. + """ + + provider_id: ClassVar[str] = "" + """Stable identifier recorded in every annotation header.""" + + provider_format_version: ClassVar[int] = 1 + """Bumped when this provider changes how it stores a record.""" + + # --- required operations ------------------------------------------------ + + @abc.abstractmethod + async def append(self, key: StreamKey, record: StreamRecord) -> StreamRecord: + """Appends one record, idempotently on ``(session_id, sequence)``. + + Returns: + The record as stored, with its assigned offset. For a repeat append + of byte-identical content this is the **original** offset and + nothing new is written. + + Raises: + AppendConflictError: The key was used before with different bytes. + """ + + @abc.abstractmethod + async def read_range( + self, key: StreamKey, first: Offset, last: Offset + ) -> list[StreamRecord]: + """Reads ``[first, last]`` -- **inclusive of both endpoints**. + + This is the replay read. The marker already names the range, so this + must never consult "what comes next"; it returns exactly what is + present in the closed interval, in increasing offset order, and says + nothing about what is missing. Detecting a missing record is the + caller's job, and it can only do it if this read is honest about what + it found. + """ + + @abc.abstractmethod + async def read_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = DEFAULT_WATCH_BLOCK, + ) -> list[StreamRecord]: + """Reads records **strictly after** ``after``, blocking up to ``block``. + + This is the live read. ``after`` is a boundary, so ``BEGINNING`` means + the start of the stream and ``AFTER(x)`` means "whatever follows the + record at ``x``" -- which must work whether or not such a record exists + yet, because that is exactly the state a consumer parked at the tail is + in. + + Returns an empty list when the block elapses with nothing new. Blocking + is a provider concern and never reaches the Workflow thread. + """ + + @abc.abstractmethod + def compare_offsets(self, left: Offset, right: Offset) -> int: + """Three-way comparison under **this provider's** ordering rule. + + Synchronous and pure: replay validation calls it once per adjacent pair + of a range, and an ordering that needed I/O would put backend latency + inside a validation loop. + """ + + async def tail_cursor(self, key: StreamKey, *, before_last: int = 0) -> Cursor: + """The boundary the newest ``before_last`` records begin after. + + With ``before_last=0`` it is the boundary after the newest record, so a + read from it sees only what is appended after this call. ``BEGINNING`` + when the stream holds fewer records than asked for. Resolved on the + Worker, never on the Workflow thread, and recorded with the + subscription so replay reads it from the marker instead of asking + again. A provider that cannot answer leaves this as it is, and a + subscription that asks for a tail start fails when the Worker resolves + it. + """ + raise NotImplementedError( + f"{type(self).__name__} cannot resolve a start {before_last} records " + f"before the tail of {key}; subscribe with a cursor instead" + ) + + # --- waking ------------------------------------------------------------- + + wake_transport: WakeTransport = "auto" + """How wakes for this backend's streams reach the server. + + Held on the backend because it is the one object both the producer and the + consuming Worker are configured with. A provider may set it per instance. + ``"auto"`` notifies the stream's channel and falls back to the Signal on a + server without channels. + """ + + def wake_counter_for(self, offset: Offset) -> int: + """The order of ``offset`` as a positive integer, or 0 when unknown. + + While a wake for a stream is pending, the server folds later wakes into + it and keeps the highest counter's position, so the receiving task sees + the latest one. A counter derived from the store's own order ranks + positions the same way for producers and Workers, where a clock-derived + one does not. Implement it when offsets map to integers that increase + with the provider's order. + """ + del offset + return 0 + + def wake_counter_now(self) -> int: + """The counter for a wake that reports no position, from the clock. + + A Worker's shutdown sweep wakes a subscription it leaves behind without + knowing where the store stands. Its counter still has to sit above every + counter the store has handed out so far, or the server folds the wake + into a pending one and the reader stays parked. A provider whose + counters follow its offsets derives this from the current time by the + same rule, with the largest sequence the rule allows. The default is the + clock in nanoseconds, which is the order the base counter has. + """ + return time.time_ns() + + # --- parking (P2b) ------------------------------------------------------ + + supports_leased_claims: ClassVar[bool] = False + """Whether :meth:`claim_park_generation` expires claims and permits takeover. + + An unleased claim introduces a failure mode with no recovery: a producer + that crashes between claiming a generation and sending its wake Signal + strands that generation, and every other producer concludes the wake is + already handled -- so the parked Workflow waits forever with data sitting + in the stream. + + A provider that cannot lease must declare ``False`` and expose + **observe-only** semantics instead: :meth:`claim_park_generation` always + grants, and every producer signals idempotently. That costs duplicate + Signals, which are harmless, rather than lost ones, which are not. + """ + + @abc.abstractmethod + async def install_park_intent(self, key: StreamKey, intent: ParkIntent) -> None: + """Records that one **subscription** intends to park. + + Keyed ``(stream key, wait_id)``, never by stream key alone: two + subscriptions in one Workflow to the same stream are two independent + waits, and a stream-keyed intent would have one overwrite the other -- + after which only one of them could ever be woken. + + The current Run ID is part of the intent's *value*, not its key, so a + new Run's intent deterministically replaces its predecessor's rather + than accumulating beside it. + """ + + @abc.abstractmethod + async def remove_park_intent(self, key: StreamKey, wait_id: int) -> None: + """Removes one subscription's intent. Idempotent.""" + + @abc.abstractmethod + async def remove_park_intent_if_matches( + self, + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + """Atomically removes an intent only when its identity still matches. + + The comparison and removal must be one backend operation. A delayed + cleanup can overlap a Continue-As-New successor installing a new intent + at the same ``(stream key, wait_id)``; implementing this as + :meth:`park_intent` followed by :meth:`remove_park_intent` can delete + that successor's live park. + + All three outcomes must be reported distinctly. Collapsing + :attr:`ParkIntentRemoval.ABSENT` into + :attr:`ParkIntentRemoval.MISMATCH` tells the consumer that someone + else's park is in the way when in fact the key is clear -- which is what + a retry after a lost reply sees, the reply to a delete this provider + already performed. + """ + + @abc.abstractmethod + async def park_intent(self, key: StreamKey, wait_id: int) -> ParkIntent | None: + """The installed intent, if any.""" + + @abc.abstractmethod + async def recheck(self, key: StreamKey, wait_id: int) -> bool: + """Whether any record now sits past the installed intent's cursor. + + Called after every intent is installed, and it is this recheck -- not + the intent -- that closes the append/park race: a producer appends + before it observes the park generation, so an append is either seen + here or paired with a wake Signal. + """ + + @abc.abstractmethod + async def claim_park_generation( + self, + key: StreamKey, + wait_id: int, + park_generation: int, + *, + claimant: str, + lease: timedelta, + ) -> bool: + """Tries to take responsibility for waking ``park_generation``. + + Returns ``True`` when this caller now holds the claim -- including when + it is renewing a claim it already held, so a producer can extend rather + than losing it mid-flight. + + A provider that declares ``supports_leased_claims = False`` must always + return ``True``: observe-only semantics, where every producer signals. + """ + + @abc.abstractmethod + async def parked_wait_ids(self, key: StreamKey) -> list[int]: + """Every subscription with an installed intent on this stream. + + A producer knows the stream it published to and nothing about the + Workflow's subscriptions -- ``wait_id`` is allocated by a per-Run counter + inside ``subscribe()``, which no producer can see. The other five + parking operations all take a ``wait_id``, so without this one a producer + that has just appended has no way to address a wake at all. + + Enumeration rather than a new concept: intents are already keyed + ``(stream key, wait_id)``, and this is the ``wait_id`` half of that key + for one stream. + """ + + @abc.abstractmethod + async def current_park_generation(self, key: StreamKey, wait_id: int) -> int | None: + """The generation a producer should name in its wake Signal. + + ``None`` means no confirmed park is installed for this subscription, in + which case a producer sends an *unparked* wake -- generation 0 -- rather + than staying silent. + """ + + # --- provided on top of the required operations ------------------------- + + async def watch( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int = 100, + block: timedelta | None = DEFAULT_WATCH_BLOCK, + ) -> AsyncIterator[StreamRecord]: + """Yields records forever, starting strictly after ``after``. + + A convenience over :meth:`read_after` for prefetch loops. The cursor + advances to ``AFTER(last delivered)``, so a resumption after any number + of empty blocks continues from where delivery stopped rather than from + where the stream currently is. + """ + cursor = after + while True: + batch = await self.read_after( + key, cursor, max_records=max_records, block=block + ) + for record in batch: + yield record + if batch: + last = batch[-1].offset + assert last is not None, "a stored record always has an offset" + cursor = AFTER(last) + + def is_before(self, left: Offset, right: Offset) -> bool: + """Whether ``left`` precedes ``right`` in this provider's order.""" + return self.compare_offsets(left, right) < 0 + + def strictly_increasing(self, offsets: list[Offset]) -> bool: + """Whether ``offsets`` ascend under this provider's ordering rule.""" + return all(self.compare_offsets(a, b) < 0 for a, b in zip(offsets, offsets[1:])) + + +def _validate_backend( # pyright: ignore[reportUnusedFunction] + backend: object, +) -> StreamBackend: + """Validate the backend before a Worker can consume through it.""" + if not isinstance(backend, StreamBackend): + raise TypeError( + f"external stream backend is a {type(backend).__name__}, which is not " + f"a {StreamBackend.__name__}" + ) + + declared = type(backend).guarantees_immutability + if declared is not True: + raise ValueError( + f"external stream backend {type(backend).__name__} declares " + f"guarantees_immutability = {declared!r}. Configuration requires True: " + "the provider must guarantee that a record's bytes cannot change " + "once written, because replay validates presence, count, order, and " + "control positions only. A provider that cannot make that guarantee " + "does not satisfy the backend contract." + ) + + if not type(backend).provider_id: + raise ValueError( + f"external stream backend {type(backend).__name__} declares no " + "provider_id; every replay annotation header records it, so replay " + "could not verify which provider wrote a marker" + ) + return backend diff --git a/temporalio/contrib/external_workflow_streams/_codec.py b/temporalio/contrib/external_workflow_streams/_codec.py new file mode 100644 index 0000000000..6e64ef2620 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_codec.py @@ -0,0 +1,206 @@ +"""Payload encoding for stream records (P4). + +A record's ``payload`` bytes are a serialized +:py:class:`temporalio.api.common.v1.Payload`, not a bare value encoding. Keeping +the envelope means the payload's own ``encoding`` and ``messageType`` metadata +travel with it, so a consumer decodes exactly what the producer wrote rather +than having to be told out of band. + +Producer and consumer must use the **same** ``DataConverter``, including any +codec. A mismatch is detected on the consumer, and is classified as a decode +failure rather than as stream integrity loss -- the stream is fine; the +configuration is not. + +The one thing this module classifies itself is an **external payload store that +cannot be reached**. That is row one of the taxonomy, not row three: the bytes +in the stream are a reference, the value does not exist until the reference is +fetched, and a driver that cannot fetch it raises whatever its client raises. +Left unlabelled it reaches the consumer as an unclassified failure on a record +whose range validated, which the classification rule turns into a decode +failure -- sending an operator to change a converter during a storage outage. +Nothing further down can tell the two apart, so the label is applied at the call +that knows. + +Decoding on the consumer is **two halves, run in two different places**: + +- :py:meth:`StreamPayloadCodec.prepare`, on the **Worker's loop**: + external-payload retrieval and the user's ``PayloadCodec``. Arbitrary + asynchronous work, and none of it depends on the value's type. +- :py:meth:`StreamPayloadCodec.convert`, on the **Workflow thread**: + ``from_payloads`` with the topic's declared type. Synchronous, performs no + I/O, and is the only half that needs to know the type at all. + +That is the same split the Worker already applies to every other payload an +activation carries: ``decode_activation`` is awaited before ``activate()`` is +handed to the executor, and the payload converter runs inside it. A record whose +codec ran on the Workflow thread would perform real I/O inside a deterministic +event loop, turn the codec's awaits into Workflow commands, and put a +multi-second KMS round trip under a 2-second deadlock timeout. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Generic + +import temporalio.api.common.v1 +import temporalio.converter +from temporalio.contrib.external_workflow_streams._errors import ( + StreamError, + StreamStorageError, +) +from temporalio.types import AnyType + +__all__ = ["StreamPayloadCodec"] + + +@dataclass(frozen=True) +class StreamPayloadCodec(Generic[AnyType]): + """Encodes and decodes one topic's values. + + The type is carried by the topic -- ``topic("tokens", type=str)`` -- and is + passed to the converter as a decode hint, so a topic declared as ``str`` + yields ``str`` rather than whatever the default converter happens to infer. + """ + + data_converter: temporalio.converter.DataConverter + value_type: type[AnyType] | None = None + + async def encode(self, value: AnyType) -> bytes: + """Serializes one value to a record's payload bytes.""" + payloads = await self.data_converter.encode([value]) + if len(payloads) != 1: + # `DataConverter.encode` is documented as free to return fewer + # payloads than values. One value must still produce one payload, + # and a converter that collapses it would silently drop a record. + raise ValueError( + f"encoding one stream value produced {len(payloads)} payloads; " + "exactly one is required" + ) + return payloads[0].SerializeToString() + + async def decode(self, payload: bytes) -> AnyType: + """Parses a record's payload bytes back into a value. + + Both halves at once. Correct anywhere a coroutine may await arbitrary + work -- the producer side, and tests -- and **wrong on the Workflow + thread**, which is why the consumer side calls :meth:`prepare` and + :meth:`convert` separately instead. + + Raises whatever the converter or codec raises. Classification into the + failure taxonomy belongs to the caller, which is the only side that + knows whether the record's range validated first. + """ + return self.convert(await self.prepare(payload)) + + async def prepare(self, payload: bytes) -> temporalio.api.common.v1.Payload: + """The asynchronous half of decoding, and the half that has no type. + + ``DataConverter.decode`` is three steps: external-payload retrieval, the + user's ``PayloadCodec``, and the payload *converter*. The first two are + arbitrary asynchronous work -- a network fetch, a KMS round trip -- and + neither of them depends on the topic's declared type. They belong on the + Worker's loop, which is exactly where the Worker already awaits them for + every other payload an activation carries. + + Returns the payload as the payload converter will see it. What is left + is :meth:`convert`, which is synchronous and needs no I/O. + """ + parsed = temporalio.api.common.v1.Payload() + parsed.ParseFromString(payload) + # A deliberate reach into two private helpers rather than an oversight: + # this split is the whole point, and `DataConverter` exposes no public + # "everything except the payload converter" entry point. `decode()` + # itself is these two calls followed by `from_payloads`, so preparing + # here and converting in `convert` is byte-for-byte the same work in the + # same order -- only on two different threads. + try: + retrieved = await self.data_converter._external_retrieve_payload_sequence( + [parsed] + ) + except StreamError: + raise + except Exception as err: + # Row one of the taxonomy, and it has to be labelled *here*. An + # external-storage driver raises whatever its client raises -- a + # bare `ConnectionError` for an unreachable payload store -- and the + # converter does not wrap it. Everything downstream sees an + # unclassified exception on a record whose range validated, and the + # classification rule for that is row three: the operator is told to + # align a converter that is in fact fine while the payload store is + # the thing that is down. Nothing below this call can tell the two + # apart, because by then the only evidence is the exception type the + # driver chose. + # + # A payload store is as much a stream read as the stream backend is: + # the record's bytes are a reference, and the value does not exist + # until they are fetched. Transient by the same argument, and clears + # the same way. + raise StreamStorageError( + "an external stream record's payload could not be retrieved from " + f"external storage: {err}" + ) from err + decoded = await self.data_converter._decode_payload_sequence(retrieved) + if len(decoded) != 1: + raise ValueError( + f"preparing one stream record produced {len(decoded)} payloads; " + "exactly one is required" + ) + return decoded[0] + + def parse_unprepared(self, payload: bytes) -> temporalio.api.common.v1.Payload: + """The envelope parse alone, for a converter with nothing to prepare. + + A ``DataConverter`` with no codec and no external storage has an empty + asynchronous half, so a record that never passed through + :meth:`prepare` still converts correctly. One with either of them does + not, and the difference is not detectable from the bytes: a codec's + output is just another payload. Rather than return a plausible wrong + value, this refuses -- an unprepared record on the Workflow thread means + the record reached it by a path that does not prepare, which is a + routing defect and not a user's converter mismatch. + """ + # Asked of the two public members rather than through the converter's + # `_decode_payload_has_effect`, which is the same condition but is marked + # in its own source as a temporary shortcircuit to be removed. This + # refusal is a safety property; it should not stop working the day an + # unrelated cleanup lands upstream. + if ( + self.data_converter.payload_codec is not None + or self.data_converter.external_storage is not None + ): + raise RuntimeError( + "an external stream record reached the Workflow thread without " + "its DataConverter's asynchronous half having been applied. " + "Preparing it here would run a payload codec or an external " + "payload fetch inside activate(); every delivery path must " + "prepare records on the Worker's loop instead." + ) + parsed = temporalio.api.common.v1.Payload() + parsed.ParseFromString(payload) + return parsed + + def convert(self, prepared: temporalio.api.common.v1.Payload) -> AnyType: + """The synchronous half: the topic's declared type, applied. + + The only step that needs the type, and the only one a Workflow thread + may run: ``from_payloads`` performs no I/O and awaits nothing, which is + what every ordinary activation payload's conversion already does on this + same thread. + """ + type_hints: list[type] | None = ( + None if self.value_type is None else [self.value_type] + ) + values = self.data_converter.payload_converter.from_payloads( + [prepared], type_hints + ) + if len(values) != 1: + raise ValueError( + f"decoding one stream record produced {len(values)} values; " + "exactly one is required" + ) + return values[0] + + def with_type(self, value_type: type[Any] | None) -> StreamPayloadCodec[Any]: + """This codec bound to a different topic's declared type.""" + return StreamPayloadCodec(self.data_converter, value_type) diff --git a/temporalio/contrib/external_workflow_streams/_continuation.py b/temporalio/contrib/external_workflow_streams/_continuation.py new file mode 100644 index 0000000000..e721cee25c --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_continuation.py @@ -0,0 +1,161 @@ +"""The Continue-As-New cursor (P15). + +A stream spans a whole Continue-As-New chain, so a new Run must resume from the +position its predecessor committed -- on replay as well as live. + +**A cursor is never derived from mutable backend state, on any Run.** Reading the +current position from the backend, or from coordination state keyed by chain, +would give replay whatever the backend holds *now* rather than what the Run +originally started from, and two replays of one history could then diverge. The +position therefore travels in a reserved internal header on the Continue-As-New +command, persisted in the new Run's ``WorkflowExecutionStarted`` and restored +before any subscription is established (ADR-022). + +It populates the same annotation-header ``start_cursor`` field that a first +execution fills with ``BEGINNING``, so replay reads an explicit starting boundary +in **every** case -- including the case where the stream was empty for the +subscription's entire life. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import temporalio.api.common.v1 +from temporalio.contrib.external_workflow_streams._annotation import ( + AnnotationDecodeError, + _put_cursor, + _put_str, + _put_uvarint, + _Reader, +) +from temporalio.contrib.external_workflow_streams._record import Cursor + +__all__ = [ + "CONTINUATION_HEADER", + "Continuation", + "decode_continuation", + "encode_continuation", + "read_continuation_header", + "write_continuation_header", +] + +CONTINUATION_HEADER = "__temporal_external_stream_continuation" +"""Reserved, and in this feature's namespace rather than ``workflow_stream``'s. + +``temporalio.contrib.workflow_streams`` reserves ``__temporal_workflow_stream_*`` +for a different feature that coexists with this one (ADR-001). +""" + +CONTINUATION_ENCODING = b"binary/plain" +"""Serialized by this module, not by the user's ``DataConverter``. + +The header is written by one Run and read by the next, and a chain whose Runs +were deployed with different converter configuration would otherwise restart at +an unreadable cursor -- silently, since an unreadable cursor looks exactly like +no cursor at all. +""" + +_SCHEMA_VERSION = 1 +"""The only continuation schema this unreleased feature understands.""" + + +@dataclass(frozen=True) +class Continuation: + """Where each subscription had got to when the Run continued as new. + + Keyed by ``wait_id`` so two subscriptions to one stream restore + independently: they are separate waits with their own cursors, and a + stream-keyed continuation would restart one of them at the other's position. + + The three maps beside ``cursors`` are the **binding the cursor was produced + under**, and each is there because without it a cursor can be restored into + something it does not describe: + + - ``stream_names`` so a renumbered subscription is detectable. Restoring a + cursor onto a differently-numbered wait would resume one stream at + another's offset, which the backend would accept and no later check would + catch. + - ``provider_ids`` and ``provider_format_versions`` because a Worker can + be reconfigured to a different implementation. Marker replay checks both + before interpreting recorded offsets; the first live read in a successor + Run happens before that Run has written a marker that could perform the + check, so the continuation has to carry them itself. + """ + + cursors: dict[int, Cursor] + stream_names: dict[int, str] + provider_ids: dict[int, str] = field(default_factory=dict) + provider_format_versions: dict[int, int] = field(default_factory=dict) + + +def encode_continuation(continuation: Continuation) -> bytes: + """Encode current continuation state deterministically.""" + out = bytearray() + _put_uvarint(out, _SCHEMA_VERSION) + _put_uvarint(out, len(continuation.cursors)) + # Sorted, so the same state always encodes to the same bytes. The header is + # re-derived from live state every time the terminal command is built, and + # bytes that depended on dict insertion order would put a different cursor + # in front of the successor on a Workflow Task the server retried. + for wait_id in sorted(continuation.cursors): + _put_uvarint(out, wait_id) + _put_cursor(out, continuation.cursors[wait_id]) + _put_str(out, continuation.stream_names.get(wait_id, "")) + _put_str(out, continuation.provider_ids[wait_id]) + _put_uvarint(out, continuation.provider_format_versions[wait_id]) + return bytes(out) + + +def decode_continuation(raw: bytes) -> Continuation: + """Decode and validate a supported continuation schema.""" + reader = _Reader(raw) + version = reader.uvarint() + if version != _SCHEMA_VERSION: + raise AnnotationDecodeError( + f"the Continue-As-New cursor header is schema version {version}, but " + f"this Worker understands {_SCHEMA_VERSION}. The previous Run of this " + "chain was executed by a newer SDK." + ) + cursors: dict[int, Cursor] = {} + stream_names: dict[int, str] = {} + provider_ids: dict[int, str] = {} + provider_format_versions: dict[int, int] = {} + for _ in range(reader.uvarint()): + wait_id = reader.uvarint() + cursors[wait_id] = reader.cursor() + stream_names[wait_id] = reader.string() + provider_ids[wait_id] = reader.string() + provider_format_versions[wait_id] = reader.uvarint() + return Continuation( + cursors=cursors, + stream_names=stream_names, + provider_ids=provider_ids, + provider_format_versions=provider_format_versions, + ) + + +def write_continuation_header( + continuation: Continuation, +) -> temporalio.api.common.v1.Payload: + """Wrap encoded continuation state in its reserved Payload envelope.""" + return temporalio.api.common.v1.Payload( + metadata={"encoding": CONTINUATION_ENCODING}, + data=encode_continuation(continuation), + ) + + +def read_continuation_header( + headers: dict[str, temporalio.api.common.v1.Payload] | None, +) -> Continuation | None: + """The continuation a predecessor Run left, or ``None`` on a first execution. + + ``None`` is not a failure: it is what every chain's first Run sees, and the + subscription then starts at ``BEGINNING``. + """ + if not headers: + return None + payload = headers.get(CONTINUATION_HEADER) + if payload is None: + return None + return decode_continuation(payload.data) diff --git a/temporalio/contrib/external_workflow_streams/_errors.py b/temporalio/contrib/external_workflow_streams/_errors.py new file mode 100644 index 0000000000..6025ca3f72 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_errors.py @@ -0,0 +1,269 @@ +"""The external stream failure taxonomy (P18). + +Four outcomes, deliberately not collapsed, because each has a different operator +response. All four reach the server through the same channel -- a Workflow +activation completion is ``oneof status { Success | Failure }`` -- so what +distinguishes them is the **error type and the metric**, never the retry +behavior of the Workflow Task. + +**There is no protocol-level "non-retryable Workflow Task failure": the server +retries Workflow Task failures regardless of cause.** Alerting must therefore +key on the metrics here rather than on Workflow Task failure counts, which a +transient backend outage also increments. + +:: + + ============================ ========================== ==================== + Condition Error type Operator response + ============================ ========================== ==================== + Backend unreachable/erroring ``StreamStorageError`` None -- clears when + the backend recovers + Recorded offset missing, ``StreamIntegrityError`` Repair or restore + expired, reordered, or the backend, or + miscounted terminate the Run + Bytes intact but undecodable ``StreamDecodeError`` Align the consumer's + converter with the + producer's + Annotation does not match ordinary nondeterminism Fix or version the + the subscriptions made Workflow code + ============================ ========================== ==================== + +There is deliberately **no** ``workflow_failure_exception_types`` registration +here: integrity loss *blocks* a Workflow rather than terminating it (ADR-014). +Retention loss is usually an operational error and usually repairable, and a +blocked Workflow can be resumed after repair where a failed one cannot. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +import temporalio.common +import temporalio.exceptions + +__all__ = [ + "ConcurrentStreamConsumerError", + "ExternalStreamCapacityError", + "StreamDecodeError", + "StreamError", + "StreamIntegrityError", + "StreamStorageError", + "classify_read_failure", +] + + +class StreamError(Exception): + """Base class for every external stream failure this module classifies.""" + + +class StreamStorageError(StreamError): + """The backend was unreachable, timed out, or returned an error. + + Transient by definition: it clears when the backend recovers, so an + operator has nothing to do. This is the one row of the taxonomy that a + Workflow Task failure count alone describes adequately. + """ + + +class StreamIntegrityError(StreamError): + """A recorded range no longer reads back as it was written. + + Raised when an offset the annotation names is missing, when a range + contains the wrong number of records, when ordering is not strictly + increasing, or when control positions do not match. Those four checks are + the whole of replay validation, and they are sufficient precisely because + every provider guarantees a record's bytes cannot change (ADR-003) -- so + the only damage replay must detect is a record that is no longer there. + + **This must never resolve to an alternate stream result.** Substituting a + later record for a deleted one would hand Workflow code a different history + than the one its commands were derived from. + """ + + +class StreamDecodeError(StreamError): + """A record was present and intact but could not be decoded. + + Separated from ``StreamIntegrityError`` because it is a configuration + error on the *consumer* -- a DataConverter or codec that does not match the + producer's -- and reporting it as integrity loss sends an operator to + restore a backend that was never damaged (ADR-015). + """ + + +class ExternalStreamCapacityError(temporalio.exceptions.ApplicationError): + """This Workflow's subscription set cannot be recorded in an annotation. + + Deliberately **not** one of the four taxonomy rows above. Those describe a + stream or a converter behaving unexpectedly at read time; this describes + Workflow code asking for more than the marker format can carry, and it is + known at ``subscribe()`` rather than at read time. The remedy is in the + Workflow: fewer streams, or shorter names. + + Non-retryable, and raised from ``subscribe()`` rather than from the + completion path, because both halves of that matter. A capacity limit + discovered while encoding a completion fails the *Workflow Task*, the server + retries Workflow Task failures regardless of cause, and the same subscription + set encodes to the same oversized header every time -- a Workflow stuck + forever with nothing durable to show why. Raised where the subscription is + made, it is an error in the Workflow's own code path, deterministic under + replay, and reported once. + """ + + def __init__(self, message: str) -> None: + """Create a deterministic, non-retryable capacity failure.""" + super().__init__( + message, type="ExternalStreamCapacityError", non_retryable=True + ) + + +class ConcurrentStreamConsumerError(temporalio.exceptions.ApplicationError): + """A second coroutine tried to wait on a subscription already being waited on. + + A subscription is **one consumer**. It has one cursor, one readiness future, + and one entry in the runtime's pending map, so a second coroutine blocking on + it replaces the first everywhere the first can be found: readiness resolves + the newer waiter, its ``finally`` clears the entry, and the older future + becomes unreachable by the readiness activation *and* by ``close()``. The + Workflow is then permanently stuck on a future nothing can resolve -- and + because the blocked-set state is shared too, Core may not even be retaining a + Workflow Task on that coroutine's behalf. + + Not one of the four taxonomy rows above, and for the same reason as + :class:`ExternalStreamCapacityError`: this is Workflow code asking for + something the design does not offer, not a stream or a converter misbehaving. + Non-retryable for the same reason too -- a second consumer is deterministic, + so a Workflow Task failure would be retried into the identical failure forever + with nothing durable to show why. + + Iterating a subscription again *after* the previous consumer has stopped is + not this: the check is on a live waiter rather than on the iterator, so the + ordinary shape of taking some records, doing something else, and coming back + to the same subscription keeps working (ADR-037). + """ + + def __init__(self, message: str) -> None: + """Create a deterministic, non-retryable consumer failure.""" + super().__init__( + message, type="ConcurrentStreamConsumerError", non_retryable=True + ) + + +#: Metric names. Operators are expected to alert on the integrity metric +#: specifically, since the transient row increments Workflow Task failures too. +METRIC_INTEGRITY: Final = "temporal_external_stream_integrity_failure" +METRIC_DECODE: Final = "temporal_external_stream_decode_failure" +METRIC_STORAGE: Final = "temporal_external_stream_storage_failure" +METRIC_SHUTDOWN_WAKE_FAILED: Final = "temporal_external_stream_shutdown_wake_failed" + + +@dataclass(frozen=True) +class StreamMetrics: + """The counters this taxonomy reports through. + + Each failure class gets its own counter rather than one counter with a + ``reason`` attribute, so an alert on integrity loss cannot be diluted by a + backend outage sharing the same series. + """ + + integrity: temporalio.common.MetricCounter + decode: temporalio.common.MetricCounter + storage: temporalio.common.MetricCounter + shutdown_wake_failed: temporalio.common.MetricCounter + + @staticmethod + def create(meter: temporalio.common.MetricMeter) -> StreamMetrics: + return StreamMetrics( + integrity=meter.create_counter( + METRIC_INTEGRITY, + "External stream replay found a recorded range that no longer reads back " + "as written. Repair the backend or terminate the Run; this does not clear " + "on its own.", + ), + decode=meter.create_counter( + METRIC_DECODE, + "External stream record bytes were intact but could not be decoded. The " + "stream is fine; the consumer's converter or codec does not match the " + "producer's.", + ), + storage=meter.create_counter( + METRIC_STORAGE, + "External stream backend was unreachable, timed out, or errored. " + "Transient; clears when the backend recovers.", + ), + shutdown_wake_failed=meter.create_counter( + METRIC_SHUTDOWN_WAKE_FAILED, + "A Worker shutdown wake Signal was still unacknowledged when the graceful " + "shutdown grace period expired.", + ), + ) + + def counter_for( + self, error_type_name: str + ) -> temporalio.common.MetricCounter | None: + """The counter for an error class *name*, or ``None`` for anything else. + + By name, because the side that reports these has only a name: a failure + raised on the Workflow thread is converted inside ``activate()``, and + what the Worker gets back is a completion whose application failure + carries the exception's class name. Keeping the mapping here rather than + at that call site is what stops the two from drifting apart -- the same + reason :meth:`record` exists for the side that still holds the + exception. + + ``None`` for every other failure, which is what keeps row four -- + ordinary nondeterminism -- and plain Workflow bugs out of these series. + """ + return { + StreamIntegrityError.__name__: self.integrity, + StreamDecodeError.__name__: self.decode, + StreamStorageError.__name__: self.storage, + }.get(error_type_name) + + def record(self, error: BaseException) -> None: + """Increments the counter matching ``error``'s class, if any.""" + if isinstance(error, StreamIntegrityError): + self.integrity.add(1) + elif isinstance(error, StreamDecodeError): + self.decode.add(1) + elif isinstance(error, StreamStorageError): + self.storage.add(1) + + +def classify_read_failure( + *, range_validated: bool, cause: BaseException +) -> StreamError: + """The mechanical classification rule, in one place. + + Unconditional, because structural immutability is required of every + provider: + + **If the range validated, the bytes are the bytes that were written**, + so any subsequent failure is a decode failure. Only a missing offset or + a range that fails validation is integrity loss. + + Args: + range_validated: Whether the four range checks passed for the range the + failing record belongs to. + cause: What actually went wrong, attached as the new error's cause. + + Returns: + The error to raise. Never raises on its own -- the caller decides + whether to raise, record a metric, or both. + """ + # A storage failure is about reaching the backend at all, so it is never + # reclassified by whether a range validated: there was no range to read. + if isinstance(cause, StreamStorageError): + return cause + + if range_validated: + err: StreamError = StreamDecodeError( + f"external stream record could not be decoded: {cause}" + ) + else: + err = StreamIntegrityError( + f"external stream recorded range failed validation: {cause}" + ) + err.__cause__ = cause + return err diff --git a/temporalio/contrib/external_workflow_streams/_manager.py b/temporalio/contrib/external_workflow_streams/_manager.py new file mode 100644 index 0000000000..2323b42e16 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_manager.py @@ -0,0 +1,2877 @@ +"""The per-Worker subscription and watcher manager (P8). + +**Any design in which ``_apply`` reads the backend is wrong by construction, not +merely slow.** ``_WorkflowInstanceImpl.activate()`` is synchronous, runs on a +thread-pool executor under a 2-second deadlock timeout, and drives a custom +deterministic event loop in which real network I/O cannot be awaited at all. + +So the manager lives out here, on the Worker's own asyncio loop, owns every +backend connection and watcher task, and communicates with the Workflow thread +through a **bounded, thread-safe buffer per subscription**: + +=========== ====================== ===================================== +Step Where What +=========== ====================== ===================================== +Register Workflow thread Records the wait. Non-blocking. +Prefetch Manager loop Reads ahead into the bounded buffer. +Readiness Manager loop Reports **only after** a record is + buffered -- never on a bare socket + event. +Drain Workflow thread Pops from the buffer. Nothing else. +=========== ====================== ===================================== + +That "only after buffered" rule is what makes the resulting activation +guaranteed non-blocking. Readiness for an unbuffered record would produce an +activation whose drain must block, reintroducing exactly the deadlock hazard +this structure exists to remove. + +Three cursors, and conflating them is how a speculative read becomes a durable +claim: + +============= ==================== ============ ================= +Cursor Advances on Owner Survives eviction +============= ==================== ============ ================= +``committed`` marker commit only the marker yes +``delivery`` hand-off to workflow the instance no +``prefetch`` buffering the manager no +============= ==================== ============ ================= + +``prefetch`` is speculative: reading a record is not consuming it, and consuming +it is not committing it. The manager may only move it *backwards* to +``committed``, never forwards past what a marker has committed. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import threading +from collections import deque +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field, replace +from datetime import timedelta +from typing import Any + +import temporalio.converter +from temporalio.contrib.external_workflow_streams._annotation import StreamBinding +from temporalio.contrib.external_workflow_streams._backend import ( + ParkIntent, + ParkIntentRemoval, + StreamBackend, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._errors import ( + StreamError, + StreamStorageError, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._replay import ( + ReplayPlan, + ReplaySegment, + build_replay_plan, +) +from temporalio.contrib.external_workflow_streams._wake import new_sender_identity + +__all__ = [ + "PreparedRecord", + "ReadinessResult", + "StreamSubscriptionManager", + "Subscription", +] + +logger = logging.getLogger(__name__) + + +@contextlib.contextmanager +def _as_storage_failure(what: str) -> Iterator[None]: + """Reports a backend failure on an activation path as row one. + + The park handshake is a backend transaction an activation is waiting on, and + a backend that is unreachable or erroring is the taxonomy's *transient* + row: nothing for an operator to do, and it clears when the backend + recovers. Left as whatever the provider raised, it reaches the server as an + anonymous Workflow Task failure with no cause and no counter, + indistinguishable from a bug in the Workflow's own code. + + Not used for the intent removal a resolve performs, which deliberately logs + rather than raises: that one is cleanup on the delivery path, and failing a + Workflow Task for it would trade a stale intent for a repeated Workflow + Task. + + Anything already classified passes through untouched, so an integrity + failure is never relabelled as a transient one. + """ + try: + yield + except asyncio.CancelledError: + raise + except StreamError: + raise + except Exception as err: + raise StreamStorageError(f"{what} failed: {err}") from err + + +DEFAULT_BUFFER_SIZE = 256 +"""How many records one subscription may hold ahead of the Workflow. + +Backpressure *is* this bound: a full buffer stops prefetch. It never drops a +record and never blocks the Workflow thread. +""" + +DEFAULT_WATCH_BLOCK = timedelta(seconds=5) + + +class ReadinessResult: + """The subset of Core's readiness answers the manager reacts to. + + Mirrors :py:class:`temporalio.bridge.worker.ExternalStreamReadyResult` but + is matched structurally, so the manager can be driven in tests without a + Core worker behind it. + """ + + ACCEPTED = "Accepted" + STALE = "Stale" + PARKED = "Parked" + NO_OPEN_WORKFLOW_TASK = "NoOpenWorkflowTask" + RUN_NOT_FOUND = "RunNotFound" + + +class RunStatus: + """The read-only answer to "what state is this Run's wait set in?". + + Mirrors :py:class:`temporalio.bridge.worker.ExternalStreamRunStatus` and is + matched structurally for the same reason :class:`ReadinessResult` is. + """ + + WFT_OPEN = "WftOpen" + PARKED = "Parked" + NO_OPEN_WORKFLOW_TASK = "NoOpenWorkflowTask" + RUN_NOT_FOUND = "RunNotFound" + + +READINESS_ATTEMPTS = 3 +"""How many times a failing readiness report is retried before a wake is owed. + +A raising notifier means the record is buffered and Core has not been told. That +is indistinguishable, from here, from a Run with no open Workflow Task -- so +after these attempts it is treated as exactly that. +""" + +READINESS_RETRY_DELAY = timedelta(milliseconds=200) + +STALE_REPORT_ATTEMPTS = 3 +"""How many times a stale readiness report is re-sent against a newer generation. + +Bounded rather than open-ended: a generation that keeps moving is a Workflow +consuming records happily, and owing a wake at the end of that costs one empty +Workflow Task, where giving up silently costs the record. +""" + +STALE_RETRY_DELAY = timedelta(milliseconds=50) + +SHUTDOWN_WAKE_ATTEMPTS = 3 +"""How many times one owed wake is attempted before it is given up on. + +More than one because a Worker shutting down is often shutting down *because* +something is unhealthy, so the first attempt is the one most likely to land in +the middle of it. Bounded because the alternative to giving up is holding +shutdown open, and the metric exists precisely so that giving up is visible. + +Governs the live path too, which owes wakes for exactly the same reason and has +a far worse backstop: a wake owed *there* is the only thing that will ever +produce a Workflow Task, so nothing re-attempts it -- see `_send_owed_wake`. +The name is the shutdown sweep's because that is where the retry started. +""" + +SHUTDOWN_WAKE_RETRY_DELAY = timedelta(milliseconds=200) +"""Short enough that three attempts fit comfortably inside the grace period.""" + +PARK_REMOVAL_ATTEMPTS = 3 +"""How many times an owed park-intent removal is retried in place. + +The cheap first line against a momentary backend blip, and *only* that: what +makes giving up here survivable is the ledger the failure is recorded in, which +outlives the Subscription and has its own retry loop. Bounded because these +inline retries sit on the close path and inside a registration, neither of +which may wait on a backend indefinitely. +""" + +PARK_REMOVAL_RETRY_DELAY = timedelta(milliseconds=100) +"""Shorter than the wake delays: this backs off *between* park-lock holds. + +The lock is a Run's park handshake serialization, and `prepare_park` runs inside +an activation under Core's deadlock timeout, so every millisecond spent backing +off is a millisecond that budget may have to absorb. +""" + +PARK_REMOVAL_MAX_RETRY_DELAY = timedelta(seconds=5) +"""Caps autonomous removal backoff while still retrying indefinitely.""" + +PARK_REMOVAL_CALL_TIMEOUT = timedelta(milliseconds=500) +"""How long park-intent work off the activation path may hold a Run's park lock. + +Bounds the *call*, which counting attempts cannot. A backend that hangs rather +than raises returns nothing to retry and raises nothing to catch, so the retry +loop, the reconciliation and the close each stop where they are -- and hold this +Run's park lock while they do, which the next park, resolve or eviction takes. + +The asymmetry is what makes giving up right: these holders retry, so an attempt +abandoned costs them a backoff, while the activation waiting behind them is +spending Core's deadlock timeout. Well inside that timeout, because waiting this +out is only the first thing the activation then has to do. + +Holds an activation takes *itself* are deliberately not bounded. Those are the +same backend exposure the park handshake already has -- ``install_park_intent`` +can hang exactly as a removal can -- so a bound there would move the wait rather +than remove it. + +Not a latency limit on the provider, though it becomes one for anything slower +than it: these are single-key operations, and a coordination backend that cannot +answer one in half a second cannot serve the park handshake either, which does +two of them per wait inside an activation. A deployment that far out is failing +Workflow Tasks before this timeout is what it notices. +""" + +DEFAULT_SHUTDOWN_GRACE = timedelta(seconds=10) +"""How long the sweep may hold shutdown open. + +Bounded on purpose: a Worker that could not reach the server would otherwise +hang on the way out, and a wake that has not been acknowledged by now is better +reported than waited on indefinitely. +""" + +DEFAULT_PROBE_GRACE = timedelta(seconds=2) +"""How long the probe phase may delay Core's stop-polling step. + +Much shorter than the sweep's grace, and for a different reason: the probe is +purely local -- one message on Core's own serialized input lane per Run -- so it +is either quick or wedged, and everything it delays is a Worker that has already +been asked to stop. A Run it does not reach is not lost; it is probed again by +the sweep, on whatever Core has left to say. +""" + + +def _status_value(status: Any) -> str: + return getattr(status, "value", status) + + +#: What the manager calls to tell Core a record is buffered. Returns one of the +#: five readiness results as a plain value (or an enum whose ``value`` is one). +ReadinessNotifier = Callable[[str, int, int], Awaitable[Any]] + +#: What the manager calls when local readiness could not be delivered. Filled in +#: by the producer wake-signal path (P14); until then a subscription simply +#: records that a wake was owed. +#: +#: Takes a :data:`WakeTarget` rather than a `Subscription`, because the wake a +#: retired stale intent owes outlives the subscription the record was buffered +#: on -- see :class:`_CleanupWake`. +WakeSender = Callable[["WakeTarget"], Awaitable[None]] + +#: The read-only Run-status probe (C4), returning one of the four run statuses. +RunStatusProbe = Callable[[str], Awaitable[Any]] + + +def _result_value(result: Any) -> str: + return getattr(result, "value", result) + + +@dataclass +class Subscription: + """One Workflow subscription, as the manager sees it.""" + + run_id: str + wait_id: int + stream_key: StreamKey + backend: StreamBackend + buffer_size: int = DEFAULT_BUFFER_SIZE + + committed_cursor: Cursor = BEGINNING + """Advances only when a marker commits. Reconstructed from History.""" + + delivery_cursor: Cursor = BEGINNING + """Advances when a record is handed to Workflow code. Rebuilt by replay.""" + + prefetch_cursor: Cursor = BEGINNING + """Advances on buffering. Speculative, and discarded outright on eviction.""" + + wait_generation: int = 0 + """Increments each time this wait re-enters the blocked state. + + Written by :meth:`note_wait_generation` from the Workflow thread and read by + the watcher on the Worker's loop, so both go through ``_lock``. Only the + runtime knows when a wait re-blocks, so nothing out here can derive it. + """ + + #: Records read ahead but not yet delivered. Appended by the manager loop, + #: popped by the Workflow thread, so every touch is under `_lock`. + _buffer: deque[StreamRecord] = field(default_factory=deque, repr=False) + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + _has_room: asyncio.Event = field(default_factory=asyncio.Event, repr=False) + _watcher: asyncio.Task[None] | None = field(default=None, repr=False) + _cancelled: bool = field(default=False, repr=False) + #: Bumped whenever every speculative read is discarded. A ``read_after`` + #: already in flight started from the cursor being discarded, so its records + #: are speculative too; the watcher compares this across the await and drops + #: them rather than appending them on top of the reset -- which would put the + #: buffer straight back where it was and re-advance `prefetch_cursor`. + _prefetch_epoch: int = field(default=0, repr=False) + + #: Wakes owed because local readiness could not be delivered. Counted rather + #: than merely logged, so a test can tell "no wake was needed" from "a wake + #: was needed and dropped". + wakes_owed: int = 0 + + #: The sender's sequence number for the wake this subscription currently + #: owes, and what the unparked wake's request ID is derived from. + #: + #: Drawn from the *manager* rather than counted here, because this object + #: does not outlive its Run: an evicted Run that comes back gets a new + #: `Subscription` with `wakes_owed` back at zero, and a counter taken from + #: that would re-derive the request ID the previous incarnation's first + #: wake already used. The server deduplicates it, no Workflow Task is + #: created, and the Run waits on records it is already holding. The + #: derivation calls this "a per-sender monotonic counter", and the sender is + #: the manager. + wake_counter: int = 0 + + installed_park_generation: int | None = None + """The generation of the park intent this manager has in the backend, if any. + + The manager's mirror of one piece of backend state, kept so the invariant + *an intent exists only while that park is outstanding* is enforceable from + here: the manager is the only installer, so it is the only thing that can + know an intent is owed a removal. ``None`` means no park *this manager + confirmed* is outstanding, and the removal path is then free -- which + matters because a resolve activation is the ordinary live-delivery path, not + a rare one. + + It is deliberately **not** the only thing that can say a removal is owed. + An intent installed by a previous Worker is mirrored nowhere -- the mirror + went with the Worker -- and a removal that was attempted and failed is owed + after the Subscription holding this field has been dropped. Both live in the + manager's per-Run ledger instead; see + :meth:`StreamSubscriptionManager._remove_park_intent`. + """ + + def __post_init__(self) -> None: + """Mark a new subscription as ready to accept buffered records.""" + self._has_room.set() + + # --- the Workflow thread's half (synchronous, never blocks) ------------- + + @property + def buffered(self) -> int: + """Return the number of records currently buffered for delivery.""" + with self._lock: + return len(self._buffer) + + def drain(self, max_records: int | None = None) -> list[StreamRecord]: + """Pops buffered records. **The only thing ``_apply`` may call.** + + Bounded, non-blocking, and performs no I/O -- which is what makes the + activation it serves safe under the 2-second deadlock timeout. + """ + with self._lock: + limit = len(self._buffer) if max_records is None else max_records + popped = [ + self._buffer.popleft() for _ in range(min(limit, len(self._buffer))) + ] + if popped: + last = popped[-1].offset + assert last is not None + # Inside the lock with the pop, not after it: `reposition_to` + # runs on this same thread but between two activations, and a + # cursor advanced outside the lock could be written on top of + # the boundary a reposition had just committed. + self.delivery_cursor = AFTER(last) + return popped + + def note_wait_generation(self, generation: int) -> None: + """Takes the wait generation the runtime just moved to. + + Called from the Workflow thread the moment the wait re-enters the + blocked state. Without it this stays 0 for the life of the + subscription, every readiness report after the first block names a + generation Core has already left behind, and Core answers ``Stale`` -- + which the watcher treats as "re-probe later" while its prefetch cursor + is already past the record. The append is then never re-announced and + the Workflow is never woken. + """ + with self._lock: + self.wait_generation = generation + + def current_wait_generation(self) -> int: + """The generation to report readiness under, read on the Worker's loop.""" + with self._lock: + return self.wait_generation + + def blocked_cursor(self) -> Cursor: + """Where this subscription's deliveries stopped. + + The terminal's boundary. Fixed the moment the last activation returned, + so it is never refreshed against the backend -- doing that could name a + position replay must not reproduce. + """ + return self.delivery_cursor + + # --- the manager loop's half -------------------------------------------- + + def _append(self, records: list[StreamRecord], epoch: int) -> bool: + """Buffers a read's result unless a reposition has retracted it. + + The epoch is compared **here, under the lock that the reposition also + takes**, rather than by the watcher before it calls: repositioning is + synchronous on the Workflow thread (see + :meth:`StreamSubscriptionManager.reposition_to_committed`), so a check + made outside this lock could pass and then have the reposition land + before the append -- putting the retracted records straight back into + the buffer and re-advancing ``prefetch_cursor`` past them. + + Returns whether the records were buffered. ``False`` means they were + read from a position the marker already accounts for and the next pass + reads from the new cursor. + """ + with self._lock: + if self._prefetch_epoch != epoch: + return False + self._buffer.extend(records) + full = len(self._buffer) >= self.buffer_size + last = records[-1].offset + assert last is not None + self.prefetch_cursor = AFTER(last) + if full: + self._has_room.clear() + return True + + def _room(self) -> int: + with self._lock: + return max(0, self.buffer_size - len(self._buffer)) + + def note_drained(self) -> None: + """Tells the watcher there is room again. + + Called from the Workflow thread through the manager, which hops it onto + the loop -- an asyncio.Event is not thread-safe to set directly. + """ + self._has_room.set() + + def reset_to_committed(self) -> None: + """Discards every speculative read, as eviction and failure require. + + Reading is not consuming and consuming is not committing, so nothing + here was ever a claim. This is exactly why "no cursor advances unless + the marker commits" is safe to state. + """ + self._retract_to_committed() + self._has_room.set() + + def _retract_to_committed(self) -> None: + """The state half of :meth:`reset_to_committed`, safe on either thread. + + Every write is under ``_lock`` and nothing here touches an + ``asyncio`` primitive, which is what lets the Workflow thread call it + directly through :meth:`reposition_to`. Waking the watcher is the part + that must run on the manager's loop, so it stays with the callers. + """ + with self._lock: + self._retract_locked() + + def _retract_locked(self) -> None: + """Discards the speculative state. The caller holds ``_lock``.""" + self._buffer.clear() + self._prefetch_epoch += 1 + self.delivery_cursor = self.committed_cursor + self.prefetch_cursor = self.committed_cursor + + def commit(self, cursor: Cursor) -> None: + """Advances the committed cursor when a marker commits.""" + with self._lock: + self.committed_cursor = cursor + + def reposition_to(self, cursor: Cursor) -> None: + """Commits a marker's boundary and restarts every read from it. + + What replay needs afterwards. While replay was handing this Workflow the + records the marker recorded, the watcher was independently reading the + *same* records from the subscription's start cursor into this buffer -- + nothing had told it the marker exists. Leaving it there hands Workflow + code every replayed record a second time the moment the next live drain + happens. + + The boundary comes from the marker rather than from what the buffer + happens to hold, because the marker is the only durable statement of + where consumption reached. + + **Called synchronously from the Workflow thread**, and deliberately not + hopped onto the manager's loop. The drain that immediately follows + replay is on the Workflow thread too, so a reposition merely *posted* + to the loop leaves the retracted records in the buffer for it to hand + over a second time -- the reposition has to have happened by the time + this returns, not merely be scheduled. Nothing here touches an + ``asyncio`` primitive, and the epoch bump is what fences a read already + in flight; see :meth:`_append`. + """ + # One lock hold for both halves: a watcher append landing between them + # would be dropped by the epoch bump anyway, but a reader that saw the + # new committed cursor next to the old delivery cursor would see a state + # that never existed. + with self._lock: + self.committed_cursor = cursor + self._retract_locked() + + +class PreparedRecord(StreamRecord): + """A record whose payload has already had retrieval and codec applied. + + A subclass rather than a field on :py:class:`StreamRecord`, because being + prepared is not a property of a record -- it is a property of *this + Worker's* handling of one, and a record read by a producer, written by a + backend, or compared for idempotency has no such half-state. Everything that + reads a record reads the same fields; only the delivery path looks for the + two added here. + + ``payload`` is deliberately left as the backend's bytes. Replacing it would + make the record no longer equal to what the stream holds, which is what + idempotency comparison and integrity validation are expressed in. + """ + + #: The payload as the payload converter will see it, or ``None`` if + #: preparing it raised. + prepared_payload: Any = None + + #: What preparing raised, carried rather than raised on the Worker's loop. + #: The watcher has no Workflow Task to fail and no Workflow to tell, and a + #: record that is never delivered must not fail anything at all -- so the + #: error travels with the record and is raised by the delivery that would + #: have yielded its value. + prepare_error: BaseException | None = None + + def __eq__(self, other: object) -> bool: + """Equal to the record it was built from, and to any equal record. + + A frozen dataclass's generated ``__eq__`` compares classes exactly, so + without this a prepared record would be unequal to the identical + ``StreamRecord`` a caller built to compare against -- and being prepared + is a property of this Worker's handling, not of the record. Preparation + is deliberately not part of the comparison for the same reason. + """ + if isinstance(other, StreamRecord): + return self._fields(self) == self._fields(other) + return NotImplemented + + def __hash__(self) -> int: + """Hash the underlying record fields, excluding preparation state.""" + return hash(self._fields(self)) + + @staticmethod + def _fields(record: StreamRecord) -> tuple[Any, ...]: + return ( + record.kind, + record.payload, + record.producer_session_id, + record.sequence, + record.offset, + ) + + @classmethod + def of( + cls, + record: StreamRecord, + prepared: Any, + error: BaseException | None, + ) -> PreparedRecord: + """Create a prepared view of ``record`` with its decode outcome.""" + out = cls( + kind=record.kind, + payload=record.payload, + producer_session_id=record.producer_session_id, + sequence=record.sequence, + offset=record.offset, + ) + # `StreamRecord` is frozen, and these two are this subclass's own + # fields rather than dataclass fields, so they are set the same way a + # frozen dataclass sets anything. + object.__setattr__(out, "prepared_payload", prepared) + object.__setattr__(out, "prepare_error", error) + return out + + +@dataclass(frozen=True) +class _OwedRemoval: + """One park intent this manager knows is installed and no park sits behind. + + Not a Subscription field, and that is the whole point of it. Every removal + path this feature has reaches its intent *through* a Subscription -- the + resolve iterates the registered ones, the withdrawal walks the ones it just + installed, the close works on the one being dropped -- so a failure recorded + on the Subscription is a failure recorded on the very object the next step + throws away. Carrying the removal separately is what makes "still owed" + outlive the close, the eviction, and the Worker that installed nothing. + + The generation and Run ID are what the intent looked like when it was + recorded, kept so a delayed retry can tell it apart from a *different* + intent that has since taken the same key -- see + :meth:`StreamSubscriptionManager._drain_owed_removals`. + """ + + backend: StreamBackend + park_generation: int + run_id: str + + #: Whether a generation-0 handoff has already been made for this entry. Set + #: by the shutdown sweep, which wakes the subscriptions of a Run it is about + #: to tear down and composes those wakes as unparked precisely because this + #: entry says the installed intent is stale. The wake the cleanup would owe + #: afterwards is then the same wake, and sending it again would ask the + #: server for a second, empty Workflow Task. + #: + #: Out of the comparison on purpose: it is a note about a handoff, not part + #: of the identity of the removal, and `_owe_removal` treats an entry that + #: compares equal as debt it has already recorded. + announced: bool = field(default=False, compare=False) + + +@dataclass +class _CleanupWake: + """The wake a retired stale intent owes, with no Subscription left to owe it. + + A stale intent does not only leak, it *silences*: every wake sent while it + was installed named the generation behind it, and Core discards a non-zero + generation that is not the park it holds. Retiring the intent ends the + suppression and announces nothing, so the removal carries a wake obligation + with it -- and :meth:`StreamSubscriptionManager._reannounce_after_cleanup` + can only discharge that obligation through local readiness, which needs a + registered `Subscription` and a buffer to read. + + Eviction and close deliberately take that Subscription away while leaving + the ledger and its retry alive, which is the whole point of the ledger. So + the obligation is carried here instead: enough to compose the one wake that + is always correct after a confirmed removal -- the unparked one, which + carries no generation for Core to discard. It costs one empty Workflow Task + when no record arrived during the suppression window, and that is the + cheaper side of the trade: from out here, with no buffer left to consult, + "a record arrived" and "none did" look identical, and only one of them is + silent if guessed wrong. + + Shaped like the part of a `Subscription` a wake is composed from, because + the sender takes one target and does not care which of the two it is. + """ + + run_id: str + wait_id: int + stream_key: StreamKey + backend: StreamBackend + + #: Counted and drawn by `_count_owed_wake`, exactly as a subscription's is: + #: the request ID derives from it, so re-drawing between attempts would ask + #: for a second Workflow Task rather than re-send this one. + wakes_owed: int = 0 + wake_counter: int = 0 + + +#: What one owed wake is composed from: a `Subscription` while the wait is still +#: registered on this Worker, and a `_CleanupWake` once it is not. +WakeTarget = Subscription | _CleanupWake + + +@dataclass +class _PendingWake: + """One Run-wide wake cycle, correlated to its current send attempt.""" + + wait_id: int + attempt_started_at: int + acknowledged: bool = False + completed_terminal: bool | None = None + + +class StreamSubscriptionManager: + """Every subscription on one Worker, keyed by Run. + + Keying by Run ID is what keeps a stale Run from leaking connections: the + teardown path for eviction, Workflow Task failure, and shutdown is the same + one, and it is reached by Run. + """ + + def __init__( + self, + *, + backend: StreamBackend | None, + notify_ready: ReadinessNotifier, + send_wake: WakeSender | None = None, + run_status: Callable[[str], Awaitable[Any]] | None = None, + shutdown_wake_failed_metric: Callable[[Any], None] | None = None, + client_identity: str = "", + data_converter: temporalio.converter.DataConverter | None = None, + buffer_size: int = DEFAULT_BUFFER_SIZE, + watch_block: timedelta = DEFAULT_WATCH_BLOCK, + ) -> None: + """Create a manager for the external stream backend on one Worker.""" + self._backend = backend + #: The Worker's own converter, used for the **asynchronous half** of + #: decoding only -- retrieval and codec, never the payload converter, + #: which needs a type only Workflow code knows. Optional so a manager + #: can be constructed in isolation; a Worker always supplies it, and a + #: record that reaches the Workflow thread unprepared under a converter + #: that had something to do is refused there rather than decoded late. + self._data_converter = data_converter + self._notify_ready = notify_ready + self._send_wake = send_wake + self._run_status = run_status + self._metric = shutdown_wake_failed_metric + #: This Worker's sender identity for unparked wakes. Drawn once here + #: rather than taken from the client identity, because two Workers in + #: one process share a `Client`: with the client identity their first + #: unparked wakes derive the same request ID, the server deduplicates + #: the second, and the Run that the surviving Worker picked up never + #: gets a Workflow Task. Fixed for this manager's lifetime, so the + #: shutdown sweep's retry stays the same wake rather than a new one. + self.wake_sender_identity = new_sender_identity(client_identity) + #: The sequence the unparked wake counter is drawn from, monotonic for + #: this manager's lifetime -- which is what makes two wakes from one + #: sender distinct request IDs. Never reset: a repeat of any value this + #: sender has already used is a wake the server deduplicates away. + self._wake_sequence = 0 + #: Runs for which a wake is in flight or acknowledged, mapped to the + #: latest activation sequence when its send began. Only a later + #: activation can have been caused by that wake, so completion of a task + #: already in flight cannot prematurely release the gate. Failed tasks + #: are evicted and replayed before a successful completion, so keeping + #: this across eviction coalesces reconstructed readiness too. + self._pending_wake_runs: dict[str, _PendingWake] = {} + self._activation_sequences: dict[str, int] = {} + #: Wakes the shutdown sweep could not get acknowledged. Reported through + #: ``external_stream_shutdown_wake_failed``; kept here so a test can tell + #: "no wake was needed" from "a wake was needed and lost". + self.shutdown_wake_failures = 0 + self._buffer_size = buffer_size + self._watch_block = watch_block + self._runs: dict[str, dict[int, Subscription]] = {} + #: Replay plans read and validated ahead of the delivering activation. + self._replay_plans: dict[str, ReplayPlan] = {} + # The loop the watchers run on, captured at construction because that is + # the Worker's loop. `register` is called from the *Workflow executor + # thread*, which has no loop of its own and must never touch this one + # directly. + self._loop = asyncio.get_event_loop() + self._shutting_down = False + #: Whether the sweep has already run, so `shutdown` can tell "nobody + #: swept" -- which it must then do itself -- from "already swept". + self._swept = False + #: Subscriptions the sweep has not yet decided anything about. Filled + #: when the sweep starts and drained as each Run's status is read; what + #: is left when the sweep stops -- cancelled by the grace period, or + #: stopped by a probe it could not complete -- is counted as a lost wake + #: rather than passed over, because a wake this Worker abandoned is + #: silent by nature and the counter is the only thing that says otherwise. + self._unaccounted: list[Subscription] = [] + #: What the probe phase heard, per Run. Recorded rather than acted on, + #: because the two halves of the sweep belong at different points of the + #: Worker's shutdown -- see `probe_runs`. + self._probed: dict[str, str] = {} + #: One Run's park-intent work, serialized. See `_park_lock`. + self._park_locks: dict[str, asyncio.Lock] = {} + #: Strong references to the in-flight registration-time reconciliations. + self._reconciliations: set[asyncio.Task[None]] = set() + #: Removals this manager decided on and did not get confirmed, per Run + #: and then per ``(stream key, wait_id)``. The durable half of the intent + #: invariant: a removal that failed is a claim about the *backend*, and + #: recording it anywhere that a close or an eviction takes away is the + #: same as not recording it at all. Drained under `_park_lock`. + self._owed_removals: dict[str, dict[tuple[StreamKey, int], _OwedRemoval]] = {} + #: One autonomous retry loop per Run with outstanding removals. The + #: tasks are held strongly here because the ledger, rather than any + #: Subscription, owns their lifetime and can outlive cache eviction. + self._owed_removal_retries: dict[str, asyncio.Task[None]] = {} + #: Wakes a sleeping retry when another removal is added to its Run or + #: the last one is retired. A new debt should not wait behind an old + #: entry's maximum backoff, and an empty loop should stop promptly. + self._owed_removal_wakeups: dict[str, asyncio.Event] = {} + self._stopping_owed_removal_retries = False + #: In-flight cleanup wakes, and what each one is for. Strong references, + #: because the loop keeps only a weak one to a running task and this + #: one's whole job is a Signal; and a mapping rather than a set, because + #: a flush that runs out of grace has to be able to say *which* handoff + #: it abandoned. Deliberately not among the tasks + #: `_stop_background_tasks` cancels: these are not retries that can be + #: resumed later, they are the wakes themselves. + self._cleanup_wakes: dict[asyncio.Task[None], _CleanupWake] = {} + + # --- registration ------------------------------------------------------- + + def register( + self, + *, + run_id: str, + wait_id: int, + stream_key: StreamKey, + start_cursor: Cursor = BEGINNING, + ) -> Subscription: + """Registers a wait and starts its watcher. Non-blocking. + + Called from the Workflow thread, so it must not await: the watcher task + is scheduled onto the manager's loop rather than started here. + """ + backend = self._backend + if backend is None: + raise RuntimeError( + "external streams are not configured on this Worker; pass " + "external_stream_backend=... to the Worker" + ) + subscription = Subscription( + run_id=run_id, + wait_id=wait_id, + stream_key=stream_key, + backend=backend, + buffer_size=self._buffer_size, + committed_cursor=start_cursor, + delivery_cursor=start_cursor, + prefetch_cursor=start_cursor, + ) + replaced = self._runs.setdefault(run_id, {}).get(wait_id) + if replaced is not None: + # A wait re-registered under a key that already has one. The old + # subscription is unreachable from here on, and its watcher would go + # on polling the backend for the life of the process -- a leak that + # only shows up as a connection that never closes. + replaced._cancelled = True + self._loop.call_soon_threadsafe(self._cancel_watcher, replaced) + self._runs[run_id][wait_id] = subscription + # `create_task` is not thread-safe, and this runs on the Workflow + # executor thread. Scheduling the start onto the manager's loop is the + # difference between a watcher that runs and one that is silently never + # scheduled -- which looks exactly like a stream that never delivers. + self._loop.call_soon_threadsafe(self._start_watcher, subscription) + return subscription + + def _cancel_watcher(self, subscription: Subscription) -> None: + watcher = subscription._watcher + if watcher is not None and not watcher.done(): + watcher.cancel() + + def _start_watcher(self, subscription: Subscription) -> None: + """Starts a watcher, and reconciles the park state it inherited. + + Both on the manager's own loop, because both are ``create_task`` calls and + `register` runs on the Workflow executor thread. + """ + if subscription._cancelled or subscription._watcher is not None: + return + reconcile = self._loop.create_task(self._reconcile_inherited_park(subscription)) + # Held, because the loop keeps only a weak reference to a running task: + # an unreferenced one can be collected mid-await, and this one's whole + # job is a backend round trip. + self._reconciliations.add(reconcile) + reconcile.add_done_callback(self._reconciliations.discard) + subscription._watcher = self._loop.create_task(self._watch(subscription)) + + async def _reconcile_inherited_park(self, subscription: Subscription) -> None: + """Removes a park intent this Worker inherited rather than installed. + + `installed_park_generation` is a *mirror* of backend state, and it lives + on the Worker that installed the park. The intent it mirrors is durable: + it survives eviction, a Workflow Task that moved to another Worker, and + shutdown, all of which take the mirror with them. Removal keyed on the + mirror alone therefore cannot reach the one class of intent that most + needs reaching -- the intents whose installer is gone -- and no later + resolve can repair it, because the resolve looks at the same empty + mirror. + + Registration is where a Worker learns such an intent exists, and it is + also the moment its status is unambiguous. A subscription is registered + by user Workflow code running, and no user code runs inside a park + (``wft-lifecycle.md``), so an intent found here belongs to a park that is + over: the Core that confirmed it has either moved on or gone with the + Worker that held it. What leaving it costs is the invariant's whole + point -- ``current_park_generation`` keeps answering a generation Core has + discarded, every producer wake names that generation and Core discards + it as stale, and because a parked wake's request ID ignores sender + identity the second such wake is byte-identical to the first and the + server deduplicates it away. The Workflow then waits forever on a record + that is durably present. + + Best-effort in the sense that no failure here reaches a Workflow Task -- + this is the registration path of a Run that is already running, and a + momentary backend failure must not fail one. It is *not* best-effort in + the sense of one attempt, and neither half of the job may end with the + burst of cheap ones below. + + Once the intent has been *read* it is recorded in this Run's + owed-removal ledger, whose own retry makes backend recovery sufficient; + this loop hands the entry over rather than draining it twice. Until it + has been read there is nothing to record -- the ledger holds an + identity, and the identity is what could not be obtained -- so + discovery keeps retrying here, with backoff, for as long as this Worker + holds the subscription. Both halves answer the same argument: waiting + for "the next time this wait is registered" is a coincidence of + eviction, not a mechanism, and the Run that most needs the removal -- + one cached and blocked on something other than this stream -- is + precisely the one that never registers it again. + + Later parks, resolves, registrations and evictions remain eager fast + paths for whatever is in the ledger. + """ + run_id = subscription.run_id + key = (subscription.stream_key, subscription.wait_id) + attempt = 0 + delay = PARK_REMOVAL_RETRY_DELAY.total_seconds() + while True: + if subscription._cancelled: + # Eviction does not cancel this task -- it is held in + # `_reconciliations`, not on the Subscription -- and shutdown + # cancels it only after Run teardown, so this flag stops it in + # either path. Anything read by now is in the ledger and its + # autonomous retry. + # + # It is also what keeps this loop *safe* to run indefinitely. + # Removing whatever is installed at this key is justified by + # this Worker holding the Run and this wait being registered on + # it: any intent there belongs to a park that is over. Once the + # subscription is gone that justification is gone with it, and + # the intent this would read could be a park a *different* + # Worker is sitting in. + return + if await self._reconcile_inherited_park_once(subscription, attempt): + return + if key in self._owed_removals.get(run_id, {}): + # Discovery worked and the removal did not, which is the case + # the ledger exists for. It owns the retry from here; a second + # loop draining the same entry would only double the round + # trips. + logger.warning( + "Could not remove the inherited park intent for %s wait %s " + "in %s attempts; its autonomous owed-removal retry " + "continues in the background", + subscription.stream_key, + subscription.wait_id, + attempt + 1, + ) + return + attempt += 1 + if attempt == PARK_REMOVAL_ATTEMPTS: + # Past the cheap burst with nothing read, so there is no ledger + # entry to hand this to -- the ledger records an intent's + # identity, and that is precisely what could not be read. Giving + # up here is the liveness gap the ledger closed for removals, + # reopened for discovery: a backend down for the burst and + # healthy after it would leave the obsolete intent installed + # until this wait happens to be registered again, which for a + # Run that stays cached is never. + logger.warning( + "Could not read the park intent for %s wait %s in %s " + "attempts; this reconciliation keeps retrying it in the " + "background, with backoff, until it can say whether one is " + "installed", + subscription.stream_key, + subscription.wait_id, + PARK_REMOVAL_ATTEMPTS, + ) + if attempt >= PARK_REMOVAL_ATTEMPTS: + delay = min(PARK_REMOVAL_MAX_RETRY_DELAY.total_seconds(), delay * 2) + await asyncio.sleep(delay) + + async def _reconcile_inherited_park_once( + self, subscription: Subscription, attempt: int + ) -> bool: + """One read-and-remove pass. Returns whether nothing is left to do. + + The lock is taken per attempt rather than held across the backoff + between them, because `prepare_park` runs inside an activation under + Core's deadlock timeout: a reconciliation that kept this Run's lock + while it slept would spend that budget on cleanup for a park that is + already over. + + Bounding the hold itself is the same argument carried one step further. + This runs on a task of its own, not inside the activation, so the calls + below can hang without anything above them noticing -- and this Run's + park lock hangs with them, which is what the next park, resolve or + eviction takes, and what the owed-removal retry loop takes. An attempt + that runs out of time is treated as a failed attempt, because that is + what it is; the entry was recorded before the call, so nothing read is + lost by giving up on it. + """ + try: + async with self._park_lock(subscription.run_id): + return await asyncio.wait_for( + self._reconcile_inherited_park_locked(subscription, attempt), + PARK_REMOVAL_CALL_TIMEOUT.total_seconds(), + ) + except asyncio.TimeoutError: + logger.warning( + "Reconciling the inherited park intent for %s wait %s did not " + "finish within %s on attempt %s/%s; the park lock is released " + "and anything already read stays owed", + subscription.stream_key, + subscription.wait_id, + PARK_REMOVAL_CALL_TIMEOUT, + attempt + 1, + PARK_REMOVAL_ATTEMPTS, + ) + return False + + async def _reconcile_inherited_park_locked( + self, subscription: Subscription, attempt: int + ) -> bool: + """The pass itself, with this Run's park lock already held.""" + run_id = subscription.run_id + key = (subscription.stream_key, subscription.wait_id) + # Drained first, because a previous attempt of this same loop may + # already have read the intent and failed only to remove it. The + # ledger is that retry; re-reading would find exactly what it holds. + await self._drain_owed_removals(run_id) + if key in self._owed_removals.get(run_id, {}): + return False + if ( + subscription._cancelled + or subscription.installed_park_generation is not None + ): + # A park confirmed since this was scheduled is *this* manager's, + # and `resolve_park` owns it. Removing it here would take the + # intent out from under a park that really is outstanding. + return True + try: + inherited = await subscription.backend.park_intent( + subscription.stream_key, subscription.wait_id + ) + except asyncio.CancelledError: + raise + except Exception: + logger.warning( + "Reading the inherited park intent for %s wait %s failed on attempt %s", + subscription.stream_key, + subscription.wait_id, + attempt + 1, + exc_info=True, + ) + return False + if inherited is None: + # The ordinary case, and the reason this is a read before it + # is a write: a Run that never parked owes the backend + # nothing. + return True + if ( + subscription._cancelled + or subscription.installed_park_generation is not None + ): + # Asked again across the read, because this Run's state can + # have moved on entirely while it was in flight: evicted and + # picked up again, with a *new* park confirmed for the same + # key. Removing what was found before that would strand the + # park that replaced it. + return True + logger.info( + "Removing an inherited external stream park intent for %s " + "wait %s: park generation %s was confirmed by a Worker that " + "no longer holds this Run", + subscription.stream_key, + subscription.wait_id, + inherited.park_generation, + ) + # Owed from the moment it is known to exist, not from the moment a + # removal fails: this is the only mirror an inherited intent ever + # gets, and without it `_remove_park_intent` goes on short-circuiting + # on an empty `installed_park_generation` and every removal path -- + # the resolve, the withdrawal, the close -- stays disabled for it. + self._owe_removal( + run_id, + key, + _OwedRemoval( + backend=subscription.backend, + park_generation=inherited.park_generation, + run_id=inherited.run_id, + ), + ) + try: + outcome = await subscription.backend.remove_park_intent_if_matches( + subscription.stream_key, + subscription.wait_id, + run_id=inherited.run_id, + park_generation=inherited.park_generation, + ) + except asyncio.CancelledError: + raise + except Exception: + logger.warning( + "Removing the inherited park intent for %s wait %s failed on " + "attempt %s", + subscription.stream_key, + subscription.wait_id, + attempt + 1, + exc_info=True, + ) + return False + self._forget_owed_removal(run_id, key) + # This path is the *ordinary* one for an inherited intent -- the removal + # succeeding first time -- and it silences records exactly as a retried + # one does. `_start_watcher` runs this reconciliation and the watcher + # concurrently, so the watcher can read a record, find no open Workflow + # Task and send its wake naming the inherited generation while this is + # still in flight. Core discards that wake, and the watcher will not + # come back through here without a *new* append. + self._reannounce_after_cleanup(subscription, outcome) + return True + + def _owe_removal( + self, run_id: str, key: tuple[StreamKey, int], record: _OwedRemoval + ) -> None: + owed = self._owed_removals.setdefault(run_id, {}) + previous = owed.get(key) + owed[key] = record + retry = self._owed_removal_retries.get(run_id) + if retry is None or retry.done(): + if not self._stopping_owed_removal_retries: + self._owed_removal_wakeups.setdefault(run_id, asyncio.Event()) + retry = self._loop.create_task(self._retry_owed_removals(run_id)) + self._owed_removal_retries[run_id] = retry + elif previous != record: + # Do not make newly discovered debt wait behind the capped backoff + # of an older failing entry for this Run. + self._owed_removal_wakeups[run_id].set() + + def _forget_owed_removal(self, run_id: str, key: tuple[StreamKey, int]) -> None: + owed = self._owed_removals.get(run_id) + if owed is None: + return + owed.pop(key, None) + if not owed: + # Dropped rather than left empty, so a Run that owes nothing costs + # nothing to carry and `evict_run` can tell the two apart cheaply. + self._owed_removals.pop(run_id, None) + wakeup = self._owed_removal_wakeups.get(run_id) + if wakeup is not None: + wakeup.set() + + async def _retry_owed_removals(self, run_id: str) -> None: + """Retries one Run's ledger without needing another lifecycle event.""" + this_task = asyncio.current_task() + initial = PARK_REMOVAL_RETRY_DELAY.total_seconds() + delay = initial + maximum = PARK_REMOVAL_MAX_RETRY_DELAY.total_seconds() + wakeup = self._owed_removal_wakeups[run_id] + try: + while self._owed_removals.get(run_id): + try: + await asyncio.wait_for(wakeup.wait(), delay) + delay = initial + except asyncio.TimeoutError: + pass + wakeup.clear() + + before = len(self._owed_removals.get(run_id, {})) + await self._bounded_drain(run_id) + after = len(self._owed_removals.get(run_id, {})) + if after == 0: + return + delay = initial if after < before else min(maximum, delay * 2) + finally: + if self._owed_removal_retries.get(run_id) is this_task: + self._owed_removal_retries.pop(run_id, None) + self._owed_removal_wakeups.pop(run_id, None) + if run_id not in self._runs and not self._owed_removals.get(run_id): + self._park_locks.pop(run_id, None) + + async def _bounded_drain( + self, run_id: str, budget: timedelta = PARK_REMOVAL_CALL_TIMEOUT + ) -> None: + """Drains under this Run's park lock, and lets go of it either way. + + The retry loop and the last pass at shutdown are the drains that take + this lock with no activation above them and nothing else bounding them -- + the reconciliation's drain is bounded by the hold it sits inside, and + every other one is inside the activation whose budget a slow backend + spends regardless. A backend that hangs rather than raises would hold the + lock for as long as the call lasts, hence a timeout and not merely a + bounded number of attempts: attempts that never return are not bounded by + counting them. + + A timeout leaves the entry owed, which is what the loop above does with a + failure -- back off and come round again. + """ + try: + async with self._park_lock(run_id): + await asyncio.wait_for( + self._drain_owed_removals(run_id), budget.total_seconds() + ) + except asyncio.TimeoutError: + logger.warning( + "A background owed-removal drain for run %s did not finish " + "within %s; the park lock is released and it stays owed", + run_id, + budget, + ) + + async def _drain_owed_removals(self, run_id: str) -> None: + """Retries every removal this Run still owes. Never raises. + + A ledger entry is not "an intent exists"; it is a removal this manager + has already decided on and not had confirmed. That is what makes + draining safe from any holder of this Run's park lock rather than only + from the path that recorded it. The entry retains nothing -- no watcher, + no buffer, no connection beyond the backend the removal has to go + through. + + A Continue-As-New successor re-uses the stream key with wait ids that + start again at 1, so an entry a predecessor Run left could name a live + park's key. The backend therefore compares both the Run ID and park + generation as part of the delete. A read followed by an unconditional + removal would leave a cross-Run window in which this retry could delete + the successor's park. + """ + owed = self._owed_removals.get(run_id) + if not owed: + return + for key, record in list(owed.items()): + stream_key, wait_id = key + try: + outcome = await record.backend.remove_park_intent_if_matches( + stream_key, + wait_id, + run_id=record.run_id, + park_generation=record.park_generation, + ) + except asyncio.CancelledError: + raise + except Exception: + logger.warning( + "Retrying the owed park intent removal for %s wait %s " + "failed; it stays owed", + stream_key, + wait_id, + exc_info=True, + ) + continue + self._forget_owed_removal(run_id, key) + subscription = self.subscription(run_id, wait_id) + if subscription is not None and subscription.stream_key == stream_key: + # The mirror is only cleared once the backend agrees, here for + # the same reason `_remove_park_intent` does it in that order. + subscription.installed_park_generation = None + self._reannounce_after_cleanup(subscription, outcome) + else: + # No subscription to announce through, which is the *ordinary* + # case for this ledger rather than a corner of it: eviction and + # close both drop the Subscription and deliberately leave the + # entry and its retry alive, so by the time a recovered backend + # lets the removal through there is frequently nothing left + # holding the record the intent silenced. Announcing only when + # the wait happens to still be registered put the record-loss + # half of the suppression straight back. + self._wake_after_cleanup(record, stream_key, wait_id, outcome) + + def _wake_after_cleanup( + self, + record: _OwedRemoval, + stream_key: StreamKey, + wait_id: int, + outcome: ParkIntentRemoval, + ) -> None: + """Hands off a record a retired intent may have silenced, with no wait left. + + The Subscription-bound half of this is + :meth:`_reannounce_after_cleanup`, and it is the better one where it + applies: local readiness is free, and it fires only for a wait that is + actually holding records. Neither is knowable from here. The buffer went + with the Subscription, and whether a *remote* producer appended during + the suppression window was never visible from this Worker at all -- so + the choice is one unparked wake or silence, and silence is the failure + this whole path exists to prevent. + + ``MISMATCH`` is the one outcome that stays silent, for the reason + :meth:`_reannounce_after_cleanup` gives: an intent is still installed at + that key, so the suppression has not ended and it belongs to a park this + entry knows nothing about. + + Posted as a task rather than awaited, because every caller reaches the + drain holding this Run's park lock -- and one bounded by + `PARK_REMOVAL_CALL_TIMEOUT`, which is a budget for a single-key backend + call and not for a Signal with retries behind it. `shutdown` awaits what + is still in flight; see :meth:`_flush_cleanup_wakes`. + """ + if outcome is ParkIntentRemoval.MISMATCH: + return + if record.announced: + # The shutdown sweep already made this exact handoff -- an unparked + # wake for this wait, composed that way *because* this entry said the + # installed intent was stale. A second one is a second empty + # Workflow Task, not a second chance. + return + if self._send_wake is None: + return + target = _CleanupWake( + run_id=record.run_id, + wait_id=wait_id, + stream_key=stream_key, + backend=record.backend, + ) + task = self._loop.create_task(self._send_cleanup_wake(target)) + self._cleanup_wakes[task] = target + task.add_done_callback(self._cleanup_wakes.pop) + + async def _send_cleanup_wake(self, target: _CleanupWake) -> None: + """Sends one cleanup handoff. Never raises except on cancellation. + + Counted through `_count_owed_wake` like any other owed wake, so the + retries inside `_send_owed_wake` are the same wake rather than three + separate asks. + + A wake that never lands is reported, and while shutting down it is also + *counted*: the sweep's promise is that a handoff this Worker did not make + cannot be silent, and a removal confirmed during the final pass is as + much a part of that shutdown as a wake the sweep sent itself. On the live + path there is no counter with that meaning -- the metric is the shutdown + sweep's -- so the warning is what is left, and it says plainly that + nothing retries this. + """ + self._count_owed_wake(target) + if await self._send_owed_wake(target): + return + logger.warning( + "The wake owed for the record a retired park intent silenced was " + "not acknowledged for %s wait %s; nothing else announces it", + target.stream_key, + target.wait_id, + ) + if self._shutting_down: + self._record_shutdown_wake_failure(target) + + async def _flush_cleanup_wakes(self, budget: timedelta) -> None: + """Waits out the cleanup handoffs still in flight as the Worker exits. + + The last one of these is created by :meth:`_final_owed_removal_pass`, + which runs after everything else shutdown does, so without this the + process leaves with the Signal task merely scheduled -- which is the same + as not having sent it, and worse, because the failure counter would say a + clean shutdown. + + Bounded by what is left of the grace period, for the reason every other + step of shutdown is: cleanup may not hold the exit open. What the bound + cuts off is counted rather than dropped, because the wake it abandons is + silent by nature. + """ + pending = set(self._cleanup_wakes) + if not pending: + return + if budget.total_seconds() > 0: + _, pending = await asyncio.wait(pending, timeout=budget.total_seconds()) + for task in pending: + target = self._cleanup_wakes.get(task) + task.cancel() + if target is None: + continue + logger.warning( + "External stream shutdown abandoned the wake owed for a retired " + "park intent on %s wait %s", + target.stream_key, + target.wait_id, + ) + self._record_shutdown_wake_failure(target) + + def _reannounce_after_cleanup( + self, subscription: Subscription, outcome: ParkIntentRemoval + ) -> None: + """Announces again a record the retired intent had already silenced. + + A stale intent does not only leak. While it was installed every wake for + this stream named the generation behind it -- the producer reads + ``current_park_generation``, and so does the Worker's own sender -- and + Core discards a non-zero generation that is not the park it is holding. + A record that arrived in that window was therefore announced to nobody, + and *removing the intent does not announce it*: `_report_ready` counted + its wake as sent, the watcher has moved `prefetch_cursor` past it and + returns on the next empty read, and `_send_owed_wake` re-sends only a + wake that **raised**. Without this, a Run can sit blocked on a record + this Worker is already holding until some unrelated event happens to + create a Workflow Task. + + Both outcomes that clear the key announce, and that is why the provider + contract does not answer with a Boolean. ``ABSENT`` is not "someone else's + intent is in the way", it is "the intent this entry named is gone" -- + which is what a retry sees after the reply to a delete that in fact + succeeded was lost, and what one cleanup owner sees after another + finished the job. Reading that as a mismatch would leave the record the + intent silenced silent for good. ``MISMATCH`` announces nothing: an intent + is still installed there, the suppression it causes has not ended, and it + belongs to a park this entry knows nothing about. + + Narrow otherwise. Only a wait still holding records, so a healthy park + costs no Workflow Task; and never during the sweep, which owns every wake + from that point and accounts for each one. Posted rather than awaited, + because callers hold the Run's park lock and `_report_ready` goes out to + Core. + + Every one of those narrowings reads the Subscription, which is why this + is only half the cleanup. Once the wait is closed or its Run evicted + there is no buffer to consult and no readiness channel to use, and the + obligation is discharged by :meth:`_wake_after_cleanup` instead. + """ + if outcome is ParkIntentRemoval.MISMATCH: + return + if subscription._cancelled or not subscription.buffered: + return + if self._shutting_down: + return + pending = self._pending_wake_runs.get(subscription.run_id) + if pending is not None and pending.wait_id == subscription.wait_id: + # The pending wake for this wait named the intent just retired, so + # Core discarded it. It cannot gate the unparked reannouncement + # whose purpose is to replace that silenced wake. + del self._pending_wake_runs[subscription.run_id] + self._loop.create_task(self._report_ready(subscription)) + + def _park_lock(self, run_id: str) -> asyncio.Lock: + """Serializes one Run's park-intent work on the manager's loop. + + The install/recheck handshake, the resolve, and the reconciliation above + all read-then-write the same ``(stream key, wait_id)`` objects, and the + reconciliation is scheduled from another thread, so their interleaving + is not otherwise constrained. Without this, a reconciliation that + overlapped a confirming park could remove the intent that park had just + installed -- an unwakeable Run produced by the very code that exists to + prevent one. + """ + lock = self._park_locks.get(run_id) + if lock is None: + lock = self._park_locks[run_id] = asyncio.Lock() + return lock + + def subscription(self, run_id: str, wait_id: int) -> Subscription | None: + """Return one registered subscription, if present.""" + return self._runs.get(run_id, {}).get(wait_id) + + def subscriptions(self, run_id: str) -> list[Subscription]: + """Return all subscriptions currently registered for a Run.""" + return list(self._runs.get(run_id, {}).values()) + + def runs_with_subscriptions(self) -> list[str]: + """Return the Run IDs that currently own at least one subscription.""" + return [run_id for run_id, subs in self._runs.items() if subs] + + # --- the Workflow thread's entry points --------------------------------- + + def drain( + self, run_id: str, wait_id: int, max_records: int | None = None + ) -> list[StreamRecord]: + """Pops from one subscription's buffer. Performs no I/O.""" + subscription = self.subscription(run_id, wait_id) + if subscription is None: + return [] + popped = subscription.drain(max_records) + if popped: + # Freeing room restarts prefetch. Hopped onto the loop because an + # asyncio.Event may not be set from another thread. + self._loop.call_soon_threadsafe(subscription.note_drained) + return popped + + def note_wait_generation(self, run_id: str, wait_id: int, generation: int) -> None: + """Hands one wait's current generation to the subscription reporting it. + + Called from the Workflow thread, and deliberately *not* hopped onto the + manager's loop: the next readiness report may be issued by a watcher + before a queued callback would run, and it would then name the stale + generation this exists to replace. The write is a single guarded field + assignment, so it is safe to make directly. + """ + subscription = self.subscription(run_id, wait_id) + if subscription is not None: + subscription.note_wait_generation(generation) + + def rearm_ready(self, run_id: str) -> None: + """Re-reports readiness for every buffer this Run left non-empty. + + The per-activation delivery budget stops an iterator with records still + buffered. Nothing else will announce them: readiness is reported once, + when the watcher buffers a record, and that watcher has long since moved + its prefetch cursor past these. Without this the Workflow blocks forever + on records already sitting in front of it -- and worse, it blocks + *quiescently*, so Core would start the idle timer and eventually park a + Workflow Task whose data had already arrived. + + Called from the Workflow thread at activation completion, so the work is + hopped onto the manager's loop: ``create_task`` is not thread-safe, and a + task created from the Workflow executor thread is silently never + scheduled -- indistinguishable from a stream that never delivers. + """ + self._loop.call_soon_threadsafe(self._rearm_ready, run_id) + + def _rearm_ready(self, run_id: str) -> None: + """The manager-loop half of :meth:`rearm_ready`.""" + for subscription in self.subscriptions(run_id): + if subscription._cancelled or not subscription.buffered: + continue + self._loop.create_task(self._report_ready(subscription)) + + def note_workflow_task_started(self, run_id: str) -> None: + """Records an activation boundary used to correlate an outstanding wake.""" + self._activation_sequences[run_id] = ( + self._activation_sequences.get(run_id, 0) + 1 + ) + + def note_workflow_task_completed(self, run_id: str, *, terminal: bool) -> None: + """Releases a coalesced wake after Core accepts a successful completion. + + Signal acknowledgement only says the wake entered History. The task it + creates can still fail or have its completion rejected; both paths are + replayed, and readiness reconstructed during that replay belongs behind + the same outstanding wake. The Worker's successful completion callback + is the first evidence that cycle is over. + + A non-terminal task may leave another generation buffered. Re-report it + after releasing the gate because a report queued by + :meth:`rearm_ready` while completion was in flight was deliberately + coalesced. A terminal task has no next activation and must not re-arm. + """ + pending = self._pending_wake_runs.get(run_id) + if pending is None: + return + # A completion for an activation already running when the send began is + # not evidence that the wake was processed. This exact interleaving is + # common in the closing window: readiness sends while the old task is + # reporting, then that old completion wins the race to Core. + if self._activation_sequences.get(run_id, 0) <= pending.attempt_started_at: + return + if not pending.acknowledged: + # The service call is still in flight. Its response decides whether + # this completion can belong to the attempt: on failure the next + # retry resets both the activation boundary and this observation. + pending.completed_terminal = terminal + return + self._release_pending_wake(run_id, pending, terminal=terminal) + + def _release_pending_wake( + self, run_id: str, pending: _PendingWake, *, terminal: bool + ) -> None: + """Ends ``pending`` if it is still this Run's current wake cycle.""" + if self._pending_wake_runs.get(run_id) is not pending: + return + del self._pending_wake_runs[run_id] + if not terminal: + self._rearm_ready(run_id) + + def _note_wake_attempt_started(self, target: WakeTarget) -> None: + """Moves a live wake's correlation boundary to this retry attempt.""" + if not isinstance(target, Subscription): + return + pending = self._pending_wake_runs.get(target.run_id) + if pending is None or pending.wait_id != target.wait_id: + return + pending.attempt_started_at = self._activation_sequences.get(target.run_id, 0) + pending.acknowledged = False + pending.completed_terminal = None + + def _note_wake_attempt_acknowledged(self, target: WakeTarget) -> None: + """Marks the current attempt delivered and applies any racing completion.""" + if not isinstance(target, Subscription): + return + pending = self._pending_wake_runs.get(target.run_id) + if pending is None or pending.wait_id != target.wait_id: + return + pending.acknowledged = True + if pending.completed_terminal is not None: + self._release_pending_wake( + target.run_id, + pending, + terminal=pending.completed_terminal, + ) + + def reposition_to_committed( + self, run_id: str, cursors: Mapping[int, Cursor] + ) -> None: + """Moves each named wait to the boundary a replayed marker committed. + + Called from the Workflow thread once replay has delivered a marker's + recorded ranges, and **completed before it returns** -- unlike + :meth:`rearm_ready`, which only has to happen eventually. The very next + thing the Workflow thread does is drain, and a reposition merely posted + to the manager's loop leaves the marker-covered records sitting in the + buffer for that drain to hand over a second time. Posting it and + returning made the fix depend on the manager loop winning a race that + nothing ordered; the observed symptom was a Workflow receiving + ``['alpha', 'alpha', 'beta']``. + + Only the watcher's wakeup is hopped, because that is the one part that + touches an ``asyncio`` primitive. The retraction itself is under the + subscription's lock and the epoch bump fences a read already in flight, + so a watcher appending concurrently is either cleared by this or + rejected by :meth:`Subscription._append`. + """ + repositioned: list[Subscription] = [] + for wait_id, cursor in cursors.items(): + subscription = self.subscription(run_id, wait_id) + if subscription is None or subscription._cancelled: + # A wait the marker records but this Run no longer holds. Replay + # itself reports that as nondeterminism; there is nothing to + # reposition here and nothing to say about it. + continue + subscription.reposition_to(cursor) + repositioned.append(subscription) + for subscription in repositioned: + # An `asyncio.Event` may not be set from another thread, so the one + # asynchronous consequence of the retraction -- a watcher parked on + # backpressure now having room -- is what gets hopped. + self._loop.call_soon_threadsafe(subscription.note_drained) + + def blocked_snapshot(self, run_id: str) -> dict[int, Cursor]: + """Where every active subscription's deliveries stopped. + + Read from manager state alone -- no provider call, no backend read. + The boundary is not "wherever the stream is now"; it is where this + Workflow Task's deliveries stopped, which is already fixed. + """ + return { + wait_id: sub.blocked_cursor() + for wait_id, sub in sorted(self._runs.get(run_id, {}).items()) + } + + # --- watchers ----------------------------------------------------------- + + async def _watch(self, subscription: Subscription) -> None: + """Prefetches into the buffer and reports readiness once buffered. + + Survives Workflow Task completion. Torn down only on cancellation, Run + eviction, or Worker shutdown -- because the window between tasks is + exactly when an append most needs someone watching for it. + """ + try: + while not subscription._cancelled: + room = subscription._room() + if room == 0: + # Backpressure. Stop reading; drop nothing, block nobody. + subscription._has_room.clear() + await subscription._has_room.wait() + continue + + # Captured before the read, so that a reposition landing while + # it is in flight is detected afterwards. The read started from + # a cursor that no longer describes this subscription, and + # appending its result would undo the reposition rather than + # merely race it. + with subscription._lock: + epoch = subscription._prefetch_epoch + try: + records = await subscription.backend.read_after( + subscription.stream_key, + subscription.prefetch_cursor, + max_records=room, + block=self._watch_block, + ) + except asyncio.CancelledError: + raise + except Exception: + # Watcher failures are retried within provider policy. The + # Workflow Task is not failed from out here -- it may not + # even exist. + logger.exception( + "External stream watcher failed for %s wait %s", + subscription.stream_key, + subscription.wait_id, + ) + await asyncio.sleep(0.05) + continue + + if not records: + continue + + # Prepared **before** buffering, so that what the Workflow + # thread pops is already a value waiting to be typed. This is + # the whole reason the manager holds a converter: a codec + # awaited on the Workflow thread performs I/O in a + # deterministic event loop. + # + # And before the epoch check rather than after it, because + # preparing awaits: a reposition landing while a user's codec + # runs would otherwise slip past a check that had already + # passed, and the retracted records would go into the buffer + # anyway. The check stays the last thing before the append. + prepared = await self._prepare( + [(subscription.stream_key, record) for record in records] + ) + + # The epoch is compared inside `_append`, under the lock the + # reposition also takes, because repositioning is synchronous on + # the Workflow thread: a check made out here could pass and then + # have the reposition land before the append. `False` means the + # records were read from a position a reposition has since + # retracted, so they are records the marker already accounts + # for; the next pass reads from the new cursor. + if not subscription._append(prepared, epoch): + continue + await self._report_ready(subscription) + except asyncio.CancelledError: + pass + + async def _prepare( + self, records: Sequence[tuple[StreamKey, StreamRecord]] + ) -> list[StreamRecord]: + """Runs the DataConverter's asynchronous half, here on the Worker's loop. + + Records arrive paired with the stream they came from, because this + manager serves every Run on the Worker and the converter has to be bound + to the Workflow each individual record belongs to. + + Every record is prepared, including ones the Workflow may never take: + this is the same bargain the Worker already makes for an activation's + payloads, which are decoded in full before the executor sees any of + them. The cost of preparing a record that is later discarded is one + codec call; the cost of *not* preparing it is that the Workflow thread + has to, and the Workflow thread cannot. + + **Nothing raises out of here.** A failure is carried on the record and + raised by the delivery that would have yielded its value, which is the + only point at which a Workflow exists to be told. Raising here would + kill the watcher for the Run -- taking every later record with it -- and + would report a failure for a record the Workflow might never have asked + for. + + Control records carry no payload by construction, so they pass through. + """ + if self._data_converter is None: + return [record for _, record in records] + from temporalio.contrib.external_workflow_streams._codec import ( + StreamPayloadCodec, + ) + + data_converter = self._data_converter + # One codec per Workflow, not one per manager. A converter bound at + # construction would be bound to nothing in particular: this manager + # outlives every Run on the Worker and prepares records for all of them + # at once, so the only correct context is the one the record's own + # stream carries. + # + # `workflow_id` rather than `run_id`, and both taken from the stream key + # rather than from a subscription: a stream spans the whole + # Continue-As-New chain, so a successor Run must decode its predecessor's + # records with the same key -- which is also why `decode_activation` + # keys on `workflow_id`. + # + # Memoized for this call only. Live batches are one stream, replay + # batches are one Workflow's waits, so the clone happens per batch + # rather than per record; holding the map on the manager instead would + # accumulate one entry per Workflow ID the Worker ever served. + codecs: dict[tuple[str, str], StreamPayloadCodec[Any]] = {} + + def codec_for(key: StreamKey) -> StreamPayloadCodec[Any]: + cached = codecs.get((key.namespace, key.workflow_id)) + if cached is None: + # No type: the type belongs to the topic, and the topic belongs + # to Workflow code. Nothing out here needs it, because nothing + # out here runs the payload converter. + cached = StreamPayloadCodec( + data_converter.with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=key.namespace, + workflow_id=key.workflow_id, + ) + ), + None, + ) + codecs[(key.namespace, key.workflow_id)] = cached + return cached + + prepared: list[StreamRecord] = [] + for key, record in records: + if record.is_control: + prepared.append(record) + continue + try: + prepared.append( + PreparedRecord.of( + record, await codec_for(key).prepare(record.payload), None + ) + ) + except asyncio.CancelledError: + raise + except Exception as err: + logger.debug( + "Preparing external stream record at %s failed; the error " + "travels with the record", + record.offset, + exc_info=True, + ) + prepared.append(PreparedRecord.of(record, None, err)) + return prepared + + async def _report_ready(self, subscription: Subscription) -> None: + """Tells Core a record is buffered, and acts on which answer comes back. + + **Total, except for cancellation.** Nothing but `CancelledError` may + leave this method. The watcher calls it in its loop, so an exception + escaping ends that watcher for good -- the subscription stays registered, + its buffer keeps its records, and nothing ever announces them again. + `_rearm_ready` also launches it with ``create_task``, where an exception + becomes a task result nobody retrieves and the failure is not even + logged. + """ + result = await self._notify_ready_with_retries(subscription) + + if result == ReadinessResult.ACCEPTED: + # Core will activate; the record is announced and this is done. + return + + if result == ReadinessResult.STALE: + # Core is holding a newer generation for this wait than the one just + # named, so the report was for a block that has already been + # resolved. Re-report against the current generation rather than + # returning: the watcher only calls back here after a *new* + # non-empty read, and `prefetch_cursor` is already past the record + # in the buffer, so nothing would announce it a second time. A + # record announced to nobody is a Workflow blocked forever on data + # it is already holding. + # + # `result` is **replaced** by what the retries ended on, not merely + # tested. The five answers differ in what happens to the watcher + # afterwards, and a retry can legitimately land on a different one + # than the first report did -- the Run can be evicted between the + # report that raced a generation change and the delayed retry that + # follows it. Keeping the original `Stale` there sent the owed wake + # (right) and then skipped the `RunNotFound` teardown (wrong), leaving + # a watcher, a buffer, a backend read loop and a Run-map entry alive + # for a Run this Worker no longer owns. + result = await self._retry_stale(subscription) + if result == ReadinessResult.ACCEPTED: + return + + # Everything left means local readiness could not be delivered. One + # Signal is owed for the Run, but only one until Core accepts the + # successful Workflow Task completion it caused. A second wait or a + # replayed copy of this one can report during the closing/failing window; + # sending each report creates another Workflow Task that can collide + # with the same completion and sustain the replay loop. + if subscription.run_id in self._pending_wake_runs: + return + self._pending_wake_runs[subscription.run_id] = _PendingWake( + wait_id=subscription.wait_id, + attempt_started_at=self._activation_sequences.get(subscription.run_id, 0), + ) + # + # Counted once, and then retried inside `_send_owed_wake`, because a + # single attempt here has nothing behind it. The watcher has already + # moved `prefetch_cursor` past the buffered record and returns on `if + # not records`, so it never comes back through here without a *new* + # append; `rearm_ready` needs the activation this lost wake was supposed + # to cause; and the idle timer only runs while a Workflow Task is + # retained, which `NoOpenWorkflowTask` says there is not. One failed + # attempt was a lost record. + self._count_owed_wake(subscription) + if not await self._send_owed_wake(subscription): + self._pending_wake_runs.pop(subscription.run_id, None) + logger.warning( + "External stream wake for %s wait %s was not acknowledged; it " + "stays owed and the shutdown sweep is the only backstop left", + subscription.stream_key, + subscription.wait_id, + ) + + if result == ReadinessResult.RUN_NOT_FOUND: + # The Run is gone from this Worker. Nothing here can serve it again. + # Dropped only now, after the retries above: this pop is what takes + # the subscription out of `_runs`, and the shutdown sweep iterates + # `_runs`, so a wake given up on before it has no backstop at all. + subscription._cancelled = True + self._runs.get(subscription.run_id, {}).pop(subscription.wait_id, None) + # PARKED and NO_OPEN_WORKFLOW_TASK both *keep* the watcher: the Run is + # still cached and this is the normal window between Workflow Tasks. + + async def _notify_ready_with_retries(self, subscription: Subscription) -> str: + """Reports readiness, retrying a failing call a bounded number of times. + + A raising notifier is a transport problem, not an answer. Letting it + escape kills the watcher; swallowing it and returning would claim the + record was announced. Retrying and then falling through to the + wake-owed branch treats an unreachable Core as what it is: local + readiness could not be delivered, which is the sixth case the five + answers do not name. + """ + for attempt in range(READINESS_ATTEMPTS): + try: + return _result_value( + await self._notify_ready( + subscription.run_id, + subscription.wait_id, + subscription.current_wait_generation(), + ) + ) + except asyncio.CancelledError: + raise + except Exception: + logger.warning( + "External stream readiness report %s/%s failed for %s wait %s", + attempt + 1, + READINESS_ATTEMPTS, + subscription.stream_key, + subscription.wait_id, + exc_info=True, + ) + if attempt + 1 < READINESS_ATTEMPTS: + await asyncio.sleep(READINESS_RETRY_DELAY.total_seconds()) + return ReadinessResult.NO_OPEN_WORKFLOW_TASK + + async def _retry_stale(self, subscription: Subscription) -> str: + """Re-reports a stale readiness against the generation Core now holds. + + Returns the result the retries ended on, so the caller can act on it. A + Boolean would answer only "was it announced", and the four + non-``Accepted`` answers are not interchangeable: they differ in what + happens to the watcher, and ``RunNotFound`` in particular requires the + watcher to be torn down. Discarding them left that teardown unreachable + from here. + + A generation moves when the wait re-enters the blocked state, which is + Workflow code coming back around to it -- so the report that raced it is + answered by the next one. Bounded, because a wait that keeps moving is a + Workflow consuming happily, and a wake owed at the end of that costs one + empty Workflow Task rather than a silent stall. + + ``RunNotFound`` ends the retries rather than using them up. It is the one + answer that cannot change back: the Run is gone from this Worker, so a + further report can only be answered the same way, and each attempt costs a + delay before the wake this record still needs. + """ + result = ReadinessResult.STALE + for _ in range(STALE_REPORT_ATTEMPTS): + await asyncio.sleep(STALE_RETRY_DELAY.total_seconds()) + if subscription._cancelled or not subscription.buffered: + # Nothing left to announce: either the wait is gone or an + # activation drained the buffer, which is the record having been + # delivered by the very block this report raced. Reported as + # `Accepted` because that is what the caller does with it -- stop, + # and owe no wake. + return ReadinessResult.ACCEPTED + result = await self._notify_ready_with_retries(subscription) + if result in (ReadinessResult.ACCEPTED, ReadinessResult.RUN_NOT_FOUND): + return result + return result + + # --- the runtime-only jobs' backend work (P19) -------------------------- + + async def prepare_park( + self, run_id: str, park_generation: int, blocked: Mapping[int, Cursor] + ) -> bool: + """Installs park intents, then rechecks every stream. + + Returns ``True`` if any stream became ready, which abandons this parking + generation. + + ``blocked`` **is the park set**, not a lookup table beside one: its keys + are the waits Core asked to park, taken from + ``PrepareExternalStreamPark.waits``, and its values are their cursor + boundaries. The manager's own registration list is a superset -- a + subscription that delivered a record and was not awaited again is + registered and not blocked -- and parking the superset is wrong in both + directions. A recheck for a wait outside the set finds records nothing + is waiting on and aborts a legitimate park, which then runs again on the + next idle timeout and aborts again; and an intent installed for a wait + outside the set is an intent with no park behind it, which is exactly + what ``backend-contract.md`` forbids leaving in a backend. + + The order is what closes the append/park race: a producer appends its + record *before* it observes the park generation, so an append is either + seen by the recheck below or paired with a wake Signal. Rechecking + before all the intents were installed would leave a window where it is + neither. + """ + async with self._park_lock(run_id): + with _as_storage_failure("installing external stream park intents"): + return await self._prepare_park(run_id, park_generation, blocked) + + async def _prepare_park( + self, run_id: str, park_generation: int, blocked: Mapping[int, Cursor] + ) -> bool: + """The handshake itself, under this Run's park lock.""" + # A park is the one moment this Run is certain to reach a backend, so it + # is the cheapest place to retire whatever a previous close or resolve + # failed to remove -- and it must happen *before* the installs below, + # which would otherwise re-key the entries this is meant to retire. + await self._drain_owed_removals(run_id) + subscriptions = [ + subscription + for subscription in self.subscriptions(run_id) + if subscription.wait_id in blocked + ] + missing = set(blocked) - {s.wait_id for s in subscriptions} + if missing: + # Core is parking a wait this manager no longer holds. Nothing can + # be installed for it and there is nothing to recheck; a producer on + # that stream finds no intent and sends the unparked wake, which + # Core accepts as a recheck request (ADR-023). + logger.warning( + "Core asked run %s to park waits %s, which this Worker does not " + "hold; they are left out of the park set", + run_id, + sorted(missing), + ) + + installed: list[Subscription] = [] + try: + for subscription in subscriptions: + await subscription.backend.install_park_intent( + subscription.stream_key, + ParkIntent( + wait_id=subscription.wait_id, + cursor=blocked[subscription.wait_id], + park_generation=park_generation, + run_id=run_id, + ), + ) + subscription.installed_park_generation = park_generation + # A fresh intent at this key supersedes anything owed for it: a + # removal recorded against the generation just overwritten would + # otherwise be retried against the park now sitting behind it. + self._forget_owed_removal( + run_id, (subscription.stream_key, subscription.wait_id) + ) + installed.append(subscription) + + became_ready = False + for subscription in subscriptions: + if await subscription.backend.recheck( + subscription.stream_key, subscription.wait_id + ): + became_ready = True + break + except BaseException: + # A park that failed part-way through is still a park every producer + # can see. The activation fails and Core confirms nothing, so those + # intents describe a park that does not exist -- and an eviction then + # takes the local mirror away while they stay. Rolling back is what + # keeps "all-or-nothing across the set" true of the failure path too. + # Removal failures are swallowed: the storage error that got here is + # the one worth reporting, and anything still installed stays owed + # in this Run's ledger for the next drain to retry. + # + # `BaseException`, not `Exception`, and not with the usual + # `except asyncio.CancelledError: raise` above it: cancellation is + # the *most* likely way a half-installed park is abandoned -- Core + # withdrawing the activation, the Worker shutting down -- and it + # derives from `BaseException`, so an `Exception` handler leaves + # exactly those intents behind. Deliberately not shielded: a single + # `Task.cancel()` still lets this await run to completion, whereas a + # shielded rollback would detach and land after `prepare_park` has + # released `_park_lock`, where it can strand a newer legitimate + # park's intent instead. + await self._withdraw_park(installed) + raise + + if became_ready: + # All-or-nothing: a park confirmed for a set with a ready member + # would lose that member's record until a producer happened to + # signal, so every intent installed above comes back out. + failures = await self._withdraw_park(installed) + if failures: + # Nothing may report `became_ready` while an intent for that + # generation is still installed. Failing the activation retries + # the whole Workflow Task, which commits no cursor and loses no + # record; the removal stays owed either way. + raise failures[0] + return became_ready + + async def _withdraw_park( + self, subscriptions: list[Subscription] + ) -> list[Exception]: + """Takes every intent one attempted park installed back out. + + Total: every subscription is attempted whatever the others do, because a + rollback that stopped at its first failure would leave behind exactly + the orphans it exists to prevent. The failures are returned rather than + raised so each caller can decide which error is the one worth reporting. + """ + failures: list[Exception] = [] + for subscription in subscriptions: + try: + await self._remove_park_intent(subscription) + except asyncio.CancelledError: + raise + except Exception as err: + logger.exception( + "Failed removing the park intent for %s wait %s while " + "withdrawing an unconfirmed park; its autonomous " + "owed-removal retry continues in the background", + subscription.stream_key, + subscription.wait_id, + ) + failures.append(err) + return failures + + async def resolve_park(self, run_id: str) -> None: + """Removes the intents of a park this Run is no longer sitting in. + + The other half of :meth:`prepare_park`, and the reason the invariant is + stated as *an intent exists only while that park is outstanding* rather + than as "an aborted park cleans up after itself". An aborted park was + only ever the cheaper half: a **confirmed** park ends too, when a wake + Signal or a fresh quiescent snapshot clears Core's ``park_generation``, + and Core's own state moving on is not something the backend can observe. + + A left-behind intent is not inert, because it is the answer + ``current_park_generation`` gives to everyone who asks: + + - a **producer** reads it to decide what its wake Signal names, and a + dead generation is precisely the claim Core is designed to discard as + stale -- so the producer appends, signals, and the Workflow is never + woken by it; + - the **shutdown sweep** reads it through the same call, and would send a + parked wake where P20 requires the unparked one (ADR-023). Worse than + useless there: a parked wake's request ID deliberately ignores sender + identity, so it comes out identical to the wake that already resolved + that generation and the server deduplicates it away. + + This is the half for parks *this* manager installed, plus whatever the + Run's owed-removal ledger still holds. An intent whose installer is gone + -- evicted, moved to another Worker, shut down -- leaves no mirror here + to remove it by, and is put into that ledger by + :meth:`_reconcile_inherited_park` when the wait is registered again. + + Driven by ``ResolveExternalStreamWaits``, which is Core telling this + Worker the wait set has moved on -- the one event that covers both ways a + confirmed park ends, and the aborted-in-Core-but-confirmed-in-lang case + the recheck cannot see either. + + Failure is logged rather than raised. The removal is cleanup, and turning + a momentary backend blip into a failed Workflow Task on the *delivery* + path would trade a stale intent for a repeated Workflow Task. It stays + in the ledger, so the next drain retries it. + """ + async with self._park_lock(run_id): + # Ahead of the registered waits, because the ledger is the only + # thing that still names a wait this Run has closed -- and a stale + # intent on a closed wait is not confined to it: it keeps + # `parked_wait_ids` non-empty, which suppresses the unparked wake + # for the whole stream and silences every live wait on it. + await self._drain_owed_removals(run_id) + for subscription in self.subscriptions(run_id): + try: + await self._remove_park_intent(subscription) + except Exception: + logger.exception( + "Failed removing the resolved park intent for %s wait %s; " + "its autonomous owed-removal retry continues in the " + "background", + subscription.stream_key, + subscription.wait_id, + ) + + async def _remove_park_intent(self, subscription: Subscription) -> None: + """Removes one installed intent, and forgets it only once it is gone. + + Two things can say an intent is installed, and either one is enough. + ``installed_park_generation`` is the mirror of a park *this* manager + confirmed. The ledger is a removal already decided on and not yet + confirmed -- including every inherited intent, which is mirrored nowhere + because the Worker that installed it is gone. Keying on the mirror alone + is what left `resolve_park`, `_withdraw_park` and `cancel` all silently + doing nothing for exactly the intents that most need reaching. + + No read guards this one, unlike :meth:`_drain_owed_removals`: it is + reached only with a Subscription of a Run this Worker still holds, under + that Run's park lock, so the successor Run whose intent a stale entry + could name does not exist yet. + """ + run_id = subscription.run_id + key = (subscription.stream_key, subscription.wait_id) + if key not in self._owed_removals.get(run_id, {}): + if subscription.installed_park_generation is None: + return + # Recorded before the call rather than after it fails: a removal + # that never comes back -- the backend raised, or this task was + # cancelled mid-await -- is owed either way, and the Subscription + # this was reached through is frequently dropped in the same breath. + self._owe_removal( + run_id, + key, + _OwedRemoval( + backend=subscription.backend, + park_generation=subscription.installed_park_generation, + run_id=run_id, + ), + ) + record = self._owed_removals[run_id][key] + await subscription.backend.remove_park_intent_if_matches( + subscription.stream_key, + subscription.wait_id, + run_id=record.run_id, + park_generation=record.park_generation, + ) + subscription.installed_park_generation = None + self._forget_owed_removal(run_id, key) + + async def prepare_replay(self, run_id: str, replay_annotation: bytes) -> ReplayPlan: + """Reads and validates every recorded range, before any delivery. + + Runs here rather than in ``_apply`` because an inclusive range read over + a whole recorded batch is exactly the multi-second operation the + Workflow thread's deadlock timeout cannot accommodate -- and it gets + worse the more records the marker recorded. + + Subscriptions may not exist yet: on replay the Workflow has not run far + enough to call ``subscribe()``. The annotation's own header carries the + stream key and configured provider identity for each wait, which is why + it records them. + """ + from temporalio.contrib.external_workflow_streams._annotation import ( + decode_annotation, + ) + + annotation = decode_annotation(replay_annotation) + backends: dict[int, StreamBackend] = {} + stream_keys: dict[int, StreamKey] = {} + for wait_id, binding in annotation.header.streams.items(): + stream_keys[wait_id] = binding.stream_key + resolved = self._replay_backend(wait_id, binding) + if resolved is not None: + backends[wait_id] = resolved + + plan = await build_replay_plan(replay_annotation, backends, stream_keys) + # Replay delivers through the same drain the live path does, so it has + # to arrive in the same condition: prepared. This is the only chance to + # do it -- by the time the segments are delivered, the Workflow thread + # is running, and the ranges have already been validated here, so a + # decode failure from now on is a converter mismatch rather than + # integrity loss (ADR-015). + for index, segment in enumerate(plan.segments): + plan.segments[index] = ReplaySegment( + tuple( + (wait_id, prepared) + for (wait_id, _), prepared in zip( + segment.deliveries, + # The annotation header is the only place a replayed + # record's stream is written down: the Workflow has not + # run far enough to have re-created the subscription + # that would otherwise carry it. + await self._prepare( + [ + (stream_keys[wait_id], record) + for wait_id, record in segment.deliveries + ] + ), + ) + ) + ) + self._replay_plans[run_id] = plan + return plan + + def _replay_backend( + self, wait_id: int, binding: StreamBinding + ) -> StreamBackend | None: + """The one backend a recorded wait may be read from. + + Returning ``None`` leaves the wait unresolved when no backend is + configured; :func:`build_replay_plan` then reports any recorded ranges + that cannot be read. + """ + backend = self._backend + if backend is None: + return None + + # Whether the Worker still carries the same implementation is a + # deployment question, not a Workflow one. It is raised before any read, + # so an incompatible implementation never interprets recorded offsets. + declared_id = type(backend).provider_id + if declared_id != binding.provider_id: + raise StreamStorageError( + f"external stream wait {wait_id} was recorded against provider " + f"{binding.provider_id!r}, but the backend configured on this " + f"Worker declares {declared_id!r}. This Worker cannot read what " + "that marker recorded; configure the recorded provider." + ) + declared_version = type(backend).provider_format_version + if declared_version != binding.provider_format_version: + raise StreamStorageError( + f"external stream wait {wait_id} was recorded by provider " + f"{binding.provider_id!r} format version " + f"{binding.provider_format_version}, but the configured backend " + f"implements format version " + f"{declared_version}. Reading it would interpret the recorded " + "offsets under a format they were not written in." + ) + return backend + + def take_replay_plan(self, run_id: str) -> ReplayPlan | None: + """The prepared plan, consumed once by the delivering activation.""" + return self._replay_plans.pop(run_id, None) + + # --- teardown ----------------------------------------------------------- + + def cancel_from_workflow_thread(self, run_id: str, wait_id: int) -> None: + """Schedules :meth:`cancel` from the Workflow thread. + + Workflow code closes a subscription inside the synchronous + ``activate()``, on the executor thread, where creating a task is not + merely unsafe but silently ineffective: the task is never scheduled, and + a watcher that was supposed to stop keeps running with nothing to say so. + """ + self._loop.call_soon_threadsafe( + lambda: self._loop.create_task(self.cancel(run_id, wait_id)) + ) + + async def cancel(self, run_id: str, wait_id: int) -> None: + """Cancels one subscription: remove its intent, drop its buffer, stop it. + + The subscription leaves the Run's map **first**, so a close always stops + the watcher. Keeping a cancelled subscription registered while a removal + is retried would resurrect the orphaned-watcher bug from the other + direction: it goes on prefetching into a buffer nothing can drain, for + the rest of the Worker's life. + + What makes dropping it safe is that the removal no longer lives *on* it. + It is attempted here, retried a bounded number of times, and whatever is + still owed after that stays in this Run's ledger -- which outlives the + subscription and retries autonomously. Parks, resolves, registrations + and eviction also drain it eagerly. Before that ledger existed there was + no retry at all: the resolve path iterates registered subscriptions, + eviction and the shutdown sweep remove no intents, and another wait's + park cannot touch a per-wait key, so an intent left behind here was left + behind for good. + + And it is not confined to the closed wait. A stale intent keeps + ``parked_wait_ids`` non-empty, which suppresses the unparked-wake fallback + for the **whole stream**: with no live wait parked, the producer sends + only the dead generation, Core discards it as stale, and dedup silences + every later publish -- so live waits across the Continue-As-New chain + lose their wakes too. + """ + subscription = self._runs.get(run_id, {}).pop(wait_id, None) + if subscription is None: + return + for attempt in range(PARK_REMOVAL_ATTEMPTS): + async with self._park_lock(run_id): + try: + # Bounded for the same reason the reconciliation's hold is: + # this runs on a task of its own, and a call that hangs + # rather than raises would hold this Run's park lock for as + # long as it lasts. A timeout arrives here as one more + # failed attempt, which is what it is. + await asyncio.wait_for( + self._remove_park_intent(subscription), + PARK_REMOVAL_CALL_TIMEOUT.total_seconds(), + ) + break + except Exception: + logger.warning( + "Removing the park intent for %s wait %s while closing " + "it failed on attempt %s/%s", + subscription.stream_key, + subscription.wait_id, + attempt + 1, + PARK_REMOVAL_ATTEMPTS, + exc_info=True, + ) + if attempt + 1 < PARK_REMOVAL_ATTEMPTS: + await asyncio.sleep(PARK_REMOVAL_RETRY_DELAY.total_seconds()) + else: + logger.warning( + "Could not remove the park intent for %s wait %s while closing " + "it; it stays in this Run's owed-removal ledger, and until a " + "drain retires it producers read a generation Core has " + "discarded and their wakes are ignored as stale", + subscription.stream_key, + subscription.wait_id, + ) + await self._stop(subscription) + + async def evict_run(self, run_id: str) -> None: + """Tears down every subscription for a Run. + + The same path serves eviction, Workflow Task failure mid-batch, and + shutdown, because all three leave exactly the same thing behind: + speculative reads that were never committed. + + Owed removals belong to the manager rather than its cache. Eviction + makes one eager attempt but leaves any failure in the autonomous retry + loop, so backend recovery is sufficient even if this Run never returns + to the Worker. Installed intents are deliberately untouched -- an + eviction is not the end of a park, and the intents of a park that really + is outstanding must survive it. What the removal *silenced* outlives this + too, and is handed off by :meth:`_wake_after_cleanup`: the subscriptions + dropped below are exactly what the Subscription-bound announcement needs. + + The eager attempt is skipped while shutting down, and that is not the + same as skipping cleanup. A retry loop stuck in a backend call holds this + Run's park lock, and a drain here would wait on it with no bound at all, + wedging the shutdown it is part of. The last attempt is made by + :meth:`_final_owed_removal_pass` instead, which runs after those loops + have been cancelled and awaited and is bounded by what is left of the + grace period. + """ + self._replay_plans.pop(run_id, None) + if self._owed_removals.get(run_id) and not self._shutting_down: + async with self._park_lock(run_id): + await self._drain_owed_removals(run_id) + for subscription in self._runs.pop(run_id, {}).values(): + await self._stop(subscription) + retry = self._owed_removal_retries.get(run_id) + if not self._owed_removals.get(run_id) and (retry is None or retry.done()): + self._park_locks.pop(run_id, None) + + async def probe_runs(self, *, grace: timedelta = DEFAULT_PROBE_GRACE) -> None: + """Records what state Core says each Run is in. The sweep's first half. + + Separated from the wakes because the two halves have opposite timing + requirements, and running them together means one of them is wrong: + + - the **probe** has to be asked while Core can still answer it. Its whole + value is telling ``WftOpen``, ``Parked`` and ``NoOpenWorkflowTask`` + apart, and all three are statements about a Run Core still holds. Core + keeps them only until the Worker's shutdown is initiated: an idle + cached Run has no pending work, so ``shutdown_done`` is satisfied by + the very first input after the shutdown token is cancelled and the + whole workflow-state lane ends. Every probe after that answers + ``RunNotFound`` -- which still owes a wake, so the sweep goes on + looking correct while its ``Parked`` branch (a Run that needs no wake + at all) and its ``WftOpen`` branch (a Run that must be left to C15b + rather than raced) have silently stopped being reachable. + - the **wakes** must not run there. Sending them before the pollers stop + would offer the Run to a task queue this Worker is still polling, which + is the opposite of a hand-off, and would hold the stop-polling step + open for as long as the server takes to acknowledge them. + + So this runs immediately *before* shutdown is initiated and does nothing + but ask and remember, and :meth:`sweep` acts on the answers afterwards. + The one thing that costs: a Run recorded as ``NoOpenWorkflowTask`` here + may still pick up a Workflow Task from an in-flight poll before the + pollers stop, and would then get both C15b's forced replacement and this + sweep's wake. That is one extra empty Workflow Task, which this design + permits, and it is the cheaper side of the trade -- the alternative is + never distinguishing the states at all. + + Bounded by ``grace`` because nothing may make stopping the pollers wait + indefinitely. Every Run left unprobed is simply probed again by the + sweep, where the answer is whatever Core has left to say. + """ + probe = self._run_status + if probe is None: + return + try: + await asyncio.wait_for(self._probe_runs(probe), grace.total_seconds()) + except asyncio.TimeoutError: + logger.warning( + "External stream shutdown probe did not finish within %s; the " + "Runs it did not reach are swept on whatever Core can still say", + grace, + ) + + async def _probe_runs(self, probe: RunStatusProbe) -> None: + for run_id in list(self._runs): + if not self._runs.get(run_id): + continue + try: + self._probed[run_id] = _status_value(await probe(run_id)) + except Exception: + logger.exception( + "External stream shutdown probe failed for run %s", run_id + ) + + async def sweep(self, *, grace: timedelta = DEFAULT_SHUTDOWN_GRACE) -> None: + """Sends the wakes the probe found owed. The sweep's second half. + + Separate from :meth:`shutdown` so teardown can stay where it belongs -- + after every activation has been answered, so per-Run teardown remains + driven by ``RemoveFromCache`` and a ``FinalizeExternalStreams`` in flight + is answered before the Run's state disappears. + + Never blocked past ``grace``. A wake that could not be acknowledged in + time is reported rather than dropped, and never counted as delivered. + """ + if self._swept: + return + self._swept = True + self._shutting_down = True + # Every subscription starts out unaccounted for, and each one leaves this + # set exactly once: when its Run's status says nothing is owed, or when + # its wake is acknowledged. Whatever is still in it when the sweep stops + # is a handoff this Worker did not make, and the whole point of the + # counter is that such a handoff cannot be silent. + self._unaccounted = [ + subscription + for run_id in list(self._runs) + for subscription in self._runs.get(run_id, {}).values() + ] + try: + await asyncio.wait_for(self._sweep(), grace.total_seconds()) + except asyncio.TimeoutError: + logger.warning( + "External stream shutdown sweep did not finish within %s; " + "tearing down anyway", + grace, + ) + finally: + # In a `finally`, and reached on the timeout path especially. + # `wait_for` cancels `_sweep()`, and cancellation lands wherever the + # sweep happened to be: inside a hanging Signal send, whose + # `CancelledError` `_send_owed_wake` deliberately re-raises, and + # before every subscription the serial loop had not reached yet. + # Accounting done only where the sweep managed to reach therefore + # reported a clean shutdown -- `shutdown_wake_failures == 0` -- for a + # Worker that had just abandoned every one of its handoffs. + self._account_unswept() + + async def shutdown(self, *, grace: timedelta = DEFAULT_SHUTDOWN_GRACE) -> None: + """Sweeps every Run that still holds subscriptions, then tears down. + + Per-Run teardown is normally eviction's job, and this is the backstop for + Runs eviction never reaches -- an idle cached Run gets no eviction + activation at shutdown at all, which is exactly the Run that most needs + the sweep: its records are buffered in a process that is about to exit, + and nothing else will ever tell the Workflow they arrived. + """ + deadline = self._loop.time() + max(0.0, grace.total_seconds()) + await self.sweep(grace=grace) + for run_id in list(self._runs): + await self.evict_run(run_id) + remaining = timedelta(seconds=max(0.0, deadline - self._loop.time())) + await self._stop_background_tasks(remaining) + await self._final_owed_removal_pass( + timedelta(seconds=max(0.0, deadline - self._loop.time())) + ) + # Last, because the pass above is what creates the final cleanup + # handoffs: a removal confirmed there ends a suppression window nothing + # else will ever announce, and its wake is posted as a task rather than + # awaited under the park lock. Leaving without waiting for those tasks is + # indistinguishable from never having sent them. + await self._flush_cleanup_wakes( + timedelta(seconds=max(0.0, deadline - self._loop.time())) + ) + + async def _final_owed_removal_pass(self, budget: timedelta) -> None: + """The last attempt at every removal still owed, and a count of what is left. + + Reached only after :meth:`_stop_background_tasks`, which is both what + makes this the last attempt and what makes it safe: the retry loops that + own these entries are cancelled and awaited by then, so nothing holds a + park lock behind this pass and nothing restarts a loop after it. + + Eviction is where this used to happen, and while shutting down it no + longer does -- a retry loop blocked in a backend call still held the lock + the eviction would have waited on. What was lost with it is the case this + exists for: an entry the loop failed once and is now sleeping out its + backoff, on a backend that has since recovered. Cancelling that sleep and + exiting would leave an intent installed that one call would have taken + out. + + Bounded by whatever is left of the grace the sweep was given, because + cleanup may not extend shutdown -- and silent about nothing, since an + intent still installed as the process exits suppresses the unparked wake + for whoever picks the Run up next. + """ + owed = list(self._owed_removals) + if not owed: + return + deadline = self._loop.time() + max(0.0, budget.total_seconds()) + for run_id in owed: + left = deadline - self._loop.time() + if left <= 0: + break + await self._bounded_drain( + run_id, min(PARK_REMOVAL_CALL_TIMEOUT, timedelta(seconds=left)) + ) + still = sum(len(entries) for entries in self._owed_removals.values()) + if still: + logger.warning( + "%s external stream park intent(s) are still installed as this " + "Worker exits, with no park behind them; the next Worker's " + "registration-time reconciliation is the only thing left that " + "removes them", + still, + ) + + async def _stop_background_tasks(self, grace: timedelta) -> None: + """Cancels manager-owned cleanup work without extending shutdown.""" + self._stopping_owed_removal_retries = True + tasks = set(self._reconciliations) + tasks.update(self._owed_removal_retries.values()) + if not tasks: + return + for task in tasks: + task.cancel() + # Give ordinary cancellation one turn even when the wake sweep consumed + # the whole grace period. Anything needing longer stays bounded by what + # remains rather than silently adding a second grace period to shutdown. + await asyncio.sleep(0) + pending = {task for task in tasks if not task.done()} + if pending and grace.total_seconds() > 0: + _, pending = await asyncio.wait(pending, timeout=grace.total_seconds()) + if pending: + logger.warning( + "%s external stream cleanup task(s) did not stop within %s", + len(pending), + grace, + ) + + async def _sweep(self) -> None: + """Acts on each Run's state and owes a wake only where one is owed. + + The probe is deliberately **not** the readiness call. Readiness means "a + record is buffered", so probing with it would assert something false and + manufacture a spurious Workflow Task on the way out of a Worker that is + shutting down. + + Nothing is counted as a failure in here beyond the wakes this actually + attempted. What was never reached is accounted for by + :meth:`_account_unswept`, which runs whether this returns or is cancelled + by the grace period. + """ + if self._run_status is None: + # No probe wired means there is no sweep on this manager at all -- + # the whole mechanism is defined in terms of what Core answers -- so + # there is no obligation to have failed to discharge. Resolved rather + # than counted: a metric that fires for a mechanism that was never + # configured tells an operator nothing about the deployment that has + # it. + self._resolve_unaccounted(self._unaccounted) + return + for run_id in list(self._runs): + subscriptions = list(self._runs.get(run_id, {}).values()) + if not subscriptions: + continue + status = self._probed.pop(run_id, None) + if status is None: + # Not probed while Core could still answer -- either nothing ran + # the probe phase, or this Run was cached from an in-flight poll + # after it. Ask anyway rather than guessing: the answer is + # whatever Core has left to say, and both answers it can still + # give owe a wake. + try: + status = _status_value(await self._run_status(run_id)) + except asyncio.CancelledError: + raise + except Exception: + # Left unaccounted for on purpose. A Run whose status cannot + # be read is a Run this Worker cannot say anything about, and + # "we could not tell" is not "nothing was owed": these + # subscriptions may each be holding a buffered record with + # nowhere to announce it. Sending a wake anyway would race a + # Workflow Task that may be open, so the honest outcome is to + # send nothing and let the counter say a handoff was lost. + logger.exception( + "External stream shutdown probe failed for run %s", run_id + ) + continue + + if status == RunStatus.WFT_OPEN: + # A Workflow Task is open and Core owns what happens to it + # (C15b). A wake here would race that transition and produce a + # second task for a Run already being attended to. + self._resolve_unaccounted(subscriptions) + continue + if status == RunStatus.PARKED: + # Already parked, so a producer's append will wake it through the + # ordinary path. Nothing is owed. + self._resolve_unaccounted(subscriptions) + continue + + # NoOpenWorkflowTask and RunNotFound both mean local readiness has + # nowhere to go. They differ in what happens to the Run afterwards, + # not in what is owed now. + for subscription in subscriptions: + await self._sweep_wake(subscription) + + async def _sweep_wake(self, subscription: Subscription) -> None: + """Sends one unparked wake, retrying within the grace period. + + Awaited rather than fired: an unacknowledged wake is the case this whole + sweep exists to prevent, and reporting shutdown as clean while a record + sits unannounced in a stream would make the wakeup-durability boundary + false. + + The grace period bounds the whole sweep, so the retries cannot extend + shutdown past it, and a wake that never lands is reported rather than + dropped -- which is the only thing that makes giving up acceptable. + + Counting the failure is deliberately **not** done here for the + cancellation case. The grace period expiring cancels this coroutine + wherever it is, `_send_owed_wake` re-raises `CancelledError` by design, + and no ``except`` here could both record the failure and leave the + cancellation intact for the subscriptions after this one -- which are not + reached either. The subscription stays in the unaccounted set instead and + :meth:`_account_unswept` counts it, which covers being cancelled and + being never visited with the same rule. + """ + self._count_owed_wake(subscription) + if await self._send_owed_wake(subscription): + self._note_swept_handoff(subscription) + self._resolve_unaccounted([subscription]) + else: + self._record_shutdown_wake_failure(subscription) + self._resolve_unaccounted([subscription]) + + def _note_swept_handoff(self, subscription: Subscription) -> None: + """Records that the sweep's wake also discharged an owed removal's handoff. + + Only where there is one. A stale intent on this wait means the wake just + sent was composed as unparked -- `wake_park_generation` saw the ledger + entry -- so it is the very generation-0 handoff the removal would owe + once it finally goes through, and `_final_owed_removal_pass` runs after + this. Without the note the two are one obligation sent twice, which + costs an empty Workflow Task; marked only on success, so a wake the + sweep could not get acknowledged leaves the handoff still owed. + """ + owed = self._owed_removals.get(subscription.run_id) + if not owed: + return + key = (subscription.stream_key, subscription.wait_id) + record = owed.get(key) + if record is not None and not record.announced: + owed[key] = replace(record, announced=True) + + def _resolve_unaccounted(self, subscriptions: Sequence[Subscription]) -> None: + """Marks these subscriptions as decided, however they were decided. + + Both outcomes are decisions: nothing was owed, or a wake was attempted + and its result recorded. What stays in the set is only what the sweep + never got to say anything about. + """ + decided = set(map(id, subscriptions)) + self._unaccounted = [ + subscription + for subscription in self._unaccounted + if id(subscription) not in decided + ] + + def _account_unswept(self) -> None: + """Counts every subscription the sweep never resolved as a lost handoff. + + The grace period is a bound on how long shutdown may be held open, not a + licence to stop counting: a Worker that abandons a wake and reports zero + failures makes the failure invisible in exactly the way this counter + exists to prevent. Both silent cases end up here -- the wake the + cancellation landed inside, and every subscription the serial loop never + reached. + """ + unaccounted, self._unaccounted = self._unaccounted, [] + for subscription in unaccounted: + if ( + self._runs.get(subscription.run_id, {}).get(subscription.wait_id) + is not subscription + ): + # Gone from the Run while the sweep ran, which the *live* readiness + # path does -- it drops a subscription on `RunNotFound`, and only + # after its own owed wake was acknowledged. `RunNotFound` is a + # likely answer during shutdown and the watchers keep running + # through the whole grace window, so this is an ordinary + # interleaving, not a corner. Counting it would report a lost + # handoff for one that was made, on the counter operators are told + # to alert on. + continue + logger.warning( + "External stream shutdown left a wake unresolved for %s wait %s", + subscription.stream_key, + subscription.wait_id, + ) + self._record_shutdown_wake_failure(subscription) + + async def wake_park_generation(self, target: WakeTarget) -> int | None: + """Which park one owed wake must name, or ``None`` for the unparked one. + + The Worker's sender asks this rather than reading + ``current_park_generation`` itself, because "installed" is not the same + as "live" and only the manager can tell the two apart. Two things make + the raw read wrong: + + - a `_CleanupWake` exists *because* the intent it names was just + retired. There is nothing left at that key that Core would accept, and + a generation another Worker has since installed belongs to a park this + handoff knows nothing about. + - a `Subscription` whose intent is in this Run's owed-removal ledger is + holding a generation this manager has **already decided to remove**. + Core discards a non-zero generation that is not the park it holds, so + naming it composes a wake that is ignored while the send reports + success -- which is exactly how the shutdown sweep came to count an + obsolete-generation Signal as a handoff it had made. + + Still a read and not a remembered value in the ordinary case: the park a + wake must name is whatever is installed *now*, and a generation cached + when the watcher started would name a park since abandoned and resolved. + """ + if isinstance(target, _CleanupWake): + return None + key = (target.stream_key, target.wait_id) + if key in self._owed_removals.get(target.run_id, {}): + return None + return await target.backend.current_park_generation( + target.stream_key, target.wait_id + ) + + def _count_owed_wake(self, target: WakeTarget) -> None: + """Counts one owed wake and draws the sequence number it is sent under. + + Both halves happen here, exactly once per wake, because + :meth:`_send_owed_wake` retries *the same* wake: re-drawing the counter + between attempts would derive a second request ID and ask the server for + a second Workflow Task rather than re-sending the one that may already + have arrived. + """ + target.wakes_owed += 1 + self._wake_sequence += 1 + target.wake_counter = self._wake_sequence + + async def _send_owed_wake(self, subscription: WakeTarget) -> bool: + """Sends the one wake ``wakes_owed`` already counts. Returns whether it landed. + + **The count belongs to the caller, and it is made exactly once, before + this is entered.** The wake's request ID is derived from it, so a retry + that re-counted would derive a *different* ID: the server would dedupe + nothing and answer with a second, empty Workflow Task, which is a + different thing from re-sending the wake that may in fact have arrived. + Every attempt below is therefore the same wake, which is precisely what + makes re-sending it safe. + + Retried because the common failure is a momentary one, and because both + callers have the same problem if it is not: on the live path nothing + re-attempts an owed wake at all, and on the shutdown path the process is + about to exit. + + Nothing but `CancelledError` leaves here. The live caller is reached + from the watcher loop, which an escaping exception would end for good -- + taking every later record on that subscription with it. + """ + if self._send_wake is None: + return False + for attempt in range(SHUTDOWN_WAKE_ATTEMPTS): + self._note_wake_attempt_started(subscription) + try: + await self._send_wake(subscription) + self._note_wake_attempt_acknowledged(subscription) + return True + except asyncio.CancelledError: + raise + except Exception: + # The wake sender reports failure by raising, because a wake + # counted as delivered when it was not is the failure this whole + # path exists to prevent. + logger.warning( + "External stream wake attempt %s/%s failed for %s wait %s", + attempt + 1, + SHUTDOWN_WAKE_ATTEMPTS, + subscription.stream_key, + subscription.wait_id, + exc_info=True, + ) + if attempt + 1 < SHUTDOWN_WAKE_ATTEMPTS: + await asyncio.sleep(SHUTDOWN_WAKE_RETRY_DELAY.total_seconds()) + return False + + def _record_shutdown_wake_failure(self, subscription: WakeTarget) -> None: + """Surfaces the wake that could not be acknowledged. + + Counted rather than only logged: a dropped wake is silent by nature -- + the Workflow simply waits, and nothing distinguishes that from a producer + having nothing to say. + """ + self.shutdown_wake_failures += 1 + if self._metric is not None: + self._metric(subscription) + + async def _stop(self, subscription: Subscription) -> None: + subscription._cancelled = True + subscription.reset_to_committed() + watcher = subscription._watcher + if watcher is not None and not watcher.done(): + watcher.cancel() + try: + await watcher + except asyncio.CancelledError: + pass diff --git a/temporalio/contrib/external_workflow_streams/_output_api.py b/temporalio/contrib/external_workflow_streams/_output_api.py new file mode 100644 index 0000000000..780291f61a --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_output_api.py @@ -0,0 +1,147 @@ +"""Workflow-facing publishers for external output streams. + +The publisher is deliberately a different type from the input subscription +handle. Workflow code only converts values into deterministic logical +payloads; the Worker applies codecs and performs backend I/O after the +activation has returned. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from datetime import timedelta +from typing import Any, Generic, Protocol, cast + +import temporalio.workflow +from temporalio.contrib.external_workflow_streams._api import _run_state +from temporalio.contrib.external_workflow_streams._record import RecordKind +from temporalio.types import AnyType + +__all__ = [ + "ExternalOutputStreamOptions", + "ExternalOutputStreamTopic", + "external_output_stream", +] + +DEFAULT_MAX_PUBLISH_LATENCY = timedelta(milliseconds=100) +DEFAULT_MAX_RECORDS_PER_BATCH = 256 +DEFAULT_MAX_LOGICAL_BYTES_PER_BATCH = 1024 * 1024 + + +class _OutputRuntime(Protocol): + """The deterministic half of the per-Run stream runtime.""" + + async def publish_output( + self, + *, + topic: str, + value: Any, + value_type: type | None, + kind: RecordKind, + max_publish_latency: timedelta, + max_records: int, + max_logical_bytes: int, + ) -> None: ... + + +@dataclass(frozen=True) +class ExternalOutputStreamOptions: + """Options inherited by every output topic created from this entry point.""" + + max_publish_latency: timedelta = DEFAULT_MAX_PUBLISH_LATENCY + max_records: int = DEFAULT_MAX_RECORDS_PER_BATCH + max_logical_bytes: int = DEFAULT_MAX_LOGICAL_BYTES_PER_BATCH + + def __post_init__(self) -> None: + """Validate direct construction as well as :meth:`with_options`.""" + if self.max_publish_latency <= timedelta(0): + raise ValueError("max_publish_latency must be positive") + if self.max_records < 1: + raise ValueError("max_records must be positive") + if self.max_logical_bytes < 1: + raise ValueError("max_logical_bytes must be positive") + + def with_options( + self, + *, + max_publish_latency: timedelta | None = None, + max_records: int | None = None, + max_logical_bytes: int | None = None, + ) -> ExternalOutputStreamOptions: + """Return a copy with validated latency and logical batch limits.""" + latency = ( + self.max_publish_latency + if max_publish_latency is None + else max_publish_latency + ) + records = self.max_records if max_records is None else max_records + logical_bytes = ( + self.max_logical_bytes if max_logical_bytes is None else max_logical_bytes + ) + if latency <= timedelta(0): + raise ValueError("max_publish_latency must be positive") + if records < 1: + raise ValueError("max_records must be positive") + if logical_bytes < 1: + raise ValueError("max_logical_bytes must be positive") + return replace( + self, + max_publish_latency=latency, + max_records=records, + max_logical_bytes=logical_bytes, + ) + + def topic( + self, name: str, *, type: type[AnyType] | None = None + ) -> ExternalOutputStreamTopic[AnyType]: + """Create a typed Workflow publisher for one output topic.""" + if not name: + raise ValueError("an output topic needs a non-empty name") + return ExternalOutputStreamTopic( + name=name, + value_type=type, + options=self, + ) + + +@dataclass(frozen=True) +class ExternalOutputStreamTopic(Generic[AnyType]): + """A Workflow-side topic with publishing operations and no subscription API.""" + + name: str + value_type: type[AnyType] | None + options: ExternalOutputStreamOptions + + async def publish(self, value: AnyType) -> None: + """Accept one value into the current Workflow Task's output batch.""" + await self._publish(value, RecordKind.DATA) + + async def finish(self) -> None: + """Append the ordered terminal record for this topic.""" + await self._publish(None, RecordKind.FINISH) + + async def _publish(self, value: Any, kind: RecordKind) -> None: + if temporalio.workflow.unsafe.is_read_only(): + raise temporalio.workflow.ReadOnlyContextError( + "While in read-only function, action attempted: publish external output" + ) + state = _run_state() + runtime = state.runtime + if runtime is None or not hasattr(runtime, "publish_output"): + raise RuntimeError( + "external output streams are not configured on this Worker; pass " + "external_stream_backend=... to the Worker" + ) + await cast(_OutputRuntime, cast(object, runtime)).publish_output( + topic=self.name, + value=value, + value_type=self.value_type, + kind=kind, + max_publish_latency=self.options.max_publish_latency, + max_records=self.options.max_records, + max_logical_bytes=self.options.max_logical_bytes, + ) + + +external_output_stream = ExternalOutputStreamOptions() +"""The default Workflow-facing external output stream entry point.""" diff --git a/temporalio/contrib/external_workflow_streams/_output_backend.py b/temporalio/contrib/external_workflow_streams/_output_backend.py new file mode 100644 index 0000000000..bcdb154d3b --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_output_backend.py @@ -0,0 +1,324 @@ +"""Provider contract for Workflow-originated external output streams. + +This contract is separate from :class:`StreamBackend`. Input-only providers +remain valid: output staging adds a pending/committed state machine and an +ordering barrier that an ordinary append-only input stream does not promise. + +The unit of staging is one topic sub-batch from one Workflow Task attempt. A +Worker-minted ``stage_token`` identifies the attempt, while ``sub_batch_id`` +distinguishes topic batches covered by that token. Records are invisible while +pending. A reader stops at the first unresolved batch even when later committed +Activity records already have provider offsets. +""" + +from __future__ import annotations + +import abc +import enum +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import ClassVar + +from temporalio.contrib.external_workflow_streams._backend import ( + DEFAULT_WATCH_BLOCK, + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._record import ( + Cursor, + Offset, + RecordKind, + StreamRecord, +) + +__all__ = [ + "OutputReadResult", + "OutputStage", + "OutputStageConflictError", + "OutputStageManifest", + "OutputStageNotFoundError", + "OutputStageResolutionError", + "OutputStageStatus", + "OutputStreamBackend", + "OutputStreamRecord", + "PendingOutputBarrier", + "StagedOutputRecord", +] + + +@enum.unique +class OutputStageStatus(enum.Enum): + """The durable visibility state of a staged output sub-batch.""" + + PENDING = "pending" + COMMITTED = "committed" + ABORTED = "aborted" + + +@dataclass(frozen=True) +class OutputStageManifest: + """Immutable logical identity and reconciliation proof for one sub-batch. + + Encoded record bytes are deliberately absent. A payload codec may use + randomness, so an idempotent retry compares this pre-codec manifest and + preserves the first successfully staged encoded bytes. + """ + + stream_key: StreamKey + provider_id: str + provider_format_version: int + stage_token: str + run_id: str + history_floor_event_id: int + sub_batch_id: int + fingerprint_version: int + fingerprint: bytes + record_count: int + logical_byte_count: int + + def __post_init__(self) -> None: + """Reject identities that could not safely be reconciled.""" + if self.stream_key.direction is not StreamDirection.OUTPUT: + raise ValueError("an output stage manifest requires an OUTPUT stream key") + for field_name in ("provider_id", "stage_token", "run_id"): + if not getattr(self, field_name): + raise ValueError( + f"an output stage manifest needs a non-empty {field_name}" + ) + for field_name in ( + "history_floor_event_id", + "provider_format_version", + "fingerprint_version", + "record_count", + ): + if getattr(self, field_name) < 1: + raise ValueError( + f"an output stage manifest needs a positive {field_name}" + ) + for field_name in ( + "sub_batch_id", + "logical_byte_count", + ): + if getattr(self, field_name) < 0: + raise ValueError( + f"an output stage manifest needs a non-negative {field_name}" + ) + if len(self.fingerprint) != 32: + raise ValueError("an output stage SHA-256 fingerprint must be 32 bytes") + + +@dataclass(frozen=True) +class StagedOutputRecord: + """One encoded record offered as part of a pending Workflow sub-batch. + + ``publish_index`` is local to the topic sub-batch. The backend assigns the + provider offset, and a retry of the same manifest must return the originally + stored bytes rather than replacing them with this attempt's encoding. + """ + + publish_index: int + kind: RecordKind + payload: bytes + + def __post_init__(self) -> None: + """Validate the deterministic identity and control-record shape.""" + if self.publish_index < 0: + raise ValueError("an output publish index must be non-negative") + if self.kind.is_control and self.payload: + raise ValueError(f"a {self.kind.name} output record carries no payload") + + +@dataclass(frozen=True) +class OutputStreamRecord: + """One provider-placed output record returned to a client.""" + + kind: RecordKind + payload: bytes + offset: Offset + + def __post_init__(self) -> None: + """Keep terminal and fence records payload-free after storage.""" + if self.kind.is_control and self.payload: + raise ValueError(f"a {self.kind.name} output record carries no payload") + + +@dataclass(frozen=True) +class OutputStage: + """The backend's durable view of one Workflow output sub-batch.""" + + manifest: OutputStageManifest + records: tuple[OutputStreamRecord, ...] + status: OutputStageStatus + + def __post_init__(self) -> None: + """Require the placed records to agree with the sealed manifest.""" + if len(self.records) != self.manifest.record_count: + raise ValueError( + "an output stage record count does not match its manifest: " + f"{len(self.records)} != {self.manifest.record_count}" + ) + + +@dataclass(frozen=True) +class PendingOutputBarrier: + """The unresolved stage preventing a client from reading later offsets.""" + + manifest: OutputStageManifest + offset: Offset + """The first physical position occupied by this pending sub-batch.""" + + +@dataclass(frozen=True) +class OutputReadResult: + """A committed prefix and the unresolved barrier immediately after it.""" + + records: tuple[OutputStreamRecord, ...] + pending: PendingOutputBarrier | None = None + + +class OutputStageConflictError(Exception): + """A stage token/sub-batch identity was reused with another manifest.""" + + def __init__(self, manifest: OutputStageManifest) -> None: + """Capture the immutable stage identity that conflicted.""" + super().__init__( + f"output stage {manifest.stage_token}/{manifest.sub_batch_id} was " + "already sealed with a different logical manifest" + ) + self.manifest = manifest + + +class OutputStageNotFoundError(Exception): + """A commit or abort named a stage the provider does not hold.""" + + def __init__(self, manifest: OutputStageManifest) -> None: + """Capture the immutable stage identity that was absent.""" + super().__init__( + f"output stage {manifest.stage_token}/{manifest.sub_batch_id} was not found" + ) + self.manifest = manifest + + +class OutputStageResolutionError(Exception): + """A terminal stage transition tried to reverse an existing decision.""" + + def __init__( + self, + manifest: OutputStageManifest, + *, + current: OutputStageStatus, + requested: OutputStageStatus, + ) -> None: + """Capture the attempted reversal of a terminal stage decision.""" + super().__init__( + f"output stage {manifest.stage_token}/{manifest.sub_batch_id} is " + f"already {current.value} and cannot become {requested.value}" + ) + self.manifest = manifest + self.current = current + self.requested = requested + + +class OutputStreamBackend(abc.ABC): + """A provider supporting externally visible Workflow output. + + Providers may implement this contract, :class:`StreamBackend`, or both. + Implementing the input contract alone does not imply transactional output + staging support. + """ + + guarantees_immutability: ClassVar[bool | None] = None + provider_id: ClassVar[str] = "" + provider_format_version: ClassVar[int] = 1 + + @abc.abstractmethod + async def stage_output( + self, + manifest: OutputStageManifest, + records: Sequence[StagedOutputRecord], + ) -> OutputStage: + """Durably append and seal an immutable pending sub-batch. + + The records become a barrier at their assigned positions but are not + readable. Repeating the call with the same complete manifest is an + idempotent success returning the original records and offsets, even if + a randomized codec supplied different encoded bytes on the retry. + Reusing ``(stage_token, sub_batch_id)`` with any different manifest + raises :class:`OutputStageConflictError`. The provider validates that + record count and publish indexes match the manifest before writing any + part of the batch. + """ + + @abc.abstractmethod + async def commit_output(self, manifest: OutputStageManifest) -> OutputStage: + """Make this exact pending stage visible, idempotently. + + Repeating a committed transition returns the committed stage. An + unknown stage raises :class:`OutputStageNotFoundError`; an aborted stage + raises :class:`OutputStageResolutionError` and is never resurrected. + """ + + @abc.abstractmethod + async def abort_output(self, manifest: OutputStageManifest) -> OutputStage: + """Resolve this exact pending stage as skipped, idempotently. + + Aborted positions remain coordination boundaries but yield no records. + An unknown stage raises :class:`OutputStageNotFoundError`; a committed + stage raises :class:`OutputStageResolutionError` and is never hidden. + """ + + @abc.abstractmethod + async def output_stage(self, manifest: OutputStageManifest) -> OutputStage | None: + """Inspect the exact stage, including its reconciliation metadata. + + Returns ``None`` only when the identity has never been staged. A stage's + manifest, encoded records, offsets, and terminal status are immutable. + """ + + @abc.abstractmethod + async def append_output( + self, key: StreamKey, record: StreamRecord + ) -> OutputStreamRecord: + """Append an immediately committed Activity/external producer record. + + ``key`` must have direction ``OUTPUT``. The append is idempotent on the + input record's ``(session_id, sequence)`` and byte identity exactly as + :meth:`StreamBackend.append`: an identical retry returns the original + output record and offset; different bytes under the same key raise + :class:`temporalio.contrib.external_workflow_streams.AppendConflictError`. + It may not pass an already-positioned pending stage in client reads. + """ + + @abc.abstractmethod + async def read_output_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = DEFAULT_WATCH_BLOCK, + ) -> OutputReadResult: + """Read the committed prefix strictly after ``after``. + + Records are returned in provider order and limited to ``max_records``. + Reading stops before the first pending stage. If that barrier is the + next unresolved position, ``pending`` names it; no record after it may + be returned even when already committed. Aborted stages yield nothing + and do not remain barriers. An empty result with no pending barrier is + ordinary watch idleness. + """ + + @abc.abstractmethod + async def output_tail(self, key: StreamKey) -> Cursor: + """Return the end boundary of the currently readable committed prefix. + + The boundary never advances past a pending stage, because a caller that + resumes after the returned cursor must still see that stage if it later + commits. It may advance across aborted coordination positions. This is + a position boundary only and does not correlate output to an Update. + """ + + @abc.abstractmethod + def compare_offsets(self, left: Offset, right: Offset) -> int: + """Three-way comparison under this provider's output offset order.""" diff --git a/temporalio/contrib/external_workflow_streams/_output_client.py b/temporalio/contrib/external_workflow_streams/_output_client.py new file mode 100644 index 0000000000..e9d446606b --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_output_client.py @@ -0,0 +1,517 @@ +"""External clients for Workflow-originated output streams. + +Committed output is read directly from the configured provider. Temporal is +consulted only when the provider reports a pending stage at the head of the +topic: History is then the authority that proves whether that exact stage +committed or was discarded with its Workflow Task attempt. +""" + +from __future__ import annotations + +import asyncio +import hmac +from collections.abc import AsyncIterator +from dataclasses import dataclass +from enum import Enum +from typing import Any, Final, Generic, cast + +import temporalio.client +import temporalio.converter +import temporalio.service +from temporalio.contrib.external_workflow_streams._backend import ( + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._errors import ( + StreamError, + StreamIntegrityError, + StreamStorageError, + classify_read_failure, +) +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputReadResult, + OutputStage, + OutputStageConflictError, + OutputStageManifest, + OutputStageNotFoundError, + OutputStageResolutionError, + OutputStageStatus, + OutputStreamBackend, + PendingOutputBarrier, +) +from temporalio.contrib.external_workflow_streams._producer import ( + ChainKeyMismatchError, + WorkflowChainKey, + _verify_chain_key, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + Offset, + RecordKind, +) +from temporalio.types import AnyType + +__all__ = [ + "ExternalOutputStreamClient", + "ExternalOutputStreamClientTopic", + "ExternalOutputStreamItem", +] + +_EXTERNAL_STREAM_MARKER_NAME: Final = "core_external_stream" +_EXTERNAL_STREAM_MARKER_DETAILS_KEY: Final = "external_stream" +_READ_BATCH_SIZE: Final = 100 +_PENDING_RECONCILIATION_BACKOFF_SECONDS: Final = 1.0 + + +@dataclass(frozen=True) +class ExternalOutputStreamItem(Generic[AnyType]): + """One decoded output value and its opaque provider resume offset.""" + + data: AnyType + offset: Offset + + +class ExternalOutputStreamClient: + """A verified client connection to one Workflow execution chain.""" + + def __init__( + self, + *, + backend: Any, + workflow: WorkflowChainKey, + client: temporalio.client.Client, + data_converter: temporalio.converter.DataConverter, + ) -> None: + """Create a reader after validating its output provider contract.""" + if not isinstance(backend, OutputStreamBackend): + raise TypeError("backend does not implement OutputStreamBackend") + if type(backend).guarantees_immutability is not True: + raise ValueError("an output backend must guarantee immutable record bytes") + if not type(backend).provider_id: + raise ValueError("an output backend must declare a provider_id") + self._backend = backend + self._workflow = workflow + self._client = client + self._data_converter = data_converter.with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=workflow.namespace, + workflow_id=workflow.workflow_id, + ) + ) + + @classmethod + async def connect( + cls, + *, + backend: OutputStreamBackend, + workflow: WorkflowChainKey, + client: temporalio.client.Client, + data_converter: temporalio.converter.DataConverter | None = None, + ) -> ExternalOutputStreamClient: + """Verify the chain binding and create an external output reader. + + The Temporal client remains attached only for resolving a pending + staging barrier. Ordinary reads and :meth:`tail` are backend-only. + """ + try: + await _verify_chain_key(client, workflow) + except ChainKeyMismatchError: + raise + except StreamError: + raise + except Exception as err: + raise StreamStorageError( + f"could not verify external output Workflow chain: {err}" + ) from err + return cls( + backend=backend, + workflow=workflow, + client=client, + data_converter=data_converter or client.data_converter, + ) + + @property + def workflow(self) -> WorkflowChainKey: + """The Workflow chain whose output this client reads.""" + return self._workflow + + def topic( + self, name: str, *, type: type[AnyType] | None = None + ) -> ExternalOutputStreamClientTopic[AnyType]: + """Create a typed reader for one output topic.""" + if not name: + raise ValueError("an output topic needs a non-empty name") + return ExternalOutputStreamClientTopic( + backend=self._backend, + stream_key=self._workflow.stream_key( + name, direction=StreamDirection.OUTPUT + ), + codec=StreamPayloadCodec(self._data_converter, type), + client=self._client, + workflow=self._workflow, + ) + + +class ExternalOutputStreamClientTopic(Generic[AnyType]): + """A typed, resumable external reader for one output topic.""" + + def __init__( + self, + *, + backend: OutputStreamBackend, + stream_key: StreamKey, + codec: StreamPayloadCodec[AnyType], + client: temporalio.client.Client, + workflow: WorkflowChainKey, + ) -> None: + """Create a typed reader bound to one direction-isolated topic.""" + self._backend = backend + self._stream_key = stream_key + self._codec = codec + self._client = client + self._workflow = workflow + + async def tail(self) -> Cursor: + """Return the boundary after the readable committed prefix.""" + try: + return await self._backend.output_tail(self._stream_key) + except StreamError: + raise + except Exception as err: + raise StreamStorageError( + f"could not read external output tail for {self._stream_key}: {err}" + ) from err + + async def subscribe( + self, *, after: Cursor | Offset = BEGINNING + ) -> AsyncIterator[ExternalOutputStreamItem[AnyType]]: + """Yield committed values strictly after ``after``. + + A pending stage is a hard ordering barrier. The committed prefix before + it may be yielded, but the client never reads or yields a later offset + until History has positively committed or aborted the pending stage. + """ + candidate = cast(object, after) + if isinstance(candidate, Offset): + cursor = AFTER(candidate) + elif isinstance(candidate, Cursor): + cursor = candidate + else: + raise TypeError( + f"after must be a Cursor or Offset, got {type(candidate).__name__}" + ) + while True: + result = await self._read_after(cursor) + self._validate_read_result(cursor, result) + for record in result.records: + cursor = AFTER(record.offset) + if record.kind is RecordKind.FINISH: + return + if record.kind.is_control: + continue + try: + data = await self._codec.decode(record.payload) + except asyncio.CancelledError: + raise + except Exception as err: + raise classify_read_failure( + range_validated=True, cause=err + ) from err + yield ExternalOutputStreamItem(data=data, offset=record.offset) + + if result.pending is not None: + resolved = await self._resolve_pending(result.pending) + if not resolved: + await asyncio.sleep(_PENDING_RECONCILIATION_BACKOFF_SECONDS) + + async def _read_after(self, after: Cursor) -> OutputReadResult: + try: + return await self._backend.read_output_after( + self._stream_key, + after, + max_records=_READ_BATCH_SIZE, + ) + except StreamError: + raise + except Exception as err: + raise StreamStorageError( + f"could not read external output for {self._stream_key}: {err}" + ) from err + + def _validate_read_result(self, after: Cursor, result: OutputReadResult) -> None: + """Refuse provider output that crosses or misorders a pending barrier.""" + previous = after.offset + for record in result.records: + if ( + previous is not None + and self._backend.compare_offsets(previous, record.offset) >= 0 + ): + raise StreamIntegrityError( + "external output provider returned records outside the requested " + f"strictly-after order for {self._stream_key}" + ) + previous = record.offset + + pending = result.pending + if pending is None: + return + if pending.manifest.stream_key != self._stream_key: + raise StreamIntegrityError( + "external output provider returned a pending barrier for another " + f"topic while reading {self._stream_key}" + ) + if ( + pending.manifest.provider_id != type(self._backend).provider_id + or pending.manifest.provider_format_version + != type(self._backend).provider_format_version + ): + raise StreamIntegrityError( + "external output provider returned a pending barrier carrying a " + "different provider binding" + ) + if ( + previous is not None + and self._backend.compare_offsets(previous, pending.offset) >= 0 + ): + raise StreamIntegrityError( + "external output provider returned data at or beyond an unresolved " + f"pending barrier for {self._stream_key}" + ) + + async def _resolve_pending(self, pending: PendingOutputBarrier) -> bool: + """Resolve one pending head from the exact producing Run's History. + + Returns ``False`` only when History contains neither this token nor a + later durable task/Workflow boundary, so the stage must remain pending. + """ + decision = await self._history_decision(pending.manifest) + if decision is None: + return False + await _apply_output_stage_decision( + backend=self._backend, + manifest=pending.manifest, + decision=decision, + ) + return True + + async def _history_decision( + self, manifest: OutputStageManifest + ) -> _HistoryDecision | None: + return await _history_decision( + client=self._client, + workflow_id=self._workflow.workflow_id, + manifest=manifest, + ) + + @staticmethod + def _validate_resolved_stage( + manifest: OutputStageManifest, + stage: OutputStage, + *, + expected_status: OutputStageStatus, + ) -> None: + _validate_resolved_stage( + manifest, + stage, + expected_status=expected_status, + ) + + +class _HistoryDecision(Enum): + COMMIT = "commit" + ABORT = "abort" + + +async def _history_decision( + *, + client: temporalio.client.Client, + workflow_id: str, + manifest: OutputStageManifest, +) -> _HistoryDecision | None: + """Prove one staged token's outcome from its exact producing History.""" + try: + events = [ + event + async for event in client.get_workflow_handle( + workflow_id, + run_id=manifest.run_id, + ).fetch_history_events() + ] + except temporalio.service.RPCError as err: + if err.status in ( + temporalio.service.RPCStatusCode.NOT_FOUND, + temporalio.service.RPCStatusCode.DATA_LOSS, + ): + raise StreamIntegrityError( + "History required to resolve external output stage " + f"{manifest.stage_token!r} is no longer available: {err}" + ) from err + raise StreamStorageError( + "could not read Temporal History while resolving external output " + f"stage {manifest.stage_token!r}: {err}" + ) from err + except StreamError: + raise + except Exception as err: + raise StreamStorageError( + "could not read Temporal History while resolving external output " + f"stage {manifest.stage_token!r}: {err}" + ) from err + + floor = manifest.history_floor_event_id + if floor and not any(event.event_id == floor for event in events): + raise StreamIntegrityError( + "History no longer contains the exact floor event needed to resolve " + f"external output stage {manifest.stage_token!r}: {floor}" + ) + relevant = [event for event in events if event.event_id > floor] + + # A task's marker and closing event commit in one server transaction, but + # the closing event precedes command events in History. Search the complete + # response for the positive token proof before considering any boundary + # proof of absence. + for event in relevant: + if _event_has_output_stage_token(event, manifest.stage_token): + return _HistoryDecision.COMMIT + if any(_is_deciding_boundary(event) for event in relevant): + return _HistoryDecision.ABORT + return None + + +async def _apply_output_stage_decision( + *, + backend: OutputStreamBackend, + manifest: OutputStageManifest, + decision: _HistoryDecision, +) -> None: + """Apply a positively proven History decision idempotently.""" + try: + if decision is _HistoryDecision.COMMIT: + stage = await backend.commit_output(manifest) + expected_status = OutputStageStatus.COMMITTED + else: + stage = await backend.abort_output(manifest) + expected_status = OutputStageStatus.ABORTED + except ( + OutputStageConflictError, + OutputStageNotFoundError, + OutputStageResolutionError, + ) as err: + raise StreamIntegrityError( + "external output pending stage could not be reconciled with its " + f"History decision: {err}" + ) from err + except StreamError: + raise + except Exception as err: + raise StreamStorageError( + f"could not {decision.value} external output stage " + f"{manifest.stage_token}: {err}" + ) from err + _validate_resolved_stage( + manifest, + stage, + expected_status=expected_status, + ) + + +async def _reconcile_output_stage( # pyright: ignore[reportUnusedFunction] + *, + backend: OutputStreamBackend, + client: temporalio.client.Client, + workflow_id: str, + manifest: OutputStageManifest, +) -> bool: + """Resolve after reporting, leaving ambiguous outcomes safely pending.""" + decision = await _history_decision( + client=client, + workflow_id=workflow_id, + manifest=manifest, + ) + if decision is None: + return False + await _apply_output_stage_decision( + backend=backend, + manifest=manifest, + decision=decision, + ) + return True + + +def _validate_resolved_stage( + manifest: OutputStageManifest, + stage: OutputStage, + *, + expected_status: OutputStageStatus, +) -> None: + """Refuse a provider which resolved a different immutable stage.""" + if stage.manifest != manifest or stage.status is not expected_status: + raise StreamIntegrityError( + "external output provider returned a different stage or status " + "after applying a History reconciliation decision" + ) + + +def _is_deciding_boundary(event: Any) -> bool: + """Whether this event proves that the staged token can no longer appear.""" + return any( + event.HasField(field) + for field in ( + "workflow_task_completed_event_attributes", + "workflow_task_failed_event_attributes", + "workflow_task_timed_out_event_attributes", + "workflow_execution_completed_event_attributes", + "workflow_execution_failed_event_attributes", + "workflow_execution_timed_out_event_attributes", + "workflow_execution_canceled_event_attributes", + "workflow_execution_terminated_event_attributes", + "workflow_execution_continued_as_new_event_attributes", + ) + ) + + +def _event_has_output_stage_token(event: Any, stage_token: str) -> bool: + """Read output commit proofs from the shared external-stream marker.""" + if not event.HasField("marker_recorded_event_attributes"): + return False + attributes = event.marker_recorded_event_attributes + if attributes.marker_name != _EXTERNAL_STREAM_MARKER_NAME: + return False + + payloads = attributes.details.get(_EXTERNAL_STREAM_MARKER_DETAILS_KEY) + if payloads is None or len(payloads.payloads) != 1: + raise StreamIntegrityError( + "an external stream marker has no singular marker envelope" + ) + + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + + marker = ExternalStreamMarkerData() + try: + marker.ParseFromString(payloads.payloads[0].data) + except Exception as err: + raise StreamIntegrityError( + f"an external stream marker envelope could not be decoded: {err}" + ) from err + + output = _marker_output_commit(marker) + return output is not None and _stage_tokens_equal(output.stage_token, stage_token) + + +def _marker_output_commit(marker: Any) -> Any | None: + """Return the shared marker's output proof, when this task had output.""" + if "output" not in marker.DESCRIPTOR.fields_by_name: + raise StreamIntegrityError( + "this SDK's external stream marker schema cannot inspect output " + "commit proofs, so a pending stage cannot be resolved safely" + ) + return marker.output if marker.HasField("output") else None + + +def _stage_tokens_equal(recorded: str | bytes, expected: str) -> bool: + if isinstance(recorded, bytes): + return hmac.compare_digest(recorded, expected.encode()) + return hmac.compare_digest(recorded, expected) diff --git a/temporalio/contrib/external_workflow_streams/_output_codec.py b/temporalio/contrib/external_workflow_streams/_output_codec.py new file mode 100644 index 0000000000..859888d670 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_output_codec.py @@ -0,0 +1,127 @@ +"""Canonical logical framing for Workflow-originated output. + +Workflow code runs only the deterministic payload converter. The resulting +``Payload`` is framed and fingerprinted before any asynchronous payload codec +or external-storage transform runs, so codec randomness and compression cannot +change replay identity or capacity boundaries. +""" + +from __future__ import annotations + +import hashlib +import struct +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Final + +import temporalio.api.common.v1 +from temporalio.contrib.external_workflow_streams._record import RecordKind + +__all__ = [ + "LOGICAL_FINGERPRINT_VERSION", + "LogicalOutputFingerprint", + "canonical_logical_record_frame", + "fingerprint_logical_frames", + "fingerprint_logical_records", +] + +LOGICAL_FINGERPRINT_VERSION: Final = 1 +"""The canonical frame and batch hash format implemented here.""" + +_LENGTH = struct.Struct(">Q") +"""Unsigned 64-bit network-order lengths used by fingerprint version 1.""" + + +@dataclass(frozen=True) +class LogicalOutputFingerprint: + """The replay identity and deterministic capacity cost of one batch.""" + + version: int + digest: bytes + logical_byte_count: int + record_count: int + + +def _length_prefixed(value: bytes) -> bytes: + if len(value) >= 1 << 64: + raise ValueError("a logical output frame field is too large to encode") + return _LENGTH.pack(len(value)) + value + + +def canonical_logical_record_frame( + topic: str, + kind: object, + payload: temporalio.api.common.v1.Payload | None, +) -> bytes: + """Build one fingerprint-version-1 logical record frame. + + The format is, in order: length-prefixed UTF-8 topic; length-prefixed + unsigned record kind; unsigned metadata-entry count; each metadata key and + value length-prefixed with keys sorted by their UTF-8 bytes; and + length-prefixed raw payload data. Control records use an empty metadata set + and empty data. The ``Payload`` protobuf itself is never serialized. + """ + if not topic: + raise ValueError("a logical output record needs a non-empty topic") + if not isinstance(kind, RecordKind): + raise TypeError( + f"a logical output record kind must be RecordKind, got {type(kind).__name__}" + ) + if kind is RecordKind.DATA and payload is None: + raise ValueError("a DATA logical output record needs a Payload") + if kind.is_control and payload is not None: + raise ValueError(f"a {kind.name} logical output record carries no Payload") + + topic_bytes = topic.encode("utf-8") + # RecordKind is an unsigned protocol number. Minimal big-endian encoding is + # unambiguous because the byte string itself is length-prefixed. + kind_value = int(kind) + kind_bytes = kind_value.to_bytes(max(1, (kind_value.bit_length() + 7) // 8), "big") + metadata = () if payload is None else payload.metadata.items() + sorted_metadata = sorted(metadata, key=lambda item: item[0].encode("utf-8")) + if len(sorted_metadata) >= 1 << 64: + raise ValueError("a logical output Payload has too many metadata entries") + + frame = bytearray() + frame.extend(_length_prefixed(topic_bytes)) + frame.extend(_length_prefixed(kind_bytes)) + frame.extend(_LENGTH.pack(len(sorted_metadata))) + for key, value in sorted_metadata: + frame.extend(_length_prefixed(key.encode("utf-8"))) + frame.extend(_length_prefixed(value)) + frame.extend(_length_prefixed(b"" if payload is None else payload.data)) + return bytes(frame) + + +def fingerprint_logical_frames( + frames: Iterable[bytes], +) -> LogicalOutputFingerprint: + """Hash ordered, length-prefixed canonical frames with SHA-256. + + ``logical_byte_count`` is the sum of the canonical frame lengths, excluding + the batch-level prefixes used only to prevent adjacent frames from being + repartitioned into the same hash input. + """ + digest = hashlib.sha256() + logical_byte_count = 0 + record_count = 0 + for frame in frames: + digest.update(_length_prefixed(frame)) + logical_byte_count += len(frame) + record_count += 1 + return LogicalOutputFingerprint( + version=LOGICAL_FINGERPRINT_VERSION, + digest=digest.digest(), + logical_byte_count=logical_byte_count, + record_count=record_count, + ) + + +def fingerprint_logical_records( + records: Iterable[tuple[str, RecordKind, temporalio.api.common.v1.Payload | None]], +) -> LogicalOutputFingerprint: + """Frame and fingerprint an ordered sequence of logical output records.""" + return fingerprint_logical_frames( + canonical_logical_record_frame(topic, kind, payload) + for topic, kind, payload in records + ) diff --git a/temporalio/contrib/external_workflow_streams/_output_continuation.py b/temporalio/contrib/external_workflow_streams/_output_continuation.py new file mode 100644 index 0000000000..3b75c76a3e --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_output_continuation.py @@ -0,0 +1,101 @@ +"""Must-understand Continue-As-New state for finished output topics.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import temporalio.api.common.v1 +from temporalio.contrib.external_workflow_streams._annotation import ( + AnnotationDecodeError, + _put_str, + _put_uvarint, + _Reader, +) + +__all__ = [ + "OUTPUT_CONTINUATION_HEADER", + "OutputContinuation", + "decode_output_continuation", + "encode_output_continuation", + "read_output_continuation_header", + "write_output_continuation_header", +] + +OUTPUT_CONTINUATION_HEADER = "__temporal_external_output_stream_continuation" +_OUTPUT_CONTINUATION_ENCODING = b"binary/plain" +_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class OutputContinuation: + """The output topics whose explicit terminal committed in this chain.""" + + finished_topics: frozenset[str] + provider_id: str + provider_format_version: int + + def __post_init__(self) -> None: + """Reject continuation state that cannot identify a durable provider.""" + if not self.finished_topics: + raise ValueError("output continuation state needs a finished topic") + if any(not topic for topic in self.finished_topics): + raise ValueError("an output continuation topic may not be empty") + if not self.provider_id: + raise ValueError("output continuation state needs a provider ID") + if self.provider_format_version < 1: + raise ValueError( + "output continuation state needs a positive format version" + ) + + +def encode_output_continuation(continuation: OutputContinuation) -> bytes: + """Encode finished topics and their provider binding deterministically.""" + out = bytearray() + _put_uvarint(out, _SCHEMA_VERSION) + _put_str(out, continuation.provider_id) + _put_uvarint(out, continuation.provider_format_version) + _put_uvarint(out, len(continuation.finished_topics)) + for topic in sorted(continuation.finished_topics): + _put_str(out, topic) + return bytes(out) + + +def decode_output_continuation(raw: bytes) -> OutputContinuation: + """Decode a supported output continuation or fail closed.""" + reader = _Reader(raw) + version = reader.uvarint() + if version != _SCHEMA_VERSION: + raise AnnotationDecodeError( + "the external output Continue-As-New header is schema version " + f"{version}, but this Worker understands {_SCHEMA_VERSION}" + ) + provider_id = reader.string() + provider_format_version = reader.uvarint() + topics = frozenset(reader.string() for _ in range(reader.uvarint())) + return OutputContinuation(topics, provider_id, provider_format_version) + + +def write_output_continuation_header( + continuation: OutputContinuation, +) -> temporalio.api.common.v1.Payload: + """Wrap output continuation bytes without using the user converter.""" + return temporalio.api.common.v1.Payload( + metadata={"encoding": _OUTPUT_CONTINUATION_ENCODING}, + data=encode_output_continuation(continuation), + ) + + +def read_output_continuation_header( + headers: dict[str, temporalio.api.common.v1.Payload] | None, +) -> OutputContinuation | None: + """Read predecessor output state, with absence meaning a first Run.""" + if not headers: + return None + payload = headers.get(OUTPUT_CONTINUATION_HEADER) + if payload is None: + return None + if payload.metadata.get("encoding") != _OUTPUT_CONTINUATION_ENCODING: + raise AnnotationDecodeError( + "the external output Continue-As-New header has an unsupported encoding" + ) + return decode_output_continuation(payload.data) diff --git a/temporalio/contrib/external_workflow_streams/_output_producer.py b/temporalio/contrib/external_workflow_streams/_output_producer.py new file mode 100644 index 0000000000..2e0800f723 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_output_producer.py @@ -0,0 +1,301 @@ +"""Activity and external-process producers for output streams.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Generic + +import temporalio.converter +from temporalio.contrib.external_workflow_streams._backend import ( + AppendConflictError, + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputStreamBackend, + OutputStreamRecord, +) +from temporalio.contrib.external_workflow_streams._producer import ( + WorkflowChainKey, + _default_session_id, + _verify_chain_key, +) +from temporalio.contrib.external_workflow_streams._record import ( + RecordKind, + StreamRecord, +) +from temporalio.types import AnyType + +if TYPE_CHECKING: + import temporalio.client + +__all__ = [ + "ExternalOutputStreamProducer", + "ExternalOutputStreamProducerTopic", + "OutputAppendNotAcknowledgedError", +] + + +class OutputAppendNotAcknowledgedError(Exception): + """A direct output append may or may not have reached the provider. + + Re-run :meth:`ExternalOutputStreamProducerTopic.resolve_append` with the + exact record carried here. The provider's idempotency rule makes that safe; + calling ``publish`` again is not safe because it draws another sequence. + """ + + def __init__(self, *, stream_key: StreamKey, record: StreamRecord) -> None: + """Capture the exact record required for safe ambiguity recovery.""" + super().__init__( + "the output append did not report an outcome; resolve it with the " + "same record before publishing another value" + ) + self.stream_key = stream_key + self.record = record + + +@dataclass(eq=False) +class _AppendOperation: + sequence: int + settled: asyncio.Event = field(default_factory=asyncio.Event) + failure: BaseException | None = None + + +class ExternalOutputStreamProducer: + """A chain-bound producer whose records are committed immediately.""" + + def __init__( + self, + *, + backend: Any, + workflow: WorkflowChainKey, + data_converter: temporalio.converter.DataConverter, + session_id: str, + ) -> None: + """Create a producer after validating its provider and session identity.""" + if not isinstance(backend, OutputStreamBackend): + raise TypeError("backend does not implement OutputStreamBackend") + if type(backend).guarantees_immutability is not True: + raise ValueError("an output backend must guarantee immutable record bytes") + if not type(backend).provider_id: + raise ValueError("an output backend must declare a provider_id") + if not session_id: + raise ValueError("an output producer session ID may not be empty") + self._backend = backend + self._workflow = workflow + self._data_converter = data_converter.with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=workflow.namespace, + workflow_id=workflow.workflow_id, + ) + ) + self._session_id = session_id + self._sequence = 0 + self._operations: dict[StreamKey, list[_AppendOperation]] = {} + self._unresolved: dict[StreamKey, list[StreamRecord]] = {} + self._closing: set[StreamKey] = set() + self._finished: set[StreamKey] = set() + + @classmethod + async def connect( + cls, + *, + backend: OutputStreamBackend, + workflow: WorkflowChainKey, + client: temporalio.client.Client, + data_converter: temporalio.converter.DataConverter | None = None, + session_id: str | None = None, + ) -> ExternalOutputStreamProducer: + """Verify the chain binding before allowing the first append.""" + await _verify_chain_key(client, workflow) + return cls( + backend=backend, + workflow=workflow, + data_converter=data_converter or client.data_converter, + session_id=session_id or _default_session_id(), + ) + + @property + def session_id(self) -> str: + """The retry-stable idempotency session.""" + return self._session_id + + @property + def workflow(self) -> WorkflowChainKey: + """The Workflow chain receiving these records.""" + return self._workflow + + def topic( + self, name: str, *, type: type[AnyType] | None = None + ) -> ExternalOutputStreamProducerTopic[AnyType]: + """Create a typed direct publisher for one output topic.""" + if not name: + raise ValueError("an output topic needs a non-empty name") + key = self._workflow.stream_key( + name, + direction=StreamDirection.OUTPUT, + ) + return ExternalOutputStreamProducerTopic( + producer=self, + stream_key=key, + codec=StreamPayloadCodec(self._data_converter, type), + ) + + def _next_sequence(self) -> int: + sequence = self._sequence + self._sequence += 1 + return sequence + + +class ExternalOutputStreamProducerTopic(Generic[AnyType]): + """One immediately committed output topic.""" + + def __init__( + self, + *, + producer: ExternalOutputStreamProducer, + stream_key: StreamKey, + codec: StreamPayloadCodec[AnyType], + ) -> None: + """Create a typed publisher bound to one output topic.""" + self._producer = producer + self._stream_key = stream_key + self._codec = codec + + @property + def stream_key(self) -> StreamKey: + """The direction-isolated physical stream identity.""" + return self._stream_key + + def _refuse_unresolved(self) -> None: + records = self._producer._unresolved.get(self._stream_key) + if records: + raise OutputAppendNotAcknowledgedError( + stream_key=self._stream_key, + record=records[0], + ) + + def _refuse_closed(self) -> None: + if self._stream_key in self._producer._finished: + raise RuntimeError("this output topic is already finished") + if self._stream_key in self._producer._closing: + raise RuntimeError("this output topic is already being finished") + + def _remember_unresolved(self, record: StreamRecord) -> None: + records = self._producer._unresolved.setdefault(self._stream_key, []) + if record not in records: + records.append(record) + + def _forget_unresolved(self, record: StreamRecord) -> None: + records = self._producer._unresolved[self._stream_key] + records.remove(record) + if not records: + del self._producer._unresolved[self._stream_key] + + async def _append(self, record: StreamRecord) -> OutputStreamRecord: + try: + return await self._producer._backend.append_output(self._stream_key, record) + except AppendConflictError: + raise + except asyncio.CancelledError: + self._remember_unresolved(record) + raise OutputAppendNotAcknowledgedError( + stream_key=self._stream_key, + record=record, + ) from None + except Exception as err: + # Provider-specific transport exceptions cannot reliably distinguish + # a refusal from a lost acknowledgement. Repeating the exact append + # is the only recovery safe in both cases. + self._remember_unresolved(record) + raise OutputAppendNotAcknowledgedError( + stream_key=self._stream_key, + record=record, + ) from err + + async def resolve_append(self, record: StreamRecord) -> OutputStreamRecord: + """Resolve an unknown append by repeating its exact identity and bytes.""" + outstanding = self._producer._unresolved.get(self._stream_key, ()) + if record not in outstanding: + raise ValueError("record is not this topic's unresolved output append") + try: + placed = await self._producer._backend.append_output( + self._stream_key, record + ) + except AppendConflictError: + self._forget_unresolved(record) + if record.kind is RecordKind.FINISH: + self._producer._closing.discard(self._stream_key) + raise + except (Exception, asyncio.CancelledError) as err: + raise OutputAppendNotAcknowledgedError( + stream_key=self._stream_key, + record=record, + ) from err + self._forget_unresolved(record) + if record.kind is RecordKind.FINISH: + self._producer._closing.discard(self._stream_key) + self._producer._finished.add(self._stream_key) + return placed + + async def publish(self, value: AnyType) -> OutputStreamRecord: + """Append one immediately committed data record.""" + self._refuse_unresolved() + self._refuse_closed() + sequence = self._producer._next_sequence() + operation = _AppendOperation(sequence) + operations = self._producer._operations.setdefault(self._stream_key, []) + operations.append(operation) + try: + record = StreamRecord( + kind=RecordKind.DATA, + payload=await self._codec.encode(value), + producer_session_id=self._producer.session_id, + sequence=sequence, + ) + placed = await self._append(record) + except BaseException as err: + operation.failure = err + operation.settled.set() + raise + else: + operation.settled.set() + return placed + finally: + operations.remove(operation) + + async def finish_writing(self) -> OutputStreamRecord: + """Append the ordered terminal after every earlier publish settles.""" + self._refuse_unresolved() + self._refuse_closed() + self._producer._closing.add(self._stream_key) + sequence = self._producer._next_sequence() + try: + preceding = tuple(self._producer._operations.get(self._stream_key, ())) + for operation in preceding: + await operation.settled.wait() + if operation.failure is not None: + raise RuntimeError( + f"output publish {operation.sequence} did not append; " + "refusing to place a terminal in front of it" + ) from operation.failure + record = StreamRecord( + kind=RecordKind.FINISH, + payload=b"", + producer_session_id=self._producer.session_id, + sequence=sequence, + ) + placed = await self._append(record) + except OutputAppendNotAcknowledgedError: + # The terminal may already be durable, so the topic stays closed to + # new publishes until resolve_append settles that exact record. + raise + except BaseException: + self._producer._closing.discard(self._stream_key) + raise + else: + self._producer._closing.discard(self._stream_key) + self._producer._finished.add(self._stream_key) + return placed diff --git a/temporalio/contrib/external_workflow_streams/_producer.py b/temporalio/contrib/external_workflow_streams/_producer.py new file mode 100644 index 0000000000..a33aca753b --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_producer.py @@ -0,0 +1,1501 @@ +"""The producer side (P6a binding, P6 append). + +The producer handle is a **different type** from the Workflow-side one, and not +the same object passed across a process boundary. A Workflow handle is bound to +the running Workflow's identity and backend configuration; a producer +handle is constructed explicitly from credentials the producer holds. + +Every binding input is explicit, because none of them can be inferred: + +- **The Workflow chain key**, including the first execution Run ID. + ``temporalio.activity.Info`` exposes ``workflow_run_id`` but *not* the first + execution Run ID, so an Activity cannot derive the key -- the Workflow passes + it in and the producer verifies it by describing the Workflow before its + first append. Publishing under an unverified key is a configuration error, + not a silent no-op. +- **A backend connection**, **a Temporal client** (for the wake Signal), and + **the same DataConverter** the consuming Workflow uses. +- **A stable producer session ID**, which is what makes append idempotent under + Activity retry. Activities default it to something derived from the + Activity's own identity so a retried attempt reuses it. Plain processes must + supply one: a random default would look like it worked and duplicate every + record on the first retry. + +The **stream name appears exactly once**, in :meth:`ExternalStreamProducer.topic`. +``connect()`` takes the chain key and ``topic(name)`` completes it into the full +stream identity, so one connection serves several topics and no two arguments +can disagree about the name. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Generic + +import temporalio.activity +import temporalio.converter +from temporalio.contrib.external_workflow_streams._backend import ( + AppendConflictError, + StreamBackend, + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._record import ( + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._wake import ( + WakeRequest, + send_wake, + wake_position, + wake_request_for, + wake_transport_of, +) +from temporalio.types import AnyType + +DEFAULT_WAKE_CLAIM_LEASE = timedelta(seconds=30) +"""Long enough to cover a Signal round trip, short enough to recover from a crash. + +The lease bounds how long a claim taken by a producer that then died keeps +saying a wake is in flight. It is not what makes the wake safe -- expiry alone +schedules nobody to take over, which is why a producer that loses the claim +signals anyway rather than treating the claim as an acknowledgement. +""" + +if TYPE_CHECKING: + import temporalio.client + +__all__ = [ + "AppendNotAcknowledgedError", + "ChainKeyMismatchError", + "ExternalStreamProducer", + "ExternalStreamProducerTopic", + "PrecedingWriteFailedError", + "WakeNotAcknowledgedError", + "WorkflowChainKey", +] + + +class WakeNotAcknowledgedError(Exception): + """The record is durable but the wake step did not complete. + + Raised rather than swallowed, because the two halves fail differently and the + caller can only act on one of them: the append has already succeeded and must + not be retried, while the wake must be. Retrying the wake with the same + inputs is safe -- it derives the same request ID and the server deduplicates + it -- so this is a resumable state, not a lost record. + + **Cancellation after the append arrives this way too**, with + :attr:`cancelled` set. It leaves the identical state -- a durable record and + an unsent wake -- and a bare ``CancelledError`` there would be + indistinguishable from cancellation *before* the append: the caller would not + know whether the value still had to be published, and its obvious move, + calling ``publish()`` again, draws a fresh sequence number and appends a + second record (ADR-036). + """ + + def __init__( + self, + message: str, + *, + pending: list[WakeRequest], + restart: bool = False, + cancelled: bool = False, + ) -> None: + """Capture the wakes still owed after a durable append.""" + super().__init__(message) + self.pending = pending + """The wakes still owed, ready to be retried verbatim.""" + self.cancelled = cancelled + """Whether what ended the wake was cancellation rather than a failure. + + Set because the two are the same *state* -- durable record, unsent wake, + one recovery -- and a caller that wants to honour the cancellation after + recovering the wake still has to know it was asked to stop. The recovery + itself does not depend on this: ``pending`` and ``restart`` say what to do + whatever ended the attempt (ADR-036). + """ + self.restart = restart + """Whether the caller must call ``wake()`` again instead of retrying. + + The wake is three steps -- observe the parked set, claim the generation, + Signal -- and only the third produces the requests + ``ProducerTopicHandle.retry_wake()`` re-sends. A failure in the first + two leaves nothing to re-send, so ``pending`` is empty and retrying it + would silently do nothing at all: the record would stay durable and + unannounced while the caller believed it had recovered. + + ``True`` therefore says "no wake was composed; compose one". Calling + ``ProducerTopicHandle.wake()`` again is safe and is the whole recovery + -- it re-observes the parked set, and a parked wake's request ID is + derived from the generation rather than from the sender, so a wake some + other producer already sent deduplicates against it. + """ + self.offset: Offset | None = None + """Where the record landed, when raised from :meth:`publish`. + + Set because the two halves fail differently and the caller can only act + on one of them: the append succeeded and must not be retried, the wake + did not and must be. + """ + + +class AppendNotAcknowledgedError(Exception): + """The append's outcome is unknown: the record may or may not be durable. + + Raised when :meth:`StreamBackend.append` neither returns nor *refuses* -- a + cancellation delivered while it was in flight, a dropped connection, a + provider exception that says nothing about what the server did with the + write. A remote backend commits on its own side and only then answers, so an + append that did not answer is not an append that did not happen: the Redis + provider runs an atomic script server-side and receives its result in a + separate client-side step, and losing the client between the two leaves a + durable record whose offset nobody holds. + + Reading that window as failure breaks the producer's contract in both + directions at once, which is why it gets a type of its own. Retrying + ``publish()`` draws a **new** sequence number, and therefore a new + idempotency key, so a record that did land is appended a second time. Not + retrying can leave a durable record that no wake was ever sent for, which + strands a parked Workflow for its whole idle timeout. A bare + ``CancelledError`` or a bare ``ConnectionError`` supports neither choice, + because neither carries the identity of the record whose fate is in + question. + + So the record is carried out instead -- byte-identical, still holding its + ``(session_id, sequence)`` -- and + :meth:`ExternalStreamProducerTopic.resolve_append` re-appends *that* record. + Under the backend contract that one call is correct whichever way the first + attempt went: a repeat append of byte-identical content under a used key + writes nothing and returns the original offset, and a key the backend never + saw is appended now (ADR-038). + + Until it is resolved, the stream refuses further appends from this producer + with this same error, since that is the caller move that duplicates the + record. + + Distinct from :class:`WakeNotAcknowledgedError`, which is the *next* window + along: there the offset is known and only the wake is owed. Resolving this + one produces that offset, after which a failing wake raises the wake error in + the ordinary way. + """ + + def __init__( + self, + message: str, + *, + stream_key: StreamKey, + record: StreamRecord, + wake: bool, + lease: timedelta, + cancelled: bool = False, + ) -> None: + """Capture an append whose durable outcome must be resolved.""" + super().__init__(message) + self.stream_key = stream_key + """The stream the append was for. Where it must be settled. + + Carried because a record does not name its own stream and the backend's + idempotency scope does: ``(session_id, sequence)`` is unused on every + *other* stream, so the same record handed to another topic's + ``resolve_append`` would append a second copy there rather than + deduplicate. The recovery refuses that, and this is what a caller + holding several topics matches on. + """ + self.record = record + """The exact record whose fate is unknown. What ``resolve_append`` takes. + + Carried rather than described, because idempotency is on identity: a + payload the caller re-encodes into different bytes under the same key is + an ``AppendConflictError`` rather than the no-op the recovery depends on. + """ + self.wake = wake + """Whether the interrupted call was going to wake, so recovery can too.""" + self.lease = lease + """The claim lease the interrupted call was going to use.""" + self.cancelled = cancelled + """Whether cancellation ended any attempt to settle this operation. + + Reported rather than propagated, for the reason ADR-036 already gives + about the wake: a caller that wants to honour the cancellation re-raises + *after* resolving the append, and that is the only order that leaves + nothing owed. Sticky across recovery attempts: a later transport failure + must not erase a cancellation the caller still has to honour. + """ + + +class PrecedingWriteFailedError(Exception): + """A write fence was not appended, because an earlier write was not either. + + Raised from :meth:`ExternalStreamProducerTopic.finish_writing` when a + ``publish()`` invoked earlier on the same stream ended without a durable + record. The fence means every write in this producer session preceding it + has been appended, and a consumer that drains through one may park on that + assertion, so a fence written over a hole would tell it the batch is + complete when it is short a record. + + Reachable only from *concurrent* calls: a publish that failed before the + fence was invoked is already finished and the fence covers what it says it + covers. Nothing was appended for this fence, so the recovery is the caller's + choice between the two halves -- publish the failed value again, which draws + a new sequence number and lands ahead of a later fence, or accept the batch + without it. Either way ``finish_writing()`` again appends the fence: the + failed write is no longer outstanding. + + Distinct from :class:`AppendNotAcknowledgedError`, which the fence raises + instead when the earlier append's outcome is *unknown* rather than failed. + There the record may well be durable, the stream refuses appends until it is + settled, and the recovery is that operation's rather than this call's. + """ + + def __init__(self, message: str, *, stream_key: StreamKey, sequence: int) -> None: + """Identify the earlier failed write that makes a fence unsafe.""" + super().__init__(message) + self.stream_key = stream_key + """The stream whose fence was refused.""" + self.sequence = sequence + """The sequence number of the earlier write that did not land. + + Names the operation rather than describing it: the failure itself is the + ``__cause__``, and the caller already saw it raised from its own call. + """ + + +@dataclass(frozen=True) +class _UnresolvedAppend: + """One append whose outcome the producer never learned. + + The *operation*, not just its record. What the interrupted call owed is more + than the bytes: whether a wake was due, under which lease, and whether + cancellation is still to be honoured once the state is settled. Holding only + the record meant a later refusal had to invent those three from whatever call + happened to be refused, so a caller following the refusal's own instructions + could drop a wake the unresolved record required. + + Keyed by stream because idempotency is scoped per stream: the same + ``(session_id, sequence)`` is unused on every other one, so a record settled + against the wrong topic appends a second copy rather than deduplicating. + """ + + stream_key: StreamKey + record: StreamRecord + wake: bool + lease: timedelta + cancelled: bool + operation: _StreamOperation | None = None + """The append order entry whose outcome this recovery decides, if any. + + A fence has none: nothing waits behind one. A publish has exactly one, and + it is the *only* place a resolution can be reported back to a fence that + already captured that operation while its outcome was unknown -- see + :meth:`ExternalStreamProducerTopic.resolve_append`. Without it, "the record + is no longer unresolved" is all a waiting fence can see, and both a durable + resolution and an ``AppendConflictError`` produce that. + """ + + def error(self, message: str) -> AppendNotAcknowledgedError: + """Reports this operation's current canonical recovery.""" + return AppendNotAcknowledgedError( + message, + stream_key=self.stream_key, + record=self.record, + wake=self.wake, + lease=self.lease, + cancelled=self.cancelled, + ) + + +@dataclass(eq=False) +class _StreamOperation: + """One **publish** on a stream, from the moment it draws its sequence. + + The entry in the producer's per-stream append order. It exists so that + :meth:`ExternalStreamProducerTopic.finish_writing` can tell which earlier + calls on that stream have not reached the backend yet -- which nothing else + records. ``publish()`` draws its sequence *before* awaiting the codec, so a + publish that is still encoding is invisible to :meth:`_refuse_while_unresolved` + (it has no unresolved append) and to the backend (it has appended nothing). + + **A fence is not one of these.** The order exists to hold a fence behind the + *data writes* that precede it, which is the whole of what a fence asserts; + two fences make independent assertions about the publishes each of them came + after, so neither has to wait for the other. Putting them in the same ledger + made a fence that never reached the backend -- cancelled while waiting, or + refused -- look to a later fence like a data write that went missing, and a + valid fence was then refused with ``PrecedingWriteFailedError``. + + Compared by identity rather than by field, because two operations may hold + the same values and the order removes exactly the one that settled. + """ + + sequence: int + + settled: asyncio.Event = field(default_factory=asyncio.Event) + """Set once the append has an outcome -- durable, refused, or unknown.""" + + failure: BaseException | None = None + """Why no durable record came of it, or ``None`` if one did. + + Read only after :attr:`settled`, and only by a fence waiting behind this + operation. The caller of the failed call has already been raised at. + + Not final while it holds an :class:`AppendNotAcknowledgedError`: that is the + one outcome the producer does not yet know, and + :meth:`ExternalStreamProducerTopic.resolve_append` replaces it with the + durable or refused answer recovery learned. + """ + + +@dataclass(frozen=True) +class WorkflowChainKey: + """The Workflow chain a producer publishes to. + + A *chain* key, not a Run key: the stream spans the whole Continue-As-New + chain, and ``first_execution_run_id`` is what stays stable across it while + still preventing collisions after Workflow ID reuse. + """ + + namespace: str + workflow_id: str + first_execution_run_id: str + + def __post_init__(self) -> None: + """Require every component of the Workflow chain identity.""" + for field_name in ("namespace", "workflow_id", "first_execution_run_id"): + if not getattr(self, field_name): + raise ValueError( + f"a Workflow chain key needs a non-empty {field_name}; it is " + "passed in by the Workflow rather than inferred, because an " + "Activity cannot derive the first execution Run ID" + ) + + def stream_key( + self, + stream_name: str, + *, + direction: StreamDirection = StreamDirection.INPUT, + ) -> StreamKey: + """Return the durable key for one stream in this Workflow chain.""" + return StreamKey( + namespace=self.namespace, + workflow_id=self.workflow_id, + first_execution_run_id=self.first_execution_run_id, + stream_name=stream_name, + direction=direction, + ) + + +class ChainKeyMismatchError(Exception): + """The chain key does not describe the Workflow the server knows about. + + Loud on purpose. Publishing under a wrong key writes records into a stream + no consumer is watching, and the Workflow simply waits out its idle timeout + with nothing to show for it. + """ + + +def _default_session_id() -> str: + """A session ID stable across an Activity's retries, or a clear error. + + Derived from the Activity's identity and deliberately **not** its attempt + number: the whole point is that attempt 2 reuses attempt 1's key so its + re-appends are recognised as the same records. + """ + try: + info = temporalio.activity.info() + except RuntimeError: + raise ValueError( + "a producer session ID is required and has no default outside an " + "Activity. It is what makes append idempotent under retry, so a " + "random default would look correct and duplicate every record the " + "first time the producer was retried." + ) from None + return f"activity:{info.workflow_run_id}:{info.activity_id}" + + +class ExternalStreamProducer: + """A bound producer connection. Serves any number of topics.""" + + def __init__( + self, + *, + backend: StreamBackend, + workflow: WorkflowChainKey, + data_converter: temporalio.converter.DataConverter, + session_id: str, + client: temporalio.client.Client | None = None, + ) -> None: + """Bind a backend, converter, and producer session to one chain.""" + self._backend = backend + self._workflow = workflow + #: Bound to the Workflow these records are for, which is the same + #: context the consuming Worker decodes them under. The two sides share + #: one converter and must therefore share one context: a producer that + #: encrypts under no context while the consumer decrypts under a + #: Workflow-derived key is a mismatch that the append reports as + #: success and that only surfaces on the far side, as a decode failure + #: against a configuration that is in fact correct. + #: + #: A producer is bound to exactly one chain, so this is settled once + #: here rather than per topic. `workflow_id` and not a Run: the chain + #: key spans Continue-As-New, and a record written for one Run is read + #: by its successors. + self._data_converter = data_converter.with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=workflow.namespace, + workflow_id=workflow.workflow_id, + ) + ) + self._session_id = session_id + self._client = client + self._sequence = 0 + self._wake_counter = 0 + #: Per stream, the appends that never reported an outcome. + #: + #: Kept because the operation *is* the recovery: only these exact bytes + #: under these exact ``(session_id, sequence)`` pairs re-append as a no-op + #: if the first attempt landed, and only what the interrupted call owed + #: says whether settling it still has a wake to send. A list rather than + #: a single slot because concurrent publishes to one stream are supported + #: and more than one of them can be interrupted; entries are added only + #: when an append fails to answer, so a publish that is merely still in + #: flight registers nothing and cannot block its own sibling. + #: + #: **Per producer instance, deliberately.** This is what binds recovery + #: to the object that still holds the session's sequence and wake + #: counters; a replacement producer built with the same session id has + #: both back at zero and recovers by re-running the same calls instead + #: (ADR-038). + self._unresolved: dict[StreamKey, list[_UnresolvedAppend]] = {} + #: Per stream, the operations that have drawn a sequence number and not + #: yet reached an append outcome. The ordered append coordinator a write + #: fence needs. + #: + #: A fence asserts that every preceding write in this session has been + #: appended, and nothing made that true: an earlier `publish()` could + #: still be inside its codec -- holding a lower sequence number, having + #: appended nothing, and unresolved nowhere -- while `finish_writing()` + #: appended the fence and sent the batch's only wake. A consumer then + #: parks on a fence whose data lands behind it, with the one wake that + #: would have moved it already spent. + #: + #: Keyed by stream because that is the scope of the fence's claim, and + #: held on the producer rather than the handle because `topic()` returns + #: a fresh handle per call: two handles for one name are one stream and + #: must join one order. Only the fence waits -- concurrent publishes have + #: no defined order between them to preserve, so ordering their appends + #: would serialize a batch for nothing. + self._appends: dict[StreamKey, list[_StreamOperation]] = {} + #: Per stream, the furthest offset this producer has seen acknowledged. + #: A wake reports it so its counter follows the store's order, and a + #: batch closed by a bare ``wake()`` still reports where it ended. + self._last_offsets: dict[StreamKey, Offset] = {} + + @staticmethod + async def connect( + *, + backend: StreamBackend, + workflow: WorkflowChainKey, + client: temporalio.client.Client, + data_converter: temporalio.converter.DataConverter | None = None, + session_id: str | None = None, + ) -> ExternalStreamProducer: + """Binds a producer, verifying the chain key before it can append. + + Args: + backend: A provider instance. A plain process constructs one + directly; there is no Worker registry out here to name. + workflow: The chain key, passed in by the Workflow. + client: Used to verify the chain key, and later to send the wake + Signal. Required -- an unverified binding is a configuration + error, not a degraded mode. + data_converter: Must match the consuming Workflow's, including any + codec. Defaults to the client's. + session_id: Stable across retries. Defaults inside an Activity to a + value derived from the Activity's identity; required outside + one. + + Raises: + ChainKeyMismatchError: The server's first execution Run ID for this + Workflow ID is not the one given. + """ + await _verify_chain_key(client, workflow) + return ExternalStreamProducer( + backend=backend, + workflow=workflow, + data_converter=data_converter or client.data_converter, + session_id=session_id or _default_session_id(), + client=client, + ) + + @property + def session_id(self) -> str: + """The idempotency session shared by this producer's appends.""" + return self._session_id + + @property + def workflow(self) -> WorkflowChainKey: + """The Workflow chain receiving this producer's records.""" + return self._workflow + + def topic( + self, name: str, *, type: type[AnyType] | None = None + ) -> ExternalStreamProducerTopic[Any]: + """A handle for one stream. + + The only place the stream name appears on the producer side. + """ + if not name: + raise ValueError("a topic needs a non-empty name") + return ExternalStreamProducerTopic( + producer=self, + stream_key=self._workflow.stream_key(name), + codec=StreamPayloadCodec(self._data_converter, type), + ) + + def _next_wake_counter(self) -> int: + """Distinguishes this sender's unparked wakes from each other. + + An unparked wake names generation 0, which carries no attempt identity, + so without a per-sender counter every unparked wake from one sender would + derive the same request ID and the server would deduplicate all but the + first away. Held fixed across *retries* of one attempt -- it advances per + attempt, not per send. + """ + self._wake_counter += 1 + return self._wake_counter + + def _next_sequence(self) -> int: + """The next sequence number in this producer session. + + Per *connection*, not per topic: the pair ``(session_id, sequence)`` is + the idempotency key, and restarting the count for each topic would make + two topics' first records collide. + """ + sequence = self._sequence + self._sequence += 1 + return sequence + + +class ExternalStreamProducerTopic(Generic[AnyType]): + """One stream, from the producer's side. + + Has ``publish`` and ``finish_writing`` and no ``subscribe`` -- the two sides + are mirror images, not one type used twice. + """ + + def __init__( + self, + *, + producer: ExternalStreamProducer, + stream_key: StreamKey, + codec: StreamPayloadCodec[AnyType], + ) -> None: + """Bind a typed codec and stream key to a producer.""" + self._producer = producer + self._stream_key = stream_key + self._codec = codec + + @property + def stream_key(self) -> StreamKey: + """The durable identity of this producer topic.""" + return self._stream_key + + def _refuse_while_unresolved(self) -> None: + """Refuses a new append while an earlier one's outcome is unknown. + + This is the caller move that duplicates a record, so it is the one the + producer stops. A new ``publish()`` draws a fresh sequence number and + therefore a fresh idempotency key, so if the unresolved append did land, + the value is now in the stream twice under two keys the backend has no + way to relate. Refusing also keeps the recovery in order: the unresolved + record is re-appended before anything is appended behind it. + + The error it raises is **the outstanding operation's**, not this call's. + Reporting the refused call's ``wake`` and ``lease`` beside the older + call's record would make the error's own instructions wrong: a caller + following them settles the record with no wake, and a parked Workflow + sits on a durable record nobody announced. + + Checked at entry, so a publish already past this point when a sibling + becomes unresolved still completes -- concurrent publishes have no + defined order between them to preserve. + """ + outstanding = self._producer._unresolved.get(self._stream_key) + if not outstanding: + return + pending = outstanding[0] + more = ( + "" + if len(outstanding) == 1 + else f" ({len(outstanding)} appends on this stream are unsettled)" + ) + raise pending.error( + f"the append of {pending.record.idempotency_key} on stream " + f"{self._stream_key} has never been acknowledged, so this stream " + "will not take another append: a new one draws a fresh sequence " + "number, and if that record did land the value would be in the " + f"stream twice{more}. Settle it with resolve_append(`.record`), " + "which owes the wake this error reports rather than the one the " + "refused call asked for." + ) + + def _begin(self, sequence: int) -> _StreamOperation: + """Joins this publish to its stream's append order. + + Synchronous, and called in the same uninterrupted step as + :meth:`_refuse_while_unresolved` and the sequence draw. An operation + registered after its first await would be invisible to a fence that + began in between, which is the entire race this closes. + """ + outstanding = self._producer._appends.setdefault(self._stream_key, []) + operation = _StreamOperation(sequence=sequence) + outstanding.append(operation) + return operation + + def _preceding_publishes(self) -> tuple[_StreamOperation, ...]: + """The publishes a fence invoked now has to wait behind. + + Snapshotted at invocation rather than re-read at wait time, because a + fence's claim is about the calls that came *before* it. A publish invoked + behind the fence is outside that claim -- it takes its place in the + stream after the fence, exactly as a later publish does -- and a fence + that waited for one would be held by work the caller started after + asking for the fence. + + Synchronous, and taken in the same uninterrupted step as the sequence + draw, for the reason :meth:`_begin` gives. A fence does not join the + order it reads: see :class:`_StreamOperation`. + """ + return tuple(self._producer._appends.get(self._stream_key, ())) + + def _settle( + self, operation: _StreamOperation, failure: BaseException | None = None + ) -> None: + """Reports this call's append outcome and releases anything behind it. + + Called once the append has an outcome and **not** after the wake. The + fence asserts that preceding writes are *appended*, which a durable + record with a still-owed Signal already satisfies; holding the fence for + the wake as well would stall it on a state its own claim says nothing + about, and on the failure of one leave a caller unable to fence at all. + """ + operation.failure = failure + operation.settled.set() + outstanding = self._producer._appends.get(self._stream_key) + if outstanding is None: + return + outstanding[:] = [held for held in outstanding if held is not operation] + if not outstanding: + del self._producer._appends[self._stream_key] + + async def _await_preceding_appends( + self, preceding: tuple[_StreamOperation, ...] + ) -> None: + """Holds a fence until every earlier publish on this stream has landed. + + What makes :meth:`finish_writing`'s claim true rather than merely + documented. Each earlier operation is waited on rather than polled, and + it settles as soon as its append returns, so a fence with nothing + outstanding ahead of it does not yield at all. + + Every outcome is read *after* the last wait rather than as each one + settles, because an outcome read earlier can still change: an append + whose answer was lost settles as unknown, and the + :meth:`resolve_append` that turns that into a durable record or an + ``AppendConflictError`` can happen while this fence is still waiting on a + later publish. Reading in the loop took the stale unknown and passed over + a write that recovery had by then proved absent. + + Then, in order: + + - An append still *unresolved* refuses the fence with that operation's + canonical error -- the same refusal :meth:`_refuse_while_unresolved` + gives at entry, because the stream takes no further append until it is + settled (ADR-038). Reported ahead of any outright failure, since it is + the one that blocks the recovery for the other: republishing a failed + value is itself refused while an append is unsettled. + - An earlier *failure* is propagated as + :class:`PrecedingWriteFailedError` instead of being passed over: the + caller asked for that write before it asked for the fence, and a fence + appended over the hole tells a consumer the batch is complete when it + is short a record. + """ + for earlier in preceding: + await earlier.settled.wait() + # Re-checked here and not only at entry: an append can lose its answer at + # any point while this fence is held, including one from a publish that + # began behind it, and appending the fence in front of a record that may + # yet be settled into the stream is the ordering this method exists to + # refuse. No await separates it from the scan below, so neither reads a + # state the other has already moved past. + self._refuse_while_unresolved() + for earlier in preceding: + failure = earlier.failure + if failure is None: + continue + raise PrecedingWriteFailedError( + f"the write at sequence {earlier.sequence} on stream " + f"{self._stream_key} did not produce a durable record, so this " + "fence was not appended: a fence means every preceding write in " + "this producer session has been appended, and a consumer that " + "drains through one may park on that. Publish the value again " + "-- it draws a new sequence number and lands ahead of a later " + "fence -- or accept the batch without it; either way " + "finish_writing() again appends the fence, since the failed " + "write is no longer outstanding.", + stream_key=self._stream_key, + sequence=earlier.sequence, + ) from failure + + def _remember(self, pending: _UnresolvedAppend) -> _UnresolvedAppend: + """Makes ``pending`` the canonical state for this unsettled operation. + + A recovery may deliberately change the wake or lease. If its own append + then loses the response, the *recovery* is now the interrupted operation, + so retaining the older policy makes the next defaulted attempt contradict + the error it was handed. Cancellation is different: once delivered it is + still owed after every later failure, so it accumulates rather than being + replaced. The append order entry is the *first* attempt's, for the same + reason: it is the publish a fence captured, and no recovery of it creates + another. + """ + outstanding = self._producer._unresolved.setdefault(self._stream_key, []) + for index, held in enumerate(outstanding): + if held.record != pending.record: + continue + pending = _UnresolvedAppend( + stream_key=pending.stream_key, + record=pending.record, + wake=pending.wake, + lease=pending.lease, + cancelled=held.cancelled or pending.cancelled, + operation=held.operation, + ) + outstanding[index] = pending + return pending + outstanding.append(pending) + return pending + + def _forget(self, record: StreamRecord) -> None: + """Drops a settled append from the unresolved set. + + Matched on the idempotency key, where :meth:`_remember` and + :meth:`_outstanding` match on the whole record. Not a discrepancy that + is reachable: a record with a stored key but different bytes cannot be + in the set, because `_outstanding` refuses it with a ``ValueError`` + before the backend is touched and `publish` always draws an unused key. + If either of those ever stops holding, this drops both entries for the + key while the other two treat them as distinct, and one unsettled append + disappears without being settled. + """ + outstanding = self._producer._unresolved.get(self._stream_key) + if not outstanding: + return + remaining = [ + held + for held in outstanding + if held.record.idempotency_key != record.idempotency_key + ] + if remaining: + self._producer._unresolved[self._stream_key] = remaining + else: + del self._producer._unresolved[self._stream_key] + + async def _append( + self, + record: StreamRecord, + *, + wake: bool, + lease: timedelta, + operation: _StreamOperation | None, + ) -> StreamRecord: + """The append, with its acknowledgement window made explicit. + + Everything that is neither a return nor a contractual refusal leaves as + :class:`AppendNotAcknowledgedError`, because from here the two are + genuinely indistinguishable: the backend may have committed and lost the + answer. ``AppendConflictError`` is the exception, and the only one the + contract defines -- the key was used with *different* bytes, so this + record did not land and re-appending it would raise the identical error. + + ``KeyboardInterrupt`` and ``SystemExit`` are not converted either, for + the reason ADR-036 gives: the interpreter is going away and there is no + caller left to recover. + + ``operation`` is the append order entry this record belongs to, carried + onto the unresolved state so a recovery of it can report the outcome it + learns back to a fence holding that entry. ``None`` for a fence's own + append, which nothing waits behind. + """ + producer = self._producer + try: + placed = await producer._backend.append(self._stream_key, record) + except AppendConflictError: + self._forget(record) + raise + except (Exception, asyncio.CancelledError) as err: + pending = _UnresolvedAppend( + stream_key=self._stream_key, + record=record, + wake=wake, + lease=lease, + cancelled=isinstance(err, asyncio.CancelledError), + operation=operation, + ) + pending = self._remember(pending) + raise pending.error( + f"the append of {record.idempotency_key} did not report an " + f"outcome: {err!r}. Whether it landed is unknown -- a backend " + "commits before it answers -- so settle it with " + "resolve_append(`.record`) rather than by publishing again." + ) from err + assert placed.offset is not None + self._forget(record) + self._note_offset(placed.offset) + return placed + + def _note_offset(self, offset: Offset) -> None: + """Keeps the furthest acknowledged offset; a repeat append can be older.""" + offsets = self._producer._last_offsets + held = offsets.get(self._stream_key) + if held is None or self._producer._backend.compare_offsets(held, offset) < 0: + offsets[self._stream_key] = offset + + async def _wake_for( + self, placed: StreamRecord, *, wake: bool, lease: timedelta + ) -> Offset: + """Steps 2 and 3 for a record now known to be durable.""" + assert placed.offset is not None + if wake: + try: + await self.wake(lease=lease) + except WakeNotAcknowledgedError as err: + # Re-raised carrying the offset: the caller has no other way to + # learn that the append it must *not* retry did succeed. + err.offset = placed.offset + raise + return placed.offset + + def _outstanding(self, record: StreamRecord) -> _UnresolvedAppend: + """This topic's unsettled append for ``record``, or why there is none. + + The lookup **is** the safety check, and it is three checks at once. It + binds the recovery to the stream, because a record does not name its own + stream and ``(session_id, sequence)`` is unused on every other one -- so a + record settled against the wrong topic appends a second copy of the value + rather than deduplicating, and leaves the real stream still blocked. It + binds the recovery to the exact bytes, because idempotency is on identity + and a re-encoded payload under the same key is an ``AppendConflictError`` + rather than the no-op this depends on. And it binds the recovery to the + producer *instance*, which is the one that still holds the session's + sequence and wake counters. + + Every failure is a ``ValueError`` naming which of the three it was, and + every one of them is raised before the backend is touched. + """ + if record.offset is not None: + raise ValueError( + f"record {record.idempotency_key} already carries offset " + f"{record.offset}, which means the backend acknowledged it. " + "There is nothing unresolved about it; if a wake is still owed, " + "that is what wake() and retry_wake() are for." + ) + producer = self._producer + for held in producer._unresolved.get(self._stream_key, ()): + if held.record == record: + return held + + if record.producer_session_id != producer.session_id: + raise ValueError( + f"record {record.idempotency_key} was written by producer " + f"session {record.producer_session_id!r}, not by this one " + f"({producer.session_id!r}). Only the session that drew the " + "sequence number can re-append under that key; from any other " + "session the same bytes are a different record." + ) + elsewhere = [ + key + for key, outstanding in producer._unresolved.items() + if any(held.record == record for held in outstanding) + ] + if elsewhere: + raise ValueError( + f"record {record.idempotency_key} has an unsettled append on " + f"stream {elsewhere[0]}, not on {self._stream_key}. Append " + "idempotency is scoped to the stream, so settling it here would " + "not deduplicate against the copy that may already be on the " + "other stream -- it would append the value a second time, on a " + "topic no consumer of it is watching, and leave the first " + "stream blocked. Settle it on the topic the error names." + ) + same_key = [ + held + for held in producer._unresolved.get(self._stream_key, ()) + if held.record.idempotency_key == record.idempotency_key + ] + if same_key: + raise ValueError( + f"the unsettled append under {record.idempotency_key} does not " + "hold these bytes. Idempotency is on identity, so re-appending " + "different content under that key is an AppendConflictError " + "rather than the no-op the recovery depends on. Pass the " + "`.record` the error carried, unmodified." + ) + raise ValueError( + f"this producer has no unsettled append under " + f"{record.idempotency_key} on stream {self._stream_key}, so there " + "is nothing here to settle. If the producer that made it is gone, " + "the recovery is not this call: rebuild the producer with the same " + "session id and re-run the same calls in the same order. That " + "re-derives the same sequence numbers and re-appends the same " + "bytes, which the backend deduplicates -- and unlike this call it " + "leaves the new session's own counters correct, where settling here " + "would let its next publish reuse a sequence number and its next " + "unparked wake reuse a request ID (ADR-038)." + ) + + async def resolve_append( + self, + record: StreamRecord, + *, + wake: bool | None = None, + lease: timedelta | None = None, + ) -> Offset: + """Settles an append :class:`AppendNotAcknowledgedError` left unknown. + + Takes the ``.record`` that error carried and re-appends **it**, which is + the whole of why this is not ``publish()`` again. The backend contract + makes the one call right for both possible histories: byte-identical + content under a used ``(session_id, sequence)`` writes nothing and + returns the original offset, and a key the backend never saw is appended + now. Either way the stream ends with exactly one copy of the record and + the caller ends with its offset. + + Must be called on the topic the append was for, from the producer that + made it. Both are checked before the backend is touched; see + :meth:`_outstanding` for what each one prevents. + + **This is where an unknown outcome becomes a known one**, so it is also + where a fence waiting behind that operation is told which one it became. + A durable record clears the operation's failure; an + ``AppendConflictError`` replaces it, because that is the contract's one + definite refusal and it says this record did not land. Leaving the + operation holding its original "unknown" and letting a fence infer + durability from the record no longer being unresolved read both the same + way, and a conflict then released a fence claiming a write that was + never in the stream. + + The wake runs afterwards on exactly the terms :meth:`publish` describes, + so a coordination or Signal failure here raises + :class:`WakeNotAcknowledgedError` carrying that offset. + + Safe to call repeatedly: if this attempt is itself interrupted, it raises + :class:`AppendNotAcknowledgedError` again with the same record, and the + next attempt is the same call. + + Args: + record: The ``.record`` the error carried, unmodified. + wake: Defaults to **what the interrupted call was going to do**, + rather than to ``True``. The recovery finishes that operation, so + inventing a wake policy for it is how a fence appended with + ``wake=False`` acquires a Signal, and how a record published with + ``wake=True`` loses one. Pass a value only to override + deliberately. + lease: Likewise defaults to the interrupted call's. + + Raises: + ValueError: The record is not this topic's outstanding append -- + wrong topic, wrong bytes, wrong session, or nothing outstanding + at all. Raised before any backend call. + + AppendConflictError: The key was used with different bytes, so this + record did not land and re-appending it cannot change that. The + operation is now definitively failed rather than unknown, and a + fence behind it raises :class:`PrecedingWriteFailedError` from + this. + """ + pending = self._outstanding(record) + wake = pending.wake if wake is None else wake + lease = pending.lease if lease is None else lease + try: + placed = await self._append( + record, wake=wake, lease=lease, operation=pending.operation + ) + except AppendConflictError as err: + self._resolve(pending.operation, err) + raise + # Only the append is reported: the fence's claim is about records being + # appended, which this one now is whatever the wake below does. + self._resolve(pending.operation, None) + return await self._wake_for(placed, wake=wake, lease=lease) + + def _resolve( + self, operation: _StreamOperation | None, failure: BaseException | None + ) -> None: + """Replaces an operation's unknown outcome with the recovered one. + + Already ``settled`` -- the call that lost the answer set that before it + raised -- so this only rewrites what a fence reads there. That is why + :meth:`_await_preceding_appends` reads every outcome after its last wait + rather than as each one settles. + """ + if operation is None: + return + operation.failure = failure + operation.settled.set() + + async def publish( + self, + value: AnyType, + *, + wake: bool = True, + lease: timedelta = DEFAULT_WAKE_CLAIM_LEASE, + ) -> Offset: + """Appends one record and completes only once its wake is acknowledged. + + Idempotent under Activity retry through ``(session_id, sequence)``: + re-appending byte-identical content is a no-op returning the original + offset, and the same key with *different* bytes is an error rather than + an overwrite. + + **Returning means the record is durable and a parked Workflow has been + told about it** -- told by *this* call, which sent the wake itself and + had it accepted. It never means that some other producer claimed + responsibility for sending one. That combination is what makes the + "durable producer" row of the wakeup-durability boundary true. An append + that lands but is never signalled leaves the Workflow parked on data + already sitting in the stream, and a ``publish()`` that returned success + there would report a delivery that never happened. + + The wake happens after the append and never instead of it: only + successfully appended records may trigger wakeup, since a wake for a + record that did not land produces a Workflow Task that finds nothing. + + Args: + wake: Set ``False`` to append without waking, then call :meth:`wake` + once for the batch. The wake is idempotent either way -- + every producer waking one generation derives the same request ID + and the server deduplicates -- so this saves round trips rather + than changing the outcome. The record is durable but **un-signalled** until + that call completes, which is the un-acknowledged state made + explicit rather than hidden. + + Raises: + WakeNotAcknowledgedError: The record is durable; the wake is not. + ``.offset`` says where the record landed and ``.pending`` carries + the wakes still owed, so the caller retries the half that failed + rather than re-appending the half that did not. **Every** failure + after the append arrives this way, including one from the + coordination steps that precede the Signal -- those set + ``.restart`` and leave ``.pending`` empty, meaning the recovery is + another :meth:`wake` rather than a :meth:`retry_wake`. This + matters because it is the only thing that tells the caller not to + retry ``publish()``: a second call draws a new sequence number and + therefore a new idempotency key, so it appends the record twice. + + **Cancellation delivered after the append also arrives this way**, + with ``.cancelled`` set, because it leaves the same state and has + the same recovery. A bare ``CancelledError`` there would be + indistinguishable from cancellation before the append -- it carries + no offset, no ``pending`` and no ``restart`` -- so the caller could + neither wake the record nor safely re-publish it (ADR-036). + Cancellation delivered *before* the append is reached -- while + the payload is still encoding -- still raises ``CancelledError``: + nothing was sent to the backend, so there is nothing to say about + a record. + + AppendNotAcknowledgedError: The append itself neither returned nor + refused, so whether the record is durable is **unknown**. This is + its own outcome and not a failure: a backend commits on its own + side before it answers, so a cancellation or a lost connection in + that window can leave a durable record with no offset in hand. + ``.record`` carries the exact bytes and identity to re-append, + and :meth:`resolve_append` is the only safe way forward -- + calling ``publish()`` again draws a new sequence number and + appends the value twice if the first one landed. Until it is + resolved, this stream refuses further appends with the same error + (ADR-038). + + AppendConflictError: The idempotency key was used before with + different bytes. Unlike the above this *is* a refusal: nothing + landed for this record and re-appending it cannot change that. + """ + # The sequence is drawn **before** the encode is awaited, and that + # ordering is the whole of the idempotency key's stability. The key is + # `(session_id, sequence)`, a retried Activity reuses the session id + # deliberately, and the payload codec is allowed to do real I/O -- an + # external payload store, a KMS round trip. Drawing the number after that + # await hands identities out in *encode-completion* order, so two + # concurrent publishes exchange sequence numbers whenever the store + # answers in the other order. On the same stream the backend then sees + # each stable key reused with different bytes and raises + # `AppendConflictError`; across topics, where deduplication is per stream + # key, the swap appends duplicates instead. Either way a valid concurrent + # Activity becomes permanently non-retryable on timing alone. + # + # Drawn here, identities follow *invocation* order, which is the order the + # retry re-runs the same calls in. An encode that raises leaves its number + # unused, which costs nothing: a gap in the sequence is not observable -- + # offsets come from the provider -- and the retry reuses the same number + # for the same call. + self._refuse_while_unresolved() + sequence = self._producer._next_sequence() + # Registered in the same step, and for the same reason the sequence is + # drawn in it: a concurrent `finish_writing()` has to be able to see this + # call before the encode below hands the event loop away, or its fence + # asserts durability for a record that has not been appended yet. + operation = self._begin(sequence) + try: + record = StreamRecord( + kind=RecordKind.DATA, + payload=await self._codec.encode(value), + producer_session_id=self._producer.session_id, + sequence=sequence, + ) + placed = await self._append( + record, wake=wake, lease=lease, operation=operation + ) + except (Exception, asyncio.CancelledError) as err: + # Settled with the failure rather than merely dropped: a fence behind + # this call is waiting on it, and what it does next depends on + # whether a record landed. Settled *before* the raise, so the two are + # one uninterrupted step. + self._settle(operation, err) + raise + self._settle(operation) + return await self._wake_for(placed, wake=wake, lease=lease) + + async def wake( + self, + *, + lease: timedelta = DEFAULT_WAKE_CLAIM_LEASE, + position: Offset | None = None, + ) -> list[str]: + """Step 2 and 3 of the send sequence: observe or claim, then Signal. + + Only ever called **after** a successful append -- only successfully + appended records may trigger wakeup, since a wake for a record that did + not land produces a Workflow Task that finds nothing. + + For each subscription parked on this stream, the current generation is + claimed under a renewable lease -- and the Signal is then sent **whether + or not the claim was granted**. + + Losing the claim means another producer *intends* to send. It is not + evidence that one did: a lease permits takeover after it expires, but it + schedules nobody to take over, so a producer that crashed between + claiming and signalling strands the generation until some later producer + happens to append again. Staying silent there would leave a parked + Workflow on a record already sitting in the stream while this call + reported an acknowledged wake. The only recovery open to the caller was + to send the wake itself -- exactly what :meth:`retry_wake` does with a + pending request -- so this sends it now instead of reporting a failure + whose only fix is the same send. + + The duplicate that costs is the one that creates a second Workflow Task, + and this is not one: a **parked** wake's request ID is derived from the + generation and ignores sender identity, so racing producers issue + byte-identical requests and the server deduplicates them into a single + wake. A granted claim therefore saves a round trip, not a wakeup. A + caller appending a batch saves many more of them with ``wake=False`` + followed by one :meth:`wake`. + + A provider that cannot lease declares ``supports_leased_claims = False`` + and always grants, which is now the same behaviour every provider gets. + + Subscriptions with no installed intent get an *unparked* wake rather than + nothing (ADR-023). The Workflow may be cached with no open Workflow Task, + or evicted; neither is visible from out here, and staying silent in + either case loses the record until something else happens to wake the + Workflow. + + Args: + lease: How long a granted claim on a parked generation holds. + position: An offset this stream holds that was appended outside + this producer. The wake reports the furthest of it and this + producer's own appends, so its counter follows the store. + + Returns: + What :func:`send_wake` returned for each wake sent -- one per parked + subscription, or a single unparked wake when nothing on this stream + is parked. + + Raises: + WakeNotAcknowledgedError: The wake did not complete. The record is + durable; the wake is not. A failed Signal fills ``.pending`` with + the wakes still owed, for :meth:`retry_wake`. A failure in the + observe or claim steps that precede it leaves ``.pending`` empty + and sets ``.restart``, because nothing was composed and calling + this method again is the recovery. **Cancellation is one of the + ways an attempt fails here**, not an exception to it: this method + is only ever reached after a durable append, so cancellation leaves + the same durable-but-unannounced record and gets the same + ``pending``/``restart`` recovery, with ``.cancelled`` set + (ADR-036). + """ + producer = self._producer + if producer._client is None: + raise RuntimeError( + "waking requires a Temporal client, and this producer was built " + "without one. Use ExternalStreamProducer.connect(), which " + "requires it." + ) + + backend = producer._backend + if position is not None: + self._note_offset(position) + # The coordination steps are inside the same guarantee as the Signal, and + # every caller reaches here *after* a durable append. A provider outage in + # any of them used to escape as whatever the provider raised -- a bare + # `ConnectionError` -- which told the caller nothing about the record that + # had already landed. `publish()` catches only the durable-but- + # unacknowledged error, so the raw exception passed straight through it + # and lost the offset with it; retrying `publish()` then appended a + # *second* record, because the sequence had already advanced and the + # idempotency key with it. + try: + parked = await backend.parked_wait_ids(self._stream_key) + # An unparked wake still needs a wait id for the envelope; 0 is the + # "no particular subscription" value, and Python rechecks every active + # subscription on wakeup regardless of which one the Signal named. + targets: list[tuple[int, int | None]] = [ + ( + wait_id, + await backend.current_park_generation(self._stream_key, wait_id), + ) + for wait_id in parked + ] + if not targets: + targets = [(0, None)] + + reported, reported_counter = wake_position( + backend, producer._last_offsets.get(self._stream_key) + ) + requests: list[WakeRequest] = [] + for wait_id, generation in targets: + if generation is not None: + # Claimed, and then signalled whichever way the claim went. The + # claim is how a provider learns a wake is in flight and how an + # abandoned one becomes takeable after its lease, so it is still + # taken -- but its answer is not an acknowledgement, and the + # result is deliberately unused. See this method's docstring. + await backend.claim_park_generation( + self._stream_key, + wait_id, + generation, + claimant=producer.session_id, + lease=lease, + ) + requests.append( + wake_request_for( + producer.workflow, + stream_name=self._stream_key.stream_name, + wait_id=wait_id, + park_generation=generation, + sender_identity=producer.session_id, + wake_counter=( + producer._next_wake_counter() if generation is None else 0 + ), + position=reported, + position_counter=reported_counter, + ) + ) + except WakeNotAcknowledgedError: + raise + except (Exception, asyncio.CancelledError) as err: + # `pending` is empty and `restart` is set: nothing was composed, so + # there is nothing to re-send and `retry_wake` would be a no-op that + # looked like recovery. Calling `wake()` again is the recovery. + # + # Cancellation is in here rather than re-raised ahead of it. Every + # caller reaches this method after a durable append, so a + # `CancelledError` escaping bare says nothing about the record that + # already landed and cannot be told apart from cancellation before the + # append -- the one distinction the caller needs in order to know + # whether the value still has to be published (ADR-036). The + # cancellation is not lost: it is what `cancelled` reports. + raise WakeNotAcknowledgedError( + "the record was appended but its wake could not be composed: " + f"{err!r}. No Signal was sent. Call wake() again -- it re-observes " + "the parked set, and a parked wake's request ID is derived from " + "the generation, so a wake another producer already sent " + "deduplicates against it.", + pending=[], + restart=True, + cancelled=isinstance(err, asyncio.CancelledError), + ) from err + + sent: list[str] = [] + for index, request in enumerate(requests): + try: + sent.append( + await send_wake( + producer._client, + request, + producer_session_id=producer.session_id, + transport=wake_transport_of(producer._backend), + ) + ) + except (Exception, asyncio.CancelledError) as err: + # Cancellation included, for the reason the coordination handler + # above gives: the record is durable either way, and here the + # wakes still owed are known, so the recovery is `retry_wake` with + # `pending` rather than a fresh `wake()`. + raise WakeNotAcknowledgedError( + f"the record was appended but its wake was not acknowledged: " + f"{err!r}. Retrying with the same request is safe -- it derives " + "the same request ID and the server deduplicates it.", + pending=requests[index:], + cancelled=isinstance(err, asyncio.CancelledError), + ) from err + return sent + + async def retry_wake(self, pending: list[WakeRequest]) -> list[str]: + """Re-sends the wakes a failed attempt still owed. + + Takes the requests verbatim rather than recomputing them: recomputing + would draw a fresh wake counter for an unparked wake, derive a different + request ID, and defeat the deduplication that makes the retry safe. + + Refuses an empty list rather than returning quietly. A + :class:`WakeNotAcknowledgedError` raised before any request was composed + carries no pending wakes and sets ``restart``; a caller that fed that + empty list to this method would get a successful-looking no-op while the + record stayed durable and unannounced. There is exactly one recovery from + that state and it is :meth:`wake`. + """ + if not pending: + raise ValueError( + "retry_wake() was given no wakes to re-send. A " + "WakeNotAcknowledgedError with an empty `pending` sets `restart`, " + "which means no Signal was composed and there is nothing to " + "re-send: call wake() again instead." + ) + producer = self._producer + assert producer._client is not None + sent: list[str] = [] + for index, request in enumerate(pending): + try: + sent.append( + await send_wake( + producer._client, + request, + producer_session_id=producer.session_id, + transport=wake_transport_of(producer._backend), + ) + ) + except (Exception, asyncio.CancelledError) as err: + raise WakeNotAcknowledgedError( + f"the wake retry did not complete: {err!r}", + pending=pending[index:], + cancelled=isinstance(err, asyncio.CancelledError), + ) from err + return sent + + async def finish_writing( + self, + *, + wake: bool = True, + lease: timedelta = DEFAULT_WAKE_CLAIM_LEASE, + ) -> Offset: + """Appends an ordered write fence. **Does not close the stream.** + + The fence means only: every write in *this* producer session preceding + it has been appended. It asserts nothing about other producers, and a + later record from one does not violate it -- it simply wakes the + Workflow and consumption resumes. + + A fence on one stream alone does not park the Workflow Task either; the + task parks early only when every active subscription is immediately + parkable. + + **Waits for every earlier ``publish()`` on this stream to reach a durable + append**, across every handle ``topic()`` has returned for the name. The + claim is otherwise not one this method can make: ``publish()`` draws its + sequence before awaiting the payload codec, so a publish invoked first + can still be encoding when this one is called, and a fence that overtook + it would park a consumer in front of data that had not been written -- + having spent, if the publish was a ``wake=False`` member of a batch, the + one wake that would have moved it again. + + It does **not** wait for another concurrent fence. Each fence asserts + durability of the publishes it was invoked after, and neither of two + fences is inside the other's claim, so ordering them would only let one + that never reached the backend refuse the other. + + Raises: + PrecedingWriteFailedError: An earlier ``publish()`` on this stream, + still in flight when this call was made, ended with no durable + record -- including one whose outcome was unknown and which + :meth:`resolve_append` then proved absent. Nothing was appended + for the fence; see that class for what the two recoveries are. + + WakeNotAcknowledgedError: The fence is durable; the wake is not, on + exactly the terms :meth:`publish` describes -- cancellation after + the append included. Retrying ``finish_writing()`` appends a + *second* fence, for the same reason retrying ``publish()`` appends + a second record, and a fence that reads back twice is a producer + session that ended twice. + + AppendNotAcknowledgedError: The fence's append reported no outcome, + again on exactly :meth:`publish`'s terms. :meth:`resolve_append` + takes ``.record`` and settles it, leaving one fence whichever way + the interrupted attempt went. **Also raised for an earlier + append on this stream whose outcome is unknown**, carrying that + operation's record rather than a fence: the record may be durable, + so the fence may not go in front of it, and the stream takes no + further append until it is settled (ADR-038). + """ + # Drawn before the append for the same reason as in `publish`, though a + # fence encodes nothing and so cannot be reordered by a codec: what the + # two share is that the number belongs to the *call*, so a retry that + # makes the same calls in the same order derives the same keys. + self._refuse_while_unresolved() + sequence = self._producer._next_sequence() + # Read in the same uninterrupted step as the draw, and *not* registered + # alongside them: a fence is not a write anything else waits behind, so + # it reads the order without joining it (see `_StreamOperation`). + preceding = self._preceding_publishes() + # The sequence identity is this call's, but the *append* is ordered: the + # fence enters the backend only once every earlier publish on this stream + # has one. + await self._await_preceding_appends(preceding) + fence = StreamRecord( + kind=RecordKind.WRITE_FENCE, + payload=b"", + producer_session_id=self._producer.session_id, + sequence=sequence, + ) + placed = await self._append(fence, wake=wake, lease=lease, operation=None) + # A fence is the record most likely to find the Workflow parked -- it is + # what a producer appends when it has nothing more to say -- so an + # unsignalled one strands the Workflow for its whole idle timeout at + # exactly the moment it was waiting to be told. + return await self._wake_for(placed, wake=wake, lease=lease) + + +async def _verify_chain_key( + client: temporalio.client.Client, workflow: WorkflowChainKey +) -> None: + """Describes the Workflow and checks the first execution Run ID matches.""" + if client.namespace != workflow.namespace: + raise ChainKeyMismatchError( + f"the client is connected to namespace {client.namespace!r} but the " + f"chain key names {workflow.namespace!r}" + ) + + description = await client.get_workflow_handle(workflow.workflow_id).describe() + actual = description.raw_description.workflow_execution_info.first_run_id + if actual != workflow.first_execution_run_id: + raise ChainKeyMismatchError( + f"Workflow {workflow.workflow_id!r} in namespace " + f"{workflow.namespace!r} has first execution Run ID {actual!r}, not " + f"{workflow.first_execution_run_id!r}. Publishing under the wrong key " + "writes records into a stream no consumer is watching, so this is " + "refused rather than accepted silently." + ) diff --git a/temporalio/contrib/external_workflow_streams/_record.py b/temporalio/contrib/external_workflow_streams/_record.py new file mode 100644 index 0000000000..8b4d96b46c --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_record.py @@ -0,0 +1,323 @@ +"""The record model: offsets, cursor boundaries, and stream records (P1). + +Nothing here touches Temporal or a provider. These are the value types every +other piece of the feature is expressed in. +""" + +from __future__ import annotations + +import enum +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Final, Protocol + + +@dataclass(frozen=True) +class StartAtTail: + """A subscription start the Worker resolves against the store: the tail. + + ``last`` is how many of the newest records the read begins with; zero + means the boundary after the newest record, so the read sees only what + is appended after it was opened. The Workflow thread cannot ask the store + where the tail is, so the runtime records the request and the Worker + resolves it before the watcher starts, writing the boundary into the + marker beside the subscription as it does a cursor the Workflow named. + """ + + last: int = 0 + + def __post_init__(self) -> None: + """Refuse a negative count.""" + if self.last < 0: + raise ValueError(f"last must not be negative, got {self.last}") + + +__all__ = [ + "StartAtTail", + "AFTER", + "BEGINNING", + "Cursor", + "IdempotencyKey", + "Offset", + "OffsetComparator", + "RecordKind", + "StreamRecord", +] + + +@enum.unique +class RecordKind(enum.IntEnum): + """What a record is, from the runtime's point of view. + + Data and control records share one offset sequence -- a fence occupies an + offset exactly as a data record does, and advances the cursor the same way. + They differ only in whether the runtime yields them to Workflow code. + """ + + DATA = 1 + """Carries a payload for Workflow code.""" + + WRITE_FENCE = 2 + """A producer session declaring its preceding writes are all appended. + + Consumed by the runtime, never yielded to Workflow code. It does not close + the stream and asserts nothing about other producers. + """ + + FINISH = 3 + """An output topic's ordered terminal record. + + Unlike a write fence, this closes the topic across its Continue-As-New + chain. It is explicit because Workflow failure or termination cannot be + relied on to execute cleanup code. + """ + + @property + def is_control(self) -> bool: + """Whether this record advances the cursor without yielding a value.""" + return self is not RecordKind.DATA + + +@dataclass(frozen=True, order=False) +class Offset: + """A provider's position token: opaque, stable, and serializable. + + Deliberately **not** comparable with ``<``. Offsets are totally ordered, + but by their provider's rule rather than lexically -- Redis stream IDs are + ``(milliseconds, sequence)`` tuples, and string comparison is wrong as soon + as the millisecond component changes width. Ordering goes through the + provider's ``compare_offsets``; see :class:`OffsetComparator`. + """ + + token: str + + def __post_init__(self) -> None: + """Reject an empty provider position token.""" + if not self.token: + raise ValueError("an offset token may not be empty") + + def __str__(self) -> str: + """Return the provider position token.""" + return self.token + + def serialize(self) -> str: + """Encode this offset as its opaque provider token.""" + return self.token + + @classmethod + def deserialize(cls, token: str) -> Offset: + """Decode an offset from its opaque provider token.""" + return cls(token) + + +class OffsetComparator(Protocol): + """A provider's total order over its own offsets. + + Returns a negative number, zero, or a positive number, in the manner of a + three-way comparison. + """ + + def __call__(self, left: Offset, right: Offset, /) -> int: + """Compare two offsets using the provider's total order.""" + ... + + +@dataclass(frozen=True, order=False) +class Cursor: + """A position **boundary**, never the identity of a record (ADR-002). + + Two forms only:: + + BEGINNING # the provider's beginning-of-stream boundary + AFTER(offset) # immediately following the record at ``offset`` + + ``AFTER(x)`` names a boundary whether or not a record after ``x`` exists + yet, which is what lets a consumer park at the tail without naming the id + of a record nobody has written. + """ + + offset: Offset | None = None + """``None`` is the beginning-of-stream boundary.""" + + @property + def is_beginning(self) -> bool: + """Whether this cursor is the beginning-of-stream boundary.""" + return self.offset is None + + def __str__(self) -> str: + """Return the cursor in the documented boundary grammar.""" + return "BEGINNING" if self.is_beginning else f"AFTER({self.offset})" + + def serialize(self) -> str: + """Round-trips through :meth:`deserialize`. + + The two forms are distinguished by a prefix rather than by emptiness so + a provider whose beginning sentinel is a real token (Redis uses + ``0-0``) cannot be confused for ``BEGINNING``. + """ + if self.is_beginning: + return "B" + assert self.offset is not None + return f"A{self.offset.serialize()}" + + @classmethod + def deserialize(cls, encoded: str) -> Cursor: + """Decode the beginning or exclusive-after cursor form.""" + if encoded == "B": + return BEGINNING + if encoded.startswith("A"): + return AFTER(Offset.deserialize(encoded[1:])) + raise ValueError(f"not a serialized cursor: {encoded!r}") + + def follows(self, other: Offset, compare: OffsetComparator) -> bool: + """Whether this boundary lies at or after the record at ``other``. + + ``BEGINNING`` follows nothing; ``AFTER(x)`` follows every offset up to + and including ``x``. + """ + if self.is_beginning: + return False + assert self.offset is not None + return compare(other, self.offset) <= 0 + + def excludes(self, candidate: Offset, compare: OffsetComparator) -> bool: + """Whether a record at ``candidate`` sits at or before this boundary. + + This is the predicate a watch applies: it reads records strictly after + the boundary, so anything this excludes has already been consumed. + """ + return self.follows(candidate, compare) + + +BEGINNING: Final[Cursor] = Cursor() +"""The beginning-of-stream boundary.""" + + +def AFTER(offset: Offset) -> Cursor: # noqa: N802 -- reads as the grammar's form + """The boundary immediately following the record at ``offset``.""" + return Cursor(offset) + + +@dataclass(frozen=True) +class IdempotencyKey: + """What makes an append idempotent under Activity retry. + + Idempotency is on ``(session_id, sequence)`` **and identity**: reusing a + key with byte-identical content is a no-op returning the original offset, + and reusing it with different bytes is an error (ADR-020). + """ + + session_id: str + sequence: int + + def __post_init__(self) -> None: + """Require a non-empty session and non-negative sequence.""" + if not self.session_id: + raise ValueError("a producer session id may not be empty") + if self.sequence < 0: + raise ValueError(f"sequence must be non-negative, got {self.sequence}") + + def __str__(self) -> str: + """Return the session and sequence as a diagnostic key.""" + return f"{self.session_id}/{self.sequence}" + + +#: Provider-neutral field names. A provider stores records as fields keyed by +#: these; the prefix keeps them from colliding with anything a provider or an +#: operator adds alongside. +_FIELD_KIND: Final = "__tes_kind" +_FIELD_PAYLOAD: Final = "__tes_payload" +_FIELD_SESSION: Final = "__tes_session" +_FIELD_SEQUENCE: Final = "__tes_seq" + +RESERVED_FIELDS: Final[frozenset[str]] = frozenset( + {_FIELD_KIND, _FIELD_PAYLOAD, _FIELD_SESSION, _FIELD_SEQUENCE} +) + + +@dataclass(frozen=True) +class StreamRecord: + """One appended record, data or control. + + ``offset`` is assigned by the provider at append time and is therefore + absent until then; the producer side builds a record without one and the + provider returns the placed record. + """ + + kind: RecordKind + payload: bytes + producer_session_id: str + sequence: int + offset: Offset | None = field(default=None) + + def __post_init__(self) -> None: + """Validate control payloads and producer sequence numbers.""" + if self.kind.is_control and self.payload: + raise ValueError(f"a {self.kind.name} record carries no payload") + if self.sequence < 0: + raise ValueError(f"sequence must be non-negative, got {self.sequence}") + + @property + def is_control(self) -> bool: + """Control records advance the cursor but are never yielded.""" + return self.kind.is_control + + @property + def idempotency_key(self) -> IdempotencyKey: + """The producer identity under which this record is appended.""" + return IdempotencyKey(self.producer_session_id, self.sequence) + + def placed_at(self, offset: Offset) -> StreamRecord: + """This record as the provider stored it.""" + return StreamRecord( + kind=self.kind, + payload=self.payload, + producer_session_id=self.producer_session_id, + sequence=self.sequence, + offset=offset, + ) + + def to_fields(self) -> dict[str, bytes]: + """The provider-neutral field encoding, excluding the offset. + + The offset is the provider's to assign, so it is not part of what gets + written -- which is also what makes two appends under one idempotency + key byte-comparable. + """ + return { + _FIELD_KIND: str(int(self.kind)).encode(), + _FIELD_PAYLOAD: self.payload, + _FIELD_SESSION: self.producer_session_id.encode(), + _FIELD_SEQUENCE: str(self.sequence).encode(), + } + + @classmethod + def from_fields( + cls, offset: Offset, fields: Mapping[str, bytes] | Mapping[bytes, bytes] + ) -> StreamRecord: + """Rebuild a record a provider stored, tolerating bytes-keyed maps. + + Redis hands back ``bytes`` keys unless the client decodes responses, + and a provider should not have to normalize before calling this. + """ + decoded: dict[str, bytes] = {} + for raw_key, raw_value in fields.items(): + key = raw_key.decode() if isinstance(raw_key, bytes) else raw_key + value = ( + raw_value if isinstance(raw_value, bytes) else str(raw_value).encode() + ) + decoded[key] = value + + missing = RESERVED_FIELDS - decoded.keys() + if missing: + raise ValueError( + f"record at {offset} is missing field(s): {sorted(missing)}" + ) + + return cls( + kind=RecordKind(int(decoded[_FIELD_KIND])), + payload=decoded[_FIELD_PAYLOAD], + producer_session_id=decoded[_FIELD_SESSION].decode(), + sequence=int(decoded[_FIELD_SEQUENCE]), + offset=offset, + ) diff --git a/temporalio/contrib/external_workflow_streams/_redis.py b/temporalio/contrib/external_workflow_streams/_redis.py new file mode 100644 index 0000000000..c4d69b1cfd --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_redis.py @@ -0,0 +1,876 @@ +"""The Redis Streams provider (P3). + +Redis Streams qualifies as a backend because an entry's fields cannot be +rewritten in place. It can be deleted by ``XDEL`` or removed by trimming -- +which is exactly the damage replay's four range checks detect -- but never +altered, which is what makes those four checks sufficient. + +The two reads are **different Redis commands**, not one with a flag: + +- ``XRANGE first last`` is **inclusive of both endpoints**, and is the only + command replay may use. The marker already names the range. +- ``XREAD BLOCK ... STREAMS key id`` returns entries **strictly after** ``id``, + which is exactly the exclusive-after semantics a live watch needs. It cannot + serve a replay read: it would silently drop the range's first record. + +``BEGINNING`` maps to the sentinel ``0-0`` for watching, so a consumer that has +drained to the tail holds ``AFTER()`` and never has to name the id of +a record nobody has written yet. + +Offsets are Redis ids, compared as numeric ``(milliseconds, sequence)`` pairs. +String comparison is wrong the moment the millisecond component changes width. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from base64 import b64decode, b64encode +from binascii import Error as BinasciiError +from collections.abc import Sequence +from datetime import timedelta +from typing import Any, ClassVar, Final +from urllib.parse import quote + +from temporalio.contrib.external_workflow_streams._backend import ( + DEFAULT_WATCH_BLOCK, + AppendConflictError, + ParkIntent, + ParkIntentRemoval, + StreamBackend, + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._errors import StreamIntegrityError +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputReadResult, + OutputStage, + OutputStageConflictError, + OutputStageManifest, + OutputStageNotFoundError, + OutputStageResolutionError, + OutputStageStatus, + OutputStreamBackend, + OutputStreamRecord, + PendingOutputBarrier, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + Offset, + StreamRecord, +) + +__all__ = ["RedisStreamBackend"] + +DEFAULT_URL: Final = "redis://127.0.0.1:6379" +DEFAULT_KEY_PREFIX: Final = "temporal-external-stream" + +#: Redis' beginning-of-stream sentinel for `XREAD`. Not an offset: no record +#: ever has this id, which is why `BEGINNING` is a distinct cursor form rather +#: than ``AFTER(Offset("0-0"))``. +_BEGINNING_SENTINEL: Final = "0-0" +_OUTPUT_STAGE_FIELD: Final = "__tes_output_stage" + + +def _parse(offset: Offset) -> tuple[int, int]: + """A Redis id as the ``(ms, seq)`` pair it actually is.""" + ms, _, seq = offset.token.partition("-") + return int(ms), int(seq or 0) + + +def _content_hash(record: StreamRecord) -> str: + """A stable digest of exactly what would be written. + + Excludes the offset, which the provider assigns -- so a re-append of + identical content hashes identically rather than looking like a conflict. + """ + digest = hashlib.sha256() + for name, value in sorted(record.to_fields().items()): + digest.update(name.encode()) + digest.update(b"\x00") + digest.update(value) + digest.update(b"\x00") + return digest.hexdigest() + + +#: Append-if-new-or-identical, atomically. +#: +#: Split across two commands without a script, a crash between ``XADD`` and the +#: idempotency write would leave a record no retry could recognise as its own, +#: so the retry would append a duplicate. +_APPEND_LUA: Final = """ +local existing = redis.call('HGET', KEYS[2], ARGV[1]) +if existing then + local sep = string.find(existing, '|') + local offset = string.sub(existing, 1, sep - 1) + local digest = string.sub(existing, sep + 1) + if digest == ARGV[2] then + return {'reused', offset} + end + return {'conflict', ''} +end +local id = redis.call('XADD', KEYS[1], '*', unpack(ARGV, 3)) +redis.call('HSET', KEYS[2], ARGV[1], id .. '|' .. ARGV[2]) +return {'appended', id} +""" + + +_STAGE_OUTPUT_LUA: Final = """ +local existing = redis.call('HGET', KEYS[2], ARGV[1]) +if existing then + if existing ~= ARGV[2] then + return {'conflict', '', ''} + end + return {'reused', redis.call('HGET', KEYS[3], ARGV[1]), redis.call('HGET', KEYS[4], ARGV[1])} +end +local count = tonumber(ARGV[3]) +local cursor = 4 +local offsets = {} +for i = 1, count do + local field_count = tonumber(ARGV[cursor]) + cursor = cursor + 1 + local fields = {} + for j = 1, field_count * 2 do + fields[j] = ARGV[cursor] + cursor = cursor + 1 + end + offsets[i] = redis.call('XADD', KEYS[1], '*', unpack(fields)) +end +local joined = table.concat(offsets, ',') +redis.call('HSET', KEYS[2], ARGV[1], ARGV[2]) +redis.call('HSET', KEYS[3], ARGV[1], 'pending') +redis.call('HSET', KEYS[4], ARGV[1], joined) +return {'staged', 'pending', joined} +""" + + +_RESOLVE_OUTPUT_LUA: Final = """ +local existing = redis.call('HGET', KEYS[1], ARGV[1]) +if not existing then + return {'missing', '', ''} +end +if existing ~= ARGV[2] then + return {'conflict', '', ''} +end +local current = redis.call('HGET', KEYS[2], ARGV[1]) +if current == ARGV[3] then + return {'resolved', current, redis.call('HGET', KEYS[3], ARGV[1])} +end +if current ~= 'pending' then + return {'reversed', current, redis.call('HGET', KEYS[3], ARGV[1])} +end +redis.call('HSET', KEYS[2], ARGV[1], ARGV[3]) +return {'resolved', ARGV[3], redis.call('HGET', KEYS[3], ARGV[1])} +""" + + +#: Compare-and-remove one park intent, atomically. The claim belongs to the +#: intent generation, so a successful removal retires both keys just like the +#: unconditional operation does. +#: +#: Returns 1 removed, 2 nothing installed, 0 someone else's intent. The absent +#: case is answered before the comparison, so a key this script already cleared +#: on a call whose reply was lost does not come back as a mismatch. +_REMOVE_PARK_INTENT_IF_MATCHES_LUA: Final = """ +local run_id = redis.call('HGET', KEYS[1], 'run_id') +if not run_id then + return 2 +end +local generation = redis.call('HGET', KEYS[1], 'generation') +if run_id == ARGV[1] and generation == ARGV[2] then + redis.call('DEL', KEYS[1], KEYS[2]) + return 1 +end +return 0 +""" + + +#: The script's return values, in the order it documents them. +_REMOVAL_OUTCOMES: Final = { + 0: ParkIntentRemoval.MISMATCH, + 1: ParkIntentRemoval.REMOVED, + 2: ParkIntentRemoval.ABSENT, +} + + +class RedisStreamBackend(StreamBackend, OutputStreamBackend): + """Redis Streams as an external workflow stream provider.""" + + guarantees_immutability: ClassVar[bool | None] = True + """``XADD`` entries cannot be rewritten in place -- only deleted or trimmed.""" + + provider_id = "redis-streams" + provider_format_version = 1 + supports_leased_claims = True + + def __init__( + self, + *, + url: str = DEFAULT_URL, + client: Any | None = None, + key_prefix: str = DEFAULT_KEY_PREFIX, + ) -> None: + """Create a Redis-backed external workflow stream provider. + + Args: + url: Connection URL, used when ``client`` is not supplied. + client: An existing ``redis.asyncio.Redis``. It **must** have been + created with ``decode_responses=False`` -- payloads are + arbitrary bytes, and a decoding client would corrupt them. + key_prefix: Prepended to every key this backend creates, so one + Redis can serve several deployments. + """ + if client is None: + import redis.asyncio # imported lazily so redis stays optional + + client = redis.asyncio.from_url(url, decode_responses=False) + self._client = client + self._key_prefix = key_prefix + self._append_script = client.register_script(_APPEND_LUA) + self._stage_output_script = client.register_script(_STAGE_OUTPUT_LUA) + self._resolve_output_script = client.register_script(_RESOLVE_OUTPUT_LUA) + self._remove_park_intent_if_matches_script = client.register_script( + _REMOVE_PARK_INTENT_IF_MATCHES_LUA + ) + + # --- key layout --------------------------------------------------------- + + def stream_key(self, key: StreamKey) -> str: + """The physical Redis key for one stream identity. + + Injective: distinct :class:`StreamKey` values always render as distinct + Redis keys, because :func:`_escaped` removes the delimiter from every + component before they are joined. See that function for why joining the + raw fields is not merely untidy but a correctness failure. + + An identity with no reserved character in it renders byte-identically to + the delimiter-joined layout this replaced, and the feature is private and + unreleased, so there is nothing written under the old rendering worth + migrating and no compatibility path here to maintain. + """ + return f"{self._key_prefix}:{_escaped(key)}" + + def _idempotency_key(self, key: StreamKey) -> str: + return f"{self.stream_key(key)}:idem" + + def _intent_key(self, key: StreamKey, wait_id: int) -> str: + # `(stream key, wait_id)`, never the stream alone: two subscriptions to + # one stream are two independent waits. + return f"{self.stream_key(key)}:park:{wait_id}" + + def _claim_key(self, key: StreamKey, wait_id: int) -> str: + return f"{self.stream_key(key)}:claim:{wait_id}" + + def _output_manifest_key(self, key: StreamKey) -> str: + return f"{self.stream_key(key)}:output:manifest" + + def _output_status_key(self, key: StreamKey) -> str: + return f"{self.stream_key(key)}:output:status" + + def _output_offsets_key(self, key: StreamKey) -> str: + return f"{self.stream_key(key)}:output:offsets" + + # --- required operations ------------------------------------------------ + + async def append(self, key: StreamKey, record: StreamRecord) -> StreamRecord: + """Append a record idempotently and return it with its Redis offset.""" + fields = record.to_fields() + args: list[Any] = [ + str(record.idempotency_key).encode(), + _content_hash(record).encode(), + ] + for name, value in sorted(fields.items()): + args.append(name.encode()) + args.append(value) + + outcome, offset = await self._append_script( + keys=[self.stream_key(key), self._idempotency_key(key)], + args=args, + ) + if _text(outcome) == "conflict": + raise AppendConflictError(record.idempotency_key) + return record.placed_at(Offset(_text(offset))) + + async def read_range( + self, key: StreamKey, first: Offset, last: Offset + ) -> list[StreamRecord]: + """Read the inclusive offset range used during replay.""" + # redis-py's overload permits response modes that this client + # configuration cannot produce, so retain the runtime response shape. + entries: Any = await self._client.xrange( + self.stream_key(key), first.serialize(), last.serialize() + ) + return [_to_record(entry_id, fields) for entry_id, fields in entries] + + async def read_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = DEFAULT_WATCH_BLOCK, + ) -> list[StreamRecord]: + """Read up to ``max_records`` strictly after a live cursor.""" + start = ( + _BEGINNING_SENTINEL if after.is_beginning else after.offset.serialize() # type: ignore[union-attr] + ) + block_ms = None if block is None else int(block.total_seconds() * 1000) + if block_ms is not None and block_ms <= 0: + # `XREAD BLOCK 0` blocks *forever*, which is the opposite of what a + # zero timeout asks for, so a non-blocking read must omit BLOCK. + block_ms = None + + streams: Any = await self._client.xread( + {self.stream_key(key): start}, count=max_records, block=block_ms + ) + if not streams: + return [] + _, entries = streams[0] + return [_to_record(entry_id, fields) for entry_id, fields in entries] + + def compare_offsets(self, left: Offset, right: Offset) -> int: + """Compare Redis stream IDs numerically rather than lexically.""" + a, b = _parse(left), _parse(right) + return (a > b) - (a < b) + + # --- workflow-originated output --------------------------------------- + + async def stage_output( + self, + manifest: OutputStageManifest, + records: Sequence[StagedOutputRecord], + ) -> OutputStage: + """Atomically append an immutable pending output sub-batch.""" + self._validate_output_manifest(manifest) + if len(records) != manifest.record_count or any( + record.publish_index != index for index, record in enumerate(records) + ): + raise ValueError( + "staged output records must be contiguous from publish index zero " + "and match the manifest record count" + ) + stage_id = _output_stage_id(manifest) + encoded_manifest = _encode_output_manifest(manifest) + args: list[Any] = [stage_id, encoded_manifest, len(records)] + for record in records: + stream_record = StreamRecord( + kind=record.kind, + payload=record.payload, + producer_session_id=f"stage:{manifest.stage_token}", + sequence=record.publish_index, + ) + fields = { + **stream_record.to_fields(), + _OUTPUT_STAGE_FIELD: stage_id.encode(), + } + args.append(len(fields)) + for name, value in sorted(fields.items()): + args.extend((name.encode(), value)) + outcome, status, offsets = await self._stage_output_script( + keys=[ + self.stream_key(manifest.stream_key), + self._output_manifest_key(manifest.stream_key), + self._output_status_key(manifest.stream_key), + self._output_offsets_key(manifest.stream_key), + ], + args=args, + ) + if _text(outcome) == "conflict": + raise OutputStageConflictError(manifest) + return await self._output_stage_from_offsets( + manifest, + _output_stage_status(status), + _text(offsets), + ) + + async def commit_output(self, manifest: OutputStageManifest) -> OutputStage: + """Promote an exact pending stage, idempotently.""" + return await self._resolve_output(manifest, OutputStageStatus.COMMITTED) + + async def abort_output(self, manifest: OutputStageManifest) -> OutputStage: + """Skip an exact pending stage, idempotently.""" + return await self._resolve_output(manifest, OutputStageStatus.ABORTED) + + async def _resolve_output( + self, + manifest: OutputStageManifest, + requested: OutputStageStatus, + ) -> OutputStage: + self._validate_output_manifest(manifest) + outcome, current, offsets = await self._resolve_output_script( + keys=[ + self._output_manifest_key(manifest.stream_key), + self._output_status_key(manifest.stream_key), + self._output_offsets_key(manifest.stream_key), + ], + args=[ + _output_stage_id(manifest), + _encode_output_manifest(manifest), + requested.value, + ], + ) + result = _text(outcome) + if result == "missing": + raise OutputStageNotFoundError(manifest) + if result == "conflict": + raise OutputStageConflictError(manifest) + if result == "reversed": + raise OutputStageResolutionError( + manifest, + current=_output_stage_status(current), + requested=requested, + ) + return await self._output_stage_from_offsets( + manifest, + requested, + _text(offsets), + ) + + async def output_stage(self, manifest: OutputStageManifest) -> OutputStage | None: + """Inspect one exact staged sub-batch.""" + self._validate_output_manifest(manifest) + stage_id = _output_stage_id(manifest) + stored, status, offsets = await _gather_redis( + self._client.hget(self._output_manifest_key(manifest.stream_key), stage_id), + self._client.hget(self._output_status_key(manifest.stream_key), stage_id), + self._client.hget(self._output_offsets_key(manifest.stream_key), stage_id), + ) + if stored is None: + return None + if stored != _encode_output_manifest(manifest): + raise OutputStageConflictError(manifest) + if status is None or offsets is None: + raise StreamIntegrityError( + "an output stage is missing its status or offsets" + ) + return await self._output_stage_from_offsets( + manifest, + _output_stage_status(status), + _text(offsets), + ) + + async def append_output( + self, key: StreamKey, record: StreamRecord + ) -> OutputStreamRecord: + """Append an immediately committed Activity/external record.""" + self._require_output_key(key) + placed = await self.append(key, record) + assert placed.offset is not None + return OutputStreamRecord(placed.kind, placed.payload, placed.offset) + + async def read_output_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = DEFAULT_WATCH_BLOCK, + ) -> OutputReadResult: + """Return a committed prefix without crossing a pending stage.""" + self._require_output_key(key) + if max_records < 1: + raise ValueError("max_records must be positive") + cursor = after + first_read = True + committed: list[OutputStreamRecord] = [] + while len(committed) < max_records: + start = ( + _BEGINNING_SENTINEL + if cursor.is_beginning + else cursor.offset.serialize() # type: ignore[union-attr] + ) + block_for = block if first_read else timedelta(0) + block_ms = ( + None if block_for is None else int(block_for.total_seconds() * 1000) + ) + if block_ms is not None and block_ms <= 0: + block_ms = None + streams: Any = await self._client.xread( + {self.stream_key(key): start}, + count=max(64, max_records - len(committed)), + block=block_ms, + ) + first_read = False + if not streams: + break + _, entries = streams[0] + if not entries: + break + stage_ids = sorted( + { + _text(fields[_OUTPUT_STAGE_FIELD.encode()]) + for _, fields in entries + if _OUTPUT_STAGE_FIELD.encode() in fields + } + ) + statuses = ( + dict( + zip( + stage_ids, + await self._client.hmget( + self._output_status_key(key), list(stage_ids) + ), + ) + ) + if stage_ids + else {} + ) + for entry_id, fields in entries: + offset = Offset(_text(entry_id)) + cursor = AFTER(offset) + raw_stage_id = fields.get(_OUTPUT_STAGE_FIELD.encode()) + if raw_stage_id is not None: + stage_id = _text(raw_stage_id) + raw_status = statuses.get(stage_id) + if raw_status is None: + raise StreamIntegrityError( + f"output record at {offset} has no stage status" + ) + status = _output_stage_status(raw_status) + if status is OutputStageStatus.PENDING: + stored = await self._client.hget( + self._output_manifest_key(key), stage_id + ) + if stored is None: + raise StreamIntegrityError( + f"pending output record at {offset} has no manifest" + ) + return OutputReadResult( + tuple(committed), + PendingOutputBarrier( + _decode_output_manifest(key, _text(stored)), offset + ), + ) + if status is OutputStageStatus.ABORTED: + continue + record = _to_record(entry_id, fields) + assert record.offset is not None + committed.append( + OutputStreamRecord(record.kind, record.payload, record.offset) + ) + if len(committed) == max_records: + break + if len(entries) < max(64, max_records - len(committed)): + break + return OutputReadResult(tuple(committed)) + + async def output_tail(self, key: StreamKey) -> Cursor: + """Return the readable boundary without moving past pending output.""" + self._require_output_key(key) + cursor = BEGINNING + while True: + result = await self.read_output_after( + key, + cursor, + max_records=256, + block=timedelta(0), + ) + if result.records: + cursor = AFTER(result.records[-1].offset) + if result.pending is not None or not result.records: + return cursor + + def _validate_output_manifest(self, manifest: OutputStageManifest) -> None: + self._require_output_key(manifest.stream_key) + if manifest.provider_id != self.provider_id or ( + manifest.provider_format_version != self.provider_format_version + ): + raise ValueError("an output manifest names a different provider format") + + @staticmethod + def _require_output_key(key: StreamKey) -> None: + if key.direction is not StreamDirection.OUTPUT: + raise ValueError("an output operation requires an OUTPUT stream key") + + async def _output_stage_from_offsets( + self, + manifest: OutputStageManifest, + status: OutputStageStatus, + encoded_offsets: str, + ) -> OutputStage: + offsets = [Offset(value) for value in encoded_offsets.split(",") if value] + if len(offsets) != manifest.record_count: + raise StreamIntegrityError( + "an output stage's stored offsets do not match its manifest" + ) + entries: Any = await self._client.xrange( + self.stream_key(manifest.stream_key), + offsets[0].serialize(), + offsets[-1].serialize(), + ) + by_offset = {_text(entry_id): fields for entry_id, fields in entries} + records = [] + for offset in offsets: + fields = by_offset.get(offset.serialize()) + if fields is None: + raise StreamIntegrityError( + f"staged output record at {offset} is missing" + ) + record = StreamRecord.from_fields(offset, fields) + records.append(OutputStreamRecord(record.kind, record.payload, offset)) + return OutputStage(manifest, tuple(records), status) + + # --- parking (P3b) ------------------------------------------------------ + + async def install_park_intent(self, key: StreamKey, intent: ParkIntent) -> None: + """Persist the intent for one Workflow wait to remain parked.""" + await self._client.hset( + self._intent_key(key, intent.wait_id), + mapping={ + b"cursor": intent.cursor.serialize().encode(), + b"generation": str(intent.park_generation).encode(), + b"run_id": intent.run_id.encode(), + }, + ) + + async def remove_park_intent(self, key: StreamKey, wait_id: int) -> None: + """Remove a wait's park intent and any associated producer claim.""" + await self._client.delete( + self._intent_key(key, wait_id), self._claim_key(key, wait_id) + ) + + async def remove_park_intent_if_matches( + self, + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + """Remove a park intent only when its Run and generation still match.""" + outcome = await self._remove_park_intent_if_matches_script( + keys=[self._intent_key(key, wait_id), self._claim_key(key, wait_id)], + args=[run_id, str(park_generation)], + ) + return _REMOVAL_OUTCOMES[int(outcome)] + + async def park_intent(self, key: StreamKey, wait_id: int) -> ParkIntent | None: + """Return the currently installed park intent for one wait, if any.""" + stored = await self._client.hgetall(self._intent_key(key, wait_id)) + if not stored: + return None + stored = {_text(name): value for name, value in stored.items()} + return ParkIntent( + wait_id=wait_id, + cursor=Cursor.deserialize(_text(stored["cursor"])), + park_generation=int(_text(stored["generation"])), + run_id=_text(stored["run_id"]), + ) + + async def recheck(self, key: StreamKey, wait_id: int) -> bool: + """Return whether a parked wait now has at least one available record.""" + intent = await self.park_intent(key, wait_id) + if intent is None: + return False + found = await self.read_after(key, intent.cursor, max_records=1, block=None) + return bool(found) + + async def claim_park_generation( + self, + key: StreamKey, + wait_id: int, + park_generation: int, + *, + claimant: str, + lease: timedelta, + ) -> bool: + """Acquire or renew the leased producer claim for one park generation.""" + claim_key = self._claim_key(key, wait_id) + value = f"{park_generation}|{claimant}".encode() + lease_ms = max(1, int(lease.total_seconds() * 1000)) + + # SET NX gives exclusion; the TTL is what makes a crashed producer's + # claim recoverable instead of stranding the generation forever. + if await self._client.set(claim_key, value, nx=True, px=lease_ms): + return True + + held = await self._client.get(claim_key) + if held is None: + # It expired between the SET and the GET -- take it. + return bool(await self._client.set(claim_key, value, nx=True, px=lease_ms)) + if held == value: + # Renewal by the holder, which must not read as contention. + await self._client.pexpire(claim_key, lease_ms) + return True + + held_generation = _text(held).split("|", 1)[0] + if held_generation != str(park_generation): + # A claim for a different generation says nothing about this one. + await self._client.set(claim_key, value, px=lease_ms) + return True + return False + + async def parked_wait_ids(self, key: StreamKey) -> list[int]: + """Return all wait IDs with installed park intents for a stream.""" + prefix = f"{self.stream_key(key)}:park:" + found = [] + # SCAN rather than KEYS: this runs on the producer's hot path after every + # append, and KEYS blocks the whole server for the length of the keyspace. + # + # The prefix is a *literal* here, so it is escaped before the trailing + # `*` makes it a pattern. `_escaped` already leaves no glob character in + # the identity half, but the operator-supplied `key_prefix` is not + # escaped, and a pattern that quietly widened would enumerate another + # stream's intents rather than fail. + async for name in self._client.scan_iter(match=f"{_as_glob_literal(prefix)}*"): + suffix = _text(name)[len(prefix) :] + if suffix.isdigit(): + found.append(int(suffix)) + return sorted(found) + + async def current_park_generation(self, key: StreamKey, wait_id: int) -> int | None: + """Return the currently parked generation for one wait, if any.""" + intent = await self.park_intent(key, wait_id) + return None if intent is None else intent.park_generation + + # --- lifecycle ---------------------------------------------------------- + + async def aclose(self) -> None: + """Close the underlying Redis client.""" + await self._client.aclose() + + async def delete_for_test(self, key: StreamKey, offset: Offset) -> None: + """Removes one record, standing in for trimming or retention expiry.""" + await self._client.xdel(self.stream_key(key), offset.serialize()) + + +#: What Redis' glob matcher treats as more than itself. ``]`` and ``^`` are special +#: only inside a class, but escaping them too costs nothing and keeps the rule +#: one line long. +_GLOB_METACHARACTERS: Final = frozenset("*?[]\\") + + +def _as_glob_literal(text: str) -> str: + """A literal string, made safe to embed in a ``SCAN MATCH`` pattern.""" + return "".join( + f"\\{character}" if character in _GLOB_METACHARACTERS else character + for character in text + ) + + +def _escaped(key: StreamKey) -> str: + r"""One stream identity as a single, unambiguous key component. + + Percent-encoded per field, then joined -- **not** joined raw. A Workflow ID + and a stream name are user-chosen strings in which ``:`` is an ordinary + character, so joining the raw fields is not injective: + + ("ns", "wf", r1, f"{r2}:tokens") and ("ns", f"wf:{r1}", r2, "tokens") + + both render as ``ns:wf:r1:r2:tokens``. Two unrelated Workflows would then share + one stream, one idempotency hash, one park intent and one claim -- delivering + each other's records, and each concluding the other's claim had already taken + its wake. Same reasoning as ``_wake.py``'s length-prefixed request-ID material, + applied to a key rather than to a digest. + + Percent-encoding rather than length prefixes because a key is read by humans: + an ordinary identity still renders verbatim in ``redis-cli``, and only a field + that actually contains a delimiter pays for it. It buys one property the + length prefix does not -- the encoded form contains no ``:``, ``*``, ``?``, ``[`` or + ``\``, so the derived ``:idem``, ``:park:`` and ``:claim:`` suffixes stay + unambiguous and ``parked_wait_ids``' pattern cannot be widened by a stream name. + + Not reversible in practice, and not meant to be: ``key_prefix`` is + operator-supplied and unescaped, so only the identity half round-trips. + """ + components: tuple[str, ...] = ( + key.namespace, + key.workflow_id, + key.first_execution_run_id, + key.stream_name, + ) + if key.direction is StreamDirection.OUTPUT: + # Existing input records must remain addressable under their original + # four-component keys. Adding a reserved fifth-component layout only + # for output is still injective because every user component is escaped. + components = ( + key.namespace, + key.workflow_id, + key.first_execution_run_id, + key.direction.value, + key.stream_name, + ) + return ":".join(quote(component, safe="") for component in components) + + +def _output_stage_id(manifest: OutputStageManifest) -> str: + """An injective Redis hash field for one token/topic sub-batch.""" + return f"{len(manifest.stage_token)}:{manifest.stage_token}:{manifest.sub_batch_id}" + + +def _output_stage_status(value: Any) -> OutputStageStatus: + """Decode coordination metadata without downgrading corruption to outage.""" + try: + return OutputStageStatus(_text(value)) + except (TypeError, ValueError) as err: + raise StreamIntegrityError( + f"an output stage has an invalid stored status: {_text(value)!r}" + ) from err + + +def _encode_output_manifest(manifest: OutputStageManifest) -> bytes: + """Stable immutable metadata compared by the atomic stage script.""" + return json.dumps( + { + "provider_id": manifest.provider_id, + "provider_format_version": manifest.provider_format_version, + "stage_token": manifest.stage_token, + "run_id": manifest.run_id, + "history_floor_event_id": manifest.history_floor_event_id, + "sub_batch_id": manifest.sub_batch_id, + "fingerprint_version": manifest.fingerprint_version, + "fingerprint": b64encode(manifest.fingerprint).decode("ascii"), + "record_count": manifest.record_count, + "logical_byte_count": manifest.logical_byte_count, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + + +def _decode_output_manifest( + key: StreamKey, encoded: str | bytes +) -> OutputStageManifest: + """Rebuild reconciliation metadata stored beside a pending barrier.""" + try: + values = json.loads(encoded) + return OutputStageManifest( + stream_key=key, + provider_id=values["provider_id"], + provider_format_version=int(values["provider_format_version"]), + stage_token=values["stage_token"], + run_id=values["run_id"], + history_floor_event_id=int(values["history_floor_event_id"]), + sub_batch_id=int(values["sub_batch_id"]), + fingerprint_version=int(values["fingerprint_version"]), + fingerprint=b64decode(values["fingerprint"], validate=True), + record_count=int(values["record_count"]), + logical_byte_count=int(values["logical_byte_count"]), + ) + except ( + BinasciiError, + KeyError, + TypeError, + ValueError, + json.JSONDecodeError, + ) as err: + raise StreamIntegrityError( + "an output stage has invalid stored reconciliation metadata" + ) from err + + +async def _gather_redis(*awaitables: Any) -> tuple[Any, ...]: + """Retain precise tuple shape for the three independent hash reads.""" + return tuple(await asyncio.gather(*awaitables)) + + +def _text(value: Any) -> str: + return value.decode() if isinstance(value, (bytes, bytearray)) else str(value) + + +def _to_record(entry_id: Any, fields: Any) -> StreamRecord: + return StreamRecord.from_fields(Offset(_text(entry_id)), fields) diff --git a/temporalio/contrib/external_workflow_streams/_replay.py b/temporalio/contrib/external_workflow_streams/_replay.py new file mode 100644 index 0000000000..ac293c1720 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_replay.py @@ -0,0 +1,266 @@ +"""The replay read path (P13). + +Replay reads the **exact ranges the marker recorded** and delivers them from +memory in the recorded order. It never asks the backend what comes next: the +answer is already in the annotation, and consulting current stream timing would +be reproducing something other than what happened. + +Three kinds of waiting exist, and only one of them occurs here: + +=============================================== ========================== +Waiting for *new* records -- watchers, idle Never. Core starts no +timer, park handshake timers and no watcher runs. +Reading *recorded* offsets Occurs. Ordinary blocking + provider I/O. +Blocking on backend latency or unavailability Not a determinism failure -- + a transient storage failure. +=============================================== ========================== + +**The four range checks are the whole of validation**, and they are sufficient +precisely because every provider guarantees a record's bytes cannot change +(ADR-003). Given that, the only damage replay has to detect is a record that is +no longer *there* -- and deletion, trimming, and retention expiry are all caught +by one of the four: + +1. both endpoints present, +2. the range contains exactly ``count`` records, +3. ordering is strictly increasing under the provider's comparator, +4. ``control_positions`` match. + +A first, middle, or last deletion each fails a different one of them. + +What these checks are **not** for is a marker that names a subscription this +Workflow never created. That is row four of the taxonomy -- ordinary +nondeterminism -- because nothing is wrong with the backend: the ranges are +exactly where they were written and the Workflow code changed underneath them. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import temporalio.workflow +from temporalio.contrib.external_workflow_streams._annotation import ( + Annotation, + Run, + decode_annotation, +) +from temporalio.contrib.external_workflow_streams._backend import ( + StreamBackend, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._errors import ( + StreamIntegrityError, + StreamStorageError, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + Cursor, + StreamRecord, +) + +__all__ = ["ReplayPlan", "ReplaySegment", "build_replay_plan", "validate_run"] + + +@dataclass(frozen=True) +class ReplaySegment: + """One original activation's deliveries, in the order they happened. + + Segments are kept apart rather than concatenated because each one was a + separate event-loop drain. Collapsing them would reproduce the record order + while changing how many drains occurred, and ``wait_condition`` predicates + would then fire a different number of times (ADR-018). + """ + + deliveries: tuple[tuple[int, StreamRecord], ...] + + +@dataclass +class ReplayPlan: + """Everything one marker's replay needs, already read and validated.""" + + annotation: Annotation + segments: list[ReplaySegment] = field(default_factory=list) + + @property + def total_records(self) -> int: + """The number of records prepared across all replay segments.""" + return sum(len(s.deliveries) for s in self.segments) + + @property + def committed_boundaries(self) -> dict[int, Cursor]: + """Where this marker left each wait it recorded anything for. + + What live delivery has to resume from once replay has handed the + recorded ranges over. The terminal is the authority, and every marker + has one: Core asks for a terminal on the boundaries it decides, and the + completion path supplies one on the boundaries Python decides -- a + Workflow Task that ended carrying server-bound commands included + (ADR-008). + + The fallback to the last recorded delivery per wait is kept for a marker + that somehow arrives without one, and it is not an approximation: both + name the last record the Workflow Task handed over. What only the + terminal can say is where a wait that received *nothing* in its final + activation stopped. + + A wait the marker recorded nothing for is deliberately absent rather + than mapped to its start cursor. Nothing was delivered for it, so there + is nothing to resume past. + """ + boundaries: dict[int, Cursor] = {} + for segment in self.annotation.segments: + for run in segment.runs: + boundaries[run.wait_id] = AFTER(run.last_offset) + # The terminal wins wherever it exists: it is where deliveries stopped + # at the end of the Workflow Task, which can be past the last run for a + # wait whose final activation delivered nothing. + for wait_id, cursor in (self.annotation.terminal or {}).items(): + boundaries[wait_id] = cursor + return boundaries + + +def validate_run(run: Run, records: list[StreamRecord], backend: StreamBackend) -> None: + """Applies the four range checks. Raises :class:`StreamIntegrityError`. + + **Integrity loss must never resolve to an alternate stream result.** Every + check below reports what is wrong rather than repairing it, because + substituting a later record for a deleted one would hand Workflow code a + different history than the one its commands were derived from -- and that + would surface much later as an unrelated nondeterminism error. + """ + offsets = [r.offset for r in records] + if any(o is None for o in offsets): + raise StreamIntegrityError( + f"the provider returned a record with no offset for range " + f"[{run.first_offset}, {run.last_offset}]" + ) + + # 1. Both endpoints present. A first- or last-record deletion fails here, + # and nothing else would catch it: the count check alone cannot tell a + # missing endpoint from a missing interior record. + if not records or records[0].offset != run.first_offset: + raise StreamIntegrityError( + f"the record at {run.first_offset} is missing from the stream; the " + f"marker records it as the first of a run of {run.count}" + ) + if records[-1].offset != run.last_offset: + raise StreamIntegrityError( + f"the record at {run.last_offset} is missing from the stream; the " + f"marker records it as the last of a run of {run.count}" + ) + + # 2. Exact count. An interior deletion fails here. + if len(records) != run.count: + raise StreamIntegrityError( + f"range [{run.first_offset}, {run.last_offset}] contains " + f"{len(records)} record(s) but the marker records {run.count}; " + "records have been deleted, trimmed, or have expired" + ) + + # 3. Strictly increasing under the *provider's* comparator, never lexically. + if not backend.strictly_increasing(offsets): # type: ignore[arg-type] + raise StreamIntegrityError( + f"range [{run.first_offset}, {run.last_offset}] did not read back in " + "strictly increasing offset order" + ) + + # 4. Control positions. A control record replaced by a data record at the + # same position would otherwise be delivered to Workflow code, which never + # saw it the first time. + actual = tuple(i for i, record in enumerate(records) if record.is_control) + if actual != tuple(run.control_positions): + raise StreamIntegrityError( + f"range [{run.first_offset}, {run.last_offset}] has control records at " + f"{actual} but the marker records them at {tuple(run.control_positions)}" + ) + + +async def build_replay_plan( + annotation_bytes: bytes, + backends: dict[int, StreamBackend], + stream_keys: dict[int, StreamKey], +) -> ReplayPlan: + """Reads and validates every recorded range, up front. + + All of it before any delivery, deliberately: a range that fails validation + must stop replay before Workflow code has seen anything from the marker, + rather than part-way through with some records already delivered. + + Args: + annotation_bytes: The marker's opaque annotation. + backends: The provider for each ``wait_id`` in the annotation. + stream_keys: The stream each ``wait_id`` was subscribed to. + + Raises: + StreamStorageError: The backend was unreachable or errored. Transient; + it clears when the backend recovers. + StreamIntegrityError: A recorded range no longer reads back as written. + Keeps failing until an operator repairs the backend. + """ + annotation = decode_annotation(annotation_bytes) + plan = ReplayPlan(annotation=annotation) + + # One read per run rather than one per record: replay I/O cost is a function + # of the consumed range, not of how the batch was segmented. + for segment in annotation.segments: + deliveries: list[tuple[int, StreamRecord]] = [] + for run in segment.runs: + backend = backends.get(run.wait_id) + key = stream_keys.get(run.wait_id) + if key is not None and backend is None: + # The wait is in the annotation, so the Workflow did create it. + # The deployment is missing the store needed to read it. + raise StreamStorageError( + f"the marker records external stream wait {run.wait_id} " + "but no external stream backend is configured on this " + "Worker; pass external_stream_backend=... to the Worker" + ) + if backend is None or key is None: + # Row four of the failure taxonomy, and **not** integrity loss. + # Nothing is wrong with the backend: the recorded ranges are + # exactly where they were written, and the Workflow code has + # changed underneath them. Reporting this as integrity loss + # would send an operator to repair a backend that is fine, when + # the fix is to version the Workflow code. + # + # Reported rather than skipped either way, because skipping + # would silently deliver a different stream result. + raise temporalio.workflow.NondeterminismError( + f"the marker records external stream wait {run.wait_id}, " + "which this Workflow did not create. A subscribe() call was " + "inserted, removed, or reordered, which renumbers every " + "later wait; gate the change behind workflow.patched() " + "exactly as an inserted timer would be." + ) + records = await _read_range(backend, key, run) + validate_run(run, records, backend) + deliveries.extend((run.wait_id, record) for record in records) + plan.segments.append(ReplaySegment(tuple(deliveries))) + + return plan + + +async def _read_range( + backend: StreamBackend, key: StreamKey, run: Run +) -> list[StreamRecord]: + """The inclusive read, with provider failures classified as transient. + + ``read_range`` and not ``read_after``: the range is closed on both ends and + the marker already names it. An exclusive read here would silently drop each + run's first record, and nothing before the first replay would notice. + """ + try: + return await backend.read_range(key, run.first_offset, run.last_offset) + except (StreamStorageError, StreamIntegrityError): + # A backend that can tell the range is gone reports the loss itself; + # wrapping it would file a permanent loss under a transient failure. + raise + except Exception as err: + # Not integrity loss: nothing has been shown to be missing, only + # unreadable right now. An operator sent to repair a backend that was + # merely unreachable would find nothing wrong with it. + raise StreamStorageError( + f"could not read recorded range [{run.first_offset}, {run.last_offset}] " + f"from {key}: {err}" + ) from err diff --git a/temporalio/contrib/external_workflow_streams/_runtime.py b/temporalio/contrib/external_workflow_streams/_runtime.py new file mode 100644 index 0000000000..d7b6317589 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_runtime.py @@ -0,0 +1,2237 @@ +"""The per-Workflow-instance runtime handle (P10a, P10b). + +One of these is created per Run and handed to Workflow code as an opaque handle. +It is the only thing that crosses the sandbox boundary, and it exposes only +stream operations. No provider instance is reachable through it. + +It also builds the **observation delta** -- the thing that makes replay possible +-- because it is the only component that sees deliveries in the order Workflow +code actually received them. Neither the manager (which sees buffering order) +nor Core (which is annotation-blind) can reconstruct that. + +Two rules here are easy to get subtly wrong: + +- **Emission is not conditional on records having been consumed.** A + subscription's first observation, an activation that drained nothing, and the + boundary an activation returned on *all* produce deltas. A subscription to an + empty stream must still record its provider, stream key, and start cursor, or + replay has no starting point at all. +- **Control records are recorded even though they are never yielded.** They + occupy offsets inside a run, so a run's ``count`` includes them and + ``control_positions`` marks which relative indices they were. Leaving them out + would make every replay range read find more records than the marker claims. +""" + +from __future__ import annotations + +import asyncio +import secrets +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import Any + +import temporalio.api.common.v1 +import temporalio.api.notification.v1 +import temporalio.converter +import temporalio.workflow +from temporalio.contrib.external_workflow_streams._annotation import ( + MAX_ANNOTATION_BYTES, + AnnotationAccumulator, + AnnotationHeader, + Run, + Segment, + SegmentEndReason, + StreamBinding, + encode_bindings, + encode_header, + encode_segment, + encode_terminal, + encoded_run_size, + encoded_segment_size, +) +from temporalio.contrib.external_workflow_streams._api import ( + MAX_RECORDS_PER_ACTIVATION, +) +from temporalio.contrib.external_workflow_streams._backend import ( + StreamBackend, + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._continuation import Continuation +from temporalio.contrib.external_workflow_streams._errors import ( + ExternalStreamCapacityError, + StreamIntegrityError, + StreamStorageError, +) +from temporalio.contrib.external_workflow_streams._manager import ( + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputStageConflictError, + OutputStageManifest, + OutputStageStatus, + OutputStreamBackend, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._output_codec import ( + LOGICAL_FINGERPRINT_VERSION, + canonical_logical_record_frame, + fingerprint_logical_frames, +) +from temporalio.contrib.external_workflow_streams._output_continuation import ( + OutputContinuation, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + RecordKind, + StartAtTail, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._replay import ReplayPlan + +__all__ = ["QuiescentWait", "StagedWorkflowOutput", "WorkflowStreamRuntime"] + +_RUN_COST_FLOOR = 64 +"""The per-record annotation cost assumed until a real run has been measured. + +Pessimistic on purpose, and only ever a *starting* price: the largest run the +current annotation has actually encoded replaces it as soon as there is one, and +that measurement survives the segment it was taken in. A floor alone is not a +bound -- a provider chooses how long its offsets are, and a run costs two of +them -- which is why :data:`_SEGMENT_SPILL_CAP` exists behind it. + +Well above what a typical provider's two offsets cost (a Redis stream ID is +fifteen bytes), so in the ordinary case this bounds nothing: the per-activation +record cap is reached long first. +""" + +_SEGMENT_SPILL_CAP = 4096 +"""The largest margin a segment frame may overrun the affordability line into. + +The runtime prices a record it has not seen yet, so it can misprice one -- and a +segment frame cannot be moved to the next annotation, because its deliveries +happened in this Workflow Task and that task's marker is where replay has to find +them. The margin is what makes a misprice a *rollover* instead of a refused +frame: the segment spills into it, the same completion asks Core to end the +Workflow Task, and the terminal is still reserved behind it. + +Capped rather than a plain fraction so that a large budget does not hold back +kilobytes it will never need; floored by the fraction so that a small budget -- +which only tests use -- still has a margin at all. +""" + +_REPLAY_UNBOUNDED = 2**63 - 1 +"""The budget reported while a recorded segment is being delivered. + +A number rather than ``None`` so that every caller keeps one code path: a +segment is finite and already recorded, so "as many as the segment holds" and +"no limit" are the same answer. +""" + +_OUTPUT_MANIFEST_SCHEMA_VERSION = 1 +_MAX_OUTPUT_MANIFEST_BYTES = 64 * 1024 +"""Hard ceiling for output metadata placed in the compact History marker.""" + + +@dataclass(frozen=True) +class QuiescentWait: + """One member of the complete blocked snapshot Core is asked to retain for.""" + + wait_id: int + generation: int + immediately_parkable: bool + + +@dataclass(frozen=True) +class _LogicalOutputRecord: + topic: str + kind: RecordKind + payload: temporalio.api.common.v1.Payload | None + frame: bytes + + +@dataclass(frozen=True) +class StagedOutputTopic: + """One topic sub-batch staged under a Workflow Task's shared token.""" + + manifest: OutputStageManifest + finished: bool + + +@dataclass(frozen=True) +class StagedWorkflowOutput: + """Compact marker material for one durably staged Workflow output batch.""" + + schema_version: int + fingerprint_version: int + stage_token: str + history_floor_event_id: int + run_id: str + provider_id: str + provider_format_version: int + topics: tuple[StagedOutputTopic, ...] + segment_record_counts: tuple[tuple[int, ...], ...] + + +@dataclass +class _SubscriptionState: + """What the runtime tracks per subscription, on the Workflow thread.""" + + wait_id: int + stream_key: StreamKey + start_cursor: Cursor + idle_timeout: timedelta + + delivery_cursor: Cursor = BEGINNING + """How far records have been handed to the *subscription's* buffer. + + What the annotation records, and what replay must reproduce. + """ + consumption_cursor: Cursor = BEGINNING + """How far Workflow code has actually taken records. + + Trails :attr:`delivery_cursor` by whatever is still sitting unread in the + buffer. That buffer dies with the Run, so this -- not delivery -- is what a + successor Run must resume from. + """ + generation: int = 0 + """Increments each time this wait re-enters the blocked state.""" + + start_pending: bool = False + """The start is a tail the Worker has yet to resolve against the store. + + Until it does, the cursors hold ``BEGINNING`` as a placeholder, no watcher + runs, and the binding stays out of every header and bindings frame, so the + marker never records the placeholder as where this wait began. + """ + + announced: bool = False + """Whether the *current* annotation has carried this subscription's binding. + + The first observation must carry provider identity, stream key, and start + cursor even if no record was ever delivered -- otherwise replay of a + subscription to an empty stream has nowhere to begin. + + Read rather than merely set, because ``register`` accepts a subscription at + any activation of a retained Workflow Task and the header frame for that + task has usually already gone to Core. A wait that joined after it is bound + by its own bindings frame, and this is what says which waits still need + one. + """ + + fence_reached: bool = False + """Set by a write fence, cleared by any later record. + + A fence means only that *this* producer session's preceding writes are all + appended. A later record does not violate it; it simply clears it. + """ + + blocked: bool = True + """Whether Workflow code is currently waiting on this subscription.""" + + ready_records: int = 0 + """Records delivered to this subscription's ready list and not yet consumed. + + What separates :attr:`delivery_cursor` from :attr:`consumption_cursor`, as a + count rather than as two boundaries -- until a close drops the ready list, + which zeroes this while leaving both cursors where they were. Kept + because the per-activation delivery budget has to be charged for records + *already in Workflow code's hands* at the start of an activation as well as + for the ones it delivers during it: a ready list carried across an activation + boundary is consumed with no drain and therefore no budget check, and n + subscriptions each carrying one would otherwise let one activation run for n + times the cap. + """ + + closed: bool = False + """Whether Workflow code has ended this subscription. + + A closed wait keeps its binding and its cursor -- replay and a successor Run + both still need them -- but can never be blocked again, so it can never put + Core back in the position of retaining a Workflow Task for a wait nothing is + awaiting. + """ + + +class WorkflowStreamRuntime: + """The handle Workflow code holds, and the observation delta's author.""" + + def __init__( + self, + *, + manager: StreamSubscriptionManager, + backend: StreamBackend | None, + run_id: str, + namespace: str, + workflow_id: str, + first_execution_run_id: str, + data_converter: temporalio.converter.DataConverter, + default_idle_timeout: timedelta, + max_annotation_bytes: int = MAX_ANNOTATION_BYTES, + continuation: Continuation | None = None, + output_continuation: OutputContinuation | None = None, + ) -> None: + """Create the per-Run runtime and restore any continuation state.""" + self._manager = manager + self._backend = backend + self._run_id = run_id + self._namespace = namespace + self._workflow_id = workflow_id + self._first_execution_run_id = first_execution_run_id + #: Already bound to this Workflow's `WorkflowSerializationContext` by + #: the Worker that built this runtime, so `codec_for` hands Workflow + #: code a converter carrying the same context every other payload in the + #: activation was converted with. Bound out there rather than in here + #: because ``with_context`` runs user code, and this object lives on the + #: far side of the sandbox boundary. + self._data_converter = data_converter + self._default_idle_timeout = default_idle_timeout + self._max_annotation_bytes = max_annotation_bytes + #: The margin a segment frame may overrun the affordability line into. + #: See :data:`_SEGMENT_SPILL_CAP`. + self._spill_bytes = min(_SEGMENT_SPILL_CAP, max(64, max_annotation_bytes // 8)) + #: What the predecessor Run committed, or None on a first execution. + #: Restored from History before any subscription is established, never + #: read from the backend -- a cursor derived from mutable backend state + #: would give replay whatever the stream holds now (ADR-022). + self._continuation = continuation + self._subscriptions: dict[int, _SubscriptionState] = {} + #: Tail starts registered in the activation under way, resolved by the + #: Worker once it returns. See :meth:`resolve_pending_starts`. + self._pending_starts: dict[int, StartAtTail] = {} + #: Where the *current* annotation begins, per wait. Captured when the + #: annotation begins rather than when its header is first needed -- + #: lazily reading the delivery cursor would let a record delivered + #: before the first emission slip in front of the start cursor, and + #: replay of that marker would then never deliver it. + self._annotation_start: dict[int, Cursor] = {} + self._accumulator: AnnotationAccumulator | None = None + #: Whether this Workflow Task's annotation has already been closed with + #: its terminal. Distinct from "no accumulator": one means nothing has + #: been written yet and a terminal must create the header for it, the + #: other that the annotation is finished and a second terminal would + #: append a whole second annotation to the marker. + self._annotation_closed = False + self._pending_deltas: list[bytes] = [] + #: Runs recorded since the current segment opened, in delivery order. + self._runs: list[Run] = [] + #: What each of those runs costs encoded, parallel to `_runs`. Kept + #: because the open segment's frame size is what decides whether another + #: record can be *delivered* at all: a segment frame that no longer fits + #: the annotation cannot be deferred to the next one -- its deliveries + #: happened in this Workflow Task and that task's marker is where replay + #: has to find them -- so the only place left to act is before the record + #: that would grow it is handed over. Measured, never estimated: a run's + #: cost is two provider-supplied offset strings whose length this side + #: does not choose. + self._run_sizes: list[int] = [] + #: The largest run *this annotation* has encoded, in bytes. Unlike + #: `_run_sizes` this survives `close_segment`, which is the whole point: + #: a record is priced before it is delivered, and pricing the first record + #: of each activation at the bare floor -- as a per-segment maximum + #: does -- hands over a record the closing segment then cannot record. + self._max_run_bytes = 0 + #: How many segments this annotation already holds. Zero means the next + #: record would be its first, which is the one case delivery is never + #: refused for: a fresh annotation is the most room there will ever be, so + #: refusing there rolls over to an annotation that refuses identically. + self._segments_in_annotation = 0 + #: A late first subscription must preserve the earlier empty drains in + #: the same task, without making unrelated workflows write markers. + self._unobserved_segments = 0 + self._segment_pending = True + self._observed_this_activation = False + #: ``wait_id -> Future``, awaited by Workflow code and resolved by the + #: readiness activation. It lives here rather than on either side alone + #: because the two halves are in different modules and a second map + #: would mean the side that resolves is never the side that registered. + self._pending: dict[int, asyncio.Future[None]] = {} + #: Per channel, the latest notification a Workflow Task of this Run was + #: woken with. Positions only, never records: the reads that follow a + #: notification come from the Worker's watcher. + self._notifications: dict[str, temporalio.api.notification.v1.Notification] = {} + #: Non-``None`` only while a recorded segment is being delivered. + self._replay_ready: list[tuple[int, StreamRecord]] | None = None + #: The bindings of the marker currently being replayed. Non-``None`` + #: only for the length of one replay job, which is what makes a + #: registration made during it checkable against what was recorded. + self._replay_bindings: dict[int, StreamBinding] | None = None + #: ``wait_id -> the converter a *recorded* wait's records convert with``. + #: Installed by the Worker before a replay job reaches this thread; see + #: :meth:`install_replay_converters` for why it exists and why it is not + #: torn down when the replay ends. + self._replay_converters: dict[int, temporalio.converter.DataConverter] = {} + #: Records this activation has put into Workflow code's hands, whether by + #: delivering them or by starting with them already in a ready list. + #: Counted here rather than per subscription because the cap is an + #: *activation* budget: `merge()` consumes from several subscriptions + #: inside one activation, and a per-subscription counter would let n + #: streams run n times as long. + self._delivered_this_activation = 0 + + # Workflow-originated output is logical Payload data until the Worker + # stages it after the activation. Keeping it here makes backend handles + # and codecs unreachable from Workflow code while still letting replay + # compare the exact deterministic pre-codec identity. + self._output_records: list[_LogicalOutputRecord] = [] + self._output_topics: list[str] = [] + if output_continuation is not None: + if not isinstance(backend, OutputStreamBackend): + raise StreamStorageError( + "Workflow History contains external output continuation state, " + "but the configured backend does not support output streams" + ) + if ( + output_continuation.provider_id != type(backend).provider_id + or output_continuation.provider_format_version + != type(backend).provider_format_version + ): + raise StreamStorageError( + "the configured output provider does not match the provider " + "recorded by the preceding Run" + ) + self._output_finished: set[str] = set( + () if output_continuation is None else output_continuation.finished_topics + ) + self._output_segments: list[dict[str, int]] = [] + self._output_history_floor_event_id: int | None = None + self._output_max_records: int | None = None + self._output_max_logical_bytes: int | None = None + self._output_max_publish_latency: timedelta | None = None + self._output_logical_bytes = 0 + self._output_rollover_requested = False + self._output_capacity_waiters: list[asyncio.Future[None]] = [] + self._output_stage_token: str | None = None + self._staged_output: StagedWorkflowOutput | None = None + self._output_replay_manifest: Any = None + self._output_replay_finished_before: set[str] | None = None + + # --- the per-activation delivery budget --------------------------------- + + def begin_activation(self, history_floor_event_id: int | None = None) -> None: + """Resets the delivery budget. Called once per activation. + + The budget is per activation because that is the unit the deadlock + timeout applies to: what must be bounded is how long one ``activate()`` + call can run, not how much a Run receives over its life. + + **Reset to what is already in Workflow code's hands, not to zero.** A + batch is delivered whole and consumed one record at a time, so an + activation that stops iterating part-way through leaves the rest in the + subscription's ready list -- where the *next* activation consumes it with + no drain, and so with no budget check of any kind. Zeroing here would make + that carried-over remainder free, and it accumulates: one subscription can + drain a full batch, consume one record and block elsewhere on every + activation in turn, so n subscriptions arrive at an activation holding + roughly n times the cap between them and hand all of it over in one + ``activate()`` call. Starting the count at the carry-over makes what an + activation may hand over -- carried-over plus newly delivered -- exactly + the cap, whatever the schedule. + """ + self._delivered_this_activation = sum( + state.ready_records for state in self._subscriptions.values() + ) + if history_floor_event_id is not None: + if ( + self._output_history_floor_event_id is not None + and self._output_history_floor_event_id != history_floor_event_id + and self._output_records + ): + raise RuntimeError( + "a new Workflow Task began before the preceding output batch " + "was staged" + ) + if self._output_history_floor_event_id != history_floor_event_id: + self._output_segments = [] + self._unobserved_segments = 0 + self._output_history_floor_event_id = history_floor_event_id + for waiter in self._output_capacity_waiters: + if not waiter.done(): + waiter.set_result(None) + self._output_capacity_waiters = [] + self._output_segments.append({}) + self._segment_pending = True + + def delivery_budget_remaining(self) -> int: + """How many more records this activation may hand to Workflow code. + + Two budgets, and the smaller wins. The record cap bounds how long one + ``activate()`` call can run; the **annotation budget** bounds what the + marker for this Workflow Task can record. The second belongs here for the + same reason as the first: a record handed to Workflow code has to be + recorded, a segment frame that no longer fits cannot be moved to the next + annotation -- its deliveries happened in *this* Workflow Task and that + task's marker is where replay must find them -- and there is no third + option. So the budget is spent before the record is delivered rather than + checked after the segment is built. + + Unbounded during replay. Delivery then comes from the recorded segments + rather than a live producer, so it is already finite, the recorded + boundaries already say how many records each activation received -- + re-cutting them here would deliver a different schedule than the one in + History -- and nothing is being written to a new annotation at all. + """ + if self._replay_ready is not None: + return _REPLAY_UNBOUNDED + return min( + max(0, MAX_RECORDS_PER_ACTIVATION - self._delivered_this_activation), + self._annotation_records_affordable(), + ) + + def delivery_budget_exhausted(self) -> bool: + """Whether this activation stopped delivering because of a budget. + + The completion path asks, because records left buffered by a budget have + no readiness notification coming: the watcher moved its prefetch cursor + past them when it buffered them. Their readiness has to be re-reported or + the Workflow waits forever on records already in front of it. + + Either budget counts. The annotation one leaves records buffered in + exactly the same way, and its rollover ends the Workflow Task rather than + the activation -- so the successor task has to be told the buffer is not + empty just as the next activation would have been. + + **Conservative rather than exact**, in both of the ways the count can + reach the cap: an activation whose last drain happened to empty the buffer + did not stop *because* of the budget, and an activation pre-charged for a + carried-over ready list may not have tried to deliver at all. Both + over-report, and over-reporting is the safe direction -- a re-arm for an + empty buffer is skipped by the manager, and a wait wrongly withheld from + immediate parkability is retained for its idle timeout instead of parked. + Under-reporting loses records: nothing else announces what a budget left + behind. + """ + return self._replay_ready is None and ( + self._delivered_this_activation >= MAX_RECORDS_PER_ACTIVATION + or self.annotation_budget_exhausted + ) + + # --- the annotation byte budget (ADR-007) -------------------------------- + + @property + def annotation_budget_exhausted(self) -> bool: + """Whether the annotation can no longer afford another record. + + Read by the completion path, which turns it into a rollover request, and + by :meth:`_segment_end_reason`, which records it as the reason this + segment ended. Not a failure: approaching the budget is a runtime event + and rollover is the mechanism for it (ADR-007). + + Deliberately **not** conditioned on this activation having observed + anything. That is a property of the activation and this is a property of + the annotation, and conflating the two wedges the Workflow: the completion + path reads this *after* ``take_observation_delta`` -- which it must, since + that call is what closes the crossing segment -- and that call clears the + observed flag. The rollover would then go unrequested, the next activation + would begin against the same full annotation, and + :meth:`delivery_budget_remaining` would hand it a budget of zero. Nothing + delivered means nothing observed, which means no rollover, forever. + """ + return self._replay_ready is None and self._annotation_records_affordable() <= 0 + + def _annotation_records_affordable(self) -> int: + """How many more records this activation's segment can afford to record. + + Bytes converted into records by the most expensive run **this annotation** + has encoded, floored at :data:`_RUN_COST_FLOOR`. Annotation-wide and not + per-segment, which is the difference between a price and a guess: the open + segment is emptied at the end of every activation, so a per-segment + maximum prices the first record of *every* activation at the bare floor, + and a real run costs two provider-chosen offset strings. Delivering on + that price hands over a record the closing segment then cannot record. + + Still only a price, never a proof: nothing here has seen the offsets of + the record it is pricing. What makes the arithmetic safe is that a + misprice spills into :data:`_SEGMENT_SPILL_CAP` and becomes a rollover, + and that the reserve behind the margin keeps the terminal affordable + regardless. + + **One record is always affordable while the annotation holds no segment.** + A fresh annotation is the most room there will ever be, so refusing there + would roll over to an annotation that refuses identically -- a Workflow + that delivers nothing, observes nothing, and therefore never even asks for + the rollover that was supposed to save it. + """ + headroom = self._annotation_headroom() - self._segment_bytes() + price = max(_RUN_COST_FLOOR, self._max_run_bytes) + if headroom >= price: + return headroom // price + if self._segments_in_annotation == 0 and not self._runs: + return 1 + return 0 + + def _annotation_headroom(self) -> int: + """Bytes this annotation still has for frames that are not its closers. + + The reserve is re-priced here rather than read off the accumulator, which + holds whatever it was told last. A terminal entry costs one byte while a + wait sits at ``BEGINNING`` and more once it has a cursor, so the figure + the accumulator is carrying goes stale on the first delivery -- and a + stale reserve on this path is an *under*-reserve, which is the direction + that matters. + + When there is no accumulator yet, the first observation is about to create + one and its header comes out of the same budget, so the header is priced + here rather than discovered to be unaffordable after the fact. + """ + size = ( + self._accumulator.size + if self._accumulator is not None + else len(encode_header(self._header_preview())) + ) + return max( + 0, + self._max_annotation_bytes + - size + - self._reserve_bytes() + - self._spill_bytes, + ) + + def _segment_bytes(self) -> int: + """What the open segment would cost as a frame, right now.""" + if not self._run_sizes: + return 0 + return encoded_segment_size(self._run_sizes) + + def _reserve_bytes(self) -> int: + """What closing this annotation will cost: the terminal, plus bindings. + + Held back rather than checked, because neither frame may ever be refused: + both record something that already happened, and an annotation Core + writes without a terminal is durable and cannot be decoded past the frame + after it. See :attr:`AnnotationAccumulator.reserved`. + """ + reserve = len( + encode_terminal( + { + wait_id: state.delivery_cursor + for wait_id, state in sorted(self._subscriptions.items()) + } + ) + ) + late = { + wait_id: self._binding(state) + for wait_id, state in sorted(self._subscriptions.items()) + if not state.announced and not state.start_pending + } + if late and self._accumulator is not None: + # Only with an accumulator: without one the header is about to carry + # every one of these, and `_annotation_headroom` prices it there. + reserve += len(encode_bindings(late)) + return reserve + + def _update_reserve(self) -> None: + """Re-prices the closing frames on the accumulator that holds them.""" + if self._accumulator is not None: + self._accumulator.reserve(self._reserve_bytes(), spill=self._spill_bytes) + + def _check_segment_recordable(self) -> None: + """Refuses a segment that has grown past even the spill margin. + + The last line, and the one thing the arithmetic above cannot rule out: a + record is priced before its offsets are seen, so a provider whose offsets + are far longer than anything measured can make one run cost more than the + whole budget has left. Rollover does not help -- a fresh annotation still + has to carry this run -- so the boundary is genuinely unrecordable. + + Raised **here**, where the record has been drained but not yet handed to + Workflow code, and as the same non-retryable capacity error + ``subscribe()`` raises. Two things follow, and both are the point: + ``AnnotationBudgetExceeded`` is not what surfaces, so the message names + the provider's offsets rather than an internal byte budget; and the + Workflow fails rather than its Workflow Task, so the encoding that cannot + fit is not retried forever (ADR-007). + """ + # Headroom plus the margin: what a segment may still spend, counting the + # header and every earlier frame this annotation already holds. Measured + # against the reserve alone it would miss the header entirely, and the + # header is the largest frame most annotations carry. + emittable = self._annotation_headroom() + self._spill_bytes + segment = self._segment_bytes() + if segment <= emittable: + return + largest = max(self._run_sizes) if self._run_sizes else 0 + raise ExternalStreamCapacityError( + f"this Workflow Task's external stream deliveries no longer fit the " + f"replay annotation: the segment recording them needs {segment} bytes " + f"against {emittable} available, and its largest single run costs " + f"{largest}. A run is two provider-supplied offsets, so this means the " + "backend's offsets are far longer than the marker format is sized for. " + "Rolling the Workflow Task over cannot help -- the next annotation has " + "to carry the same run. Use a backend with shorter offsets, or " + "subscribe to fewer streams from one Workflow." + ) + + def _check_annotation_capacity(self, wait_id: int) -> None: + """Refuses a subscription set no annotation could ever record. + + A header is one indivisible frame, and a binding carries four + caller-chosen strings -- namespace, Workflow ID, first-execution Run ID, + stream name -- plus the provider id. Enough + subscriptions, or long enough valid names, and the header alone is larger + than the whole budget. Nothing downstream can recover from that: rollover + writes a *fresh* header, so the next annotation is the same size and the + one after that too, and the Workflow Task fails identically on every + retry with no marker ever written. ADR-007 exists to keep the budget from + producing exactly that. + + So the capacity question is asked where the answer is still actionable: + inside the Workflow's ``subscribe()`` call. It is deterministic -- the + same subscriptions in the same order give the same answer -- so replay + reproduces the refusal rather than diverging on it. + + Priced against an **empty** annotation, and against everything such an + annotation is nonetheless obliged to carry: its header, its terminal, one + segment frame -- an activation that drained and observed nothing still + encodes one, and it is meaningful (ADR-018) -- and the spill margin a + mispriced record overruns into. Leaving any of those out accepts a + subscription set that clears the check and then cannot encode its very + first completion, which is the failure this exists to prevent rather than + to relocate. + """ + floor = ( + len(encode_header(self._header_preview())) + + len( + encode_terminal( + { + other: state.delivery_cursor + for other, state in sorted(self._subscriptions.items()) + } + ) + ) + + len(encode_segment(Segment((), SegmentEndReason.NO_DATA_AVAILABLE))) + + self._spill_bytes + ) + if floor <= self._max_annotation_bytes: + return + raise ExternalStreamCapacityError( + f"subscribing external stream wait {wait_id} would make this " + f"Workflow's replay annotation need {floor} bytes before recording a " + f"single record -- its header, its terminal, one segment frame, and " + f"the {self._spill_bytes}-byte margin -- past the " + f"{self._max_annotation_bytes}-byte budget. " + "Rolling the Workflow Task over cannot help: every annotation begins " + "with a header of this size. Subscribe to fewer streams from one " + "Workflow, or shorten the stream and provider names." + ) + + def rearm_readiness(self) -> None: + """Re-reports readiness for every buffer this Run left non-empty. + + Called on the Workflow thread, so the hop onto the manager's loop happens + inside the manager -- the same reason ``register`` hops. + """ + self._manager.rearm_ready(self._run_id) + + # --- the ExternalStreamRuntime protocol --------------------------------- + + def stream_key(self, stream_name: str) -> StreamKey: + """Build one stream's durable key in this Workflow chain.""" + return StreamKey( + namespace=self._namespace, + workflow_id=self._workflow_id, + first_execution_run_id=self._first_execution_run_id, + stream_name=stream_name, + ) + + # --- Workflow-originated output --------------------------------------- + + async def publish_output( + self, + *, + topic: str, + value: Any, + value_type: type | None, + kind: RecordKind, + max_publish_latency: timedelta, + max_records: int, + max_logical_bytes: int, + ) -> None: + """Convert and buffer one deterministic logical output record. + + This method runs on the Workflow thread. It deliberately performs only + the synchronous payload-converter step; codecs, external payload + storage, and the output provider run later on the Worker's loop. + """ + del value_type # A decode hint for readers; encoding depends on the value. + if self._output_replay_manifest is None and not isinstance( + self._backend, OutputStreamBackend + ): + raise RuntimeError( + "the configured external stream backend does not support output staging" + ) + if topic in self._output_finished: + raise temporalio.workflow.NondeterminismError( + f"external output topic {topic!r} was already finished" + ) + + payload: temporalio.api.common.v1.Payload | None + if kind is RecordKind.DATA: + payloads = self._data_converter.payload_converter.to_payloads([value]) + if len(payloads) != 1: + raise ValueError( + f"converting one output value produced {len(payloads)} Payloads; " + "exactly one is required" + ) + payload = payloads[0] + else: + payload = None + frame = canonical_logical_record_frame(topic, kind, payload) + replaying = self._output_replay_manifest is not None + if not replaying and len(frame) > max_logical_bytes: + raise ExternalStreamCapacityError( + f"one record for output topic {topic!r} has {len(frame)} logical " + f"bytes, exceeding the configured maximum {max_logical_bytes}" + ) + + while not replaying: + effective_records = min( + max_records, + self._output_max_records or max_records, + ) + effective_bytes = min( + max_logical_bytes, + self._output_max_logical_bytes or max_logical_bytes, + ) + if not self._output_records or ( + len(self._output_records) + 1 <= effective_records + and self._output_logical_bytes + len(frame) <= effective_bytes + ): + break + # Awaiting is the deterministic backpressure point. The activation + # returns with the current batch, the Worker stages it, and Core's + # forced replacement task resolves this future from begin_activation. + self._output_rollover_requested = True + waiter = self.new_readiness_future() + self._output_capacity_waiters.append(waiter) + await waiter + + if topic not in self._output_topics: + self._output_topics.append(topic) + if not self._output_segments: + self._output_segments.append({}) + self._output_segments[-1][topic] = self._output_segments[-1].get(topic, 0) + 1 + self._output_records.append(_LogicalOutputRecord(topic, kind, payload, frame)) + self._output_logical_bytes += len(frame) + # Replay validates the batch History recorded. It must not retain the + # currently configured capacity or latency as policy for the next live + # batch: those options may have changed since this marker was written, + # and neither is active while replaying. + if not replaying: + self._output_max_records = min( + max_records, + self._output_max_records or max_records, + ) + self._output_max_logical_bytes = min( + max_logical_bytes, + self._output_max_logical_bytes or max_logical_bytes, + ) + self._output_max_publish_latency = min( + max_publish_latency, + self._output_max_publish_latency or max_publish_latency, + ) + if kind is RecordKind.FINISH: + self._output_finished.add(topic) + + @property + def has_output(self) -> bool: + """Whether this Workflow Task has logical output awaiting staging.""" + return bool(self._output_records) + + @property + def output_rollover_requested(self) -> bool: + """Whether capacity backpressure is waiting for a replacement task.""" + return self._output_rollover_requested + + @property + def output_max_publish_latency(self) -> timedelta | None: + """The minimum latency configured by non-empty topics in this batch.""" + return self._output_max_publish_latency + + async def stage_output(self, history_floor_event_id: int) -> StagedWorkflowOutput: + """Apply outbound transforms and durably stage the current batch.""" + if history_floor_event_id < 1: + raise RuntimeError( + "Core did not provide the exact predecessor of this Workflow " + "Task's Scheduled event; refusing to stage output" + ) + if not self._output_records: + raise RuntimeError("there is no Workflow output to stage") + if self._output_history_floor_event_id not in (None, history_floor_event_id): + raise RuntimeError("the output batch belongs to a different Workflow Task") + backend = self._backend + if not isinstance(backend, OutputStreamBackend): + raise RuntimeError("the configured backend does not support output staging") + + token = self._output_stage_token + if token is None: + token = secrets.token_urlsafe(24) + self._output_stage_token = token + + records_by_topic = { + topic: [record for record in self._output_records if record.topic == topic] + for topic in self._output_topics + } + topic_manifests: list[StagedOutputTopic] = [] + for sub_batch_id, topic in enumerate(self._output_topics): + logical_records = records_by_topic[topic] + fingerprint = fingerprint_logical_frames( + record.frame for record in logical_records + ) + key = StreamKey( + namespace=self._namespace, + workflow_id=self._workflow_id, + first_execution_run_id=self._first_execution_run_id, + stream_name=topic, + direction=StreamDirection.OUTPUT, + ) + manifest = OutputStageManifest( + stream_key=key, + provider_id=type(backend).provider_id, + provider_format_version=type(backend).provider_format_version, + stage_token=token, + run_id=self._run_id, + history_floor_event_id=history_floor_event_id, + sub_batch_id=sub_batch_id, + fingerprint_version=fingerprint.version, + fingerprint=fingerprint.digest, + record_count=fingerprint.record_count, + logical_byte_count=fingerprint.logical_byte_count, + ) + topic_manifests.append( + StagedOutputTopic( + manifest, + any(record.kind is RecordKind.FINISH for record in logical_records), + ) + ) + + staged = StagedWorkflowOutput( + schema_version=_OUTPUT_MANIFEST_SCHEMA_VERSION, + fingerprint_version=LOGICAL_FINGERPRINT_VERSION, + stage_token=token, + history_floor_event_id=history_floor_event_id, + run_id=self._run_id, + provider_id=type(backend).provider_id, + provider_format_version=type(backend).provider_format_version, + topics=tuple(topic_manifests), + segment_record_counts=tuple( + tuple(segment.get(topic, 0) for topic in self._output_topics) + for segment in self._output_segments + ), + ) + # Bound the exact protobuf that will enter History before running a + # payload codec, external payload store, or provider operation. Topic + # names and a many-activation segment schedule can otherwise make a + # payload-free marker exceed the server event budget after external + # side effects have already happened. + from temporalio.bridge.proto.external_data import ExternalOutputStreamManifest + + marker_manifest = ExternalOutputStreamManifest( + schema_version=staged.schema_version, + fingerprint_version=staged.fingerprint_version, + stage_token=staged.stage_token, + history_floor_event_id=staged.history_floor_event_id, + run_id=staged.run_id, + provider_id=staged.provider_id, + provider_format_version=staged.provider_format_version, + ) + for staged_topic in staged.topics: + source = staged_topic.manifest + marker_topic = marker_manifest.topics.add() + marker_topic.topic = source.stream_key.stream_name + marker_topic.record_count = source.record_count + marker_topic.logical_byte_count = source.logical_byte_count + marker_topic.logical_fingerprint = source.fingerprint + marker_topic.finished = staged_topic.finished + for counts in staged.segment_record_counts: + marker_manifest.segments.add().record_counts_by_topic.extend(counts) + manifest_bytes = marker_manifest.ByteSize() + if manifest_bytes > _MAX_OUTPUT_MANIFEST_BYTES: + raise ExternalStreamCapacityError( + "this Workflow Task's external output manifest needs " + f"{manifest_bytes} bytes, exceeding the hard " + f"{_MAX_OUTPUT_MANIFEST_BYTES}-byte marker budget; shorten output " + "topic names or publish from fewer activation segments" + ) + + # Save the immutable attempt before the first provider await. A lost + # acknowledgement retries the same token and manifest; encoded bytes may + # change under a randomized codec, and the provider retains the first. + if self._staged_output is None: + self._staged_output = staged + elif self._staged_output != staged: + raise RuntimeError( + "one stage token was reused for different logical output" + ) + + encoded_by_topic: dict[str, list[StagedOutputRecord]] = {} + for topic in self._output_topics: + encoded_records: list[StagedOutputRecord] = [] + for publish_index, record in enumerate(records_by_topic[topic]): + if record.payload is None: + encoded = b"" + else: + copied = temporalio.api.common.v1.Payload() + copied.CopyFrom(record.payload) + transformed = await self._data_converter._encode_payload_sequence( + [copied] + ) + transformed = ( + await self._data_converter._external_store_payload_sequence( + transformed + ) + ) + if len(transformed) != 1: + raise ValueError( + "transforming one output Payload did not produce exactly one" + ) + encoded = transformed[0].SerializeToString() + encoded_records.append( + StagedOutputRecord(publish_index, record.kind, encoded) + ) + encoded_by_topic[topic] = encoded_records + + for topic in self._output_topics: + topic_stage = next( + value + for value in staged.topics + if value.manifest.stream_key.stream_name == topic + ) + try: + provider_stage = await backend.stage_output( + topic_stage.manifest, encoded_by_topic[topic] + ) + except OutputStageConflictError as err: + raise StreamIntegrityError( + "the output provider rejected an idempotent stage retry with " + f"a conflicting manifest for topic {topic!r}" + ) from err + except (StreamIntegrityError, StreamStorageError): + raise + except Exception as err: + raise StreamStorageError( + f"failed staging external output topic {topic!r}" + ) from err + if ( + provider_stage.manifest != topic_stage.manifest + or provider_stage.status is not OutputStageStatus.PENDING + ): + raise StreamIntegrityError( + "the output provider did not retain the exact stage as pending " + f"for topic {topic!r}" + ) + return staged + + def output_stage_recorded(self) -> None: + """Start the next batch after its compact commit command was emitted.""" + self._output_records = [] + self._output_topics = [] + self._output_segments = [] + self._output_logical_bytes = 0 + self._output_max_records = None + self._output_max_logical_bytes = None + self._output_max_publish_latency = None + self._output_rollover_requested = False + self._output_stage_token = None + self._staged_output = None + + def begin_output_replay(self, manifest: Any) -> None: + """Install the marker's output expectations without touching a backend.""" + if self._output_replay_manifest is not None: + raise RuntimeError("an external output replay is already in progress") + if manifest.schema_version != _OUTPUT_MANIFEST_SCHEMA_VERSION: + raise RuntimeError( + "external output marker schema version " + f"{manifest.schema_version} is unsupported" + ) + if manifest.fingerprint_version != LOGICAL_FINGERPRINT_VERSION: + raise RuntimeError( + "external output logical fingerprint version " + f"{manifest.fingerprint_version} is unsupported" + ) + self._output_replay_manifest = manifest + self._output_replay_finished_before = set(self._output_finished) + self._output_records = [] + self._output_topics = [] + self._output_segments = [] + self._output_logical_bytes = 0 + self._output_max_records = None + self._output_max_logical_bytes = None + self._output_max_publish_latency = None + self._output_rollover_requested = False + self._output_stage_token = None + + def begin_output_replay_segment(self) -> None: + """Attach output expectations to the shared input/output drain frame.""" + self._output_segments.append({}) + + def verify_output_replay(self) -> None: + """Match replayed publish calls against the compact logical manifest.""" + expected = self._output_replay_manifest + if expected is None: + return + actual_topics = [] + for topic in self._output_topics: + records = [ + record for record in self._output_records if record.topic == topic + ] + fingerprint = fingerprint_logical_frames(record.frame for record in records) + actual_topics.append( + ( + topic, + fingerprint.record_count, + fingerprint.logical_byte_count, + fingerprint.digest, + any(record.kind is RecordKind.FINISH for record in records), + ) + ) + expected_topics = [ + ( + topic.topic, + topic.record_count, + topic.logical_byte_count, + bytes(topic.logical_fingerprint), + topic.finished, + ) + for topic in expected.topics + ] + expected_segments = [ + tuple(segment.record_counts_by_topic) for segment in expected.segments + ] + actual_segments = [ + tuple(segment.get(topic, 0) for topic in self._output_topics) + for segment in self._output_segments + ] + if actual_topics != expected_topics or actual_segments != expected_segments: + raise temporalio.workflow.NondeterminismError( + "Workflow output publish calls do not match the external output " + "manifest recorded in History" + ) + self._output_replay_manifest = None + self._output_replay_finished_before = None + self._output_records = [] + self._output_topics = [] + self._output_segments = [] + self._output_logical_bytes = 0 + self._output_max_records = None + self._output_max_logical_bytes = None + self._output_max_publish_latency = None + self._output_rollover_requested = False + + def abandon_output_replay(self) -> None: + """Drop a failed marker's expectations without validating partial output.""" + if self._output_replay_manifest is None: + return + if self._output_replay_finished_before is not None: + self._output_finished = self._output_replay_finished_before + self._output_replay_manifest = None + self._output_replay_finished_before = None + self._output_records = [] + self._output_topics = [] + self._output_segments = [] + self._output_logical_bytes = 0 + self._output_max_records = None + self._output_max_logical_bytes = None + self._output_max_publish_latency = None + self._output_rollover_requested = False + + def register( + self, + *, + wait_id: int, + stream_key: StreamKey, + idle_timeout: timedelta | None = None, + start_cursor: Cursor | None = None, + start_at_tail: StartAtTail | None = None, + ) -> None: + """Registers a wait with the Worker's manager. Non-blocking, no I/O. + + The start cursor defaults to what the predecessor Run committed for this + ``wait_id``, so a chain resumes where it left off without the Workflow + code saying anything about it -- and a first execution gets ``BEGINNING`` + from the same path. A ``start_at_tail`` is resolved by the Worker after + this activation and recorded with the subscription; on replay the + recorded boundary is taken from the marker and the store is not asked. + + Raises: + ExternalStreamCapacityError: This subscription set cannot be recorded + in an annotation at all -- see + :meth:`_check_annotation_capacity`. Raised *here*, out of the + Workflow's own ``subscribe()`` call, because that is the only + point at which the answer is still "do not make this + subscription" rather than "this Workflow Task cannot be + completed". + temporalio.workflow.NondeterminismError: This wait's stream is not + the one the predecessor Run recorded -- see + :meth:`restored_start`. + StreamStorageError: The configured backend no longer uses the + provider the predecessor read through. + """ + if self._backend is None: + raise RuntimeError( + "external streams are not configured on this Worker; pass " + "external_stream_backend=... to the Worker" + ) + if start_at_tail is not None and start_cursor is not None: + raise ValueError( + "a subscription starts at a cursor or at the tail, not at both" + ) + pending = False + if start_at_tail is not None: + recorded = (self._replay_bindings or {}).get(wait_id) + if recorded is not None: + # The live run resolved it and the marker carries the answer. + start_cursor = recorded.start_cursor + else: + pending = True + start_cursor = BEGINNING + elif start_cursor is None: + start_cursor = self.restored_start(wait_id, stream_key.stream_name) + if self._replay_bindings is not None: + # A subscription made while a marker is being replayed -- which is + # every subscription, on the activation that both starts the + # Workflow and replays its first marker. Checked before the state + # exists, so a mismatch is reported before a single recorded record + # can be handed through the wrong subscription. + binding = self._replay_bindings.get(wait_id) + if binding is not None: + self._verify_binding(wait_id, stream_key, binding) + state = _SubscriptionState( + wait_id=wait_id, + stream_key=stream_key, + start_cursor=start_cursor, + delivery_cursor=start_cursor, + consumption_cursor=start_cursor, + idle_timeout=idle_timeout or self._default_idle_timeout, + start_pending=pending, + ) + self._subscriptions[wait_id] = state + # A subscription created part-way through an annotation begins at its + # own start cursor, not at wherever the others happen to be. + self._annotation_start[wait_id] = start_cursor + try: + self._check_annotation_capacity(wait_id) + except Exception: + # Rolled back so the refusal leaves no half-registered wait behind: + # a state entry with no manager registration would reach the next + # annotation's header as a binding for a wait nothing is watching. + del self._subscriptions[wait_id] + del self._annotation_start[wait_id] + raise + self._update_reserve() + if pending: + assert start_at_tail is not None + self._pending_starts[wait_id] = start_at_tail + else: + self._manager.register( + run_id=self._run_id, + wait_id=wait_id, + stream_key=stream_key, + start_cursor=start_cursor, + ) + # A registration alone is replay-visible: it is what puts the stream in + # the annotation header, without which replay cannot start. + self._observed_this_activation = True + + async def resolve_pending_starts(self) -> None: + """Resolves every tail start the activation just finished registered. + + On the Worker's loop, between the activation and its completion. The + Workflow thread can do no I/O, and the boundary has to be fixed before + anything is delivered on the wait, so the marker records it beside the + subscription and replay reads it there instead of asking the store, + which by then holds something else. + """ + if not self._pending_starts: + return + backend = self._backend + if backend is None: + raise RuntimeError( + "external streams are not configured on this Worker; pass " + "external_stream_backend=... to the Worker" + ) + pending, self._pending_starts = self._pending_starts, {} + for wait_id, tail in pending.items(): + state = self._subscriptions.get(wait_id) + if state is None or not state.start_pending: + continue + cursor = await backend.tail_cursor(state.stream_key, before_last=tail.last) + state.start_cursor = cursor + state.delivery_cursor = cursor + state.consumption_cursor = cursor + state.start_pending = False + self._annotation_start[wait_id] = cursor + self._check_annotation_capacity(wait_id) + self._update_reserve() + self._manager.register( + run_id=self._run_id, + wait_id=wait_id, + stream_key=state.stream_key, + start_cursor=cursor, + ) + + def drain(self, wait_id: int, max_records: int | None = None) -> list[StreamRecord]: + """Pops buffered records. Performs no I/O and never blocks. + + During replay the records come from the segment currently being + delivered rather than from the live buffer. ``_apply`` cannot tell the + difference, which is the point: replay and live delivery run the same + Workflow code down the same path. + """ + if self._replay_ready is not None: + # Deliberately *not* the live buffer, even if the manager happens to + # hold something: a watcher that ran before the Run was evicted may + # have prefetched past where the marker stops, and delivering that + # would replay records this Workflow Task never saw. + # + # Taken from the **front**, and only while the front belongs to this + # wait. A segment is the recorded global order across every stream -- + # the order `record_delivery` saw, which is the order Workflow code + # took records in -- so a drain that searched past a record belonging + # to another wait would hand this one a record that came *after* it + # live. A segment recorded as (wait 2, wait 1) replays through + # `merge` as (wait 1, wait 2) under that reading, because `merge` + # asks in `wait_id` order and the search obliges every time. + # + # Live, that drain would simply have found nothing: the record was + # not in this wait's buffer yet. Returning nothing here is the same + # answer, and it leaves the record where the drain that recorded it + # will find it. + taken: list[StreamRecord] = [] + while ( + self._replay_ready + and self._replay_ready[0][0] == wait_id + and (max_records is None or len(taken) < max_records) + ): + taken.append(self._replay_ready.pop(0)[1]) + return taken + return self._manager.drain(self._run_id, wait_id, max_records) + + # --- replay delivery ---------------------------------------------------- + + def take_replay_plan(self) -> ReplayPlan | None: + """The plan prepared for this Run, if a replay job is being delivered.""" + return self._manager.take_replay_plan(self._run_id) + + def install_replay_converters( + self, converters: Mapping[int, temporalio.converter.DataConverter] + ) -> None: + """The context a recorded wait's records must be *converted* with. + + Called on the Worker's loop, before the replay job reaches this thread, + because the converters are produced by ``with_context`` -- user code + that clones the component converters, and that this side of the sandbox + boundary must not run. See ``_handle_external_stream_jobs``. + + The asynchronous half of decoding is already bound this way: the manager + prepares a replayed record under the *annotation's* stream key rather + than under anything the running Run knows, because the Workflow has not + run far enough to have re-created the subscription. Leaving the + synchronous half on the runtime's own converter converts one record + under two different Workflow identities, which is not hypothetical: an + offline ``Replayer`` runs under ``ReplayNamespace`` by default, so a + converter keyed on the namespace decodes a valid history to a different + value or refuses it outright. The same accommodation + :meth:`_verify_binding` makes for a replay harness's namespace, applied + to conversion. + + Merged rather than replaced, and deliberately **not** cleared when the + replay ends. Delivery is not consumption: a batch drained during replay + can be consumed over several later activations, and the record's context + is a property of where it was recorded rather than of the marker that + happened to still be open. Keeping the entry costs nothing on a live + Worker -- a wait's stream key is the Run's own identity, so the recorded + context *is* the runtime's -- and the two only differ under a replay + harness, which delivers nothing live to mis-convert. + """ + self._replay_converters.update(converters) + + def begin_replay(self, bindings: Mapping[int, StreamBinding]) -> None: + """Holds the marker's bindings open, and checks the ones already made. + + Delivery joins a recorded run to a subscription by ``wait_id`` alone, + and an integer is not an identity: it says nothing about *what* the wait + was subscribed to. Without this the same wait number pointing at a + different stream quietly delivers the recorded stream's bytes through + the new subscription -- the failure taxonomy's row four turned into a + silently different stream result, which is the one outcome the design + says must never happen. + + Checked here as well as in :meth:`register` because the two moments are + different Workflow Tasks' worth of history: a marker replayed after the + Workflow has already run past its ``subscribe()`` calls finds the + subscriptions in place, and one replayed in the same activation that + starts the Workflow finds none of them yet. + """ + self._replay_bindings = dict(bindings) + for wait_id, state in sorted(self._subscriptions.items()): + binding = self._replay_bindings.get(wait_id) + if binding is not None: + self._verify_binding(wait_id, state.stream_key, binding) + + def _verify_binding( + self, + wait_id: int, + stream_key: StreamKey, + binding: StreamBinding, + ) -> None: + """Row four, and deliberately not integrity loss. + + The stream compared here was chosen by Workflow code, so a difference + means the code moved, not that anything is wrong with the backend. + + Only the **stream name** is compared, not the whole key. The other three + components -- namespace, Workflow id, first execution Run id -- are the + Run's identity rather than anything the code chose, and a replay harness + legitimately supplies its own: `Replayer` runs under ``ReplayNamespace``, + so comparing the full key would report every replayed history as + nondeterministic. The key is still *recorded* whole, because replay has + to read the ranges it names. + + The start cursor is **not** compared. It is the position the wait stood + at when that Workflow Task opened, not a property of the subscription, + so for every marker after a Run's first it legitimately differs from the + cursor the subscription was registered with. What guards the cursor + across Runs is ADR-022's continuation check in :meth:`restored_start`. + """ + if stream_key.stream_name != binding.stream_key.stream_name: + raise temporalio.workflow.NondeterminismError( + f"the marker records external stream wait {wait_id} on stream " + f"{binding.stream_key.stream_name!r}, but this Workflow " + f"subscribes it to {stream_key.stream_name!r}. A subscribe() " + "call was inserted, removed, or reordered, which renumbers " + "every later wait; gate the change behind workflow.patched() " + "exactly as an inserted timer would be." + ) + + def begin_replay_segment( + self, deliveries: Sequence[tuple[int, StreamRecord]] + ) -> None: + """Makes one recorded segment the only thing a drain can see.""" + self._verify_replay_consumed() + self._replay_ready = list(deliveries) + + def verify_replay_consumed(self) -> None: + """Everything the marker recorded must have happened again. + + Two things, checked once the last segment has been delivered. + + A record in a run was handed to Workflow code during the activation the + run was recorded in, so a replay that leaves one behind is running + different code. The common way to get here is a removed ``subscribe()`` + call: its wait is never registered, nothing ever drains it, and the + marker's records for it would otherwise be discarded in silence -- the + Workflow reaching its next command having consumed less than History + says it consumed. + + And every wait the marker *bound* must have been recreated, which is a + strictly larger claim: a binding is written for a subscription whether + or not anything was ever delivered through it. The first observation has + to carry provider identity, stream key, and start cursor even for a + stream that stayed quiet for the whole Workflow Task, so a binding with + no runs behind it is the ordinary shape of a quiet subscription rather + than an exotic one -- and it is invisible to every check that reasons + from deliveries. + """ + self._verify_replay_consumed() + # The one check that reads in the other direction. Every other one walks + # what the code *did*: :meth:`begin_replay` iterates the subscriptions + # that exist and verifies only those the marker also names, + # :meth:`register` verifies a wait only when it is registered at all, + # and :meth:`_verify_replay_consumed` has nothing to report unless + # records were left undelivered. A recorded wait that the code no longer + # creates and that the marker holds no records for is therefore reached + # by none of them. + # + # Two removals escape without this, and the second is the worse one: + # + # - the **last** ``subscribe()`` removed, on a stream that was quiet. + # Nothing renumbers, no records go undelivered, and the replay is + # accepted although the Workflow now holds one subscription fewer than + # History says it did -- with its own live reads never made. + # - a **middle** ``subscribe()`` removed where the later waits name the + # same stream and backend. Every survivor renumbers down by one, so + # the binding comparison compares equal for all of them, and the + # records the marker recorded for wait *k* are consumed by what was + # subscription *k+1*: a different cursor, a different consumer, and + # nothing left over for the delivery check to notice. + # + # Deliberately **not** symmetric, and deliberately not in + # :meth:`_verify_replay_consumed`, which :meth:`begin_replay_segment` + # also calls before each segment. "every bound wait was registered" is + # the invariant; the converse is not, because replay runs the Workflow + # forward past the Workflow Task the marker covers, and the + # subscriptions it makes there belong to the *next* marker's header. An + # added ``subscribe()`` at the end is a supported change and must stay + # one. + missing = sorted(set(self._replay_bindings or {}) - set(self._subscriptions)) + if missing: + # Row four, not integrity loss: the recorded ranges are exactly + # where they were written, and it is the Workflow code that moved. + raise temporalio.workflow.NondeterminismError( + f"the marker records external stream wait(s) {missing} that this " + "Workflow never created. A subscribe() call was removed or " + "reordered, which renumbers every later wait and can hand one " + "wait's recorded records to another wait's subscription; gate " + "the change behind workflow.patched() exactly as a removed " + "timer would be." + ) + + def _verify_replay_consumed(self) -> None: + if not self._replay_ready: + return + wait_ids = sorted({wait_id for wait_id, _ in self._replay_ready}) + raise temporalio.workflow.NondeterminismError( + f"the marker records {len(self._replay_ready)} delivery(s) for " + f"external stream wait(s) {wait_ids} that this Workflow never took. " + "A subscribe() call was removed, reordered, or is no longer " + "consumed, which leaves recorded records undelivered; gate the " + "change behind workflow.patched() exactly as a removed timer would " + "be." + ) + + def reposition_after_replay(self, boundaries: Mapping[int, Cursor]) -> None: + """Moves the manager's cursors to what the replayed marker committed. + + Replay delivers from the annotation, not from the manager's buffer -- + but the watcher has been filling that buffer from the subscription's + start cursor the whole time, because nothing out there knows a marker + for these records exists. Without this the first live drain after a + replay hands Workflow code every replayed record a second time; the + symptom is a Workflow that received ``['alpha', 'alpha', 'beta']``. + + The boundaries come from the marker rather than from wherever the + replay's deliveries happened to stop -- the marker is History's own + statement of what was committed -- and only the waits it recorded + something for are named. + + Performed **synchronously**, before this returns: the drain that + follows replay runs on this same thread, so a reposition merely posted + to the manager's loop would still have the marker-covered records in the + buffer when that drain reaches it. Only the watcher's wakeup is hopped; + see :meth:`StreamSubscriptionManager.reposition_to_committed`. + """ + self._manager.reposition_to_committed(self._run_id, boundaries) + + def end_replay(self) -> None: + """Hands drains back to the live buffer. + + Called from a ``finally``: a partial replay that left this set would + make every later drain on this Run return nothing at all, turning one + marker's failure into a Workflow that silently never receives again. + """ + self._replay_ready = None + self._replay_bindings = None + + def codec_for( + self, value_type: type | None, wait_id: int + ) -> StreamPayloadCodec[Any]: + """Return the wait's typed codec under its recorded serialization context.""" + # The recorded context wins wherever one was installed, so that both + # halves of a replayed record's decoding see the same Workflow identity. + # Everything else -- every live record, and every wait no marker bound -- + # converts with this Run's own converter, which is the same object the + # activation's other payloads used. + converter = self._replay_converters.get(wait_id, self._data_converter) + return StreamPayloadCodec(converter, value_type) + + def new_readiness_future(self) -> asyncio.Future[None]: + """A future belonging to the Workflow's own deterministic event loop.""" + return asyncio.get_event_loop().create_future() + + def register_pending(self, wait_id: int, future: asyncio.Future[None]) -> None: + """Register the future readiness will resolve for one blocked wait.""" + self._pending[wait_id] = future + + def discard_pending(self, wait_id: int) -> None: + """Forget a wait that is no longer awaiting readiness.""" + self._pending.pop(wait_id, None) + + def resolve_all_pending(self) -> int: + """Resumes every waiting subscription. Returns how many were resumed. + + *Every* one, not only those a readiness job named: the job's hints are + hints, not an exhaustive availability claim, so each subscription is + left to find out for itself whether anything arrived. Resolving only the + hinted ones would strand a record whose notification was coalesced away. + """ + resumed = 0 + for wait_id in sorted(self._pending): + future = self._pending.pop(wait_id, None) + if future is not None and not future.done(): + future.set_result(None) + resumed += 1 + return resumed + + def clear_pending(self) -> None: + """Drops every waiting future, as eviction requires.""" + self._pending.clear() + + def notifications_received( + self, notifications: Iterable[temporalio.api.notification.v1.Notification] + ) -> None: + """Notes the notifications a Workflow Task's scheduled event carried. + + Core resumes the parked waits for them; this side only remembers where + each channel's store stood, keeping the highest counter per channel the + way the server folds. Read from History, so a replay notes the same. + """ + for notification in notifications: + known = self._notifications.get(notification.channel) + if known is None or notification.counter >= known.counter: + self._notifications[notification.channel] = notification + + def latest_notification( + self, channel: str + ) -> temporalio.api.notification.v1.Notification | None: + """The latest notification this Run was woken with on ``channel``.""" + return self._notifications.get(channel) + + # --- recording what Workflow code observed ------------------------------- + + def record_delivery(self, wait_id: int, record: StreamRecord) -> None: + """Notes one record reaching the runtime, in observed global order. + + Called for **every** record including control records: they occupy + offsets inside a run, so a run's count includes them and their relative + indices go in ``control_positions``. Omitting them would make replay's + range read find more records than the marker claims and fail as + integrity loss. + + **Also where the activation's delivery budget is spent.** Delivery is the + moment a record leaves the manager's buffer for a subscription's private + ready list, and that is what has to be charged rather than the later + consumption of it, for two reasons that point the same way. The budget is + a *reservation*: a drain that checked the budget and charged nothing let + the next subscription's drain see the same room and take it again, so two + subscriptions consumed independently -- no `merge()` involved -- delivered + twice the cap in one activation, and n of them n times it. And delivery is + what the annotation records, so a cap charged at consumption bounded a + different quantity than the segment it is supposed to bound: the recorded + segment could hold more records than the cap replay will divide it by. + """ + state = self._subscriptions.get(wait_id) + if self._replay_ready is None: + # Counted before the guards below so that a record whose subscription + # has already gone still costs its budget: the cap has to bound the + # activation whatever the bookkeeping says. + # + # Replayed records are deliberately not counted, not merely not + # capped. Counting them would leave the budget spent for any live + # delivery later in the same activation, and would make the completion + # re-arm readiness on a purely replayed Workflow Task. + self._delivered_this_activation += 1 + if state is None or record.offset is None: + return + + state.delivery_cursor = AFTER(record.offset) + state.ready_records += 1 + state.fence_reached = record.is_control + + if self._replay_ready is not None: + # Replay re-delivers records the marker already recorded. Advancing + # the cursors is right -- the runtime has to end up in the state the + # live run was in -- but accumulating them into a *new* annotation is + # not: Core would be asked to write a second marker for observations + # already in History, and the command would be matched against the + # very event it was read from. + return + + self._observed_this_activation = True + + # Extend the open run when this is another consecutive delivery from the + # same stream. A run is *maximal*, which is what makes a single-stream + # batch of 100,000 records cost one run rather than 100,000. + if self._runs and self._runs[-1].wait_id == wait_id: + previous = self._runs[-1] + positions = previous.control_positions + if record.is_control: + positions = (*positions, previous.count) + self._runs[-1] = Run( + wait_id=wait_id, + first_offset=previous.first_offset, + last_offset=record.offset, + count=previous.count + 1, + control_positions=positions, + ) + # Only the run that changed is re-measured. Extending one is cheap; + # re-encoding the whole segment once per record would not be. + self._run_sizes[-1] = encoded_run_size(self._runs[-1]) + self._max_run_bytes = max(self._max_run_bytes, self._run_sizes[-1]) + self._check_segment_recordable() + return + + self._runs.append( + Run( + wait_id=wait_id, + first_offset=record.offset, + last_offset=record.offset, + count=1, + control_positions=(0,) if record.is_control else (), + ) + ) + self._run_sizes.append(encoded_run_size(self._runs[-1])) + self._max_run_bytes = max(self._max_run_bytes, self._run_sizes[-1]) + self._check_segment_recordable() + + def unsubscribe(self, wait_id: int) -> None: + """Ends a wait, and tells the Worker to stop serving it. + + The subscription's state is **kept**, not removed. Two things are built + from it after a wait ends: the annotation header, whose binding replay + needs in order to know what the closed wait was reading -- without it a + replay of unchanged code fails as though the Workflow never created that + wait -- and the Continue-As-New continuation, whose cursor is what stops + a successor Run restarting that stream from the beginning. Closing + changes exactly one thing about the state: the wait can never be blocked + again. + """ + state = self._subscriptions.get(wait_id) + if state is None or state.closed: + return + state.closed = True + state.blocked = False + # Whatever was drained but never handed over is dropped rather than + # consumed -- that is what leaves the consumption cursor short of it, so a + # Continue-As-New successor receives it. It also stops being carry-over the + # next activation's budget has to pay for: those records are gone, and a + # count left standing would shrink every later activation's budget by the + # size of a ready list nobody can consume. + state.ready_records = 0 + self._manager.cancel_from_workflow_thread(self._run_id, wait_id) + + def note_blocked(self, wait_id: int, blocked: bool) -> None: + """Records whether Workflow code is waiting on this subscription.""" + state = self._subscriptions.get(wait_id) + if state is None or (state.closed and blocked): + # A closed wait may leave the blocked set but never re-enter it: the + # coroutine that was awaiting it is gone, so retaining a Workflow + # Task for it would be retaining for nobody. + return + if blocked and not state.blocked: + # Re-entering the blocked state is what the wait generation counts, + # and it is what makes a readiness notification for the previous + # block recognisable as stale. + state.generation += 1 + # Pushed to the manager immediately. The manager is what reports + # readiness to Core, and Core compares the generation it is given + # against the one this runtime put in the quiescent snapshot. A + # manager left at the generation it registered with reports every + # block after the first under a stale number, Core answers `Stale`, + # and the watcher -- whose prefetch cursor is already past the + # record -- never re-announces it. The Workflow then waits forever + # on an append that did arrive. + self._manager.note_wait_generation(self._run_id, wait_id, state.generation) + state.blocked = blocked + + # --- the observation delta (P10b) ---------------------------------------- + + def close_segment(self, reason: SegmentEndReason | None = None) -> None: + """Closes this activation's segment, whether or not it saw anything. + + An empty segment is meaningful: an activation that drained and found + nothing still ran one event-loop drain, and replay must reproduce that + drain or ``wait_condition`` predicates fire a different number of times. + """ + if not self._observed_this_activation and not self._segment_pending: + return + self._segment_pending = False + if not self._observed_this_activation and not self._subscriptions: + self._unobserved_segments += 1 + return + if reason is None: + reason = self._segment_end_reason() + accumulator = self._ensure_accumulator() + for _ in range(self._unobserved_segments): + self._pending_deltas.append( + accumulator.add_segment(Segment((), SegmentEndReason.NO_DATA_AVAILABLE)) + ) + self._segments_in_annotation += 1 + self._unobserved_segments = 0 + self._pending_deltas.append( + accumulator.add_segment(Segment(tuple(self._runs), reason)) + ) + self._runs = [] + self._run_sizes = [] + self._segments_in_annotation += 1 + + def _segment_end_reason(self) -> SegmentEndReason: + """Why this activation stopped delivering. + + Decided here rather than passed in, because the reason has to be right + whether the caller remembered it or not: it is durable, and a reader of + the annotation has nothing else to tell it why the activation ended. + + ``BATCH_LIMIT`` outranks the two availability reasons. Both of them + assert that nothing more was available -- ``FENCE_REACHED`` additionally + that the set is immediately parkable -- and both are simply false when the + runtime stopped with records still sitting in the buffer. Recording + ``NO_DATA_AVAILABLE`` for a budget cut-off would put a claim in History + that the stream ran dry when it did not. + + ``BUDGET_ROLLOVER`` outranks even that, because it says something the + others do not: the batch continues in the *following marker*. A rollover + recorded as ``BATCH_LIMIT`` would leave replay treating this segment as + the end of the consumption. + """ + if self.annotation_budget_exhausted: + # Ranked first: it is the only one of the four that says the batch + # continues in the *next marker*, which is what tells replay to + # reassemble across the rollover boundary rather than treat this + # segment as the end of the consumption. + return SegmentEndReason.BUDGET_ROLLOVER + if self.delivery_budget_exhausted(): + return SegmentEndReason.BATCH_LIMIT + if self._subscriptions and all( + s.fence_reached for s in self._subscriptions.values() + ): + return SegmentEndReason.FENCE_REACHED + return SegmentEndReason.NO_DATA_AVAILABLE + + def take_observation_delta(self) -> bytes | None: + """The bytes to put on this completion's `WorkflowStreamProgress`. + + Once an input subscription exists, even an empty activation belongs to + the shared input/output replay schedule. Workflows that never subscribe + still emit nothing, and replay does not rewrite its existing marker. + """ + if self._replay_ready is not None: + return None + self.close_segment() + if not self._pending_deltas: + return None + delta = b"".join(self._pending_deltas) + self._pending_deltas = [] + self._observed_this_activation = False + return delta + + def add_terminal(self) -> bytes: + """Encodes the blocked snapshot that closes this Workflow Task. + + Read from in-memory state alone: the boundary is not "wherever the + stream is now", it is where this task's deliveries stopped, which was + fixed the moment the last activation returned. Refreshing it against the + backend could name a position replay must not reproduce. + + Returns **no bytes at all** when this Workflow Task's annotation is + already closed and nothing has been accumulated since. Core can decide a + boundary for a task whose last completion had already ended one -- a + rollover deadline or a shutdown landing on a completion that left no + subscription blocked -- and answering with a second header and terminal + would append a whole second annotation to the one Core is about to + write. The marker would then decode as far as the first terminal and + fail on the frame after it. + """ + if self._annotation_closed and self._accumulator is None: + return b"" + accumulator = self._ensure_accumulator() + terminal = accumulator.add_terminal( + { + wait_id: state.delivery_cursor + for wait_id, state in sorted(self._subscriptions.items()) + } + ) + # Anything still pending goes out with the terminal. Usually there is + # nothing -- each activation flushes its own delta -- but a Workflow Task + # that observed *nothing* creates its accumulator right here, and the + # header rides that creation. Returning the terminal alone would produce + # an annotation that begins at a terminal frame and cannot be decoded. + delta = b"".join([*self._pending_deltas, terminal]) + + # The annotation ends at the terminal. A later activation on this Run -- + # the Continue-As-New's own, say -- opens a fresh annotation rather than + # appending past it: Core writes one marker per finalized annotation, and + # a segment recorded after the terminal could never be read back. + self.start_new_annotation() + self._annotation_closed = True + return delta + + @property + def annotation_started(self) -> bool: + """Whether Core is holding accumulated bytes for the current annotation. + + True from the moment the header is emitted, which is what makes it the + right question to ask before closing an annotation with its terminal: an + annotation nobody has begun has nothing to terminate, and asking for a + terminal anyway would create a header and a terminal for a Workflow Task + that never touched a stream. + """ + return self._accumulator is not None + + @property + def request_rollover(self) -> bool: + """Whether Core should be asked to end this Workflow Task's annotation. + + Two conditions, and the second is not redundant. The high-water mark is a + *fraction* of the budget, and it only becomes true once a frame that + crossed it has been emitted -- but an indivisible frame can be larger than + the fraction that was left, so waiting for the mark alone is not a bound. + The second condition is the runtime having stopped delivering because the + annotation could no longer afford to record another record: that one is + true in the same activation, before anything has overflowed, and it is + what makes "an annotation can never exceed the budget" (ADR-007) hold for + a frame of any size. + """ + if self.annotation_budget_exhausted: + return True + return self._accumulator is not None and self._accumulator.request_rollover + + def start_new_annotation(self) -> None: + """Begins a fresh annotation, as the next Workflow Task requires. + + Its header records the *current* cursors rather than the original start + cursors, so consumption continues uninterrupted across a rollover or any + other Workflow Task boundary. + """ + self._accumulator = None + self._annotation_closed = False + self._pending_deltas = [] + self._runs = [] + self._run_sizes = [] + self._max_run_bytes = 0 + self._segments_in_annotation = 0 + self._unobserved_segments = 0 + self._observed_this_activation = False + self._annotation_start = { + wait_id: state.delivery_cursor + for wait_id, state in self._subscriptions.items() + } + + def record_consumption(self, wait_id: int, record: StreamRecord) -> None: + """Notes that Workflow code has been handed this record. + + Distinct from delivery, which advances by whole drained batches. The + difference is exactly the records sitting in a subscription's buffer that + the Workflow never asked for -- which die with the Run, and which a + successor must therefore still receive. + + Not where the delivery budget is spent -- :meth:`record_delivery` is, + because the budget has to be reserved by the drain that moves a whole + batch into a ready list rather than charged one record at a time + afterwards. What this does spend is the *carry-over* the next + :meth:`begin_activation` starts its count from: a record consumed here is + one the next activation no longer has in hand. + """ + state = self._subscriptions.get(wait_id) + if state is None or record.offset is None: + return + state.consumption_cursor = AFTER(record.offset) + state.ready_records = max(0, state.ready_records - 1) + + def continuation(self) -> Continuation: + """Where each subscription had got to, for the successor Run. + + The **consumption** cursor, which is the only one of the four positions + a successor may resume from. + + Not prefetch, which is speculative. Not delivery: a batch is delivered + whole, but a Workflow that stops iterating part-way through has taken + only its prefix, and the buffer holding the difference dies with this + Run -- so a successor starting from delivery would step silently over + records nothing ever showed to Workflow code. Not the committed cursor + either: by the time this is read the terminal command is being built, + and the observation delta covering these deliveries commits on that same + path, so a continuation taken from the last *marker* would restart the + successor at a stale cursor and lose the final segment. + + Carried with each cursor is the whole binding it is a position in -- the + stream and configured provider identity -- taken from :meth:`_binding`, + the same place the annotation header takes it from. An offset means + nothing outside the store that produced it. + """ + bindings = { + wait_id: self._binding(state) + for wait_id, state in self._subscriptions.items() + } + return Continuation( + cursors={ + wait_id: state.consumption_cursor + for wait_id, state in self._subscriptions.items() + }, + stream_names={ + wait_id: binding.stream_key.stream_name + for wait_id, binding in bindings.items() + }, + provider_ids={ + wait_id: binding.provider_id for wait_id, binding in bindings.items() + }, + provider_format_versions={ + wait_id: binding.provider_format_version + for wait_id, binding in bindings.items() + }, + ) + + def output_continuation(self) -> OutputContinuation | None: + """Finished output topics that a successor Run must not reopen.""" + if not self._output_finished: + return None + backend = self._backend + if not isinstance(backend, OutputStreamBackend): + raise RuntimeError( + "finished output topics have no configured output provider" + ) + return OutputContinuation( + finished_topics=frozenset(self._output_finished), + provider_id=type(backend).provider_id, + provider_format_version=type(backend).provider_format_version, + ) + + def restored_start(self, wait_id: int, stream_name: str) -> Cursor: + """The start cursor for a subscription, from the predecessor Run if any. + + ``BEGINNING`` on a first execution -- the same field, filled the same + way, so replay reads an explicit boundary in either case. + + The whole binding is compared before the cursor is handed back. The + stream is Workflow code, so a change is nondeterminism; the provider is + Worker configuration, so a change is a storage failure. Both are raised + before the cursor reaches the manager. + """ + if self._continuation is None: + return BEGINNING + restored = self._continuation.cursors.get(wait_id) + if restored is None: + # A subscription the predecessor did not have. Safe: adding one on a + # path the chain has not reached yet is the supported change. + return BEGINNING + # Row four of the taxonomy, not integrity loss: the cursor is exactly + # what the predecessor committed, and it is the Workflow code that moved. + recorded = self._continuation.stream_names.get(wait_id, "") + if recorded and recorded != stream_name: + raise temporalio.workflow.NondeterminismError( + f"the predecessor Run recorded external stream wait {wait_id} " + f"on stream {recorded!r}, but this Run subscribes it to " + f"{stream_name!r}. A subscribe() call was inserted, removed, or " + "reordered, which renumbers every later wait; gate the change " + "behind workflow.patched() exactly as an inserted timer would be." + ) + self._verify_restored_provider(wait_id) + return restored + + def _verify_restored_provider(self, wait_id: int) -> None: + """Whether the configured backend is what produced the cursor. + + A deployment question rather than a Workflow one, and classified exactly + as :meth:`StreamSubscriptionManager._replay_backend` classifies it: the + Workflow is unchanged and neither store is damaged, so it is a storage + failure -- retried, and it clears when a Worker carrying the recorded + implementation picks the task up. + + Marker replay makes this check for every recorded range it reads. A + successor Run's *first* read happens before it has written a marker that + could make it, which is why the continuation carries the provider + identity at all. + + The backend contract reserves no format-version value, so zero is + compared exactly like every other version. + """ + assert self._continuation is not None + recorded_id = self._continuation.provider_ids[wait_id] + backend = self._backend + if backend is None: + raise RuntimeError( + "external streams are not configured on this Worker; pass " + "external_stream_backend=... to the Worker" + ) + declared_id = type(backend).provider_id + if declared_id != recorded_id: + raise StreamStorageError( + f"external stream wait {wait_id} was continued from a Run that " + f"read it through provider {recorded_id!r}, but the backend " + f"configured on this Worker declares {declared_id!r}. The " + "restored cursor is a position in the store that produced it; " + "configure the recorded provider." + ) + declared_version = type(backend).provider_format_version + recorded_version = self._continuation.provider_format_versions[wait_id] + if recorded_version != declared_version: + raise StreamStorageError( + f"external stream wait {wait_id} was continued from a Run that " + f"read it through provider {recorded_id!r} format version " + f"{recorded_version}, but the configured backend implements " + f"format version {declared_version}. " + "Resuming would interpret the restored cursor under a format it " + "was not written in." + ) + + def _ensure_accumulator(self) -> AnnotationAccumulator: + if self._accumulator is None: + self._accumulator = AnnotationAccumulator( + self._header(), max_bytes=self._max_annotation_bytes + ) + # The header has to ride the *first delta*. Core accumulates by byte + # append and never parses, so anything the runtime holds but does + # not emit simply never reaches the marker -- and a marker whose + # annotation starts at a segment frame cannot be decoded at all. + self._pending_deltas.append(self._accumulator.accumulated()) + else: + self._announce_late_subscriptions() + # Re-priced after any late binding went out, so what is held back is what + # is still owed rather than what was owed when the accumulator was made. + self._update_reserve() + return self._accumulator + + def _announce_late_subscriptions(self) -> None: + """Binds every subscription the emitted header does not already carry. + + ``register`` accepts a subscription at any activation of a retained + Workflow Task, and only the first of those activations gets to write the + header -- Core appends the deltas it is handed and never rewrites what it + already holds, so the header cannot be extended in place. A wait that + joined later is bound by its own frame instead. + + Called from :meth:`_ensure_accumulator`, which is on the path of both + things that can follow a registration: the segment that records the + wait's first run, and the terminal that records where it stopped. The + binding therefore always precedes both, and no wait can reach the marker + as a run or a terminal entry with no stream key, no backend, and no start + cursor -- which replay reports as a wait "this Workflow did not create" + even when the code is unchanged. + """ + assert self._accumulator is not None + late = { + wait_id: state + for wait_id, state in sorted(self._subscriptions.items()) + if not state.announced and not state.start_pending + } + if not late: + return + for state in late.values(): + state.announced = True + self._pending_deltas.append( + self._accumulator.add_bindings( + {wait_id: self._binding(state) for wait_id, state in late.items()} + ) + ) + + def _header(self) -> AnnotationHeader: + """One binding per subscription, each naming **its own** backend. + + Marks every subscription announced, which is also how the flag is reset + for the *next* annotation: a header is written from scratch whenever an + accumulator is created, so whatever exists when it is carries its + binding there rather than in a bindings frame. + + The provider identity is read from the backend this subscription is + actually registered against rather than from a single annotation-wide + one. The API lets every topic name a different backend, so a shared + label would be right for at most one wait and would send replay to the + wrong store for all the others. + """ + for state in self._subscriptions.values(): + if not state.start_pending: + state.announced = True + return AnnotationHeader( + streams={ + wait_id: self._binding(state) + for wait_id, state in sorted(self._subscriptions.items()) + if not state.start_pending + }, + ) + + def _header_preview(self) -> AnnotationHeader: + """The header a fresh annotation would carry, without claiming it. + + :meth:`_header` marks every subscription announced, which is right when + the header is actually being emitted and wrong when the question is only + what it would cost. + """ + return AnnotationHeader( + streams={ + wait_id: self._binding(state) + for wait_id, state in sorted(self._subscriptions.items()) + if not state.start_pending + }, + ) + + def _binding(self, state: _SubscriptionState) -> StreamBinding: + backend = self._backend + if backend is None: + raise RuntimeError( + "external streams are not configured on this Worker; pass " + "external_stream_backend=... to the Worker" + ) + return StreamBinding( + stream_key=state.stream_key, + start_cursor=self._annotation_start.get(state.wait_id, state.start_cursor), + provider_id=type(backend).provider_id, + provider_format_version=type(backend).provider_format_version, + ) + + # --- quiescence (P10a) ---------------------------------------------------- + + def quiescent_snapshot(self) -> list[QuiescentWait] | None: + """The **complete** set of waits Workflow code is blocked on. + + ``None`` when nothing is blocked, which is the signal not to ask for + retention at all. A *partial* set would be worse than none: it would let + one idle stream park a Workflow Task another stream is still driving. + """ + blocked = [s for s in self._subscriptions.values() if s.blocked] + if not blocked: + return None + # A set the delivery budget stopped is blocked but not quiescent: records + # are still sitting in the local buffer, and the only reason nobody is + # reading them is that this activation ran out of budget. Calling any of + # it immediately parkable would ask Core to park a Workflow Task whose + # data has already arrived at the Worker. + exhausted = self.delivery_budget_exhausted() + return [ + QuiescentWait( + wait_id=s.wait_id, + generation=s.generation, + immediately_parkable=s.fence_reached and not exhausted, + ) + for s in sorted(blocked, key=lambda s: s.wait_id) + ] + + def effective_idle_timeout(self) -> timedelta: + """The quiescent set's timeouts reduced to one value. + + The reduction is ``min``, applied in ``wait_id`` order over the blocked + set and nothing else, so the result is deterministic and reproduces on + replay. It has to reduce at all because the timeout is a property of the + *set*: subscriptions configured differently still share one timer. + """ + blocked = sorted( + (s for s in self._subscriptions.values() if s.blocked), + key=lambda s: s.wait_id, + ) + if not blocked: + return self._default_idle_timeout + reduced = blocked[0].idle_timeout + for state in blocked[1:]: + reduced = min(reduced, state.idle_timeout) + return reduced + + def listened_stream_keys(self) -> list[StreamKey]: + """The stream of every reader still open, in ``wait_id`` order. + + What the run's channel report is built from. A reader that has not + blocked yet is open: records appended before its first iteration have + to wake the run as much as later ones. A closed reader has left, so its + stream is not listened on even though its state is kept for replay and + for the Continue-As-New cursor. + """ + return [ + state.stream_key + for _, state in sorted(self._subscriptions.items()) + if not state.closed + ] + + # --- teardown ------------------------------------------------------------- + + def subscriptions(self) -> list[int]: + """Return the registered wait IDs in deterministic order.""" + return sorted(self._subscriptions) + + def blocked_snapshot(self) -> dict[int, Cursor]: + """Return each registered wait's latest delivery boundary.""" + return { + wait_id: state.delivery_cursor + for wait_id, state in sorted(self._subscriptions.items()) + } diff --git a/temporalio/contrib/external_workflow_streams/_wake.py b/temporalio/contrib/external_workflow_streams/_wake.py new file mode 100644 index 0000000000..e1b9ade227 --- /dev/null +++ b/temporalio/contrib/external_workflow_streams/_wake.py @@ -0,0 +1,670 @@ +"""The producer wake path (P14). + +The server-visible wakeup, used whenever no open Workflow Task can accept local +readiness. It carries no stream payload. It goes out as a notification on the +stream's channel when the server has channels, see :func:`send_wake` and +:func:`channel_for`, and otherwise as the reserved Signal described below, +which carries only enough identity for Core to decide whether it is a wake, a +stale claim, or someone else's chain. + +Two properties are load-bearing and neither is available from the public Signal +API, which is why this path does not reuse it: + +**Codec bypass.** The envelope goes out through a raw ``SignalWorkflowExecution`` +built with the protocol's own serialization rather than the user's +``DataConverter`` (ADR-025). Core is the component that must *read* this Signal, +and Core has no access to a user codec -- a codec that encrypts payloads would +make the envelope unreadable to the only reader that matters. Nothing user-owned +is in it, so bypassing the codec leaks nothing. + +**Stable request ID.** The Temporal ``request_id`` is derived deterministically +from the wake's identity, so a producer retrying after an ambiguous failure sends +the identical request and the server deduplicates it. The public Signal path +generates a fresh UUID per attempt, which would turn every retry into a second +wake. +""" + +from __future__ import annotations + +import enum +import hashlib +import time +import uuid +import weakref +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal, get_args +from urllib.parse import quote + +import temporalio.api.common.v1 +import temporalio.api.notification.v1 +import temporalio.api.workflowservice.v1 +import temporalio.common +import temporalio.service +from temporalio.bridge.proto.external_stream.external_stream_pb2 import WakeSignal +from temporalio.client._channel import ChannelAddress +from temporalio.contrib.external_workflow_streams._record import Offset + +if TYPE_CHECKING: + import temporalio.client + from temporalio.contrib.external_workflow_streams._backend import StreamKey + from temporalio.contrib.external_workflow_streams._producer import WorkflowChainKey + +__all__ = [ + "ChannelAddress", + "ChannelSupport", + "WakeRequest", + "WakeTransport", + "channel_for", + "channel_support", + "new_sender_identity", + "send_wake", + "send_wake_signal", + "server_has_channels", + "wake_request_id", +] + +WakeTransport = Literal["auto", "channel", "signal"] +"""How a wake reaches the server. + +``"channel"`` notifies the stream's channel with ``NotifyChannel``. Every +listener of the channel is woken, and the only record is the notification on +the scheduled event of the Workflow Task it causes. ``"signal"`` uses the +reserved Signal. ``"auto"`` tries the channel first and steps down to the +Signal on ``UNIMPLEMENTED``, remembering per client that the server lacks +channels. +""" + +CHANNEL_PREFIX = "external-stream" +"""The first segment of every channel this package names. + +Keeps these channels apart from any a Workflow subscribes to by hand under a +name that happens to look like a stream's. +""" + + +def channel_for(key: StreamKey) -> ChannelAddress: + """The channel a stream's writers notify and its readers listen on. + + One formatting for both sides, so a producer process and the consuming + Worker never disagree on the name. The namespace is left out because the + server scopes a channel to the namespace of the call. Every segment is + percent-encoded, so a Workflow ID or stream name containing ``/`` cannot + collide with another stream's segments. + + A stream key names a workflow chain, so the channel is linked to that + workflow: it is the stream's owner and the one reader of its input. A + server with the linked kind keeps the channel in the owner's state, where + a notification is one write and no subscription is needed. A server + without it ignores the owner and serves the independent channel of the + same name, which the reader then subscribes to by command. + """ + channel = "/".join( + quote(part, safe="") + for part in ( + CHANNEL_PREFIX, + key.workflow_id, + key.first_execution_run_id, + key.direction.value, + key.stream_name, + ) + ) + return ChannelAddress( + channel=channel, + execution=( + temporalio.common.Execution.workflow(key.workflow_id) + if key.workflow_id + else None + ), + ) + + +WAKE_SIGNAL_NAME = "__temporal_external_stream_wake" +"""Fixed, and versioned by the envelope rather than by the name. + +Distinct from every ``__temporal_workflow_stream_*`` name already reserved by +``temporalio.contrib.workflow_streams`` (ADR-001), which is a different feature +that coexists with this one. +""" + +WAKE_SIGNAL_MESSAGE_TYPE = "coresdk.external_stream.WakeSignal" +WAKE_SIGNAL_ENCODING = b"binary/protobuf" +WAKE_SIGNAL_ENVELOPE_VERSION = 1 + +UNPARKED_WAKE_GENERATION = 0 +"""No confirmed park is known; ask for a Workflow Task anyway (ADR-023). + +Not silence. The runtime rechecks every active subscription on wakeup regardless +of which stream the Signal named, so an unnecessary unparked wake costs at most +one empty Workflow Task -- while staying silent because no park was observed +would lose the record until something else happened to wake the Workflow. +""" + +_REQUEST_ID_NAMESPACE = uuid.UUID("6f1f8a1e-6f26-5f3f-9f2b-4b0a2c7d8e10") +"""A fixed namespace, so the derivation is stable across processes and versions. + +Deriving a UUID rather than passing the tuple through keeps the request ID +inside the length the server accepts while staying a pure function of its +inputs. +""" + + +def new_sender_identity(client_identity: str = "") -> str: + """One sender instance's identity for unparked wakes. + + Unique per *sender*, not per client identity. Two Workers in one process + share a ``Client`` and therefore one client identity, so deriving from that + alone gives their first unparked wakes byte-identical request IDs -- the + server deduplicates the second, no Workflow Task is created, and the Run + stalls. That is precisely the loss the sender identity exists to prevent. + + Drawn once and held for the sender's lifetime, which is what "fixed across + retries of that one attempt" requires: a value redrawn per attempt would + turn the shutdown sweep's in-grace retry into a second wake. + + Random, which is safe here: the wake Signal is sent from the Worker's own + event loop and never from Workflow code, so it is not replay-visible and + cannot affect determinism. The client identity is kept as a prefix so a + request ID stays traceable to a Worker in server-side logs. + """ + return f"{client_identity}#{uuid.uuid4()}" + + +@dataclass(frozen=True) +class WakeRequest: + """One wake attempt's complete identity. + + Everything the request ID is derived from lives here, so "the same attempt" + is a property of a value rather than of how carefully a caller re-passed six + arguments. + """ + + namespace: str + workflow_id: str + first_execution_run_id: str + stream_name: str + wait_id: int + park_generation: int + sender_identity: str = "" + wake_counter: int = 0 + position: bytes = b"" + """The store position the wake reports, opaque to the server.""" + position_counter: int = 0 + """The store's order for ``position``; 0 when the store cannot give one. + + Orders positions within one pending burst of wakes, so the task that takes + them sees the latest position. It is not a dedupe key: once a task has + received a source's wake, the server accepts the next one whatever its + counter, because only the receiver knows what it read. + """ + channel: str = "" + """The channel the stream's readers listen on, see :func:`channel_for`. + + Empty for a request composed by code that predates the channel, which the + transport then wakes by Signal. Not part of the request ID: the ID names + the wake, and the channel is only where it is delivered. + """ + channel_execution: temporalio.common.Execution | None = None + """The execution the channel is linked to, ``None`` for an independent channel. + + Set with ``channel`` for a workflow-owned stream, so the notification + addresses the owner's linked channel on a server that has the kind. + """ + + @property + def channel_address(self) -> ChannelAddress: + """The channel and its owner as one address.""" + return ChannelAddress(channel=self.channel, execution=self.channel_execution) + + @property + def is_unparked(self) -> bool: + """Whether this wake requests a task outside a confirmed park.""" + return self.park_generation == UNPARKED_WAKE_GENERATION + + def __post_init__(self) -> None: + """Validate the identity required for server-side deduplication.""" + if self.is_unparked and not self.sender_identity: + raise ValueError( + "an unparked wake needs a sender identity. Generation 0 carries " + "no attempt identity of its own, so without it two Workers " + "shutting down at different times would derive the same request " + "ID and the server would deduplicate the second wake away -- " + "turning a correct retry mechanism into silent loss." + ) + + +def wake_request_id(request: WakeRequest) -> str: + """The deterministic ``request_id`` for one wake attempt. + + A **parked** wake is identified by its generation: a generation is woken + once, so every producer that races to wake it derives the same ID and the + server keeps one. + + An **unparked** wake has no generation to identify it, so the derivation + additionally includes the sender's identity -- unique per sender *instance*, + see :py:func:`new_sender_identity` -- and a per-sender monotonic counter, + both held fixed across retries of that one attempt. Two senders' unparked + wakes must stay distinct even though neither knows about the other. + """ + parts = [ + request.namespace, + request.workflow_id, + request.first_execution_run_id, + request.stream_name, + str(request.wait_id), + str(request.park_generation), + ] + if request.is_unparked: + parts += [request.sender_identity, str(request.wake_counter)] + # Length-prefixed rather than delimiter-joined: a stream literally named + # "a\x00b" must not collide with the pair ("a", "b"). + material = b"".join( + len(encoded).to_bytes(4, "big") + encoded + for encoded in (part.encode("utf-8") for part in parts) + ) + return str(uuid.uuid5(_REQUEST_ID_NAMESPACE, hashlib.sha256(material).hexdigest())) + + +def wake_envelope(request: WakeRequest, *, producer_session_id: str = "") -> bytes: + """The Signal's single argument, serialized at the protocol level.""" + return WakeSignal( + envelope_version=WAKE_SIGNAL_ENVELOPE_VERSION, + stream_name=request.stream_name, + wait_id=request.wait_id, + park_generation=request.park_generation, + first_execution_run_id=request.first_execution_run_id, + producer_session_id=producer_session_id, + ).SerializeToString() + + +def build_signal_request( + request: WakeRequest, + *, + identity: str, + producer_session_id: str = "", +) -> temporalio.api.workflowservice.v1.SignalWorkflowExecutionRequest: + """The raw request, built without touching the user's ``DataConverter``. + + Addressed to the Workflow ID with **no Run ID**, so it always lands on the + current Run of the chain -- a wake sent while a Continue-As-New is in flight + must reach the successor, not fail against a Run that has already closed. + ``first_execution_run_id`` inside the envelope is what lets Core tell the + chain apart from a reused Workflow ID. + """ + return temporalio.api.workflowservice.v1.SignalWorkflowExecutionRequest( + namespace=request.namespace, + workflow_execution=temporalio.api.common.v1.WorkflowExecution( + workflow_id=request.workflow_id + ), + signal_name=WAKE_SIGNAL_NAME, + input=temporalio.api.common.v1.Payloads( + payloads=[ + temporalio.api.common.v1.Payload( + metadata={ + "encoding": WAKE_SIGNAL_ENCODING, + "messageType": WAKE_SIGNAL_MESSAGE_TYPE.encode(), + }, + data=wake_envelope( + request, producer_session_id=producer_session_id + ), + ) + ] + ), + identity=identity, + request_id=wake_request_id(request), + ) + + +async def send_wake_signal( + client: temporalio.client.Client, + request: WakeRequest, + *, + producer_session_id: str = "", +) -> str: + """Sends one wake and returns the request ID it was sent under. + + Retrying with the same :class:`WakeRequest` is safe and is the intended + recovery from an ambiguous failure: the server deduplicates on the request + ID, so the second attempt resolves the first rather than waking twice. + """ + signal_request = build_signal_request( + request, + identity=client.service_client.config.identity, + producer_session_id=producer_session_id, + ) + await client.workflow_service.signal_workflow_execution(signal_request) + return signal_request.request_id + + +_TRANSPORT_ORDER: tuple[str, ...] = ("channel", "signal") +"""What ``"auto"`` tries, in order. + +Every server takes the Signal, so stepping down never loses a wake, only the +folding the channel gives and the History event the Signal costs. +""" + +_BEST_TRANSPORT: weakref.WeakKeyDictionary[Any, str] = weakref.WeakKeyDictionary() +"""Per service client, the first transport its server has not refused. + +Keyed weakly by the client's service client so the answer lives as long as the +connection it describes, without adding state to the public client. Absent +means nothing has been learned yet and ``"auto"`` starts at the top. +""" + + +def _service_key(client: temporalio.client.Client) -> Any: + return getattr(client, "service_client", None) + + +def _best_transport(client: temporalio.client.Client) -> str: + key = _service_key(client) + if key is None: + return _TRANSPORT_ORDER[0] + try: + return _BEST_TRANSPORT.get(key, _TRANSPORT_ORDER[0]) + except TypeError: + return _TRANSPORT_ORDER[0] + + +def _step_down(client: temporalio.client.Client, transport: str) -> str: + """Remembers that ``transport`` is beyond this client's server. + + Returns the transport to try next. Only ever moves forward: a refusal of a + later transport may already be on record, and going back would retry a + call known to fail. + """ + following = _TRANSPORT_ORDER[_TRANSPORT_ORDER.index(transport) + 1] + key = _service_key(client) + if key is not None: + try: + known = _TRANSPORT_ORDER.index(_best_transport(client)) + if known < _TRANSPORT_ORDER.index(following): + _BEST_TRANSPORT[key] = following + except TypeError: + pass + return following + + +PROBE_CHANNEL = f"{CHANNEL_PREFIX}/probe" +"""A channel nobody writes to, described to learn whether the server has any.""" + + +class ChannelSupport(enum.Enum): + """What a server offers a stream's readers, as :func:`channel_support` finds it.""" + + NONE = "none" + """No notification channels: readers are woken by the Signal.""" + + INDEPENDENT = "independent" + """Channels, each its own execution: readers subscribe to them by command.""" + + LINKED = "linked" + """Channels linked to a workflow as well: a workflow-owned stream's reader is + the channel's listener by construction, and only the other streams need + the command.""" + + +async def _describe_probe( + client: temporalio.client.Client, + workflow_id: str | None, +) -> temporalio.api.workflowservice.v1.DescribeChannelResponse | None: + """Describes the probe channel, or returns ``None`` for a server without channels. + + A refusal is remembered the way a refused notify is, so the writers on + the same client skip the call as well. Any other failure propagates for + the caller to decide what it means. + """ + if _best_transport(client) != "channel": + return None + service = getattr(client, "workflow_service", None) + call = getattr(service, "describe_channel", None) + if call is None: + _step_down(client, "channel") + return None + request = temporalio.api.workflowservice.v1.DescribeChannelRequest( + namespace=client.namespace, channel=PROBE_CHANNEL + ) + if workflow_id: + request.execution.CopyFrom( + temporalio.common.Execution.workflow(workflow_id).to_proto() + ) + try: + return await call(request) + except temporalio.service.RPCError as err: + if err.status == temporalio.service.RPCStatusCode.UNIMPLEMENTED: + _step_down(client, "channel") + return None + raise + + +async def server_has_channels(client: temporalio.client.Client) -> bool | None: + """Whether this client's server implements notification channels. + + Asked with a ``DescribeChannel`` of a channel nobody writes to, because the + one reader-side decision that depends on the answer is a command: a + Workflow Task that subscribes to a channel on a server without them fails, + and nothing in the wake path would ever tell the reader. + + Returns ``None`` when the server could not be asked, so the caller can ask + again later rather than settle on an answer the server never gave. + """ + try: + return await _describe_probe(client, None) is not None + except temporalio.service.RPCError as err: + if err.status == temporalio.service.RPCStatusCode.NOT_FOUND: + # The server knows the call and has never seen this channel. + return True + return None + + +async def channel_support( + client: temporalio.client.Client, workflow_id: str +) -> ChannelSupport | None: + """What this client's server offers the readers of ``workflow_id``'s streams. + + Asked with a ``DescribeChannel`` of the probe channel linked to that + workflow, which has to be running: a linked channel of a running workflow + exists by construction, so a server with the kind describes it, with no + listeners and nothing retained, where a server with only independent + channels ignores the owner and answers ``NOT_FOUND`` for a name nobody has + notified, and a server without channels answers ``UNIMPLEMENTED``. + + The caller records the answer on the run's first task as a lang flag, so + a replay takes the same path whatever server replays it. Returns ``None`` + when the server could not be asked. + """ + try: + response = await _describe_probe(client, workflow_id) + except temporalio.service.RPCError as err: + if err.status == temporalio.service.RPCStatusCode.NOT_FOUND: + return ChannelSupport.INDEPENDENT + return None + if response is None: + return ChannelSupport.NONE + if response.kind == temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_LINKED: + return ChannelSupport.LINKED + return ChannelSupport.INDEPENDENT + + +def wake_transport_of(backend: object) -> WakeTransport: + """The transport a backend asks for, defaulting for objects that predate it.""" + return getattr(backend, "wake_transport", "auto") + + +def wake_position(backend: object, offset: Offset | None) -> tuple[bytes, int]: + """The position and counter a wake reports for ``offset`` in ``backend``. + + Without an offset the backend's own clock rule gives the counter, so a + wake that reports no position still orders above every position the store + handed out before now. A counter the backend cannot give is 0, which + :func:`wake_call_counter` replaces with the clock. + """ + if offset is None: + counter_now = getattr(backend, "wake_counter_now", None) + return b"", max(counter_now(), 0) if counter_now is not None else 0 + counter_for = getattr(backend, "wake_counter_for", None) + counter = counter_for(offset) if counter_for is not None else 0 + return offset.token.encode("utf-8"), max(counter, 0) + + +def wake_call_counter(request: WakeRequest) -> int: + """The counter the notification carries for ``request``. + + The server folds a notification into one still pending for the same + listener when its counter is not above the pending one, so the counter + decides which position the receiving task sees. The store's own order when + it has one. Otherwise the wall clock in nanoseconds, which keeps one + sender's wakes increasing but is not ordered across senders: a pending + notification can then end up reporting an older position than another + sender's. That costs latency only, because the task still reads from its + own cursor and the watcher reports whatever it finds. + """ + if request.position_counter > 0: + return request.position_counter + return time.time_ns() + + +def build_notify_request( + request: WakeRequest, + *, + identity: str, + metadata: Mapping[str, temporalio.api.common.v1.Payload] | None = None, +) -> temporalio.api.workflowservice.v1.NotifyChannelRequest: + """The ``NotifyChannel`` request for one wake. + + The server deduplicates a retried notification by ``request_id``, so the + Signal's derivation serves here unchanged: two producers waking one parked + generation send the identical request, and a sender retrying an attempt + that may have landed sends it again rather than waking twice. + + A channel with an owner is addressed to it, so a server with the linked + kind delivers to the owner's own channel; one without the kind ignores + the owner and serves the independent channel of the same name, which is + the one the reader subscribed to there. + """ + if not request.channel: + raise ValueError( + "a channel notification needs the stream's channel; compose the " + "request with wake_request_for() or set WakeRequest.channel" + ) + notify = temporalio.api.workflowservice.v1.NotifyChannelRequest( + namespace=request.namespace, + notification=temporalio.api.notification.v1.Notification( + channel=request.channel, + position=request.position, + counter=wake_call_counter(request), + metadata=dict(metadata or {}), + ), + identity=identity, + request_id=wake_request_id(request), + ) + if request.channel_execution is not None: + notify.execution.CopyFrom(request.channel_execution.to_proto()) + return notify + + +async def send_wake( + client: temporalio.client.Client, + request: WakeRequest, + *, + producer_session_id: str = "", + transport: WakeTransport = "auto", + metadata: Mapping[str, temporalio.api.common.v1.Payload] | None = None, +) -> str: + """Sends one wake over ``transport`` and returns the request ID it went under. + + ``"auto"`` starts at the best transport this client's server is known to + take and steps down on ``UNIMPLEMENTED``, remembering the step. A channel + with no listeners is not a reason to step down: the server retains the + notification and hands it to the next listener at registration, which is + what closes the gap between a reader's last look at the store and its + subscription taking effect. + + A ``NOT_FOUND`` from the channel propagates as the ``RPCError`` it is, + which is what a Signal to an ended Workflow raises as well, so callers keep + one meaning for "the chain is over". + + Args: + metadata: Payloads for the listeners, carried on the notification + only. The channel already names the stream, so most wakes send + none. + + Raises: + temporalio.service.RPCError: The server refused the wake. With + ``"channel"`` this includes ``UNIMPLEMENTED``, since that + transport was asked for explicitly. + """ + # Checked against the alias rather than a literal tuple so an untyped caller + # still gets a ValueError instead of a silent Signal. + if transport not in get_args(WakeTransport): + raise ValueError(f"unknown wake transport {transport!r}") + attempt = _best_transport(client) if transport == "auto" else transport + if attempt == "channel" and not request.channel and transport == "auto": + # Composed without its channel by a caller that predates it. Nothing + # about the server follows from that, so nothing is remembered. + attempt = "signal" + if attempt == "channel": + service = getattr(client, "workflow_service", None) + call = getattr(service, "notify_channel", None) + if call is None: + # An SDK build whose generated service lacks the call can no more + # send a notification than a server that lacks it can take one. + if transport != "auto": + raise RuntimeError( + "the channel transport was requested but this client has " + "no notify_channel call" + ) + _step_down(client, "channel") + else: + identity = client.service_client.config.identity + try: + await call( + build_notify_request(request, identity=identity, metadata=metadata) + ) + return wake_request_id(request) + except temporalio.service.RPCError as err: + if ( + transport != "auto" + or err.status != temporalio.service.RPCStatusCode.UNIMPLEMENTED + ): + raise + _step_down(client, "channel") + return await send_wake_signal( + client, request, producer_session_id=producer_session_id + ) + + +def wake_request_for( + workflow: WorkflowChainKey, + *, + stream_name: str, + wait_id: int, + park_generation: int | None, + sender_identity: str, + wake_counter: int = 0, + position: bytes = b"", + position_counter: int = 0, +) -> WakeRequest: + """Builds the request from a chain key and an observed generation. + + ``park_generation=None`` -- no confirmed park for this subscription -- + becomes an unparked wake rather than no wake at all. The channel is the + input stream's, which is the only direction a producer writes. + """ + address = channel_for(workflow.stream_key(stream_name)) + return WakeRequest( + namespace=workflow.namespace, + workflow_id=workflow.workflow_id, + first_execution_run_id=workflow.first_execution_run_id, + stream_name=stream_name, + wait_id=wait_id, + park_generation=( + UNPARKED_WAKE_GENERATION if park_generation is None else park_generation + ), + sender_identity=sender_identity, + wake_counter=wake_counter, + position=position, + position_counter=position_counter, + channel=address.channel, + channel_execution=address.execution, + ) diff --git a/temporalio/contrib/langgraph/_plugin.py b/temporalio/contrib/langgraph/_plugin.py index 03881ca2a3..1819c646d9 100644 --- a/temporalio/contrib/langgraph/_plugin.py +++ b/temporalio/contrib/langgraph/_plugin.py @@ -104,11 +104,13 @@ class LangGraphPlugin(SimplePlugin): inherited default of either form. streaming_topic: When set, ``langgraph.config.get_stream_writer()`` inside a node publishes to this topic on the workflow's - :class:`WorkflowStream`. The workflow must construct - ``WorkflowStream()`` in its ``@workflow.init`` (the plugin's + :class:`temporalio.contrib.workflow_streams.WorkflowStream`. The + workflow must construct ``WorkflowStream()`` in its + ``@workflow.init`` (the plugin's interceptor verifies this on workflow start). Nodes with ``execute_in='activity'`` publish through - :class:`WorkflowStreamClient` (signal); nodes with + :class:`temporalio.contrib.workflow_streams.WorkflowStreamClient` + (signal); nodes with ``execute_in='workflow'`` publish synchronously to the in-workflow stream (no signal). streaming_batch_interval: How often the activity-side stream diff --git a/temporalio/contrib/server_streams/__init__.py b/temporalio/contrib/server_streams/__init__.py new file mode 100644 index 0000000000..f6ac83556d --- /dev/null +++ b/temporalio/contrib/server_streams/__init__.py @@ -0,0 +1,460 @@ +"""Server-side streams behind the Workflow Streams API. + +This is the same surface as :mod:`temporalio.contrib.workflow_streams`, backed +by a Temporal-owned log instead of by Signals and Updates. An application swaps +the import and keeps its code: publishing from a Workflow is still a plain +call, publishing from an Activity is still a buffered handle, and a consumer +still subscribes by topic from an offset. An Activity's appends carry its own +id and attempt, so a retried Activity's repeat is deduplicated by the server +rather than written twice. + +What changes is underneath. A publish is a Workflow Command whose payload never +enters History, so History gets one fixed-size event per Workflow Task rather +than a Signal per batch. A consumer reads the log directly rather than +long-polling an Update, so there is no per-Workflow limit on how many can read +at once, and a closed Workflow stays readable until its stream's retention +expires. + +As in the shipped feature, a topic here is a label on a record in the +Workflow's one default stream, which a consumer filters on. The provider in +:mod:`temporalio.streams.providers.native` keeps one stream per topic instead; +the two do not share a log. + +Prototype support for AI-198. It needs a server built from that branch, and it +opens its own gRPC channel because sdk-core does not know the stream service +yet, which is also why it does not support TLS or API keys. +""" + +from __future__ import annotations + +import asyncio +import builtins +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Generic, TypeVar, overload + +from temporalio import activity, workflow +from temporalio.api.stream.v1 import StreamRecord, StreamRecordKind +from temporalio.client import Client, WorkflowExecutionDescription +from temporalio.client_stream import StreamClient, WorkflowStreamHandle, shared_client +from temporalio.common import RawValue +from temporalio.converter import PayloadCodec, PayloadConverter + +__all__ = [ + "RawPage", + "TopicHandle", + "WorkflowStream", + "WorkflowStreamClient", + "WorkflowStreamItem", + "WorkflowTopicHandle", +] + +T = TypeVar("T") + +DEFAULT_BATCH_INTERVAL = timedelta(milliseconds=50) + + +@dataclass +class RawPage: + """One read whose items are still the stored records. + + ``closed`` with ``next_offset >= head_offset`` is the end of the stream. + The bodies are as the server holds them, codec included, for a caller + that forwards records rather than using them. + """ + + items: list[WorkflowStreamItem[StreamRecord]] + next_offset: int + head_offset: int + closed: bool + + +@dataclass +class WorkflowStreamItem(Generic[T]): + """One item read from a workflow's stream. + + ``offset`` is where the item sits in the whole stream, so it is what a + consumer hands back to resume. A topic filter leaves gaps in it. + """ + + topic: str + data: T + offset: int = 0 + + +def _record(converter: PayloadConverter, topic: str, value: Any) -> StreamRecord: + """The record one published value becomes. + + The body is the value's payload, so the encoding metadata the consumer + needs to decode into a type travels with it and a + :class:`temporalio.common.RawValue` passes through pre-encoded. + """ + record = StreamRecord(topic=topic, kind=StreamRecordKind.STREAM_RECORD_KIND_DATA) + record.body.CopyFrom(converter.to_payloads([value])[0]) + return record + + +def _decode( + converter: PayloadConverter, record: StreamRecord, as_type: type | None +) -> Any: + if as_type is None: + return converter.from_payloads([record.body])[0] + return converter.from_payloads([record.body], [as_type])[0] + + +class WorkflowTopicHandle(Generic[T]): + """A topic on the stream the running Workflow owns.""" + + def __init__(self, topic: str, value_type: type[T]) -> None: + """Prefer :meth:`WorkflowStream.topic`.""" + self._name = topic + self._type = value_type + + @property + def name(self) -> str: + """The topic name this handle is bound to.""" + return self._name + + @property + def type(self) -> type[T]: + """The value type this handle is bound to.""" + return self._type + + def publish(self, value: T | RawValue) -> None: + """Append ``value`` to the Workflow's stream on this topic. + + Returns at once. There is nothing to await: the Workflow Task's + publishes become one command the server applies in the task's own + commit, so it costs this Workflow no round trip and no extra + transition. The Worker's payload codec applies to the body as it does + to any other payload the Workflow sends. + """ + workflow._append_stream_records( + [_record(workflow.payload_converter(), self._name, value)] + ) + + +class WorkflowStream: + """The stream the running Workflow owns, from inside it. + + Construct in ``@workflow.init``. Unlike the Signals-and-Updates + implementation this holds no state of its own: the log lives on the server, + so there is nothing here for replay to reconstruct. + """ + + def __init__(self, prior_state: Any = None) -> None: + """Take the same argument as the Workflow Streams version and ignore it. + + That version carried the log across a continue-as-new, because the log + was Workflow state. Here it is not, so there is nothing to carry. + """ + self._prior_state = prior_state + + @overload + def topic(self, name: str) -> WorkflowTopicHandle[Any]: ... + + @overload + def topic(self, name: str, *, type: type[T]) -> WorkflowTopicHandle[T]: ... + + def topic(self, name: str, *, type: type = object) -> WorkflowTopicHandle[Any]: + """Bind a topic on this Workflow's stream.""" + return WorkflowTopicHandle(name, type) + + +class TopicHandle(Generic[T]): + """A topic on a Workflow's stream, from outside that Workflow.""" + + def __init__( + self, client: WorkflowStreamClient, topic: str, value_type: type[T] + ) -> None: + """Prefer :meth:`WorkflowStreamClient.topic`.""" + self._client = client + self._name = topic + self._type = value_type + + @property + def name(self) -> str: + """The topic name this handle is bound to.""" + return self._name + + @property + def type(self) -> type[T]: + """The value type this handle is bound to.""" + return self._type + + def publish(self, value: T | RawValue, *, force_flush: bool = False) -> None: + """Buffer ``value`` for the next flush. + + Buffered rather than sent, because an append costs one transition on + the owning execution whatever its size. A token at a time would pay + that per token. + """ + self._client._buffer(self._name, value) + if force_flush: + self._client._flush_soon() + + def subscribe( + self, + *, + from_offset: int = 0, + # Spelled out because `type` in this class body is the property below. + result_type: builtins.type | None = None, + poll_cooldown: timedelta | None = None, + ) -> AsyncIterator[WorkflowStreamItem[T]]: + """Read this topic from ``from_offset`` onwards.""" + return self._client.subscribe( + topics=[self._name], + from_offset=from_offset, + result_type=result_type or self._type, + poll_cooldown=poll_cooldown, + ) + + +class WorkflowStreamClient: + """Publishes to and reads from a Workflow's stream, from outside it.""" + + def __init__( + self, + handle: WorkflowStreamHandle, + converter: PayloadConverter, + batch_interval: timedelta = DEFAULT_BATCH_INTERVAL, + *, + codec: PayloadCodec | None = None, + describe: Callable[[], Awaitable[WorkflowExecutionDescription]] | None = None, + producer_id: str = "", + ) -> None: + """Prefer :meth:`create` or :meth:`from_within_activity`. + + ``codec`` is applied to every body this client sends and receives, so + a namespace whose payloads are encoded agrees with the Worker, whose + payload visitor applies the same codec to the Workflow's publishes. + ``describe`` is how an unpinned handle learns which run it follows; + see :meth:`WorkflowStreamHandle.pin`. ``producer_id`` is who the + appends are written as; without one they are at-least-once, because + the server has nothing to deduplicate a retry against. + """ + self._handle = handle + self._converter = converter + self._codec = codec + self._batch_interval = batch_interval + self._describe = describe + self._producer_id = producer_id + self._sequence = 0 + self._pending: tuple[list[StreamRecord], int] | None = None + self._buffered: list[tuple[str, Any]] = [] + self._flusher: asyncio.Task[None] | None = None + self._wake = asyncio.Event() + self._closed = False + + @classmethod + def create( + cls, + client: Client, + workflow_id: str, + *, + owner_run_id: str = "", + batch_interval: timedelta = DEFAULT_BATCH_INTERVAL, + producer_id: str = "", + ) -> WorkflowStreamClient: + """Open the stream owned by ``workflow_id``. + + Without ``owner_run_id`` the current run is looked up on the first + call and the handle pinned to it, so a reader following across a + continue-as-new sees the run end rather than being moved to the + successor's stream at a stale offset. Without ``producer_id`` the + appends are at-least-once; inside an Activity, + :meth:`from_within_activity` supplies one. + """ + return cls( + _stream_client(client).workflow_stream( + workflow_id, owner_run_id=owner_run_id + ), + client.data_converter.payload_converter, + batch_interval, + codec=client.data_converter.payload_codec, + describe=client.get_workflow_handle(workflow_id).describe, + producer_id=producer_id, + ) + + @classmethod + def from_within_activity( + cls, *, batch_interval: timedelta = DEFAULT_BATCH_INTERVAL + ) -> WorkflowStreamClient: + """Open the stream owned by the Workflow that scheduled this Activity.""" + info = activity.info() + if info.workflow_id is None: + raise RuntimeError( + "no Workflow stream to open: this Activity was not started by a " + "Workflow" + ) + # The Activity's output belongs to the run that scheduled it, and the + # Activity already knows which run that is. Its id and attempt are + # also what lets the server drop a batch a retried Activity re-sends. + return cls.create( + activity.client(), + info.workflow_id, + owner_run_id=info.workflow_run_id or "", + batch_interval=batch_interval, + producer_id=f"{info.activity_id}#{info.attempt}", + ) + + async def __aenter__(self) -> WorkflowStreamClient: + """Start the background flusher.""" + self._flusher = asyncio.create_task(self._run_flusher()) + return self + + async def __aexit__(self, *_exc: object) -> None: + """Drain what is buffered before letting the caller go. + + An Activity that returned with a batch still buffered would have + reported work its readers never saw. The flusher is asked to stop + rather than cancelled: a cancel landing inside its append would + unwind with the batch it had already taken off the buffer. + """ + self._closed = True + self._wake.set() + if self._flusher is not None: + await self._flusher + self._flusher = None + await self.flush() + + async def _pin(self) -> None: + if self._handle.owner_run_id or self._describe is None: + return + self._handle.pin((await self._describe()).run_id) + + @overload + def topic(self, name: str) -> TopicHandle[Any]: ... + + @overload + def topic(self, name: str, *, type: type[T]) -> TopicHandle[T]: ... + + def topic(self, name: str, *, type: type = object) -> TopicHandle[Any]: + """Bind a topic on this Workflow's stream.""" + return TopicHandle(self, name, type) + + async def get_offset(self) -> int: + """Where the stream currently ends. + + A reader that wants only what comes next starts here. + """ + await self._pin() + return (await self._handle.describe()).head_offset + + async def subscribe( + self, + *, + topics: Sequence[str] = (), + from_offset: int = 0, + result_type: type | None = None, + poll_cooldown: timedelta | None = None, + ) -> AsyncIterator[WorkflowStreamItem[Any]]: + """Yield items from ``from_offset`` as they arrive. + + ``poll_cooldown`` is accepted and ignored. It paced a client that had + to re-ask; the server parks this read until something arrives. + """ + del poll_cooldown + await self._pin() + async for entry in self._handle.follow(from_offset=from_offset, topics=topics): + record = await self._decoded(entry.record) + yield WorkflowStreamItem( + topic=record.topic, + data=_decode(self._converter, record, result_type), + offset=entry.offset, + ) + + async def poll_raw( + self, + *, + topics: Sequence[str] = (), + from_offset: int = 0, + wait: bool = True, + ) -> RawPage: + """One read, with the records left as they were stored. + + For a caller that forwards items on rather than using them. A gateway + would only have to encode again what this decoded. + """ + await self._pin() + page = await self._handle.poll( + from_offset=from_offset, topics=topics, wait=wait + ) + return RawPage( + items=[ + WorkflowStreamItem( + topic=entry.record.topic, data=entry.record, offset=entry.offset + ) + for entry in page.entries + ], + next_offset=page.next_offset, + head_offset=page.head_offset, + closed=page.closed, + ) + + async def flush(self) -> None: + """Append everything buffered as one batch. + + A batch whose append failed stays pending and goes out again on the + next flush under the sequence it already had, so an append the server + did accept is deduplicated and one it never saw still lands. Nothing + comes off the buffer until there is a batch to replace it with. + + Raises: + temporalio.streams.StreamProducerError: The server holds this + producer's sequence with different content. + """ + if self._pending is not None: + records, sequence = self._pending + else: + if not self._buffered: + return + await self._pin() + # Encoded before the buffer is cleared, so a converter failure + # leaves the values where the caller can still see them. + records = [ + await self._encoded(_record(self._converter, topic, value)) + for topic, value in self._buffered + ] + sequence = self._sequence + self._buffered = [] + self._pending = (records, sequence) + await self._pin() + await self._handle.append( + *records, producer_id=self._producer_id, sequence=sequence + ) + self._sequence = sequence + len(records) + self._pending = None + + def _buffer(self, topic: str, value: Any) -> None: + self._buffered.append((topic, value)) + + def _flush_soon(self) -> None: + self._wake.set() + + async def _run_flusher(self) -> None: + """Append on a fixed cadence, so a slow producer still gets delivered.""" + while not self._closed: + try: + await asyncio.wait_for( + self._wake.wait(), self._batch_interval.total_seconds() + ) + except asyncio.TimeoutError: + pass + self._wake.clear() + await self.flush() + + async def _encoded(self, record: StreamRecord) -> StreamRecord: + if self._codec is not None and record.HasField("body"): + record.body.CopyFrom((await self._codec.encode([record.body]))[0]) + return record + + async def _decoded(self, record: StreamRecord) -> StreamRecord: + if self._codec is not None and record.HasField("body"): + record.body.CopyFrom((await self._codec.decode([record.body]))[0]) + return record + + +def _stream_client(client: Client) -> StreamClient: + return shared_client(client) diff --git a/temporalio/contrib/workflow_streams/__init__.py b/temporalio/contrib/workflow_streams/__init__.py index 41f670f0c0..c44bd25ead 100644 --- a/temporalio/contrib/workflow_streams/__init__.py +++ b/temporalio/contrib/workflow_streams/__init__.py @@ -13,12 +13,18 @@ """ from temporalio.contrib.workflow_streams._client import WorkflowStreamClient -from temporalio.contrib.workflow_streams._stream import WorkflowStream +from temporalio.contrib.workflow_streams._stream import ( + POLL_UPDATE_NAME, + PUBLISH_SIGNAL_NAME, + WorkflowStream, +) from temporalio.contrib.workflow_streams._topic_handle import ( TopicHandle, WorkflowTopicHandle, ) from temporalio.contrib.workflow_streams._types import ( + STREAM_DRAINING_ERROR_TYPE, + TRUNCATED_OFFSET_ERROR_TYPE, PollInput, PollResult, PublishEntry, @@ -29,6 +35,10 @@ ) __all__ = [ + "POLL_UPDATE_NAME", + "PUBLISH_SIGNAL_NAME", + "STREAM_DRAINING_ERROR_TYPE", + "TRUNCATED_OFFSET_ERROR_TYPE", "PollInput", "PollResult", "PublishEntry", diff --git a/temporalio/contrib/workflow_streams/_client.py b/temporalio/contrib/workflow_streams/_client.py index 605bf3f035..5c458bdab1 100644 --- a/temporalio/contrib/workflow_streams/_client.py +++ b/temporalio/contrib/workflow_streams/_client.py @@ -325,6 +325,20 @@ def topic( self._topic_types[name] = bound return TopicHandle(self, name, bound) + @property + def handle(self) -> WorkflowHandle[Any, Any]: + """The workflow handle this client publishes to and polls. + + Re-targeted when :py:meth:`subscribe` follows a continue-as-new, so + read it when needed rather than caching it. + """ + return self._handle + + @property + def payload_converter(self) -> PayloadConverter: + """The sync payload converter used for per-item encode and decode.""" + return self._payload_converter() + async def flush(self) -> None: """Flush buffered (and pending) items and wait for server confirmation. diff --git a/temporalio/contrib/workflow_streams/_stream.py b/temporalio/contrib/workflow_streams/_stream.py index ae8608c3b9..e1ebda15b8 100644 --- a/temporalio/contrib/workflow_streams/_stream.py +++ b/temporalio/contrib/workflow_streams/_stream.py @@ -50,8 +50,22 @@ _WorkflowStreamWireItem, ) -_PUBLISH_SIGNAL = "__temporal_workflow_stream_publish" -_POLL_UPDATE = "__temporal_workflow_stream_poll" +PUBLISH_SIGNAL_NAME = "__temporal_workflow_stream_publish" +"""The signal :class:`WorkflowStream` registers for external publishes. + +Public so code that sends the signal itself, with its own publisher identity, +does not have to copy the name. +""" + +POLL_UPDATE_NAME = "__temporal_workflow_stream_poll" +"""The update :class:`WorkflowStream` registers for long polls. + +Public so code that drives the poll itself, with its own retry and +cancellation rules, does not have to copy the name. +""" + +_PUBLISH_SIGNAL = PUBLISH_SIGNAL_NAME +_POLL_UPDATE = POLL_UPDATE_NAME _OFFSET_QUERY = "__temporal_workflow_stream_offset" _MAX_POLL_RESPONSE_BYTES = 1_000_000 @@ -234,6 +248,25 @@ def topic( self._topic_types[name] = bound return WorkflowTopicHandle(self, name, bound) + @property + def next_offset(self) -> int: + """The global offset the next published item will receive.""" + return self._base_offset + len(self._log) + + def items_from(self, offset: int) -> list[tuple[int, str, Payload]]: + """Return ``(offset, topic, payload)`` for every item at or past ``offset``. + + Reads the log in place, so it is safe to call from a + :func:`temporalio.workflow.wait_condition` predicate. An ``offset`` + below the truncation base starts at the base instead; the offsets in + the result say where the items actually sit. + """ + start = max(offset, self._base_offset) - self._base_offset + return [ + (self._base_offset + index, item.topic, item.data) + for index, item in enumerate(self._log[start:], start) + ] + def get_state( self, *, publisher_ttl: timedelta = timedelta(seconds=900) ) -> WorkflowStreamState: diff --git a/temporalio/nexus/_link_conversion.py b/temporalio/nexus/_link_conversion.py index e3ef3988ba..138618b2ef 100644 --- a/temporalio/nexus/_link_conversion.py +++ b/temporalio/nexus/_link_conversion.py @@ -120,6 +120,9 @@ def temporal_link_to_nexus_link( case "batch_job": raise NotImplementedError("batch_job links are not supported") + case "callback": + raise NotImplementedError("callback links are not supported") + case None: logger.warning("Invalid Temporal link: missing variant") return None diff --git a/temporalio/nexus/system/workflow_service/models.py b/temporalio/nexus/system/workflow_service/models.py index 1e81199db2..232bc91ca6 100644 --- a/temporalio/nexus/system/workflow_service/models.py +++ b/temporalio/nexus/system/workflow_service/models.py @@ -239,6 +239,11 @@ def to_transfer_type( @dataclasses.dataclass(slots=True) class UserMetadata: + """ + .. warning:: + This API is experimental and subject to change. + """ + static_summary: typing.Any | None = None static_details: typing.Any | None = None diff --git a/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py b/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py index 177666ae94..2698b110af 100644 --- a/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py +++ b/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py @@ -575,7 +575,7 @@ async def signal_with_start_workflow( args: List-form arguments for workflow. Cannot be set if positional_args are set. For typed workflow callables, list contents are not statically typechecked; pass workflow arguments positionally for precise typechecking. - id: Unique identifier for the workflow execution. + id: Unique identifier for the workflow execution. Must be nonempty. task_queue: Task queue to run the workflow on. signal: Signal name or callable to send with the start request. signal_args: Argument value, or list of argument values, for signal. For typed @@ -584,14 +584,15 @@ async def signal_with_start_workflow( pass a single signal argument that is itself a list, wrap it in another list; otherwise the list is interpreted as multiple signal arguments. execution_timeout: Total workflow execution timeout, including retries and - continue-as-new. - run_timeout: Timeout of a single workflow run. - task_timeout: Timeout of a single workflow task. + continue-as-new. Defaults to unlimited. + run_timeout: Timeout of a single workflow run. Defaults to the workflow + execution timeout. + task_timeout: Timeout of a single workflow task. Defaults to 10 seconds. id_reuse_policy: Behavior when a closed workflow with the same ID exists. Default is allow-duplicate. id_conflict_policy: Behavior when a workflow is currently running with the same ID. Set to use-existing for idempotent deduplication on workflow ID. Cannot - be set if id-reuse-policy is terminate-if-running. + be set if id-reuse-policy is terminate-if-running. Defaults to use-existing. retry_policy: Retry policy for the workflow. cron_schedule: Cron schedule for recurring workflow executions. See https://docs.temporal.io/cron-job. diff --git a/temporalio/streams/__init__.py b/temporalio/streams/__init__.py new file mode 100644 index 0000000000..8b1d35f1c7 --- /dev/null +++ b/temporalio/streams/__init__.py @@ -0,0 +1,165 @@ +"""Streams: a channel a workflow reads, decides on, and writes. + +.. warning:: + This module is experimental and may change in future versions. The + design is meant to be the shape that goes GA; the label is the SDK's + release convention, not a licence to break it. + +The contract, in five statements: + +1. **A workflow publishes only to topics of its own stream, and it publishes + transactionally.** ``temporalio.workflow.StreamWriter.publish`` + returns at once. The record is visible when the Workflow Task is accepted, + and never if the task fails, so no reader can see a decision the workflow + did not commit. +2. **Reading is an observation, and the SDK records it.** What + ``temporalio.workflow.StreamReader`` handed to workflow code, + including the boundary where it found nothing, is committed with the + commands that reading produced. Recovery re-supplies the same records in + the same order. +3. **Anything that does I/O publishes on its own account.** An activity or an + outside process writes through a :class:`StreamProducer` with a producer + id, an attempt and a sequence, and its records are visible as soon as the + store accepts them. Those three let a reader tell a retry from a new + generation. A retry that carries the same content is written once; one that + carries different content at the same sequence is refused with + :class:`StreamProducerError`, so a divergent retry is never dropped in + silence. +4. **A cursor is opaque and belongs to its provider.** Hand it back to resume + strictly after the record it names; :meth:`StreamHandle.latest` positions a + follower. Do not compare two cursors or do arithmetic on one. A read with + no cursor yet starts at :data:`BEGINNING`, at :data:`END`, or at the last + ``N`` records with ``last=N``. +5. **A workflow addresses its streams relative to itself, by topic.** A topic + can be written by the workflow and by outside producers, and read by the + workflow and by outside consumers; which of those happen is the + application's business. A topic is defined once with :func:`topic`, with + the type its records decode to, and that definition is shared by the + workflow, its activities and the backend; a plain string names a topic + decided at runtime. A call that names no topic addresses the workflow's + default topic, :data:`DEFAULT_TOPIC`. + +A provider is an object, registered once as a plugin: +``Client.connect(plugins=[provider])``; workers built from that client inherit +it, and ``Worker(plugins=[provider])`` or ``Replayer(plugins=[provider])`` +registers it on a worker alone. Each context then asks for its stream the +same way. Workflow code uses :func:`temporalio.workflow.stream_reader` and +:func:`temporalio.workflow.stream_writer`. An activity uses +:func:`temporalio.activity.stream_handle`: an activity a workflow scheduled +reaches that workflow's stream pinned to its run, a standalone activity +reaches its own, and ``scope="activity"`` gives the first kind its own +streams too. Any process holding a client uses +:meth:`temporalio.client.Client.get_stream_handle`, which mirrors +``get_workflow_handle`` and takes an ``activity_id`` for an activity's +streams. The explicit form, +``provider.get_stream_handle(client, workflow_id)``, stays for a process that +talks to two stores. This module keeps the shared types, the errors and the +protocols a provider implements; nothing here that workflow code imports does +I/O. + +A handle is bound to its client and provider, so a stream is handed to another +process as a :class:`StreamRef`: the owner and the topic as plain data, with +no cursor and no provider name. :meth:`StreamHandle.ref` makes one, the +default data converter carries it as JSON, and the receiver opens it with +``client.get_stream_handle(ref)`` or ``activity.stream_handle(ref)`` on +whatever provider its client has. + +A stream can also stand alone, with an id of its own and no owner. +``client.create_stream(stream_id, retention=...)`` creates it with a retention +policy and returns its handle, ``client.get_stream_handle(stream_id=...)`` +reaches an existing one, and the handle's ``close()`` seals it, after which +appends are refused with :class:`StreamClosedError` and the retained records +stay readable. A provider whose store cannot hold an ownerless stream raises +:class:`StreamUnsupportedError` for both. + +What the contract does not promise: that a :attr:`RecordKind.FINISH` record +means the writing activity succeeded, that a superseded attempt's records can +be withdrawn, or that a stream outlives the retention its provider is +configured for. Reading somebody else's stream is out of scope for this +release. + +The record on the wire is ``temporal.api.stream.v1.StreamRecord`` on every +provider, with the user's value in ``body`` as an ordinary payload, so a +reader in any language decodes the same bytes and a payload codec applies. A +provider owes that body what the SDK gives every payload it sends: it encodes +it through the client's data converter, so the codec and the +:class:`temporalio.converter.ExternalStorage` drivers apply, it takes the +retry fingerprint over the converted bytes before either runs and leaves the +plaintext hash on the record under :data:`CONTENT_HASH_KEY`, and it offloads a +workflow's own publish off the workflow thread. :func:`encode_body`, +:func:`decode_body` and :func:`content_fingerprint` are the shared code for +that; :class:`StreamProvider` states the rule. +""" + +from __future__ import annotations + +from temporalio.streams._body import ( + CONTENT_HASH_KEY, + content_fingerprint, + content_hash, + decode_body, + encode_body, +) +from temporalio.streams._errors import ( + StreamClosedError, + StreamCursorError, + StreamError, + StreamNotFoundError, + StreamProducerError, + StreamUnsupportedError, +) +from temporalio.streams._provider import ( + ReadSource, + StreamHandle, + StreamProducer, + StreamProvider, + WorkflowStreamProvider, + WriteSink, +) +from temporalio.streams._record import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamRecord, + Supersession, +) +from temporalio.streams._ref import StreamOwnerKind, StreamRef +from temporalio.streams._topic import ( + DEFAULT_TOPIC, + StreamTopic, + resolve_topic, + topic, +) + +__all__ = [ + "BEGINNING", + "CONTENT_HASH_KEY", + "DEFAULT_TOPIC", + "END", + "Cursor", + "ReadSource", + "RecordKind", + "StreamClosedError", + "StreamCursorError", + "StreamError", + "StreamHandle", + "StreamNotFoundError", + "StreamOwnerKind", + "StreamProducer", + "StreamProducerError", + "StreamProvider", + "StreamRecord", + "StreamRef", + "StreamTopic", + "StreamUnsupportedError", + "Supersession", + "WorkflowStreamProvider", + "WriteSink", + "content_fingerprint", + "content_hash", + "decode_body", + "encode_body", + "resolve_topic", + "topic", +] diff --git a/temporalio/streams/_body.py b/temporalio/streams/_body.py new file mode 100644 index 0000000000..2f10b6d4a9 --- /dev/null +++ b/temporalio/streams/_body.py @@ -0,0 +1,116 @@ +"""What a provider owes a record's body between the converter and its store. + +:func:`temporalio.streams._wire.to_wire` converts a value into the body with +the payload converter and stops there. Every other payload the SDK sends then +passes through the payload codec and external storage, and a stream body owes +the same, or a codec-protected deployment would leak plaintext through its +streams and a claim-check deployment would push oversized bodies at its store. +A provider runs the body through :func:`encode_body` before it stores or ships +a record and through :func:`decode_body` after it reads one back, off the +workflow thread in both directions. + +The order inside :func:`encode_body` is the point. The plaintext hash is taken +first and stamped on the record, and :func:`content_fingerprint` is taken over +the converted records too, before the codec runs, because a codec that +encrypts with a fresh nonce makes every retry's bytes differ, and a store that +fingerprinted those bytes would refuse the retry as a divergent write. The +store keeps the plaintext hash under :data:`CONTENT_HASH_KEY` and can compare +retries by it without ever seeing the plaintext. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Sequence + +from temporalio.api.common.v1 import Payload +from temporalio.api.stream.v1 import StreamRecord as WireRecord +from temporalio.converter import DataConverter + +__all__ = [ + "CONTENT_HASH_KEY", + "content_fingerprint", + "content_hash", + "decode_body", + "encode_body", +] + +CONTENT_HASH_KEY = "temporal.io/content-hash" +"""The record metadata key the plaintext hash of the body is stored under. + +Its value is a payload with ``encoding`` ``binary/plain`` whose data is the +hex digest :func:`content_hash` returns. A ``FINISH`` record carries no body +and no hash. +""" + +_HASH_ENCODING = b"binary/plain" + + +def content_hash(payload: Payload) -> str: + """The hex SHA-256 of ``payload`` as the converter produced it. + + Taken over the deterministic serialization of the whole payload, metadata + included, so two payloads that differ only in their encoding hash apart. + """ + return hashlib.sha256(payload.SerializeToString(deterministic=True)).hexdigest() + + +def content_fingerprint(records: Sequence[WireRecord]) -> bytes: + """The identity of one append, taken over its converted records. + + Length-delimited, so a batch split differently cannot collide with this + one. Take it before :func:`encode_body`, while the bodies are still what + the converter produced; that is what makes a retry through a + nondeterministic codec match its original. + """ + digest = hashlib.sha256() + for record in records: + body = record.SerializeToString(deterministic=True) + digest.update(len(body).to_bytes(8, "big")) + digest.update(body) + return digest.digest() + + +async def encode_body(converter: DataConverter, record: WireRecord) -> WireRecord: + """Stamp the plaintext hash on ``record`` and encode its body for the store. + + In place, and returned for convenience. The hash goes under + :data:`CONTENT_HASH_KEY` first; then the body passes through + ``converter``'s payload codec and external storage in the order + :meth:`temporalio.converter.DataConverter.encode` uses, so a body above + the external storage threshold is replaced by a claim and the claim is + what the store holds. A record without a body is returned untouched. + """ + if not record.HasField("body"): + return record + record.metadata[CONTENT_HASH_KEY].CopyFrom( + Payload( + metadata={"encoding": _HASH_ENCODING}, + data=content_hash(record.body).encode(), + ) + ) + encoded = await converter._encode_payload_sequence([record.body]) + stored = await converter._external_store_payload_sequence(encoded) + record.body.CopyFrom(stored[0]) + return record + + +async def decode_body(converter: DataConverter, record: WireRecord) -> WireRecord: + """Undo :func:`encode_body` on a record read back from the store. + + In place, and returned for convenience. The body is retrieved from + external storage when it is a claim and then run through the payload + codec, in the order :meth:`temporalio.converter.DataConverter.decode` + uses, leaving the payload the converter can turn back into a value. The + hash stays on the record. + + Raises: + RuntimeError: The body is a claim and ``converter`` has no external + storage to redeem it with. + """ + if not record.HasField("body"): + return record + retrieved = await converter._external_retrieve_payload_sequence([record.body]) + decoded = await converter._decode_payload_sequence(retrieved) + record.body.CopyFrom(decoded[0]) + return record diff --git a/temporalio/streams/_errors.py b/temporalio/streams/_errors.py new file mode 100644 index 0000000000..a2f4fecf5d --- /dev/null +++ b/temporalio/streams/_errors.py @@ -0,0 +1,48 @@ +"""The errors a stream call raises. + +Every stream condition is a :class:`StreamError`, so a caller can catch by +meaning the way it catches other :class:`temporalio.exceptions.TemporalError` +subclasses. Argument mistakes stay ``ValueError``. A provider's transport +failure surfaces as :class:`temporalio.service.RPCError`, never as the +transport's own exception type. +""" + +from __future__ import annotations + +import temporalio.exceptions + +__all__ = [ + "StreamClosedError", + "StreamCursorError", + "StreamError", + "StreamNotFoundError", + "StreamProducerError", + "StreamUnsupportedError", +] + + +class StreamError(temporalio.exceptions.TemporalError): + """Base for stream conditions.""" + + +class StreamNotFoundError(StreamError): + """The workflow, chain or topic does not exist or is past retention.""" + + +class StreamCursorError(StreamError): + """The cursor was minted by another provider or names a record no longer retained.""" + + +class StreamProducerError(StreamError): + """The producer attempt or sequence conflicts with what the store holds.""" + + +class StreamClosedError(StreamError): + """The standalone stream was sealed, so it takes no more records. + + Its retained records stay readable; only appends are refused. + """ + + +class StreamUnsupportedError(StreamError): + """This provider does not offer the requested capability.""" diff --git a/temporalio/streams/_ids.py b/temporalio/streams/_ids.py new file mode 100644 index 0000000000..2a76dcf196 --- /dev/null +++ b/temporalio/streams/_ids.py @@ -0,0 +1,26 @@ +"""The store key a provider derives from a workflow id and a topic. + +A workflow id may contain any character, ``:`` included, so joining the pair +with a bare ``:`` is ambiguous: ``("a:b", "c")`` and ``("a", "b:c")`` would +land in one store. Every provider that keys a store by the pair goes through +:func:`topic_key`, so they all agree and none of them collides. +""" + +from __future__ import annotations + +__all__ = ["topic_key"] + + +def _escape(component: str) -> str: + # Percent first, so an escaped component cannot be mistaken for one that + # already contained the escape. + return component.replace("%", "%25").replace(":", "%3A") + + +def topic_key(workflow_id: str, topic: str) -> str: + """The store key for ``topic`` of ``workflow_id``'s stream. + + Both components are percent-encoded before joining, so the only bare ``:`` + in the result is the separator. + """ + return f"{_escape(workflow_id)}:{_escape(topic)}" diff --git a/temporalio/streams/_policy.py b/temporalio/streams/_policy.py new file mode 100644 index 0000000000..b27c839e50 --- /dev/null +++ b/temporalio/streams/_policy.py @@ -0,0 +1,79 @@ +"""Turning a producer's newer attempt into something a reader can act on. + +An activity that streams half an answer and then fails leaves those records in +the stream. Its retry calls the model again and writes different words. No +provider can undo the first half, and a workflow that already acted on it has +committed that decision, so the honest thing is to tell the reader that a new +attempt began and let the application decide. + +This runs in the reader over records it already observed, so it costs no round +trip and replays without the provider being involved. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from temporalio.streams._record import ( + Cursor, + RecordKind, + StreamRecord, + Supersession, +) + +__all__ = ["AttemptTracker"] + + +class AttemptTracker: + """Watches producer attempts on one subscription.""" + + def __init__(self, warn: Callable[[str], None] | None = None) -> None: + """Start with no producer seen, saying anything odd through ``warn``.""" + self._attempts: dict[str, int] = {} + self._warn = warn + + def note( + self, producer_id: str, attempt: int, *, topic: str, previous: Cursor + ) -> StreamRecord[Any] | None: + """A supersession record when this record starts a newer attempt. + + ``previous`` is the cursor of the last record delivered before the one + being noted, or the cursor the read started from. The synthesized + record carries it, so a consumer that checkpoints the supersession and + resumes after it is handed the new attempt's first record next rather + than skipping it. + + A producer that declares no attempt supersedes nothing, because there + is no generation to compare. That is the same answer as an unnumbered + record: the interface reports what it was told and invents nothing. + + An attempt that goes backwards supersedes nothing either, and is said + rather than passed off as ordinary data: attempts only ever rise on one + producer, so a lower one means a store reordered two generations, and a + consumer reading it as the current answer would render a stale one. + """ + if not producer_id or attempt <= 0: + return None + seen = self._attempts.get(producer_id, 0) + if attempt < seen and self._warn is not None: + self._warn( + f"stream record on {topic!r} at {previous} is from attempt {attempt} of " + f"producer {producer_id!r}, behind attempt {seen}, which this reader has " + "already delivered: the store handed back two generations out of order" + ) + if attempt <= seen: + return None + self._attempts[producer_id] = attempt + if seen == 0: + return None + return StreamRecord( + kind=RecordKind.SUPERSEDED, + cursor=previous, + topic=topic, + producer_id=producer_id, + attempt=attempt, + supersession=Supersession( + producer_id=producer_id, previous_attempt=seen, attempt=attempt + ), + ) diff --git a/temporalio/streams/_provider.py b/temporalio/streams/_provider.py new file mode 100644 index 0000000000..7ef3a6b6fb --- /dev/null +++ b/temporalio/streams/_provider.py @@ -0,0 +1,478 @@ +"""What a provider implements, in two halves. + +:class:`WorkflowStreamProvider` runs on the workflow thread and must keep the +contract's first two rules: publishes commit with the Workflow Task, and reads +are recorded observations. Nothing it needs may do I/O. :class:`StreamProvider` +is the half a process holds: it makes the workflow half for a worker and hands +out :class:`StreamHandle` objects to code outside a workflow. A Python provider +usually implements both on one class; the split is what lets a language whose +workflow code is bundled separately name the two halves in two packages. + +A provider only moves ``temporal.api.stream.v1.StreamRecord`` protos. The +handles around it convert values, synthesize supersession and mint cursors, +and turn a :class:`temporalio.streams.StreamTopic` into the plain name the +provider sees, through :func:`temporalio.streams.resolve_topic`. What it owes +a record's body on the way to and from its store, :class:`StreamProvider` +lists and :func:`temporalio.streams.encode_body` and +:func:`temporalio.streams.decode_body` do. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Protocol, TypeVar, overload + +from temporalio.api.stream.v1 import StreamRecord as WireRecord +from temporalio.streams._record import BEGINNING, Cursor, StreamRecord +from temporalio.streams._topic import StreamTopic + +if TYPE_CHECKING: + from temporalio.client import Client + from temporalio.streams._ref import StreamRef + +__all__ = [ + "ReadSource", + "StreamHandle", + "StreamProducer", + "StreamProvider", + "WorkflowStreamProvider", + "WriteSink", +] + +T = TypeVar("T") +T_contra = TypeVar("T_contra", contravariant=True) + + +class StreamProducer(Protocol[T_contra]): + """Appends to one topic from outside workflow code. + + Every append is visible as soon as the store accepts it, and carries the + producer id, attempt and sequence that let a reader tell a retried append + from a new generation. The type parameter is the topic definition's + value type; a producer on a string-named topic takes any value. + """ + + @property + def producer_id(self) -> str: + """Who this producer writes as.""" + ... + + @property + def attempt(self) -> int: + """The generation this producer is writing, or 0 when undeclared.""" + ... + + async def append(self, *values: T_contra) -> Cursor | None: + """Append ``values`` and return the cursor of the last record as the store holds it. + + A repeat of an earlier append (same producer, attempt and sequence) + carrying the same content is written once and returns the position the + original landed at. A repeat carrying different content is a conflict, + not a retry: it raises :class:`StreamProducerError` and writes nothing, + because the store cannot tell which of the two the reader was meant to + see. A provider that cannot compare content says so in its own + documentation rather than picking one silently. + + An empty call writes nothing and returns the same value a repeat would: + the position of this producer's last record, or ``BEGINNING`` when it + has written none. ``None`` means one thing only: this provider learns + positions at read time, and a caller that needs one positions itself + with :meth:`StreamHandle.latest`. + + Raises: + StreamProducerError: The attempt or sequence conflicts with what + the store holds. + """ + ... + + async def finish(self) -> None: + """Write ``FINISH`` for this producer on this topic. + + Says this producer has nothing more to send. It does not say the + activity behind it succeeded, and it does not end anyone's read. + """ + ... + + +class StreamHandle(Protocol): + """One owner's stream, addressed by topic, from outside workflow code. + + The owner is a workflow, an activity, or a standalone stream that has an + id of its own and no owner. A handle on a workflow follows its execution + chain unless it was opened with a ``run_id``, in which case it is pinned + to that run. A topic is a :class:`temporalio.streams.StreamTopic` + definition, which carries the record type, or a plain string with + ``result_type=`` for a name decided at runtime. A transport failure + surfaces as :class:`temporalio.service.RPCError`, never as the + transport's own exception type. + + A handle is bound to its client and provider. To hand a stream to another + process, :meth:`ref` names it as a :class:`temporalio.streams.StreamRef`, + which is plain data; the receiver opens it with + ``temporalio.client.Client.get_stream_handle`` or + ``temporalio.activity.stream_handle``, and calls that name no topic + on that handle address the ref's topic. + """ + + @overload + def read( + self, + *, + topic: StreamTopic[T], + after: Cursor = ..., + last: int | None = None, + ) -> AsyncGenerator[StreamRecord[T], None]: ... + + @overload + def read( + self, + *, + topic: str | None = None, + after: Cursor = ..., + last: int | None = None, + result_type: type[T], + ) -> AsyncGenerator[StreamRecord[T], None]: ... + + @overload + def read( + self, + *, + topic: str | None = None, + after: Cursor = ..., + last: int | None = None, + result_type: None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: ... + + def read( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor = BEGINNING, + last: int | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield the records on ``topic`` after ``after`` as they arrive. + + Without ``topic`` it reads :data:`temporalio.streams.DEFAULT_TOPIC`. + ``BEGINNING`` yields everything the topic retains, starting at the + oldest record it still holds. ``END`` yields only what is appended + after the read starts. ``last=N`` starts at the newest ``N`` records, + or at all of them when there are fewer; it counts records of every + kind, so a ``FINISH`` among them leaves fewer than ``N`` values, and + it is exclusive with a cursor. Any other cursor came from a record a + reader saw, and reading resumes just past it, so a reader that stores + the last cursor it handled and hands it back sees every record + exactly once; that is the only way to resume. The read ends when the owning + execution, or its chain, is closed and every retained record after + ``after`` has been delivered; until then it waits. The result is a + generator, so a caller that stops early should ``aclose()`` it. How + much that releases is the provider's to say: one that holds only + local state lets go at once, and one that parked something on a + store it cannot un-park says in its own documentation what it + releases and when. Read the provider's ``read`` before relying on an + immediate release. + + Raises: + ValueError: ``result_type`` was passed with a topic definition, + the topic is empty, ``last`` is not positive, or ``last`` was + passed with a cursor. + StreamCursorError: ``after`` came from another provider or names + a record no longer retained. Raised by this call, not by the + first iteration. + StreamUnsupportedError: The provider cannot start a read where + ``END`` or ``last=`` asks. A provider that raises it says so + in its own documentation. + StreamNotFoundError: The workflow or topic does not exist or is + past retention. + """ + ... + + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: + """The cursor of the newest record on ``topic``, or ``BEGINNING`` when empty. + + Without ``topic`` it answers for :data:`temporalio.streams.DEFAULT_TOPIC`. + + For a reader that wants to follow from now: ``read(after=latest())`` + yields only what is published after this call returned, which is how + a client that is about to send a message positions itself before + sending, without the workflow having to report a position. + """ + ... + + @overload + def producer( + self, *, topic: StreamTopic[T], producer_id: str = ..., attempt: int = ... + ) -> StreamProducer[T]: ... + + @overload + def producer( + self, *, topic: str | None = None, producer_id: str = ..., attempt: int = ... + ) -> StreamProducer[Any]: ... + + def producer( + self, + *, + topic: str | StreamTopic[Any] | None = None, + producer_id: str = "", + attempt: int = 0, + ) -> StreamProducer[Any]: + """A producer on ``topic``, or on the default topic without one. + + Inside an activity, leave ``producer_id`` and ``attempt`` unset: the + activity's own id and attempt are the right answer, and they are what + let a reader tell a retry from a new generation. Outside one, + ``producer_id`` is required and an empty one raises ``ValueError``. + """ + ... + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A :class:`temporalio.streams.StreamRef` to ``topic`` of this owner. + + Without ``topic`` it names :data:`temporalio.streams.DEFAULT_TOPIC`, + or the topic this handle was opened from a ref with. The ref carries + the owner exactly as this handle addresses it, a ``run_id`` included + when the handle is pinned, and no cursor or provider name, so it can + travel as a workflow argument, an activity result or a Nexus + operation input or result and be opened wherever a client is. + """ + ... + + async def close(self) -> None: + """Seal the standalone stream this handle is on. + + A sealed stream takes no more records: a later ``append`` raises + :class:`temporalio.streams.StreamClosedError`, while everything it + retains stays readable and a read on it ends once that tail has been + delivered. Idempotent. Only a standalone stream can be closed here, + because an owned stream ends with its owner. + + Raises: + ValueError: This handle is on a workflow's or an activity's + stream. + """ + ... + + +class ReadSource(Protocol): + """One subscription, as a provider supplies it to the workflow thread.""" + + async def next_batch(self) -> list[tuple[Cursor, WireRecord]]: + """The next records with their positions, waiting until there is at least one. + + A batch rather than a record because delivery boundaries are what a + provider actually records, and flattening them here keeps that out of + the contract. A record that cannot be parsed into a ``StreamRecord`` + proto is the provider's to skip. + + Raises: + StopAsyncIteration: This subscription has ended. + """ + ... + + def close(self) -> None: + """End the subscription. Idempotent.""" + ... + + +class WriteSink(Protocol): + """One topic of the running workflow's stream, as a provider binds it.""" + + def publish(self, record: WireRecord) -> None: + """Take one record into this Workflow Task's output. + + Synchronous: there is nothing to wait for inside a task, because the + task is the visibility boundary. The provider commits what it buffered + when the task completes and drops it when the task fails. A record + the provider cannot stage raises :class:`temporalio.streams.StreamError` + and fails the task, loudly. + """ + ... + + +class WorkflowStreamProvider(Protocol): + """The half of a provider that runs on the workflow thread. + + Imports nothing that does I/O. The worker creates one per workflow + instance through :meth:`StreamProvider.workflow_provider`, so state kept + here dies with the instance the way handlers do. It sees topics by name; + the definitions are resolved before it is called. + """ + + def open_reader( + self, topic: str, *, after: Cursor, last: int | None = None + ) -> ReadSource: + """Subscribe the running workflow to ``topic`` of its own stream. + + ``after`` and ``last`` mean what they mean on + :meth:`StreamHandle.read`, and arrive already checked. Where a start + is resolved has to be something replay reproduces, so a provider + resolves it in the store and records the result, never by reading + the store from the workflow thread. + + Raises: + StreamCursorError: ``after`` was minted by another provider. + StreamUnsupportedError: The provider cannot start where ``END`` + or ``last`` asks. + """ + ... + + def open_writer(self, topic: str) -> WriteSink: + """Bind ``topic`` of the running workflow's stream for publishing.""" + ... + + def on_workflow_start(self) -> None: + """Called before the workflow function runs, and before the first task's handlers. + + After the workflow's own ``__init__`` and before any Signal or Update + of the first task is handled, which the SDK does ahead of the + workflow function. A provider that serves outside readers through + handlers on the workflow registers them here, so an Update that + arrives with the first task finds them. + """ + ... + + async def on_workflow_finish(self) -> None: + """Called after the workflow function returns, raises or continues as new. + + A provider that parked an outside reader against the run lets go + here, so the workflow can close. + """ + ... + + +class StreamProvider(Protocol): + """What a store ships. Also a :class:`temporalio.worker.Plugin` when it serves workers. + + Construct one, pass it to ``Client.connect(plugins=[provider])`` so the + client and the workers built from it carry it, or to + ``Worker(plugins=[provider])`` and ``Replayer(plugins=[provider])`` for a + worker alone, and open handles from it anywhere else. Nothing is global: + two workers in one process may hold two providers. + + **What a provider owes a record's body.** The handles convert a value + into the body with the payload converter and no more; what the SDK does + to every other payload it sends, the codec and external storage, the + provider owes the body too, through the client's data converter, so the + :class:`temporalio.converter.ExternalStorage` drivers an application + configured apply to stream bodies as well. It does that in one order. + First it takes the retry fingerprint, the identity a repeated append is + matched by, over the converted bytes, before the codec and before any + offload, so a codec that encrypts with a fresh nonce cannot turn a retry + into a divergent write; the plaintext hash also rides the record under + :data:`temporalio.streams.CONTENT_HASH_KEY`, where the store can read it. + Then it encodes the body and offloads it, and on a read it does the + reverse before the record reaches a reader. A workflow's own publish is + converted on the workflow thread and no further: the codec and the offload + run when the provider commits the task's batch, off that thread. + :func:`temporalio.streams.encode_body`, + :func:`temporalio.streams.decode_body` and + :func:`temporalio.streams.content_fingerprint` are that rule in code. + + **Standalone streams.** A stream can have an id of its own and no owner. + It is created on purpose, with :meth:`create_standalone_stream` and a + retention policy, and sealed on purpose, with the handle's ``close``. It + is addressed by topic like an owner's streams; how a provider lays its + topics out in the store is its own. A provider whose store cannot hold a + stream without an owner raises + :class:`temporalio.streams.StreamUnsupportedError` from both standalone + calls. + """ + + def workflow_provider(self) -> WorkflowStreamProvider: + """The half that serves one workflow instance on its thread.""" + ... + + def get_stream_handle( + self, client: Client, workflow_id: str, *, run_id: str | None = None + ) -> StreamHandle: + """A handle on ``workflow_id``'s stream. + + Without ``run_id`` it follows the execution chain, so a consumer keeps + reading across continue-as-new; with one it is pinned to that run. + """ + ... + + def get_activity_stream_handle( + self, + client: Client, + activity_id: str, + *, + workflow_id: str | None = None, + run_id: str | None = None, + ) -> StreamHandle: + """A handle on the streams an activity owns. + + Without ``workflow_id`` the activity is a standalone one, an execution + of its own, and ``run_id`` pins one run of it. With ``workflow_id`` it + is an activity that workflow scheduled, and ``run_id`` pins the + workflow's run. Either way these streams are apart from any + workflow's: a topic here and the same topic on the workflow's handle + are two streams. A retry of the activity writes to the same streams, + and a read ends when the activity reaches a terminal status, not when + an attempt fails. + + Raises: + StreamUnsupportedError: The provider's store cannot hold a stream + an activity owns. + """ + ... + + async def create_standalone_stream( + self, + client: Client, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> StreamHandle: + """Create the standalone stream ``stream_id`` and return a handle on it. + + The three policy arguments bound what the stream retains: records + older than ``retention``, beyond the newest ``max_records``, or past + ``max_bytes`` of stored records are dropped, and ``None`` leaves that + bound to the provider's default. Creating a stream that exists with + the same policy returns a handle on it, so a retried create is + harmless. + + Raises: + ValueError: ``stream_id`` is empty, a bound is not positive, or + the stream exists with a different policy. + StreamUnsupportedError: The provider's store cannot hold a stream + without an owner. + """ + ... + + def get_standalone_stream_handle( + self, client: Client, stream_id: str + ) -> StreamHandle: + """A handle on the standalone stream ``stream_id``, which must exist. + + Nothing here creates the stream: the first ``read``, ``latest`` or + ``producer`` on a stream that does not exist raises + :class:`temporalio.streams.StreamNotFoundError`, unless the provider + can wait for the stream to be created, in which case a ``read`` parks + until the first write and says so in its own documentation. + + Raises: + StreamUnsupportedError: The provider's store cannot hold a stream + without an owner. + """ + ... + + async def close(self) -> None: + """Release what this provider holds for the process. + + A provider that keeps a connection pool or an HTTP session open needs + a moment where the process says it is done; this is it. A provider + that holds nothing returns at once. + + The application calls this, not the worker and not the client. One + provider serves the workers built from a client and every handle + opened outside them, so no single one of those owns its lifetime and + a worker shutting down would close a connection its siblings are + still reading through. A provider that outlives the process it was + made in is the application's to close. + """ + ... diff --git a/temporalio/streams/_record.py b/temporalio/streams/_record.py new file mode 100644 index 0000000000..c31050ede1 --- /dev/null +++ b/temporalio/streams/_record.py @@ -0,0 +1,146 @@ +"""The value types the stream contract is expressed in. + +Nothing here touches Temporal or a provider, so every provider shares it +unchanged. +""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass +from typing import Generic, TypeVar + +__all__ = [ + "BEGINNING", + "END", + "Cursor", + "RecordKind", + "StreamRecord", + "Supersession", +] + +T = TypeVar("T") + + +@enum.unique +class RecordKind(enum.IntEnum): + """What a record is. + + Mirrors ``temporal.api.stream.v1.StreamRecordKind`` value for value, so a + record's kind crosses the wire as the integer the proto holds. + """ + + UNSPECIFIED = 0 + """The proto's zero value. + + A stored record whose writer set no kind is read as :attr:`DATA`, as the + proto defines it, so a reader never sees this kind on a record. + """ + + DATA = 1 + """Carries a value published by a workflow or a producer.""" + + FINISH = 2 + """The producer named in ``producer_id`` will write nothing more on this topic. + + An empty ``producer_id`` names the owning workflow. It does not end a + read, which ends when the owning execution or its chain is closed and the + retained tail has been delivered, and it says nothing about the producer's + outcome: an activity can still time out after writing it. + """ + + SUPERSEDED = 3 + """A later attempt of the same producer started writing. + + Synthesized by the reader from what it observed, never stored, so every + provider delivers it identically and replay reproduces it without the + provider's help. Its cursor is the position before the new attempt's + first record, so resuming after it delivers that record next. + """ + + +@dataclass(frozen=True) +class Cursor: + """A position in a stream, ordered by its provider rather than by value. + + Opaque on purpose. One provider numbers records with integers and another + with a millisecond-and-sequence pair, so comparing tokens here would be + right for one and wrong for the other. Hand a cursor back to resume after + the record it names; nothing here advances one. The token starts with the + name of the provider that minted it, and a provider refuses a token from + another with :class:`temporalio.streams.StreamCursorError`. + """ + + token: str + + def __str__(self) -> str: + """The provider's position token.""" + return self.token + + +BEGINNING = Cursor("") +"""Read from the oldest record the stream still retains.""" + +END = Cursor("$end") +"""Read only what is appended after the read starts. + +Provider-neutral, like :data:`BEGINNING`. It is resolved when the read +starts, not when it is called, so it cannot position a client before it +sends something; :meth:`temporalio.streams.StreamHandle.latest` does that. +""" + + +def check_read_start(after: Cursor, last: int | None) -> None: + """Refuse a read start that names two places, or a count that names none. + + ``after=`` resumes a read and ``last=`` starts one, so a call gives one or + the other. ``BEGINNING`` is the default for ``after=``, and passing it + alongside ``last=`` is the same as passing ``last=`` alone. + + Raises: + ValueError: ``last`` is not a positive int, or it was given together + with a cursor. + """ + if last is None: + return + if isinstance(last, bool) or not isinstance(last, int) or last <= 0: + raise ValueError(f"last must be a positive int, got {last!r}") + if after != BEGINNING: + raise ValueError( + "pass either after= or last=, not both: after= resumes a read and " + "last= starts one" + ) + + +@dataclass(frozen=True) +class Supersession: + """What a :attr:`RecordKind.SUPERSEDED` record reports.""" + + producer_id: str + previous_attempt: int + attempt: int + + +@dataclass(frozen=True) +class StreamRecord(Generic[T]): + """One record as a reader sees it. + + ``value`` is set on a :attr:`RecordKind.DATA` record and ``supersession`` + on a :attr:`RecordKind.SUPERSEDED` one; every other kind carries neither. + Each field means one thing, so a consumer narrows on ``kind`` and reads + the field that kind promises. + """ + + kind: RecordKind + cursor: Cursor + topic: str + producer_id: str = "" + """Who wrote it, or empty when the owning workflow wrote it itself.""" + attempt: int = 0 + """The producer's attempt, or 0 when it did not declare one.""" + sequence: int = 0 + """The producer's position within its attempt, or 0 when it does not number.""" + value: T | None = None + """The published value. Set on ``DATA`` only.""" + supersession: Supersession | None = None + """The attempt change being reported. Set on ``SUPERSEDED`` only.""" diff --git a/temporalio/streams/_ref.py b/temporalio/streams/_ref.py new file mode 100644 index 0000000000..b4dcd126c3 --- /dev/null +++ b/temporalio/streams/_ref.py @@ -0,0 +1,229 @@ +"""A reference to one stream that crosses a process boundary as data. + +A handle is bound to a client and a provider, so it cannot be a workflow +argument, an activity result or a Nexus operation result. A +:class:`StreamRef` can: it names the owner and the topic, nothing more, and +whoever receives it opens the stream on the provider its own client carries. +It carries no cursor, because a position belongs to a reader, and no provider +name, because the same owner and topic name the same stream on every provider +a deployment runs. +""" + +from __future__ import annotations + +import dataclasses +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +from temporalio.streams._record import BEGINNING, Cursor, StreamRecord +from temporalio.streams._topic import DEFAULT_TOPIC, StreamTopic, resolve_topic + +if TYPE_CHECKING: + from temporalio.client import Client + from temporalio.streams._provider import ( + StreamHandle, + StreamProducer, + StreamProvider, + ) + +__all__ = ["StreamOwnerKind", "StreamRef", "open_ref"] + +StreamOwnerKind = Literal["workflow", "activity", "standalone"] +"""What owns a stream: a workflow, an activity, or the stream itself.""" + + +@dataclass(frozen=True) +class StreamRef: + """One stream, named by its owner and its topic. + + ``kind`` says what owns the stream. A ``"workflow"`` ref carries + ``workflow_id`` and, when pinned to one run, ``run_id``. An ``"activity"`` + ref carries ``activity_id``, plus ``workflow_id`` (and its ``run_id``) + when a workflow scheduled the activity; without ``workflow_id`` it is a + standalone activity, and ``run_id`` then pins one run of it. A + ``"standalone"`` ref carries ``stream_id`` and nothing else. ``topic`` is + the topic's name, and a handle opened from a ref addresses it whenever a + call names no topic. + + A ref comes from :meth:`temporalio.streams.StreamHandle.ref`, or from + :meth:`for_workflow`, :meth:`for_activity` and :meth:`for_standalone` + when only the ids are at hand. The default data converter carries it as + JSON, so it can be a workflow argument, an activity result, or a Nexus + operation input or result, and + ``temporalio.client.Client.get_stream_handle`` and + ``temporalio.activity.stream_handle`` open one directly. + """ + + kind: StreamOwnerKind + topic: str = DEFAULT_TOPIC + workflow_id: str | None = None + run_id: str | None = None + activity_id: str | None = None + stream_id: str | None = None + + def __post_init__(self) -> None: + """Refuse a ref that names an owner its kind does not have.""" + if not self.topic: + raise ValueError("a StreamRef needs a topic name") + if self.kind == "workflow": + if not self.workflow_id: + raise ValueError("a workflow StreamRef needs a workflow_id") + if self.activity_id is not None or self.stream_id is not None: + raise ValueError( + "a workflow StreamRef carries no activity_id and no stream_id" + ) + elif self.kind == "activity": + if not self.activity_id: + raise ValueError("an activity StreamRef needs an activity_id") + if self.stream_id is not None: + raise ValueError("an activity StreamRef carries no stream_id") + elif self.kind == "standalone": + if not self.stream_id: + raise ValueError("a standalone StreamRef needs a stream_id") + if ( + self.workflow_id is not None + or self.run_id is not None + or self.activity_id is not None + ): + raise ValueError("a standalone StreamRef carries only its stream_id") + else: + raise ValueError( + f"unknown StreamRef kind {self.kind!r}; expected 'workflow', " + "'activity' or 'standalone'" + ) + + @classmethod + def for_workflow( + cls, + workflow_id: str, + *, + run_id: str | None = None, + topic: str | StreamTopic[Any] | None = None, + ) -> StreamRef: + """A ref to ``topic`` of ``workflow_id``'s stream. + + Without ``run_id`` a handle opened from it follows the execution + chain; with one it is pinned to that run. Without ``topic`` it names + :data:`temporalio.streams.DEFAULT_TOPIC`. + """ + name, _ = resolve_topic(topic) + return cls("workflow", name, workflow_id=workflow_id, run_id=run_id) + + @classmethod + def for_activity( + cls, + activity_id: str, + *, + workflow_id: str | None = None, + run_id: str | None = None, + topic: str | StreamTopic[Any] | None = None, + ) -> StreamRef: + """A ref to ``topic`` of the streams ``activity_id`` owns. + + With ``workflow_id`` the activity is one that workflow scheduled and + ``run_id`` is the workflow's run; without it the activity is a + standalone one and ``run_id`` pins one run of it. + """ + name, _ = resolve_topic(topic) + return cls( + "activity", + name, + workflow_id=workflow_id, + run_id=run_id, + activity_id=activity_id, + ) + + @classmethod + def for_standalone( + cls, stream_id: str, *, topic: str | StreamTopic[Any] | None = None + ) -> StreamRef: + """A ref to ``topic`` of the standalone stream ``stream_id``.""" + name, _ = resolve_topic(topic) + return cls("standalone", name, stream_id=stream_id) + + def with_topic(self, topic: str | StreamTopic[Any] | None) -> StreamRef: + """The same owner, naming ``topic`` instead. + + ``None`` names :data:`temporalio.streams.DEFAULT_TOPIC`, as a call + that passes no topic does. + """ + name, _ = resolve_topic(topic) + return dataclasses.replace(self, topic=name) + + +def open_ref(provider: StreamProvider, client: Client, ref: StreamRef) -> StreamHandle: + """The handle ``ref`` names, on ``provider``. + + The ref's kind picks the provider call that opens the owner, and its + topic becomes the handle's default, so a call that names none addresses + the stream the ref names. A provider that cannot host that owner kind + raises :class:`temporalio.streams.StreamUnsupportedError` from the call + that would have opened it. + """ + if ref.kind == "workflow": + assert ref.workflow_id is not None + handle = provider.get_stream_handle(client, ref.workflow_id, run_id=ref.run_id) + elif ref.kind == "activity": + assert ref.activity_id is not None + handle = provider.get_activity_stream_handle( + client, ref.activity_id, workflow_id=ref.workflow_id, run_id=ref.run_id + ) + else: + assert ref.stream_id is not None + handle = provider.get_standalone_stream_handle(client, ref.stream_id) + return _RefHandle(handle, ref) + + +class _RefHandle: + """A provider's handle whose default topic is the one a ref names. + + Every call passes through unchanged when it names a topic; one that + names none gets the ref's. The wrapper exists so a ref can address a + stream without every provider learning about refs. + """ + + def __init__(self, inner: StreamHandle, ref: StreamRef) -> None: + self._inner = inner + self._ref = ref + + def _topic(self, topic: str | StreamTopic[Any] | None) -> str | StreamTopic[Any]: + return self._ref.topic if topic is None else topic + + def read( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor = BEGINNING, + last: int | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + # The protocol's overloads each take one shape of topic and + # result_type; a passthrough hands over whatever it was given. + inner: Any = self._inner + return inner.read( + topic=self._topic(topic), after=after, last=last, result_type=result_type + ) + + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: + return await self._inner.latest(topic=self._topic(topic)) + + def producer( + self, + *, + topic: str | StreamTopic[Any] | None = None, + producer_id: str = "", + attempt: int = 0, + ) -> StreamProducer[Any]: + inner: Any = self._inner + return inner.producer( + topic=self._topic(topic), producer_id=producer_id, attempt=attempt + ) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + if topic is None: + return self._ref + return self._inner.ref(topic=topic) + + async def close(self) -> None: + await self._inner.close() diff --git a/temporalio/streams/_topic.py b/temporalio/streams/_topic.py new file mode 100644 index 0000000000..f887b6b579 --- /dev/null +++ b/temporalio/streams/_topic.py @@ -0,0 +1,109 @@ +"""Typed topic definitions. + +Temporal's idiom is to define once and refer by reference: signals, queries +and updates are decorated methods, activities and workflows are functions, +Nexus operations are typed definitions. A topic follows the same rule. It is +defined once, at module level, with the type its records decode to, and the +workflow, its activities and the backend all refer to that one definition. A +plain string names a topic too, the way a string names a signal chosen at +runtime; then the decode hint travels as ``result_type=`` on each call. + +The wire does not change: a topic is a string on the proto and in every +store, and :attr:`temporalio.streams.StreamRecord.topic` is that string. + +Every workflow also has a default topic, :data:`DEFAULT_TOPIC`, which a call +addresses by naming no topic at all. It is an ordinary name, so naming it +explicitly is the same topic, not an error. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Generic, TypeVar, overload + +__all__ = ["DEFAULT_TOPIC", "StreamTopic", "resolve_topic", "topic"] + +T = TypeVar("T") + +DEFAULT_TOPIC = "output" +"""The topic a call addresses when it names none. + +The server resolves an unnamed stream of a workflow to this same name, so on +the native provider the default topic is the server's default stream, and a +store with no default of its own holds it under this name. It stays an +ordinary name rather than a reserved one for the same reason the server does +not reserve it: refusing it would make one stream reachable under two rules. +""" + + +@dataclass(frozen=True) +class StreamTopic(Generic[T]): + """A topic of a workflow's stream, with the type its records decode to. + + Made with :func:`topic`. Hand it to ``temporalio.workflow.stream_reader``, + ``temporalio.workflow.stream_writer``, and to a handle's ``read``, + ``latest`` and ``producer``, and the record and value types follow from + it; ``result_type=`` is not passed alongside a definition. + """ + + name: str + result_type: type[T] | None = None + """The type records decode to, or ``None`` for the converter's default.""" + + +@overload +def topic(name: str, result_type: type[T]) -> StreamTopic[T]: ... + + +@overload +def topic(name: str, result_type: None = None) -> StreamTopic[Any]: ... + + +def topic(name: str, result_type: type | None = None) -> StreamTopic[Any]: + """Define a topic of a workflow's stream. + + Define it once, at module level, and share it: the workflow reads or + publishes it, an activity or a backend produces onto it or reads it, and + the type it carries is inferred wherever it is used. Use a plain string + instead only when the name is decided at runtime. + + Args: + name: The topic's name, as it appears on every record. + result_type: The type records on this topic decode to. Without one, + the payload converter's default applies. + + Raises: + ValueError: ``name`` is empty. + """ + if not name: + raise ValueError("topic name must not be empty") + return StreamTopic(name, result_type) + + +def resolve_topic( + topic: str | StreamTopic[Any] | None = None, result_type: type | None = None +) -> tuple[str, type | None]: + """The name and decode hint a call means, from either form of topic. + + Providers call this once at the top of ``read``, ``latest`` and + ``producer``, so a definition and a string are the same to the store. + ``None`` is a call that named no topic, and means :data:`DEFAULT_TOPIC`. + + Raises: + ValueError: A definition was given together with ``result_type``, + which would name two types for one topic, or the name is empty. + """ + if isinstance(topic, StreamTopic): + if result_type is not None: + raise ValueError( + f"topic {topic.name!r} already carries its type; do not pass " + "result_type= with a definition" + ) + name, result_type = topic.name, topic.result_type + elif topic is None: + name = DEFAULT_TOPIC + else: + name = topic + if not name: + raise ValueError("topic must not be empty") + return name, result_type diff --git a/temporalio/streams/_wire.py b/temporalio/streams/_wire.py new file mode 100644 index 0000000000..7d05dbe5db --- /dev/null +++ b/temporalio/streams/_wire.py @@ -0,0 +1,181 @@ +"""How a record crosses a provider: the proto is the record. + +``temporal.api.stream.v1.StreamRecord`` is the wire format on every provider. +A store that keeps bytes keeps ``SerializeToString()`` of it, the native +server stores the proto it is handed, and a reader in any language parses the +same bytes. ``body`` is the user's payload, produced and consumed through the +payload converter, so a codec applies to it like any other payload and a +pre-encoded :class:`temporalio.common.RawValue` passes through untouched. + +Cursors are self-describing: a token starts with the name of the provider that +minted it, so a provider can refuse a foreign one at the call rather than +misreading it deep in a generator. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import temporalio.converter +from temporalio.api.stream.v1 import StreamRecord as WireRecord +from temporalio.api.stream.v1 import StreamRecordKind +from temporalio.streams._errors import StreamCursorError +from temporalio.streams._policy import AttemptTracker +from temporalio.streams._record import BEGINNING, Cursor, RecordKind, StreamRecord + +__all__ = [ + "RecordDecoder", + "WireRecord", + "cursor_position", + "from_wire", + "mint_cursor", + "producer_identity", + "to_wire", +] + + +def to_wire( + converter: temporalio.converter.PayloadConverter, + *, + topic: str, + kind: RecordKind, + value: Any = None, + producer_id: str = "", + attempt: int = 0, + sequence: int = 0, +) -> WireRecord: + """Build the record a provider stores or ships. + + Only a ``DATA`` record carries a body; the converter encodes ``value`` + into it, which is where a pre-encoded ``RawValue`` passes through. + """ + record = WireRecord( + topic=topic, + kind=StreamRecordKind.ValueType(int(kind)), + producer_id=producer_id, + attempt=attempt, + sequence=sequence, + ) + if kind is RecordKind.DATA: + record.body.CopyFrom(converter.to_payloads([value])[0]) + return record + + +def from_wire( + converter: temporalio.converter.PayloadConverter, + cursor: Cursor, + wire: WireRecord, + result_type: type | None, +) -> StreamRecord[Any]: + """Turn a stored record into the record a reader yields. + + Raises: + ValueError: The kind is one no store may hold, such as a synthesized + ``SUPERSEDED`` or a value this SDK does not know. + """ + kind = RecordKind(wire.kind) + if kind is RecordKind.SUPERSEDED: + raise ValueError("a SUPERSEDED record is synthesized by readers, never stored") + if kind is RecordKind.UNSPECIFIED: + # The proto defines an unset kind as DATA, so every reader agrees. + kind = RecordKind.DATA + value: Any = None + if kind is RecordKind.DATA and wire.HasField("body"): + hints = [result_type] if result_type is not None else None + value = converter.from_payloads([wire.body], hints)[0] + return StreamRecord( + kind=kind, + cursor=cursor, + topic=wire.topic, + producer_id=wire.producer_id, + attempt=wire.attempt, + sequence=wire.sequence, + value=value, + ) + + +class RecordDecoder: + """Turns the records a provider hands over into the records a reader yields. + + One per read. It synthesizes supersession from the attempts it observes, + positions each synthesized record at the cursor before the record that + triggered it, and skips a record it cannot decode with a warning rather + than raising, so a poisoned record cannot pin a workflow on every retry + while an outside reader of the same stream moves past it. + """ + + def __init__( + self, + converter: temporalio.converter.PayloadConverter, + result_type: type | None, + *, + after: Cursor, + warn: Callable[[str], None], + ) -> None: + """Decode with ``converter`` into ``result_type``, resuming after ``after``.""" + self._converter = converter + self._result_type = result_type + self._previous = after + self._warn = warn + self._attempts = AttemptTracker(warn) + + def decode(self, cursor: Cursor, wire: WireRecord) -> list[StreamRecord[Any]]: + """The records to yield for one stored record, in order.""" + try: + record = from_wire(self._converter, cursor, wire, self._result_type) + except Exception as error: + self._warn(f"skipping stream record at {cursor}: {error}") + # The skipped record still holds its position, so a resume after + # it moves on rather than tripping over it again. + self._previous = cursor + return [] + out: list[StreamRecord[Any]] = [] + superseded = self._attempts.note( + wire.producer_id, wire.attempt, topic=wire.topic, previous=self._previous + ) + if superseded is not None: + out.append(superseded) + out.append(record) + self._previous = cursor + return out + + +def mint_cursor(provider: str, position: str) -> Cursor: + """A cursor that names ``position`` and the provider that understands it.""" + return Cursor(f"{provider}:{position}") + + +def cursor_position(cursor: Cursor, *, provider: str) -> str | None: + """The position inside a cursor ``provider`` minted, or ``None`` for BEGINNING. + + Raises: + StreamCursorError: The cursor came from another provider. + """ + if cursor == BEGINNING: + return None + prefix = f"{provider}:" + if not cursor.token.startswith(prefix): + raise StreamCursorError( + f"cursor {cursor.token!r} was not minted by the {provider} stream provider" + ) + return cursor.token[len(prefix) :] + + +def producer_identity(producer_id: str, attempt: int) -> tuple[str, int]: + """Resolve who a producer is, defaulting to the running activity. + + Imported lazily so the module workflow code imports carries no activity + machinery; the default only means anything inside an activity anyway. + """ + if producer_id: + return producer_id, attempt + import temporalio.activity + + if not temporalio.activity.in_activity(): + raise ValueError( + "producer_id is required outside an activity; inside one it defaults " + "to the activity's id and attempt" + ) + info = temporalio.activity.info() + return info.activity_id, attempt or info.attempt diff --git a/temporalio/streams/providers/__init__.py b/temporalio/streams/providers/__init__.py new file mode 100644 index 0000000000..fcefdaa1b1 --- /dev/null +++ b/temporalio/streams/providers/__init__.py @@ -0,0 +1,118 @@ +"""Stream providers, one module each. + +A provider that serves workers is a :class:`temporalio.worker.Plugin`, and +one registered on a client is a :class:`temporalio.client.Plugin` too. +:class:`ProviderPlugin` is the plugin half the providers in this tree share: +it hands the provider to the client, the worker and the replayer as their +``stream_provider`` and leaves their execution alone, so +``Client.connect(plugins=[provider])`` registers it once and every worker +built from that client inherits it. A provider that holds connections closes +them through its own ``close()``, not with the worker, because the same +provider serves handles outside any worker. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import AbstractAsyncContextManager + +import temporalio.client +import temporalio.worker +from temporalio.client import ClientConfig, WorkflowHistory +from temporalio.service import ConnectConfig, ServiceClient +from temporalio.streams._provider import StreamProvider +from temporalio.worker import ( + Replayer, + ReplayerConfig, + Worker, + WorkerConfig, + WorkflowReplayResult, +) + +__all__ = ["ProviderPlugin"] + + +class ProviderPlugin( + StreamProvider, temporalio.client.Plugin, temporalio.worker.Plugin +): + """The plugin every provider in this tree is built on. + + Subclasses implement :class:`temporalio.streams.StreamProvider`; this + class supplies the plugin hooks, so ``Client.connect(plugins=[provider])``, + ``Worker(plugins=[provider])`` and ``Replayer(plugins=[provider])`` reach + the provider through their ``stream_provider`` option. A worker built from + a client that carries the plugin inherits it, and the worker installs the + interceptor that calls the workflow half's lifecycle hooks. + """ + + def _claim(self, held: StreamProvider | None) -> StreamProvider: + """This provider, unless another one already holds the slot. + + There is one slot and it decides where every workflow on the thing + being configured reads and publishes. A last-write-wins here would + silently drop a provider the user passed by hand or a second provider + plugin, so say it instead; a process that talks to two stores opens + the other one's handles from the provider object. + + Raises: + ValueError: Another provider already holds the slot. + """ + if held is not None and held is not self: + raise ValueError( + f"stream provider {held!r} is already registered; pass one provider " + f"and open the other's handles from the object itself" + ) + return self + + def configure_client(self, config: ClientConfig) -> ClientConfig: + """Set this provider as the client's ``stream_provider``. + + Raises: + ValueError: Another provider already holds the slot. + """ + config["stream_provider"] = self._claim(config.get("stream_provider")) + return config + + async def connect_service_client( + self, + config: ConnectConfig, + next: Callable[[ConnectConfig], Awaitable[ServiceClient]], + ) -> ServiceClient: + """Connect unchanged.""" + return await next(config) + + def configure_worker(self, config: WorkerConfig) -> WorkerConfig: + """Set this provider as the worker's ``stream_provider``. + + Raises: + ValueError: Another provider already holds the slot. + """ + config["stream_provider"] = self._claim(config.get("stream_provider")) + return config + + def configure_replayer(self, config: ReplayerConfig) -> ReplayerConfig: + """Set this provider as the replayer's ``stream_provider``. + + Raises: + ValueError: Another provider already holds the slot. + """ + config["stream_provider"] = self._claim(config.get("stream_provider")) + return config + + async def run_worker( + self, worker: Worker, next: Callable[[Worker], Awaitable[None]] + ) -> None: + """Run the worker unchanged.""" + await next(worker) + + def run_replayer( + self, + replayer: Replayer, + histories: AsyncIterator[WorkflowHistory], + next: Callable[ + [Replayer, AsyncIterator[WorkflowHistory]], + AbstractAsyncContextManager[AsyncIterator[WorkflowReplayResult]], + ], + ) -> AbstractAsyncContextManager[AsyncIterator[WorkflowReplayResult]]: + """Run the replayer unchanged.""" + return next(replayer, histories) diff --git a/temporalio/streams/providers/_nexus_generated/__init__.py b/temporalio/streams/providers/_nexus_generated/__init__.py new file mode 100644 index 0000000000..d855a63814 --- /dev/null +++ b/temporalio/streams/providers/_nexus_generated/__init__.py @@ -0,0 +1,25 @@ +# Generated by nexgen v0.2.4. DO NOT EDIT! + +from __future__ import annotations + +from ._definitions import Violation +from .models import ( + AppendInput, + AppendOutput, + ReadInput, + ReadOutput, + RecordWire, + StreamRef, +) +from .services import TemporalStreams + +__all__ = [ + "Violation", + "AppendInput", + "AppendOutput", + "ReadInput", + "ReadOutput", + "RecordWire", + "StreamRef", + "TemporalStreams", +] diff --git a/temporalio/streams/providers/_nexus_generated/_definitions.py b/temporalio/streams/providers/_nexus_generated/_definitions.py new file mode 100644 index 0000000000..dc8de5bb57 --- /dev/null +++ b/temporalio/streams/providers/_nexus_generated/_definitions.py @@ -0,0 +1,542 @@ +# Generated by nexgen v0.2.4. DO NOT EDIT! + +from __future__ import annotations + +import base64 +import collections.abc +import dataclasses +import datetime +import json +import re +import typing + +import temporalio.converter +import temporalio.exceptions + +__all__ = [ + "Violation", + "_binary64", + "_check_contains", + "_check_date_time", + "_check_duration", + "_check_time", + "_check_unique_items", + "_collect", + "_json_values_equal", + "_format_base64", + "_format_base64url", + "_format_date", + "_format_date_time", + "_format_duration", + "_format_time", + "_parse_base64", + "_parse_base64url", + "_parse_date", + "_parse_date_time", + "_parse_duration", + "_parse_spec_integer", + "_parse_time", + "_quote", + "_transfer_type_convertible", +] + + +@dataclasses.dataclass(frozen=True, slots=True) +class Violation: + """A single constraint failure, located by JSON path.""" + + path: str + reason: str + + +def _quote(value: object) -> str: + """Renders a value in the JSON form every target quotes offending values in.""" + + try: + return json.dumps(value, ensure_ascii=False) + except (TypeError, ValueError): + return repr(value) + + +def _collect( + violations: list[Violation], + path: str, + error: temporalio.exceptions.ApplicationError, +) -> None: + """Re-paths a nested model's violations under `path` and appends them.""" + + if error.type != "PayloadValidationError" or not error.details: + raise error + # Generated failures retain the original list as their first detail. The + # cast is type-checker-only and performs no serialization. + nested_violations = typing.cast(list[Violation], error.details[0]) + for inner in nested_violations: + # A nested violation about the value *itself* carries no path of its own + # (a union branch's own constraint, an element-level check), so the + # prefix is the whole path -- never a dangling separator (P11). + nested = f"{path}.{inner.path}" if inner.path else path + violations.append(Violation(path=nested, reason=inner.reason)) + + +_ModelT = typing.TypeVar("_ModelT") + + +def _transfer_type_convertible( + converter: type[temporalio.converter.TransferTypeConverter[typing.Any, typing.Any]], +) -> collections.abc.Callable[[type[_ModelT]], type[_ModelT]]: + """Registers a transfer type converter on a model class. + + Wraps `temporalio.converter.transfer_type_convertible` to erase the + converter's value-type parameter. Binding it directly on the decorated class + is circular for a static type checker -- the class's type depends on the + decorator, whose value type depends on the class -- which degrades the model + to `Unknown`. Erasing it here keeps the decorator idiomatic at each model and + resolves the cycle. + """ + + return temporalio.converter.transfer_type_convertible(converter) + + +_INTEGER_CAP = (1 << 53) - 1 + + +def _parse_spec_integer( + value: object, path: str, violations: list[Violation] +) -> int | None: + """Parses a JSON number as a spec integer (`1.0` accepted, `1.5` rejected).""" + + # `bool` is a subclass of `int`, so it must be excluded before the int check. + if isinstance(value, bool) or not isinstance(value, (int, float)): + violations.append(Violation(path=path, reason="expected integer")) + return None + if isinstance(value, float): + if not value.is_integer(): + violations.append(Violation(path=path, reason="expected integer")) + return None + out = int(value) + else: + out = value + if abs(out) > _INTEGER_CAP: + violations.append(Violation(path=path, reason="expected integer")) + return None + return out + + +def _binary64(value: float) -> float: + """Narrows a `number` value to the binary64 domain shared by every target.""" + + try: + return float(value) + except OverflowError: + # Past the binary64 range there is nothing to narrow to. The caller's + # finiteness check has already recorded that violation and will raise, so + # hand the value back rather than throwing out of turn and losing the + # other violations collected alongside it. + return value + + +def _json_values_equal(left: typing.Any, right: typing.Any) -> bool: + """Compares JSON values without Python's bool-as-int equality leak.""" + + left_is_number = isinstance(left, (int, float)) and not isinstance(left, bool) + right_is_number = isinstance(right, (int, float)) and not isinstance(right, bool) + if left_is_number or right_is_number: + return left_is_number and right_is_number and left == right + if type(left) is not type(right): + return False + if isinstance(left, list): + left_list = typing.cast("list[typing.Any]", left) + right_list = typing.cast("list[typing.Any]", right) + return len(left_list) == len(right_list) and all( + _json_values_equal(a, b) for a, b in zip(left_list, right_list) + ) + if isinstance(left, dict): + left_dict = typing.cast("dict[typing.Any, typing.Any]", left) + right_dict = typing.cast("dict[typing.Any, typing.Any]", right) + return left_dict.keys() == right_dict.keys() and all( + _json_values_equal(left_dict[key], right_dict[key]) for key in left_dict + ) + return left == right + + +def _check_unique_items( + value: list[typing.Any], path: str, violations: list[Violation] +) -> None: + """Asserts an array's elements are pairwise distinct.""" + + seen: list[typing.Any] = [] + for index, element in enumerate(value): + for earlier, previous in enumerate(seen): + if _json_values_equal(previous, element): + violations.append( + Violation( + path=path, + reason=( + f"duplicate items: element at index {index} " + f"equals index {earlier}" + ), + ) + ) + break + seen.append(element) + + +def _check_contains( + value: list[typing.Any], + matches: typing.Callable[[typing.Any], bool], + min_contains: int, + max_contains: int | None, + bounded_min: bool, + path: str, + violations: list[Violation], +) -> None: + """Asserts how many of an array's elements match the `contains` schema.""" + + match_count = sum(1 for element in value if matches(element)) + if match_count < min_contains: + if bounded_min: + violations.append( + Violation( + path=path, + reason=( + f"too few matching items: at least {min_contains}, " + f"got {match_count}" + ), + ) + ) + else: + violations.append( + Violation(path=path, reason="no element matches the required schema") + ) + if max_contains is not None and match_count > max_contains: + violations.append( + Violation( + path=path, + reason=( + f"too many matching items: at most {max_contains}, " + f"got {match_count}" + ), + ) + ) + + +_TEMPORAL_DATE_TIME_RE = re.compile( + r"^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?([Zz]|[+-]([01][0-9]|2[0-3]):[0-5][0-9])\Z" +) +_TEMPORAL_DATE_RE = re.compile(r"^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])\Z") +_TEMPORAL_TIME_RE = re.compile( + r"^([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?([Zz]|[+-]([01][0-9]|2[0-3]):[0-5][0-9])?\Z" +) +_TEMPORAL_DURATION_RE = re.compile( + r"^PT(?:[0-9]+H(?:[0-9]+M(?:[0-9]+S)?)?|[0-9]+M(?:[0-9]+S)?|[0-9]+S)\Z" +) +_TEMPORAL_MAX_DURATION_SECONDS = ((1 << 63) - 1) // 1_000_000_000 +# A duration component with more digits than the cap itself is over the cap +# whatever those digits are, which is how the magnitude is bounded before `int()` +# sees it: CPython refuses to convert a string of more than 4300 digits. +_TEMPORAL_MAX_DURATION_DIGITS = len(str(_TEMPORAL_MAX_DURATION_SECONDS)) +# `datetime` resolves to microseconds, and `fromisoformat` before Python 3.11 +# parses only the fraction widths `isoformat` writes. +_TEMPORAL_FRACTION_DIGITS = 6 + + +def _days_in_month(year: int, month: int) -> int: + if month in (1, 3, 5, 7, 8, 10, 12): + return 31 + if month in (4, 6, 9, 11): + return 30 + if month == 2: + return 29 if (year % 4 == 0 and year % 100 != 0) or year % 400 == 0 else 28 + return 0 + + +def _valid_temporal_calendar(value: str) -> bool: + if len(value) < 10: + return False + try: + year, month, day = int(value[0:4]), int(value[5:7]), int(value[8:10]) + except ValueError: + return False + # `datetime.MINYEAR` is 1, which is also the shared cross-language floor. + # Year 0000 is rejected rather than shifted into range, and + # `_temporal_reason` says so. + if year < datetime.MINYEAR: + return False + maximum = _days_in_month(year, month) + return maximum > 0 and 1 <= day <= maximum + + +def _temporal_reason(name: str, value: str) -> str: + """The reason a rejected temporal string is reported under. + + Year 0000 earns its own clause so the caller sees the shared calendar floor + rather than only a generic malformed-timestamp reason. + """ + + if value[0:4] == "0000": + return ( + f"must be a valid {name}, got {_quote(value)}: year 0000 is not" + f" representable (datetime.MINYEAR is {datetime.MINYEAR})" + ) + return f"must be a valid {name}, got {_quote(value)}" + + +def _temporal_isoformat(value: str) -> str: + """Rewrites a wire temporal into the spelling `fromisoformat` accepts. + + `Z` becomes `+00:00`, and the fractional second is padded or truncated to + exactly `_TEMPORAL_FRACTION_DIGITS`: before Python 3.11 `fromisoformat` + parses only what `isoformat` writes, so an RFC 3339 `.1` or `.1234567` -- + which every other target accepts -- would otherwise raise. Digits past the + sixth are dropped, the loss at `datetime`'s own resolution that P1 allows; + the canonical output re-trims the padding, so `.1` still writes as `.1`. + """ + + normalized = value.upper() + if normalized.endswith("Z"): + normalized = normalized[:-1] + "+00:00" + dot = normalized.find(".") + if dot < 0: + return normalized + end = dot + 1 + while end < len(normalized) and normalized[end].isdigit(): + end += 1 + fraction = normalized[dot + 1 : end].ljust(_TEMPORAL_FRACTION_DIGITS, "0") + return ( + normalized[: dot + 1] + fraction[:_TEMPORAL_FRACTION_DIGITS] + normalized[end:] + ) + + +def _parse_date_time( + value: str, path: str, violations: list[Violation] +) -> datetime.datetime | None: + if _TEMPORAL_DATE_TIME_RE.match(value) is None or not _valid_temporal_calendar( + value + ): + violations.append( + Violation(path=path, reason=_temporal_reason("date-time", value)) + ) + return None + return datetime.datetime.fromisoformat(_temporal_isoformat(value)) + + +def _parse_date( + value: str, path: str, violations: list[Violation] +) -> datetime.date | None: + if _TEMPORAL_DATE_RE.match(value) is None or not _valid_temporal_calendar(value): + violations.append(Violation(path=path, reason=_temporal_reason("date", value))) + return None + return datetime.date.fromisoformat(value) + + +def _parse_time( + value: str, path: str, violations: list[Violation] +) -> datetime.time | None: + if _TEMPORAL_TIME_RE.match(value) is None: + violations.append(Violation(path=path, reason=_temporal_reason("time", value))) + return None + return datetime.time.fromisoformat(_temporal_isoformat(value)) + + +def _parse_duration( + value: str, path: str, violations: list[Violation] +) -> datetime.timedelta | None: + if _TEMPORAL_DURATION_RE.match(value) is None: + violations.append( + Violation(path=path, reason=_temporal_reason("duration", value)) + ) + return None + total = 0 + number = "" + for char in value[2:]: + if char.isdigit(): + number += char + continue + digits = number.lstrip("0") + number = "" + if len(digits) > _TEMPORAL_MAX_DURATION_DIGITS: + # Over the cap by digit count alone (see the constant), so the + # conversion `int()` would refuse is never attempted. + total = _TEMPORAL_MAX_DURATION_SECONDS + 1 + break + total += int(digits or "0") * {"H": 3600, "M": 60, "S": 1}[char] + if total > _TEMPORAL_MAX_DURATION_SECONDS: + break + if total > _TEMPORAL_MAX_DURATION_SECONDS: + violations.append( + Violation(path=path, reason=_temporal_reason("duration", value)) + ) + return None + return datetime.timedelta(seconds=total) + + +def _check_temporal_offset( + name: str, + value: datetime.datetime | datetime.time, + offset: datetime.timedelta, + path: str, + violations: list[Violation], +) -> None: + """Asserts a UTC offset is a whole number of minutes, the finest the wire + form spells (`tzinfo` allows seconds, which the offset would silently lose). + """ + + if offset % datetime.timedelta(minutes=1): + violations.append( + Violation( + path=path, + reason=( + f"must be a valid {name}, got {_quote(str(value))}: " + f"the UTC offset {offset} is not a whole number of minutes" + ), + ) + ) + + +def _check_date_time( + value: datetime.datetime, path: str, violations: list[Violation] +) -> None: + """Asserts a datetime is writable as a wire date-time (P12). + + A dataclass is constructed unchecked, so a naive datetime -- with no offset + the required wire form could carry -- reaches serialize; without this it + would emit a value this module's own parser rejects. + """ + + offset = value.utcoffset() + if offset is None: + violations.append( + Violation( + path=path, + reason=( + f"must be a valid date-time, got {_quote(str(value))}: " + "a naive datetime carries no UTC offset" + ), + ) + ) + return + _check_temporal_offset("date-time", value, offset, path, violations) + + +def _check_time(value: datetime.time, path: str, violations: list[Violation]) -> None: + """Asserts a time is writable as a wire time (P12). The offset is optional in + the grammar, so only its precision is held to anything.""" + + offset = value.utcoffset() + if offset is not None: + _check_temporal_offset("time", value, offset, path, violations) + + +def _check_duration( + value: datetime.timedelta, path: str, violations: list[Violation] +) -> None: + """Asserts a timedelta is writable as a wire duration (P12): the grammar is + unsigned, whole-second and capped, and a `timedelta` is none of those.""" + + if value < datetime.timedelta(0): + reason = "a duration cannot be negative" + elif value % datetime.timedelta(seconds=1): + reason = "a duration cannot carry a fraction of a second" + elif value.total_seconds() > _TEMPORAL_MAX_DURATION_SECONDS: + reason = f"a duration cannot exceed {_TEMPORAL_MAX_DURATION_SECONDS} seconds" + else: + return + violations.append( + Violation( + path=path, + reason=f"must be a valid duration, got {_quote(str(value))}: {reason}", + ) + ) + + +def _temporal_frac(microsecond: int) -> str: + if microsecond == 0: + return "" + return "." + f"{microsecond:06d}".rstrip("0") + + +def _temporal_offset(value: datetime.datetime | datetime.time) -> str: + offset = value.utcoffset() + if offset is None: + return "" + total = int(offset.total_seconds()) + if total == 0: + return "Z" + sign = "+" if total > 0 else "-" + total = abs(total) + return f"{sign}{total // 3600:02d}:{(total % 3600) // 60:02d}" + + +def _format_date_time(value: datetime.datetime) -> str: + return ( + f"{value.year:04d}-{value.month:02d}-{value.day:02d}" + f"T{value.hour:02d}:{value.minute:02d}:{value.second:02d}" + f"{_temporal_frac(value.microsecond)}{_temporal_offset(value)}" + ) + + +def _format_date(value: datetime.date) -> str: + return f"{value.year:04d}-{value.month:02d}-{value.day:02d}" + + +def _format_time(value: datetime.time) -> str: + return ( + f"{value.hour:02d}:{value.minute:02d}:{value.second:02d}" + f"{_temporal_frac(value.microsecond)}{_temporal_offset(value)}" + ) + + +def _format_duration(value: datetime.timedelta) -> str: + total = int(value.total_seconds()) + if total == 0: + return "PT0S" + hours, remainder = divmod(total, 3600) + minutes, seconds = divmod(remainder, 60) + out = "PT" + if hours: + out += f"{hours}H" + if minutes: + out += f"{minutes}M" + if seconds: + out += f"{seconds}S" + return out + + +_BASE64_RE = re.compile( + "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?\\Z", + re.ASCII, +) +_BASE64URL_RE = re.compile( + "^(?:[A-Za-z0-9_-]{4})*(?:[A-Za-z0-9_-][AQgw]|[A-Za-z0-9_-]{2}[AEIMQUYcgkosw048])?\\Z", + re.ASCII, +) + + +def _parse_base64(value: str, path: str, violations: list[Violation]) -> bytes | None: + if _BASE64_RE.match(value) is None: + violations.append( + Violation(path=path, reason=f"must be base64-encoded, got {_quote(value)}") + ) + return None + return base64.b64decode(value, validate=True) + + +def _format_base64(value: bytes) -> str: + return base64.b64encode(value).decode("ascii") + + +def _parse_base64url( + value: str, path: str, violations: list[Violation] +) -> bytes | None: + if _BASE64URL_RE.match(value) is None: + violations.append( + Violation( + path=path, reason=f"must be base64url-encoded, got {_quote(value)}" + ) + ) + return None + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def _format_base64url(value: bytes) -> str: + return base64.urlsafe_b64encode(value).rstrip(b"=").decode("ascii") diff --git a/temporalio/streams/providers/_nexus_generated/models.py b/temporalio/streams/providers/_nexus_generated/models.py new file mode 100644 index 0000000000..fd4c65fb67 --- /dev/null +++ b/temporalio/streams/providers/_nexus_generated/models.py @@ -0,0 +1,1029 @@ +# Generated by nexgen v0.2.4. DO NOT EDIT! + +from __future__ import annotations + +import dataclasses +import typing + +import typing_extensions + +import temporalio.converter +import temporalio.exceptions + +from ._definitions import ( + Violation, + _collect, + _format_base64, + _parse_base64, + _parse_spec_integer, + _quote, + _transfer_type_convertible, +) + +_APPEND_OUTPUT_DECLARED: frozenset[str] = frozenset({"cursor"}) + + +_READ_OUTPUT_DECLARED: frozenset[str] = frozenset({"records", "next_token", "done"}) + + +_RECORD_WIRE_DECLARED: frozenset[str] = frozenset({"token", "record"}) + + +class _AppendInputTransferTypeConverter( + temporalio.converter.TransferTypeConverter["AppendInput", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["AppendInput"] + ) -> "AppendInput": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + stream_value: StreamRef = typing.cast("typing.Any", None) + if "stream" not in raw or raw["stream"] is None: + violations.append(Violation(path="stream", reason="required")) + else: + stream_value_raw = raw["stream"] + try: + stream_value = _StreamRefTransferTypeConverter().from_transfer_type( + stream_value_raw, StreamRef + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, "stream", error) + + producer_id_value: str = typing.cast("typing.Any", None) + if "producer_id" not in raw or raw["producer_id"] is None: + violations.append(Violation(path="producer_id", reason="required")) + else: + producer_id_value_raw = raw["producer_id"] + if not isinstance(producer_id_value_raw, str): + violations.append( + Violation(path="producer_id", reason="expected string") + ) + else: + producer_id_value = producer_id_value_raw + + attempt_value: int = typing.cast("typing.Any", None) + if "attempt" not in raw or raw["attempt"] is None: + violations.append(Violation(path="attempt", reason="required")) + else: + attempt_value_raw = raw["attempt"] + attempt_value_parsed = _parse_spec_integer( + attempt_value_raw, "attempt", violations + ) + if attempt_value_parsed is not None: + attempt_value = attempt_value_parsed + + sequence_value: int = typing.cast("typing.Any", None) + if "sequence" not in raw or raw["sequence"] is None: + violations.append(Violation(path="sequence", reason="required")) + else: + sequence_value_raw = raw["sequence"] + sequence_value_parsed = _parse_spec_integer( + sequence_value_raw, "sequence", violations + ) + if sequence_value_parsed is not None: + sequence_value = sequence_value_parsed + if sequence_value < 0: + violations.append( + Violation( + path="sequence", + reason=f"must be >= 0, got {sequence_value}", + ) + ) + + batch_index_value: int = typing.cast("typing.Any", None) + if "batch_index" not in raw or raw["batch_index"] is None: + violations.append(Violation(path="batch_index", reason="required")) + else: + batch_index_value_raw = raw["batch_index"] + batch_index_value_parsed = _parse_spec_integer( + batch_index_value_raw, "batch_index", violations + ) + if batch_index_value_parsed is not None: + batch_index_value = batch_index_value_parsed + if batch_index_value < 1: + violations.append( + Violation( + path="batch_index", + reason=f"must be >= 1, got {batch_index_value}", + ) + ) + + payloads_value: list[bytes] | None = None + if "payloads" in raw: + payloads_value_raw = raw["payloads"] + if payloads_value_raw is None: + violations.append( + Violation(path="payloads", reason="explicit null not allowed") + ) + else: + if not isinstance(payloads_value_raw, list): + violations.append( + Violation(path="payloads", reason="expected array") + ) + else: + payloads_value_list: list[bytes] = [] + for payloads_value_index, payloads_value_element in enumerate( + typing.cast("list[typing.Any]", payloads_value_raw) + ): + payloads_value_item_path = f"payloads[{payloads_value_index}]" + payloads_value_item_violation_count = len(violations) + payloads_value_item: bytes = typing.cast("typing.Any", None) + if not isinstance(payloads_value_element, str): + violations.append( + Violation( + path=payloads_value_item_path, + reason="expected string", + ) + ) + else: + payloads_value_item_parsed = _parse_base64( + payloads_value_element, + payloads_value_item_path, + violations, + ) + if payloads_value_item_parsed is not None: + payloads_value_item = payloads_value_item_parsed + if len(violations) == payloads_value_item_violation_count: + payloads_value_list.append(payloads_value_item) + payloads_value = payloads_value_list + + finish_value: bool | None = None + if "finish" in raw: + finish_value_raw = raw["finish"] + if finish_value_raw is None: + violations.append( + Violation(path="finish", reason="explicit null not allowed") + ) + else: + if not isinstance(finish_value_raw, bool): + violations.append( + Violation(path="finish", reason="expected boolean") + ) + else: + finish_value = finish_value_raw + + for key in raw: + if ( + key != "stream" + and key != "producer_id" + and key != "attempt" + and key != "sequence" + and key != "batch_index" + and key != "payloads" + and key != "finish" + ): + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return AppendInput( + stream=stream_value, + producer_id=producer_id_value, + attempt=attempt_value, + sequence=sequence_value, + batch_index=batch_index_value, + payloads=payloads_value, + finish=finish_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "AppendInput") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + try: + out["stream"] = _StreamRefTransferTypeConverter().to_transfer_type( + value.stream + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, "stream", error) + out["producer_id"] = value.producer_id + if abs(value.attempt) > 9007199254740991: + violations.append( + Violation(path="attempt", reason="exceeds ±(2^53-1) integer cap") + ) + out["attempt"] = value.attempt + if abs(value.sequence) > 9007199254740991: + violations.append( + Violation(path="sequence", reason="exceeds ±(2^53-1) integer cap") + ) + if value.sequence < 0: + violations.append( + Violation(path="sequence", reason=f"must be >= 0, got {value.sequence}") + ) + out["sequence"] = value.sequence + if abs(value.batch_index) > 9007199254740991: + violations.append( + Violation(path="batch_index", reason="exceeds ±(2^53-1) integer cap") + ) + if value.batch_index < 1: + violations.append( + Violation( + path="batch_index", reason=f"must be >= 1, got {value.batch_index}" + ) + ) + out["batch_index"] = value.batch_index + if value.payloads is not None: + out["payloads"] = [_format_base64(element) for element in value.payloads] + if value.finish is not None: + out["finish"] = value.finish + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_AppendInputTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class AppendInput: + """One append call: who is writing, where, and what.""" + + stream: StreamRef + + producer_id: str + """Identifies the writer across its retries, so its attempts can be ordered. Never + empty here: the caller resolved it, from the activity context when it was not given. + """ + + attempt: int + """The generation this producer is writing. A later attempt supersedes an earlier one.""" + + sequence: int + """The producer's sequence of the first record in this batch; the batch is numbered + from it and a finish takes the next number. It has to continue where the previous + batch ended, so the handler and the store agree on every record's position within + the attempt. + """ + + batch_index: int + """Counts this producer's batches from 1. The handler answers a repeat of the last + index with the original's position, and refuses one that skips ahead, one already + behind the last, or one that resumes an attempt it never saw start. + """ + + payloads: list[bytes] | None = None + """The record bodies to write, in order, each a serialized + temporal.api.common.v1.Payload. A payload codec configured on the caller has already + run on them. Empty on a call that only finishes. + """ + + finish: bool | None = None + """Write FINISH for this producer after the batch: it will write nothing more on the + topic. Says nothing about the producer's outcome. + """ + + +class _AppendOutputTransferTypeConverter( + temporalio.converter.TransferTypeConverter["AppendOutput", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["AppendOutput"] + ) -> "AppendOutput": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + cursor_value: str | None = None + if "cursor" in raw: + cursor_value_raw = raw["cursor"] + if cursor_value_raw is None: + violations.append( + Violation(path="cursor", reason="explicit null not allowed") + ) + else: + if not isinstance(cursor_value_raw, str): + violations.append( + Violation(path="cursor", reason="expected string") + ) + else: + cursor_value = cursor_value_raw + + additional_properties: dict[str, typing.Any] = {} + for key in raw: + if key not in _APPEND_OUTPUT_DECLARED: + additional_properties[key] = raw[key] + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return AppendOutput( + cursor=cursor_value, + additional_properties=additional_properties, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "AppendOutput") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + if value.cursor is not None: + out["cursor"] = value.cursor + for key, entry in value.additional_properties.items(): + if key in _APPEND_OUTPUT_DECLARED: + violations.append( + Violation( + path=key, + reason="additional property collides with declared property", + ) + ) + else: + out[key] = entry + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_AppendOutputTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class AppendOutput: + """Where the append landed, when the store can say.""" + + cursor: str | None = None + """Opaque token naming the last record written by this call, or by the original when + the call repeated the last batch. Absent when the store learns positions only at + read time; such a caller positions itself with a latest_only read. + """ + + additional_properties: dict[str, typing.Any] = dataclasses.field( + default_factory=dict + ) + + +class _ReadInputTransferTypeConverter( + temporalio.converter.TransferTypeConverter["ReadInput", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["ReadInput"] + ) -> "ReadInput": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + stream_value: StreamRef = typing.cast("typing.Any", None) + if "stream" not in raw or raw["stream"] is None: + violations.append(Violation(path="stream", reason="required")) + else: + stream_value_raw = raw["stream"] + try: + stream_value = _StreamRefTransferTypeConverter().from_transfer_type( + stream_value_raw, StreamRef + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, "stream", error) + + after_token_value: str | None = None + if "after_token" in raw: + after_token_value_raw = raw["after_token"] + if after_token_value_raw is None: + violations.append( + Violation(path="after_token", reason="explicit null not allowed") + ) + else: + if not isinstance(after_token_value_raw, str): + violations.append( + Violation(path="after_token", reason="expected string") + ) + else: + after_token_value = after_token_value_raw + + max_records_value: int | None = None + if "max_records" in raw: + max_records_value_raw = raw["max_records"] + if max_records_value_raw is None: + violations.append( + Violation(path="max_records", reason="explicit null not allowed") + ) + else: + max_records_value_parsed = _parse_spec_integer( + max_records_value_raw, "max_records", violations + ) + if max_records_value_parsed is not None: + max_records_value = max_records_value_parsed + if max_records_value < 1: + violations.append( + Violation( + path="max_records", + reason=f"must be >= 1, got {max_records_value}", + ) + ) + if max_records_value > 1000: + violations.append( + Violation( + path="max_records", + reason=f"must be <= 1000, got {max_records_value}", + ) + ) + + wait_ms_value: int | None = None + if "wait_ms" in raw: + wait_ms_value_raw = raw["wait_ms"] + if wait_ms_value_raw is None: + violations.append( + Violation(path="wait_ms", reason="explicit null not allowed") + ) + else: + wait_ms_value_parsed = _parse_spec_integer( + wait_ms_value_raw, "wait_ms", violations + ) + if wait_ms_value_parsed is not None: + wait_ms_value = wait_ms_value_parsed + if wait_ms_value < 0: + violations.append( + Violation( + path="wait_ms", + reason=f"must be >= 0, got {wait_ms_value}", + ) + ) + if wait_ms_value > 60000: + violations.append( + Violation( + path="wait_ms", + reason=f"must be <= 60000, got {wait_ms_value}", + ) + ) + + latest_only_value: bool | None = None + if "latest_only" in raw: + latest_only_value_raw = raw["latest_only"] + if latest_only_value_raw is None: + violations.append( + Violation(path="latest_only", reason="explicit null not allowed") + ) + else: + if not isinstance(latest_only_value_raw, bool): + violations.append( + Violation(path="latest_only", reason="expected boolean") + ) + else: + latest_only_value = latest_only_value_raw + + last_n_value: int | None = None + if "last_n" in raw: + last_n_value_raw = raw["last_n"] + if last_n_value_raw is None: + violations.append( + Violation(path="last_n", reason="explicit null not allowed") + ) + else: + last_n_value_parsed = _parse_spec_integer( + last_n_value_raw, "last_n", violations + ) + if last_n_value_parsed is not None: + last_n_value = last_n_value_parsed + if last_n_value < 1: + violations.append( + Violation( + path="last_n", + reason=f"must be >= 1, got {last_n_value}", + ) + ) + + for key in raw: + if ( + key != "stream" + and key != "after_token" + and key != "max_records" + and key != "wait_ms" + and key != "latest_only" + and key != "last_n" + ): + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return ReadInput( + stream=stream_value, + after_token=after_token_value, + max_records=max_records_value, + wait_ms=wait_ms_value, + latest_only=latest_only_value, + last_n=last_n_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "ReadInput") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + try: + out["stream"] = _StreamRefTransferTypeConverter().to_transfer_type( + value.stream + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, "stream", error) + if value.after_token is not None: + out["after_token"] = value.after_token + if value.max_records is not None: + if abs(value.max_records) > 9007199254740991: + violations.append( + Violation( + path="max_records", reason="exceeds ±(2^53-1) integer cap" + ) + ) + if value.max_records < 1: + violations.append( + Violation( + path="max_records", + reason=f"must be >= 1, got {value.max_records}", + ) + ) + if value.max_records > 1000: + violations.append( + Violation( + path="max_records", + reason=f"must be <= 1000, got {value.max_records}", + ) + ) + out["max_records"] = value.max_records + if value.wait_ms is not None: + if abs(value.wait_ms) > 9007199254740991: + violations.append( + Violation(path="wait_ms", reason="exceeds ±(2^53-1) integer cap") + ) + if value.wait_ms < 0: + violations.append( + Violation( + path="wait_ms", reason=f"must be >= 0, got {value.wait_ms}" + ) + ) + if value.wait_ms > 60000: + violations.append( + Violation( + path="wait_ms", reason=f"must be <= 60000, got {value.wait_ms}" + ) + ) + out["wait_ms"] = value.wait_ms + if value.latest_only is not None: + out["latest_only"] = value.latest_only + if value.last_n is not None: + if abs(value.last_n) > 9007199254740991: + violations.append( + Violation(path="last_n", reason="exceeds ±(2^53-1) integer cap") + ) + if value.last_n < 1: + violations.append( + Violation(path="last_n", reason=f"must be >= 1, got {value.last_n}") + ) + out["last_n"] = value.last_n + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_ReadInputTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class ReadInput: + """One read call: where to resume from and how long to wait.""" + + stream: StreamRef + + after_token: str | None = None + """Opaque cursor from an earlier record or append. The read resumes strictly after the + record it names, so a caller never sees that record twice. Empty starts at the + beginning. The token is produced by whichever store sits behind the endpoint, so a + caller cannot tell which one that is, and a token from another store is refused. + """ + + max_records: int | None = None + """Return at most this many records. Defaults to 100 when omitted.""" + + wait_ms: int | None = None + """Wait at most this long for records to arrive before answering. The handler shortens + it to fit the request deadline. A call that collects nothing answers with no records + and the caller's own token. + """ + + latest_only: bool | None = None + """Answer with the newest position and no records, for a reader that wants to follow + from now. + """ + + last_n: int | None = None + """With no after_token, start at the newest this many records, or at all of them when + the stream holds fewer. Records of every kind count. Refused alongside an + after_token, which is how a read resumes. + """ + + +class _ReadOutputTransferTypeConverter( + temporalio.converter.TransferTypeConverter["ReadOutput", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["ReadOutput"] + ) -> "ReadOutput": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + records_value: list[RecordWire] | None = None + if "records" in raw: + records_value_raw = raw["records"] + if records_value_raw is None: + violations.append( + Violation(path="records", reason="explicit null not allowed") + ) + else: + if not isinstance(records_value_raw, list): + violations.append( + Violation(path="records", reason="expected array") + ) + else: + records_value_list: list[RecordWire] = [] + for records_value_index, records_value_element in enumerate( + typing.cast("list[typing.Any]", records_value_raw) + ): + records_value_item_path = f"records[{records_value_index}]" + records_value_item_violation_count = len(violations) + records_value_item: RecordWire = typing.cast("typing.Any", None) + try: + records_value_item = ( + _RecordWireTransferTypeConverter().from_transfer_type( + records_value_element, RecordWire + ) + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, records_value_item_path, error) + if len(violations) == records_value_item_violation_count: + records_value_list.append(records_value_item) + records_value = records_value_list + + next_token_value: str | None = None + if "next_token" in raw: + next_token_value_raw = raw["next_token"] + if next_token_value_raw is None: + violations.append( + Violation(path="next_token", reason="explicit null not allowed") + ) + else: + if not isinstance(next_token_value_raw, str): + violations.append( + Violation(path="next_token", reason="expected string") + ) + else: + next_token_value = next_token_value_raw + + done_value: bool | None = None + if "done" in raw: + done_value_raw = raw["done"] + if done_value_raw is None: + violations.append( + Violation(path="done", reason="explicit null not allowed") + ) + else: + if not isinstance(done_value_raw, bool): + violations.append(Violation(path="done", reason="expected boolean")) + else: + done_value = done_value_raw + + additional_properties: dict[str, typing.Any] = {} + for key in raw: + if key not in _READ_OUTPUT_DECLARED: + additional_properties[key] = raw[key] + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return ReadOutput( + records=records_value, + next_token=next_token_value, + done=done_value, + additional_properties=additional_properties, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "ReadOutput") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + if value.records is not None: + records_out: list[typing.Any] = [] + for records_index, records_element in enumerate(value.records): + try: + records_out.append( + _RecordWireTransferTypeConverter().to_transfer_type( + records_element + ) + ) + except temporalio.exceptions.ApplicationError as error: + _collect(violations, f"records[{records_index}]", error) + out["records"] = records_out + if value.next_token is not None: + out["next_token"] = value.next_token + if value.done is not None: + out["done"] = value.done + for key, entry in value.additional_properties.items(): + if key in _READ_OUTPUT_DECLARED: + violations.append( + Violation( + path=key, + reason="additional property collides with declared property", + ) + ) + else: + out[key] = entry + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_ReadOutputTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class ReadOutput: + """What one read call answered, and where to resume.""" + + records: list[RecordWire] | None = None + """The records after the caller's token, in stream order.""" + + next_token: str | None = None + """Opaque cursor to pass as after_token on the following call. Echoes the caller's own + token when the call collected nothing. + """ + + done: bool | None = None + """The store ended the read: the owning execution, or its chain, is closed and every + retained record after the caller's token has been delivered. Nothing more will + arrive, so the caller stops. + """ + + additional_properties: dict[str, typing.Any] = dataclasses.field( + default_factory=dict + ) + + +class _RecordWireTransferTypeConverter( + temporalio.converter.TransferTypeConverter["RecordWire", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["RecordWire"] + ) -> "RecordWire": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + token_value: str = typing.cast("typing.Any", None) + if "token" not in raw or raw["token"] is None: + violations.append(Violation(path="token", reason="required")) + else: + token_value_raw = raw["token"] + if not isinstance(token_value_raw, str): + violations.append(Violation(path="token", reason="expected string")) + else: + token_value = token_value_raw + + record_value: bytes = typing.cast("typing.Any", None) + if "record" not in raw or raw["record"] is None: + violations.append(Violation(path="record", reason="required")) + else: + record_value_raw = raw["record"] + if not isinstance(record_value_raw, str): + violations.append(Violation(path="record", reason="expected string")) + else: + record_value_parsed = _parse_base64( + record_value_raw, "record", violations + ) + if record_value_parsed is not None: + record_value = record_value_parsed + + additional_properties: dict[str, typing.Any] = {} + for key in raw: + if key not in _RECORD_WIRE_DECLARED: + additional_properties[key] = raw[key] + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return RecordWire( + token=token_value, + record=record_value, + additional_properties=additional_properties, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "RecordWire") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + out["token"] = value.token + out["record"] = _format_base64(value.record) + for key, entry in value.additional_properties.items(): + if key in _RECORD_WIRE_DECLARED: + violations.append( + Violation( + path=key, + reason="additional property collides with declared property", + ) + ) + else: + out[key] = entry + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_RecordWireTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class RecordWire: + """One record on the wire: its cursor and the record itself.""" + + token: str + """Opaque cursor naming this record. Pass it as after_token to resume just past it.""" + + record: bytes + """The serialized temporal.api.stream.v1.StreamRecord: topic, kind, producer_id, + attempt, sequence and body, as the store holds it. + """ + + additional_properties: dict[str, typing.Any] = dataclasses.field( + default_factory=dict + ) + + +class _StreamRefTransferTypeConverter( + temporalio.converter.TransferTypeConverter["StreamRef", typing.Any] +): + @typing_extensions.override + def from_transfer_type( + self, value: typing.Any, type_hint: type["StreamRef"] + ) -> "StreamRef": + violations: list[Violation] = [] + if not isinstance(value, dict): + raise temporalio.converter.create_payload_validation_error( + [Violation(path="", reason="expected object")] + ) + raw = typing.cast("dict[str, typing.Any]", value) + + kind_value: typing.Literal["workflow", "activity", "standalone"] = typing.cast( + "typing.Any", None + ) + if "kind" not in raw or raw["kind"] is None: + violations.append(Violation(path="kind", reason="required")) + else: + kind_value_raw = raw["kind"] + if not isinstance(kind_value_raw, str): + violations.append(Violation(path="kind", reason="expected string")) + elif kind_value_raw not in ("workflow", "activity", "standalone"): + violations.append( + Violation( + path="kind", + reason=f'must be one of ["workflow", "activity", "standalone"], got {_quote(kind_value_raw)}', + ) + ) + else: + kind_value = kind_value_raw + + workflow_id_value: str | None = None + if "workflow_id" in raw: + workflow_id_value_raw = raw["workflow_id"] + if workflow_id_value_raw is None: + workflow_id_value = None + else: + if not isinstance(workflow_id_value_raw, str): + violations.append( + Violation(path="workflow_id", reason="expected string") + ) + else: + workflow_id_value = workflow_id_value_raw + + run_id_value: str | None = None + if "run_id" in raw: + run_id_value_raw = raw["run_id"] + if run_id_value_raw is None: + run_id_value = None + else: + if not isinstance(run_id_value_raw, str): + violations.append( + Violation(path="run_id", reason="expected string") + ) + else: + run_id_value = run_id_value_raw + + activity_id_value: str | None = None + if "activity_id" in raw: + activity_id_value_raw = raw["activity_id"] + if activity_id_value_raw is None: + activity_id_value = None + else: + if not isinstance(activity_id_value_raw, str): + violations.append( + Violation(path="activity_id", reason="expected string") + ) + else: + activity_id_value = activity_id_value_raw + + stream_id_value: str | None = None + if "stream_id" in raw: + stream_id_value_raw = raw["stream_id"] + if stream_id_value_raw is None: + stream_id_value = None + else: + if not isinstance(stream_id_value_raw, str): + violations.append( + Violation(path="stream_id", reason="expected string") + ) + else: + stream_id_value = stream_id_value_raw + + topic_value: str = typing.cast("typing.Any", None) + if "topic" not in raw or raw["topic"] is None: + violations.append(Violation(path="topic", reason="required")) + else: + topic_value_raw = raw["topic"] + if not isinstance(topic_value_raw, str): + violations.append(Violation(path="topic", reason="expected string")) + else: + topic_value = topic_value_raw + + for key in raw: + if ( + key != "kind" + and key != "workflow_id" + and key != "run_id" + and key != "activity_id" + and key != "stream_id" + and key != "topic" + ): + violations.append(Violation(path=key, reason="unknown field")) + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return StreamRef( + kind=kind_value, + workflow_id=workflow_id_value, + run_id=run_id_value, + activity_id=activity_id_value, + stream_id=stream_id_value, + topic=topic_value, + ) + + @typing_extensions.override + def to_transfer_type(self, value: "StreamRef") -> typing.Any: + violations: list[Violation] = [] + out: dict[str, typing.Any] = {} + if typing.cast("object", value.kind) not in ( + "workflow", + "activity", + "standalone", + ): + violations.append( + Violation( + path="kind", + reason=f'must be one of ["workflow", "activity", "standalone"], got {_quote(value.kind)}', + ) + ) + out["kind"] = value.kind + if value.workflow_id is not None: + out["workflow_id"] = value.workflow_id + if value.run_id is not None: + out["run_id"] = value.run_id + if value.activity_id is not None: + out["activity_id"] = value.activity_id + if value.stream_id is not None: + out["stream_id"] = value.stream_id + out["topic"] = value.topic + if violations: + raise temporalio.converter.create_payload_validation_error(violations) + return out + + +@_transfer_type_convertible(_StreamRefTransferTypeConverter) +@dataclasses.dataclass(slots=True, kw_only=True) +class StreamRef: + """A stream, named by its owner and a topic: what an operation returns to hand a stream + to its caller, and what read and append take in place of an owner spelled out. It + names no cursor and no store, so the same reference is good behind any endpoint that + serves the owner. A member the owner kind does not use is absent or null; null is + how the SDK writes an unset member of its own StreamRef. + """ + + kind: typing.Literal["workflow", "activity", "standalone"] + """What owns the stream. A workflow's streams are keyed by workflow_id and, when + pinned, run_id. An activity's own streams are keyed by activity_id and, when a + workflow scheduled it, that workflow's ids. A standalone stream has its own + stream_id and no execution behind it. + """ + + workflow_id: str | None = None + """The owning workflow, or the workflow that scheduled the owning activity. Required + for a workflow owner. + """ + + run_id: str | None = None + """Pin the owner to this run. Absent follows the execution chain across + continue-as-new, which is what a producer normally wants. + """ + + activity_id: str | None = None + """The owning activity. Required for an activity owner.""" + + stream_id: str | None = None + """The stream's own id. Required for a standalone owner.""" + + topic: str + """The topic on the owner's stream.""" diff --git a/temporalio/streams/providers/_nexus_generated/services.py b/temporalio/streams/providers/_nexus_generated/services.py new file mode 100644 index 0000000000..591e310466 --- /dev/null +++ b/temporalio/streams/providers/_nexus_generated/services.py @@ -0,0 +1,44 @@ +# Generated by nexgen v0.2.4. DO NOT EDIT! + +from __future__ import annotations + +from nexusrpc import Operation, service + +from .models import ( + AppendInput, + AppendOutput, + ReadInput, + ReadOutput, +) + + +@service +class TemporalStreams: + """The stream endpoint's two operations. One Temporal-authenticated endpoint hides the + store behind it, so an operator switches storage without touching callers. Reads + hand out batches and an append carries one batch per call, because a Nexus operation + per record costs too much for token streams. A record crosses as the serialized + temporal.api.stream.v1.StreamRecord, the same bytes every store keeps, so a caller + in any language decodes it with the api protos alone. + """ + + append: Operation[ + AppendInput, + AppendOutput, + ] = Operation(name="append") + """Append one batch on the caller's account. The handler writes a batch once: a repeat + of the last batch_index answers with where the original landed, and an index that + skips ahead, one already behind the last, a sequence that does not continue, or a + producer attempt the handler has no state for is refused. Supersession records are + not transported: a reader re-synthesizes them from the attempts it observes. + """ + + read: Operation[ + ReadInput, + ReadOutput, + ] = Operation(name="read") + """Answer with the records after the caller's token, or time out. The call parks until + it has max_records records or wait_ms elapses, whichever comes first, and returns + whatever it collected. It says when the store has ended the read, so the caller can + stop. + """ diff --git a/temporalio/streams/providers/memory.py b/temporalio/streams/providers/memory.py new file mode 100644 index 0000000000..ce8a4ae10d --- /dev/null +++ b/temporalio/streams/providers/memory.py @@ -0,0 +1,814 @@ +"""The in-process reference provider. + +Exists so the conformance suite can exercise the whole surface without a +store, and to document in one file what a provider owes. Its limits, stated +so nobody mistakes it for evidence: + +- It is not replay-safe. Workflow-side state lives in plain process memory, + so run it with a warm workflow cache and do not use it to demonstrate + recovery. +- A workflow's publish becomes visible at ``publish`` time rather than at + task acceptance, and a failed task's records stay, so it only approximates + rule 1 of the contract. +- Topics are keyed by workflow id rather than by run, so a successor run's + reader from ``BEGINNING`` sees the chain's records. A ``run_id`` on a + handle only decides which run's close ends a read. +- It learns that a workflow closed by describing it, so a handle opened + without a client reads until the caller closes it. +- An activity's own streams are keyed by the activity, not by its run. A + standalone activity's read ends when describing it shows a terminal + status. A workflow's activity is described through its workflow: the read + ends once the activity has been seen pending and is no longer, or the + workflow closed, so a read opened after the activity already finished + waits for the workflow. +- It keeps every record until :meth:`MemoryStreams.truncate` drops the + oldest ones, which stands in for a store's retention in tests. +- A standalone stream lives here with its policy and a sealed flag. + ``retention``, ``max_records`` and ``max_bytes`` are applied when a record + is appended, so a stream nobody writes to keeps records past their + retention. A read on it ends when it is sealed and the tail delivered, and + a read, ``latest`` or ``producer`` on a stream id that does not exist + raises :class:`temporalio.streams.StreamNotFoundError` at the call rather + than waiting for the stream to be created. +- The outside path encodes and decodes bodies through the client's data + converter, codec and external storage included, and fingerprints a retry + over the converted bytes first. The workflow half has no client, so a + workflow's own publish is stored as the payload converter produced it and + a workflow-side read hands records over as stored. + +The outside surface (producer identity, retry deduplication, positions, +supersession, cursors) is faithful, which is what the conformance tests lean +on. One list per topic; a topic is written by the workflow and by outside +producers alike and read from either side. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Generic, TypeVar + +from google.protobuf.message import DecodeError + +import temporalio.converter +from temporalio import workflow +from temporalio.client import ActivityExecutionStatus, Client, WorkflowExecutionStatus +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams._body import content_fingerprint, decode_body, encode_body +from temporalio.streams._errors import ( + StreamClosedError, + StreamCursorError, + StreamNotFoundError, + StreamProducerError, +) +from temporalio.streams._ids import topic_key +from temporalio.streams._provider import ReadSource, WriteSink +from temporalio.streams._record import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamRecord, + check_read_start, +) +from temporalio.streams._ref import StreamRef +from temporalio.streams._topic import StreamTopic, resolve_topic +from temporalio.streams._wire import ( + RecordDecoder, + WireRecord, + cursor_position, + mint_cursor, + producer_identity, + to_wire, +) +from temporalio.streams.providers import ProviderPlugin + +__all__ = ["MemoryProducer", "MemoryStreamHandle", "MemoryStreams"] + +_PROVIDER = "memory" + +T = TypeVar("T") + +logger = logging.getLogger(__name__) + + +def _wake(future: asyncio.Future[None]) -> None: + if not future.done(): + future.set_result(None) + + +@dataclass(frozen=True) +class _Policy: + """What a standalone stream retains, applied as records are appended.""" + + retention: timedelta | None = None + max_records: int | None = None + max_bytes: int | None = None + + def __post_init__(self) -> None: + if self.retention is not None and self.retention <= timedelta(0): + raise ValueError("retention must be positive") + if self.max_records is not None and self.max_records <= 0: + raise ValueError("max_records must be positive") + if self.max_bytes is not None and self.max_bytes <= 0: + raise ValueError("max_bytes must be positive") + + +class _Standalone: + """One standalone stream: its policy, its seal and its topics.""" + + def __init__(self, policy: _Policy) -> None: + self.policy = policy + self.sealed = False + self.topics: dict[str, _Topic] = {} + + +class _Topic: + """One topic's records, and the waiters parked on its tail.""" + + def __init__(self, policy: _Policy | None = None, *, sealed: bool = False) -> None: + self.policy = policy + self.sealed = sealed + # The retained records, the first of which sits at offset ``base``. + # Offsets are never reused, so a cursor keeps naming the same record + # after truncation drops the ones before it. + self.base = 0 + self.records: list[bytes] = [] + # When each retained record landed, for a retention policy. + self.stamps: list[float] = [] + # Dedupe identity is (producer#attempt, first sequence of the append), + # the same pair the storage providers use, mapped to where the batch + # landed and a digest of what it held, so a repeat answers with the + # original position and a divergent one is told apart from it. + self.seen: dict[tuple[str, int], tuple[int, int, bytes]] = {} + # Each waiter is parked with the loop it belongs to. A workflow's + # publish runs on the workflow thread, and waking a foreign loop's + # future from there needs call_soon_threadsafe or the loop stays + # blocked in select until unrelated I/O happens to wake it. + self._waiters: list[tuple[asyncio.AbstractEventLoop, asyncio.Future[None]]] = [] + + def append( + self, + wires: list[WireRecord], + *, + writer: str | None = None, + sequence: int = 0, + content: bytes | None = None, + ) -> tuple[int, int]: + """Store ``wires`` and return where they landed as ``(first offset, count)``. + + With a ``writer``, a repeat of ``(writer, sequence)`` carrying the same + content stores nothing and returns where the original landed. + ``content`` is the fingerprint the repeat is matched by; a producer + takes it over the records before their bodies are encoded, and + without one it is taken over ``wires`` as they are. + + Raises: + StreamProducerError: ``(writer, sequence)`` is held with different + content. + StreamClosedError: The stream was sealed. + """ + if self.sealed: + raise StreamClosedError("the stream is closed and takes no more records") + key = (writer or "", sequence) + bodies = [wire.SerializeToString(deterministic=True) for wire in wires] + if content is None: + content = content_fingerprint(wires) + if writer is not None: + held = self.seen.get(key) + if held is not None: + first, count, seen_content = held + if seen_content != content: + raise StreamProducerError( + f"producer sequence {sequence} already used with different " + f"content by {writer!r}" + ) + return first, count + first = self.head + now = time.time() + self.records.extend(bodies) + self.stamps.extend([now] * len(bodies)) + if writer is not None: + self.seen[key] = (first, len(wires), content) + self._apply_policy(now) + self._wake_waiters() + return first, len(wires) + + def _wake_waiters(self) -> None: + waiters, self._waiters = self._waiters, [] + for loop, future in waiters: + loop.call_soon_threadsafe(_wake, future) + + def _apply_policy(self, now: float) -> None: + policy = self.policy + if policy is None: + return + drop = 0 + if policy.max_records is not None: + drop = max(drop, len(self.records) - policy.max_records) + if policy.max_bytes is not None: + held = sum(len(record) for record in self.records) + while drop < len(self.records) and held > policy.max_bytes: + held -= len(self.records[drop]) + drop += 1 + if policy.retention is not None: + floor = now - policy.retention.total_seconds() + while drop < len(self.records) and self.stamps[drop] < floor: + drop += 1 + if drop: + self._drop(drop) + + def seal(self) -> None: + """Take no more records, and let parked readers see the end.""" + self.sealed = True + self._wake_waiters() + + @property + def head(self) -> int: + """The offset the next record lands at.""" + return self.base + len(self.records) + + def at(self, offset: int) -> bytes: + """The retained record at ``offset``.""" + return self.records[offset - self.base] + + def truncate(self, keep: int) -> None: + """Drop all but the newest ``keep`` records.""" + self._drop(max(0, len(self.records) - keep)) + + def _drop(self, count: int) -> None: + self.base += count + del self.records[:count] + del self.stamps[:count] + + async def wait_past(self, offset: int, timeout: float | None) -> None: + """Wait until a record exists at ``offset``, or ``timeout`` passes.""" + if self.head > offset: + return + loop = asyncio.get_running_loop() + future: asyncio.Future[None] = loop.create_future() + self._waiters.append((loop, future)) + try: + await asyncio.wait_for(future, timeout) + except asyncio.TimeoutError: + pass + finally: + # Dropped on every exit, cancellation included, so a reader that + # aclose()s while parked here leaves nothing behind on the topic. + self._waiters = [w for w in self._waiters if w[1] is not future] + + +def _parse(cursor: Cursor, raw: bytes, warn: Any) -> WireRecord | None: + try: + return WireRecord.FromString(raw) + except DecodeError as error: + # Same answer as an undecodable body: skip and say so, so one bad + # record cannot pin a reader. + warn("skipping stream record at %s: %s", cursor, error) + return None + + +class _MemReadSource: + """Workflow-side read that wakes by polling a timer. + + A real provider wakes the workflow by delivering; polling is the price of + having no delivery path, and it is why this provider is for tests. + """ + + def __init__(self, store: _Topic, start: int, poll: timedelta) -> None: + self._store = store + self._offset = start + self._poll = poll + self._closed = False + + async def next_batch(self) -> list[tuple[Cursor, WireRecord]]: + while not self._closed: + head = self._store.head + if head > self._offset: + batch: list[tuple[Cursor, WireRecord]] = [] + for offset in range(max(self._offset, self._store.base), head): + cursor = mint_cursor(_PROVIDER, str(offset)) + wire = _parse( + cursor, self._store.at(offset), workflow.logger.warning + ) + if wire is not None: + batch.append((cursor, wire)) + self._offset = head + if batch: + return batch + continue + await workflow.sleep(self._poll) + raise StopAsyncIteration + + def close(self) -> None: + self._closed = True + + +class _MemWriteSink: + def __init__(self, store: _Topic) -> None: + self._store = store + + def publish(self, record: WireRecord) -> None: + # Visible at once rather than at task acceptance: the documented gap + # between this provider and rule 1. + self._store.append([record]) + + +class _MemoryWorkflowProvider: + """The workflow half. Nothing to install and nothing to release.""" + + def __init__(self, streams: MemoryStreams) -> None: + self._streams = streams + + def open_reader( + self, topic: str, *, after: Cursor, last: int | None = None + ) -> ReadSource: + check_read_start(after, last) + store = self._streams._topic(workflow.info().workflow_id, topic) + start = self._streams._start(store, after, last) + return _MemReadSource(store, start, self._streams._poll) + + def open_writer(self, topic: str) -> WriteSink: + return _MemWriteSink(self._streams._topic(workflow.info().workflow_id, topic)) + + def on_workflow_start(self) -> None: + pass + + async def on_workflow_finish(self) -> None: + pass + + +class MemoryProducer(Generic[T]): + """The outside producer, faithful to the contract.""" + + def __init__( + self, + store: _Topic, + converter: temporalio.converter.DataConverter, + topic: str, + producer_id: str, + attempt: int, + ) -> None: + """Bind this producer to ``topic``'s ``store``.""" + self._store = store + self._converter = converter + self._topic = topic + self._producer_id = producer_id + self._attempt = attempt + # One-based, because zero on the wire says the producer does not + # number its records and this one does. + self._sequence = 1 + self._last = BEGINNING + + @property + def producer_id(self) -> str: + """Who this producer writes as.""" + return self._producer_id + + @property + def attempt(self) -> int: + """The generation this producer is writing.""" + return self._attempt + + @property + def _writer(self) -> str: + return ( + f"{self._producer_id}#{self._attempt}" + if self._attempt + else self._producer_id + ) + + async def append(self, *values: T) -> Cursor: + """Append ``values`` and return the cursor of the last record as stored. + + A repeat of the same content returns where the original landed; an + empty call returns the position of this producer's last record. + + Raises: + StreamProducerError: This sequence is held with different content. + """ + if not values: + return self._last + return await self._write( + [ + to_wire( + self._converter.payload_converter, + topic=self._topic, + kind=RecordKind.DATA, + value=value, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence + index, + ) + for index, value in enumerate(values) + ] + ) + + async def finish(self) -> None: + """Write ``FINISH`` for this producer on this topic.""" + await self._write( + [ + to_wire( + self._converter.payload_converter, + topic=self._topic, + kind=RecordKind.FINISH, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence, + ) + ] + ) + + async def _write(self, wires: list[WireRecord]) -> Cursor: + # The fingerprint comes first, over the converted records, so a codec + # that encrypts with a fresh nonce cannot make a retry look divergent. + content = content_fingerprint(wires) + for wire in wires: + await encode_body(self._converter, wire) + first, count = self._store.append( + wires, writer=self._writer, sequence=self._sequence, content=content + ) + self._sequence += len(wires) + self._last = mint_cursor(_PROVIDER, str(first + count - 1)) + return self._last + + +class MemoryStreamHandle: + """One owner's stream from outside, with the shared reader rules. + + The owner is ``workflow_id``'s workflow, or with ``activity_id`` an + activity: a standalone one without ``workflow_id``, or one that workflow + scheduled. With ``stream_id`` the handle is on a standalone stream, which + has no owner. + """ + + def __init__( + self, + streams: MemoryStreams, + client: Client | None, + workflow_id: str | None, + run_id: str | None, + activity_id: str | None = None, + stream_id: str | None = None, + ) -> None: + """Address the owner's topics in ``streams``.""" + self._streams = streams + self._client = client + self._workflow_id = workflow_id + self._run_id = run_id + self._activity_id = activity_id + self._stream_id = stream_id + self._seen_pending = False + self._converter = ( + client.data_converter + if client is not None + else temporalio.converter.DataConverter.default + ) + + def read( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor = BEGINNING, + last: int | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield records on ``topic`` from where the read starts until the owner closes. + + ``END`` and ``last=`` are resolved by this call, against what the + topic holds when it is made. On a standalone stream the read ends + once the stream is sealed and the tail delivered. + """ + check_read_start(after, last) + name, result_type = resolve_topic(topic, result_type) + store = self._store(name) + # Parsed here so a foreign cursor fails this call, not the first + # iteration of the generator. + start = self._streams._start(store, after, last) + # The decoder positions a synthesized record at the one before it, so + # it is told the position before the first record this read yields. + previous = mint_cursor(_PROVIDER, str(start - 1)) if start else BEGINNING + return self._read(store, start, previous, result_type) + + async def _read( + self, + store: _Topic, + offset: int, + after: Cursor, + result_type: type | None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + decoder = RecordDecoder( + self._converter.payload_converter, + result_type, + after=after, + warn=logger.warning, + ) + closed = False + while True: + while offset < store.head: + if offset < store.base: + raise StreamCursorError( + f"offset {offset} was truncated while this read was behind; " + f"the topic now starts at {store.base}" + ) + cursor = mint_cursor(_PROVIDER, str(offset)) + wire = _parse(cursor, store.at(offset), logger.warning) + offset += 1 + if wire is None: + continue + await decode_body(self._converter, wire) + for record in decoder.decode(cursor, wire): + yield record + if closed: + return + # One more pass after learning the workflow closed, so a record + # that landed between the scan and the describe is not lost. + closed = await self._closed() + if not closed: + await store.wait_past( + offset, + None + if self._client is None + else self._streams._poll.total_seconds(), + ) + + def _store(self, topic: str) -> _Topic: + if self._stream_id is not None: + return self._streams._standalone_topic(self._stream_id, topic) + if self._activity_id is not None: + return self._streams._activity_topic( + self._workflow_id, self._activity_id, topic + ) + assert self._workflow_id is not None + return self._streams._topic(self._workflow_id, topic) + + async def _closed(self) -> bool: + if self._stream_id is not None: + return self._streams._standalone_stream(self._stream_id).sealed + if self._client is None: + return False + try: + if self._activity_id is not None and self._workflow_id is None: + activity = await self._client.get_activity_handle( + self._activity_id, run_id=self._run_id + ).describe() + return activity.status != ActivityExecutionStatus.RUNNING + assert self._workflow_id is not None + description = await self._client.get_workflow_handle( + self._workflow_id, run_id=self._run_id + ).describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + # A producer may write before the owner exists; there is + # nothing to follow yet, so keep waiting. + return False + raise + status = description.status + if self._activity_id is not None: + pending = any( + info.activity_id == self._activity_id + for info in description.raw_description.pending_activities + ) + if pending: + self._seen_pending = True + elif self._seen_pending: + return True + # The activity's streams are not the workflow's chain: a run that + # continued as new took its activities with it. + return status is not None and status != WorkflowExecutionStatus.RUNNING + if status is None or status == WorkflowExecutionStatus.RUNNING: + return False + # Following the chain, a run that continued as new is not the end: + # the next describe without a run id finds its successor. + return not ( + self._run_id is None and status == WorkflowExecutionStatus.CONTINUED_AS_NEW + ) + + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: + """The cursor of the newest record on ``topic``, for following from now.""" + name, _ = resolve_topic(topic) + head = self._store(name).head + return mint_cursor(_PROVIDER, str(head - 1)) if head else BEGINNING + + def producer( + self, + *, + topic: str | StreamTopic[Any] | None = None, + producer_id: str = "", + attempt: int = 0, + ) -> MemoryProducer[Any]: + """A producer on ``topic``; inside an activity its identity is the activity's.""" + name, _ = resolve_topic(topic) + store = self._store(name) + producer_id, attempt = producer_identity(producer_id, attempt) + return MemoryProducer(store, self._converter, name, producer_id, attempt) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A ref to ``topic`` of this owner's stream, pinned as this handle is.""" + if self._stream_id is not None: + return StreamRef.for_standalone(self._stream_id, topic=topic) + if self._activity_id is not None: + return StreamRef.for_activity( + self._activity_id, + workflow_id=self._workflow_id, + run_id=self._run_id, + topic=topic, + ) + assert self._workflow_id is not None + return StreamRef.for_workflow( + self._workflow_id, run_id=self._run_id, topic=topic + ) + + async def close(self) -> None: + """Seal a standalone stream. An owned stream ends with its owner, not by a caller.""" + if self._stream_id is None: + raise ValueError( + "only a standalone stream can be closed; this handle is on an owned " + "stream, which ends when its workflow or activity does" + ) + self._streams._seal(self._stream_id) + + +class MemoryStreams(ProviderPlugin): + """The in-memory provider, one list per topic. + + Construct one and pass the same instance to the worker and to the code + that opens handles; two instances share nothing. + """ + + def __init__( + self, *, poll_interval: timedelta = timedelta(milliseconds=100) + ) -> None: + """Create an empty provider. + + Args: + poll_interval: How often a workflow-side reader with nothing to + read checks again, and how often an outside reader asks + whether the workflow closed. + """ + self._poll = poll_interval + self._topics: dict[str, _Topic] = {} + self._activity_topics: dict[tuple[str | None, str, str], _Topic] = {} + self._standalone: dict[str, _Standalone] = {} + + def reset(self) -> None: + """Drop every topic and every standalone stream. For tests.""" + self._topics.clear() + self._activity_topics.clear() + self._standalone.clear() + + def truncate(self, workflow_id: str, topic: str, *, keep: int) -> None: + """Drop all but the newest ``keep`` records of a topic. For tests. + + Stands in for a store's retention: offsets are kept, so a cursor from + before still names its record, and a read from ``BEGINNING`` starts + at the oldest one left. + """ + self._topic(workflow_id, topic).truncate(keep) + + def workflow_provider(self) -> _MemoryWorkflowProvider: + """The workflow half, over this provider's topics.""" + return _MemoryWorkflowProvider(self) + + def get_stream_handle( + self, client: Client | None, workflow_id: str, *, run_id: str | None = None + ) -> MemoryStreamHandle: + """A handle on ``workflow_id``'s topics. + + ``client`` may be ``None`` here, unlike on a storage provider; then + the handle cannot see the workflow close and a read waits until the + caller closes it. + """ + return MemoryStreamHandle(self, client, workflow_id, run_id) + + def get_activity_stream_handle( + self, + client: Client | None, + activity_id: str, + *, + workflow_id: str | None = None, + run_id: str | None = None, + ) -> MemoryStreamHandle: + """A handle on the topics ``activity_id`` owns, apart from any workflow's. + + As on :meth:`get_stream_handle`, ``client`` may be ``None``, and then + a read waits until the caller closes it. + """ + return MemoryStreamHandle(self, client, workflow_id, run_id, activity_id) + + async def create_standalone_stream( + self, + client: Client | None, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> MemoryStreamHandle: + """Create the standalone stream ``stream_id``, or find it with the same policy. + + The policy is applied on every append to any of the stream's topics. + ``client`` may be ``None``, as on the other handles. + + Raises: + ValueError: ``stream_id`` is empty, a bound is not positive, or + the stream exists with a different policy. + """ + if not stream_id: + raise ValueError("stream_id must not be empty") + policy = _Policy(retention, max_records, max_bytes) + existing = self._standalone.get(stream_id) + if existing is None: + self._standalone[stream_id] = _Standalone(policy) + elif existing.policy != policy: + raise ValueError( + f"standalone stream {stream_id!r} exists with policy " + f"{existing.policy}, not {policy}" + ) + return MemoryStreamHandle(self, client, None, None, stream_id=stream_id) + + def get_standalone_stream_handle( + self, client: Client | None, stream_id: str + ) -> MemoryStreamHandle: + """A handle on the standalone stream ``stream_id``. + + Nothing is checked here: a ``read``, ``latest`` or ``producer`` on a + stream that was never created raises + :class:`temporalio.streams.StreamNotFoundError` at the call. + """ + return MemoryStreamHandle(self, client, None, None, stream_id=stream_id) + + async def close(self) -> None: + """Nothing to release: the provider holds no connection.""" + + def _topic(self, workflow_id: str, topic: str) -> _Topic: + if not topic: + raise ValueError("topic must not be empty") + key = topic_key(workflow_id, topic) + found = self._topics.get(key) + if found is None: + found = self._topics[key] = _Topic() + return found + + def _activity_topic( + self, workflow_id: str | None, activity_id: str, topic: str + ) -> _Topic: + # Kept apart from the workflow topics, so no workflow id can name an + # activity's stream. + if not topic: + raise ValueError("topic must not be empty") + key = (workflow_id, activity_id, topic) + found = self._activity_topics.get(key) + if found is None: + found = self._activity_topics[key] = _Topic() + return found + + def _standalone_stream(self, stream_id: str) -> _Standalone: + stream = self._standalone.get(stream_id) + if stream is None: + raise StreamNotFoundError( + f"standalone stream {stream_id!r} does not exist; create it with " + "client.create_stream" + ) + return stream + + def _standalone_topic(self, stream_id: str, topic: str) -> _Topic: + stream = self._standalone_stream(stream_id) + if not topic: + raise ValueError("topic must not be empty") + found = stream.topics.get(topic) + if found is None: + found = stream.topics[topic] = _Topic(stream.policy, sealed=stream.sealed) + return found + + def _seal(self, stream_id: str) -> None: + stream = self._standalone_stream(stream_id) + stream.sealed = True + for store in stream.topics.values(): + store.seal() + + def _start(self, store: _Topic, after: Cursor, last: int | None) -> int: + """The offset a read starts at, resolved against what ``store`` holds now.""" + if last is not None: + return max(store.base, store.head - last) + if after == END: + return store.head + position = cursor_position(after, provider=_PROVIDER) + if position is None: + return store.base + try: + start = int(position) + 1 + except ValueError: + raise StreamCursorError( + f"cursor {after.token!r} does not name a position on the memory provider" + ) from None + if start < store.base: + raise StreamCursorError( + f"cursor {after.token!r} names a record no longer retained; the " + f"topic starts at offset {store.base}" + ) + return start diff --git a/temporalio/streams/providers/native.py b/temporalio/streams/providers/native.py new file mode 100644 index 0000000000..8779c34651 --- /dev/null +++ b/temporalio/streams/providers/native.py @@ -0,0 +1,1068 @@ +"""The server-side (native) provider. + +Streams live on the Temporal server, beside the workflow that owns them. A +topic is one owned stream named after the topic, created by whoever touches it +first: the workflow publishes to it with a command the server applies in the +transaction that accepts the Workflow Task, subscribes to it by name and reads +the ranges the server delivers on its Workflow Tasks; outside code appends and +reads through the stream service, and the workflow's records and an outside +producer's land in one log in the order the server accepted them. The default +topic, :data:`temporalio.streams.DEFAULT_TOPIC`, is the server's default +stream: the server resolves an unnamed stream to that same name, so the +provider sends the name explicitly and a record's topic and its stream's name +never differ. + +An activity owns topics of its own, apart from its workflow's: a standalone +activity is its own owner on the server, and an activity a workflow scheduled +is addressed through that workflow. They are one stream per activity +execution, so a retry writes to the same stream, and the server ends them when +the activity reaches a terminal status. + +A cursor names the run as well as the offset, because an owned stream belongs +to one run and a successor's starts over at zero. A handle without a run id +reads run after run, learning from the poll that a run's stream is closed and +from the run's close event who came next; with a run id it is pinned. A run +that was reset is followed too: its close event does not name the run reset +from it, describe does, and that run's streams continue the base run's offset +space where it inherited a subscription, so the read resumes at the floor the +stream reports rather than at zero. + +Prototype support for AI-198. It needs a server built from that branch and +reaches the stream service on a channel of its own, opened with the client's +connection settings (target, TLS, API key, headers, retries), because sdk-core +does not know the service yet. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import AsyncGenerator, Callable +from datetime import timedelta +from typing import Any, Generic, TypeVar + +from temporalio import workflow +from temporalio.api.common.v1 import Payload +from temporalio.api.stream.v1 import StreamStartPosition +from temporalio.client import Client, WorkflowHistoryEventFilterType +from temporalio.client_stream import ( + Appended, + Page, + SharedKey, + StreamClient, + WorkflowStreamHandle, + close_shared_clients, + shared_client, + shared_key, +) +from temporalio.client_stream import StreamHandle as ServiceStreamHandle +from temporalio.converter import ( + DataConverter, + StorageDriverStoreContext, + StorageDriverWorkflowInfo, +) +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams._body import ( + CONTENT_HASH_KEY, + content_hash, + decode_body, + encode_body, +) +from temporalio.streams._errors import StreamCursorError, StreamNotFoundError +from temporalio.streams._provider import ReadSource, StreamHandle, WriteSink +from temporalio.streams._record import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamRecord, + check_read_start, +) +from temporalio.streams._ref import StreamRef, open_ref +from temporalio.streams._topic import StreamTopic, resolve_topic +from temporalio.streams._wire import ( + RecordDecoder, + WireRecord, + cursor_position, + mint_cursor, + producer_identity, + to_wire, +) +from temporalio.streams.providers import ProviderPlugin + +__all__ = [ + "NativeActivityStreamHandle", + "NativeProducer", + "NativeStandaloneStreamHandle", + "NativeStreamHandle", + "NativeStreams", +] + +T = TypeVar("T") + +_PROVIDER = "native" + +# Seconds between polls on a standalone stream that does not exist yet, when +# the server answers without parking. +_CREATE_WAIT_PACE = 1.0 + +logger = logging.getLogger(__name__) + + +def _require_topic(topic: str) -> None: + if not topic: + raise ValueError("topic must not be empty") + + +def _cursor(run_id: str, offset: int) -> Cursor: + # A standalone stream has no run; its id takes the run's place. + return mint_cursor(_PROVIDER, f"{run_id}:{offset}") + + +def _position(after: Cursor) -> tuple[str, int] | None: + """The ``(run id, offset)`` a cursor of this provider names, or ``None`` for BEGINNING.""" + token = cursor_position(after, provider=_PROVIDER) + if token is None: + return None + run_id, _, offset = token.rpartition(":") + try: + if not run_id: + raise ValueError + return run_id, int(offset) + except ValueError: + raise StreamCursorError( + f"cursor {after.token!r} does not name a run and an offset on the " + "native provider" + ) from None + + +def _fingerprint(record: WireRecord) -> WireRecord: + """Stamp ``record`` with the hash of its body as converted, before the worker's pass. + + The outside half gets the same stamp from :func:`encode_body`; a workflow's + own publish is encoded later, by the worker's payload pass, so the stamp + is taken here while the body is still what the converter produced. A + record without a body has nothing to compare. + """ + if not record.HasField("body"): + return record + record.metadata[CONTENT_HASH_KEY].CopyFrom( + Payload( + metadata={"encoding": b"binary/plain"}, + data=content_hash(record.body).encode(), + ) + ) + return record + + +class _NativeReadSource: + """One subscription of the running workflow, fed by delivered ranges.""" + + def __init__(self, stream_id: str, run_id: str) -> None: + self._stream_id = stream_id + self._run_id = run_id + self._closed = False + + async def next_batch(self) -> list[tuple[Cursor, WireRecord]]: + if self._closed: + raise StopAsyncIteration + delivered = await workflow._read_stream_records(self._stream_id) + if self._closed: + # Closed while this was parked; the buffer woke it with nothing. + raise StopAsyncIteration + return [(_cursor(self._run_id, item.offset), item.record) for item in delivered] + + def close(self) -> None: + """Stop reading, and stop keeping what the server keeps delivering. + + The server has no unsubscribe command, so ranges keep arriving on + every Workflow Task for the life of the run. What this ends is the + reading and the keeping: nothing further is held for this stream, so + a run that closes a reader early does not grow for the rest of its + life. The subscription itself, and the delivery it costs each task, + stay until the run ends. + """ + if self._closed: + return + self._closed = True + workflow._close_stream_records(self._stream_id) + + +class _NativeWriteSink: + def __init__(self, topic: str) -> None: + self._topic = topic + + def publish(self, record: WireRecord) -> None: + # Held by the runtime until the task completes, when the task's + # records on this topic become one command the server applies with + # the task: rule 1 through the server's own commit. The body is still + # plaintext here; the worker's payload pass runs after the task. + workflow._append_stream_records([_fingerprint(record)], stream_name=self._topic) + + +class _NativeWorkflowProvider: + """The workflow half: the server's commands and delivered ranges.""" + + def open_reader( + self, topic: str, *, after: Cursor, last: int | None = None + ) -> ReadSource: + check_read_start(after, last) + _require_topic(topic) + run_id = workflow.info().run_id + # The server resolves the position when it registers the subscription + # and records the offset on the subscribed event, so replay never + # resolves it again. + if last is not None: + start = StreamStartPosition(last_n=last) + elif after == END: + start = StreamStartPosition(tail=True) + elif after == BEGINNING: + start = StreamStartPosition(earliest=True) + else: + named = _position(after) + assert named is not None + if named[0] != run_id: + raise StreamCursorError( + f"cursor {after.token!r} names another run; a run's stream is its own" + ) + start = StreamStartPosition(offset=named[1] + 1) + workflow._subscribe_stream(topic, start=start) + return _NativeReadSource(topic, run_id) + + def open_writer(self, topic: str) -> WriteSink: + _require_topic(topic) + return _NativeWriteSink(topic) + + def on_workflow_start(self) -> None: + pass + + async def on_workflow_finish(self) -> None: + pass + + +class NativeProducer(Generic[T]): + """Appends to a topic from outside workflow code. + + Every append is visible as soon as the server accepts it. That is the + point for an activity streaming model output, and it is why an activity + carries its own identity: the retry of a failed attempt has no commit + boundary to sort it out afterwards. + """ + + def __init__( + self, + handle: WorkflowStreamHandle | _StandaloneTarget, + pin: Any, + converter: DataConverter, + store_target: Callable[[str], StorageDriverWorkflowInfo], + topic: str, + producer_id: str, + attempt: int, + ) -> None: + """Bind this producer to ``topic`` on the stream ``handle`` names. + + ``converter`` is the client's data converter, applied to every body + as the worker applies it to a workflow's own records, with the + plaintext hash stamped first. ``store_target`` names the execution an + offloaded body is stored under, given the run the producer pinned. + """ + self._handle = handle + self._pin = pin + self._converter = converter + self._store_target = store_target + self._bound: DataConverter | None = None + self._topic = topic + self._producer_id = producer_id + self._attempt = attempt + # One-based, because zero on the wire says the producer does not + # number its records and this one does. + self._sequence = 1 + self._last = BEGINNING + + @property + def producer_id(self) -> str: + """Who this producer writes as.""" + return self._producer_id + + @property + def attempt(self) -> int: + """The generation this producer is writing.""" + return self._attempt + + @property + def _writer(self) -> str: + # The server dedupes on this and the sequence. The attempt is part of + # it so a retried append is dropped while a new generation writing + # different words at the same sequence is not. + return ( + f"{self._producer_id}#{self._attempt}" + if self._attempt + else self._producer_id + ) + + async def append(self, *values: T) -> Cursor: + """Append ``values`` and return the cursor of the last record as stored. + + A repeat returns where the original landed, because the server + answers a deduplicated batch with the original offsets; an empty call + returns the position of this producer's last record. + """ + if not values: + return self._last + return await self._write( + [ + to_wire( + self._converter.payload_converter, + topic=self._topic, + kind=RecordKind.DATA, + value=value, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence + index, + ) + for index, value in enumerate(values) + ] + ) + + async def finish(self) -> None: + """Write ``FINISH`` for this producer on this topic.""" + await self._write( + [ + to_wire( + self._converter.payload_converter, + topic=self._topic, + kind=RecordKind.FINISH, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence, + ) + ] + ) + + async def _write(self, records: list[WireRecord]) -> Cursor: + # Pinned before the first write, so every cursor this producer hands + # out names the run its records landed in. + if not self._handle.owner_run_id: + self._handle.pin(await self._pin()) + if self._bound is None: + # An offloaded body is stored under the execution that owns the + # stream, as the worker stores a workflow's own. + self._bound = self._converter._with_store_context( + StorageDriverStoreContext( + target=self._store_target(self._handle.owner_run_id) + ) + ) + for record in records: + await encode_body(self._bound, record) + appended = await self._handle.append( + *records, producer_id=self._writer, sequence=self._sequence + ) + self._sequence += len(records) + self._last = _cursor(self._handle.owner_run_id, appended.next_offset - 1) + return self._last + + +class NativeStreamHandle: + """One workflow's topics from outside, over the stream service.""" + + def __init__( + self, + client: Client, + workflow_id: str, + run_id: str | None, + *, + opened: set[SharedKey] | None = None, + ) -> None: + """Address ``workflow_id``'s topics, pinned to ``run_id`` when one is given. + + ``opened`` is where this handle records the shared channel it used, so + the provider that made it closes that one and no other. + """ + self._client = client + self._workflow_id = workflow_id + self._run_id = run_id + self._opened = set() if opened is None else opened + self._data_converter = client.data_converter + self._converter = client.data_converter.payload_converter + self._streams: StreamClient | None = None + + def _service(self) -> StreamClient: + # Resolved on first use, because the shared channel belongs to the + # running loop and a handle may be made before there is one. + if self._streams is None: + self._streams = shared_client(self._client) + self._opened.add(shared_key(self._client)) + return self._streams + + def _stream(self, topic: str, run_id: str) -> WorkflowStreamHandle: + return self._service().workflow_stream( + self._workflow_id, topic, owner_run_id=run_id + ) + + def read( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor = BEGINNING, + last: int | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield records on ``topic`` from where the read starts until the chain, or the pinned run, closes. + + ``BEGINNING`` is the oldest record the chain's first retained run + still holds. ``END`` and ``last=`` start on the current run, or the + pinned one: an earlier run of a chain has ended and holds neither the + tail nor the newest records. The server resolves each on the first + poll, in the read that serves it. + + The chain is followed across continue-as-new and across a reset: a + run reset from a closed one is read next, from the floor its stream + reports. A handle pinned to a run that was reset ends with that run. + """ + check_read_start(after, last) + topic, result_type = resolve_topic(topic, result_type) + # Parsed here so a foreign cursor fails this call, not the first + # iteration of the generator. + named = None if after == END else _position(after) + if named is not None and self._run_id is not None and named[0] != self._run_id: + raise StreamCursorError( + f"cursor {after.token!r} names another run than this handle is pinned to" + ) + start: StreamStartPosition | None = None + if last is not None: + start = StreamStartPosition(last_n=last) + elif after == END: + start = StreamStartPosition(tail=True) + elif named is None: + start = StreamStartPosition(earliest=True) + return self._read(topic, named, start, after, result_type) + + async def _read( + self, + topic: str, + named: tuple[str, int] | None, + start: StreamStartPosition | None, + after: Cursor, + result_type: type | None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + decoder: RecordDecoder | None = None + offset = 0 + if named is not None: + run_id, offset = named[0], named[1] + 1 + elif start is not None and start.WhichOneof("position") != "earliest": + run_id = self._run_id or await self._current_run() + else: + run_id = self._run_id or await self._first_run() + while True: + stream = self._stream(topic, run_id) + while True: + page = await stream.poll(from_offset=offset, start=start) + if decoder is None: + decoder = RecordDecoder( + self._converter, + result_type, + after=self._previous(run_id, page, start, after), + warn=logger.warning, + ) + start = None + for entry in page.entries: + record = await decode_body(self._data_converter, entry.record) + for out in decoder.decode(_cursor(run_id, entry.offset), record): + yield out + offset = page.next_offset + # On a pinned stream the server reports the run's end as closed, + # and a closed stream is finished once its head is delivered. + if page.closed and offset >= page.head_offset: + break + if self._run_id is not None: + return + following = await self._successor(topic, run_id) + if following is None: + return + run_id, offset = following + + @staticmethod + def _previous( + run_id: str, page: Page, start: StreamStartPosition | None, after: Cursor + ) -> Cursor: + """The position before the first record a read yields. + + A synthesized record is positioned there. After ``END`` or ``last=`` + it is only known once the server resolved the start, from the first + page. It names a run so a chain-following resume stays on this one. + """ + if start is None or start.WhichOneof("position") == "earliest": + return after + first = page.entries[0].offset if page.entries else page.next_offset + return _cursor(run_id, first - 1) + + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: + """The cursor of the newest record on ``topic``, naming the run it was read from. + + An empty topic on the chain's first run is the beginning of the + stream; on a successor it is a position of its own, because + ``BEGINNING`` would send a chain-following read back to the first run. + """ + topic, _ = resolve_topic(topic) + run_id = self._run_id or await self._current_run() + try: + head = (await self._stream(topic, run_id).describe()).head_offset + except StreamNotFoundError: + # A topic nobody has written yet does not exist on the server, + # which is the same answer as an empty one. + head = 0 + if head > 0: + return _cursor(run_id, head - 1) + if self._run_id is None and await self._predecessor(run_id) is not None: + return _cursor(run_id, -1) + return BEGINNING + + def producer( + self, + *, + topic: str | StreamTopic[Any] | None = None, + producer_id: str = "", + attempt: int = 0, + ) -> NativeProducer[Any]: + """A producer on ``topic``; inside an activity its identity is the activity's.""" + topic, _ = resolve_topic(topic) + producer_id, attempt = producer_identity(producer_id, attempt) + return NativeProducer( + self._stream(topic, self._run_id or ""), + self._current_run, + self._data_converter, + self._store_target, + topic, + producer_id, + attempt, + ) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A ref to ``topic`` of this workflow's stream, pinned as this handle is.""" + return StreamRef.for_workflow( + self._workflow_id, run_id=self._run_id, topic=topic + ) + + async def close(self) -> None: + """Refuse: a workflow's stream ends with the workflow. + + Raises: + ValueError: Always; only a standalone stream is closed by hand. + """ + raise ValueError( + "a workflow's stream ends with its workflow; only a standalone stream " + "can be closed" + ) + + def _store_target(self, run_id: str) -> StorageDriverWorkflowInfo: + """The execution an offloaded body of this owner's stream is stored under.""" + return StorageDriverWorkflowInfo( + namespace=self._client.namespace, id=self._workflow_id, run_id=run_id + ) + + async def _current_run(self) -> str: + try: + description = await self._client.get_workflow_handle( + self._workflow_id + ).describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} was not found" + ) from error + raise + assert description.run_id is not None + return description.run_id + + async def _first_run(self) -> str: + """The oldest retained run of the chain, walking back from the latest.""" + run_id = await self._current_run() + while True: + previous = await self._predecessor(run_id) + if previous is None: + return run_id + run_id = previous + + async def _predecessor(self, run_id: str) -> str | None: + handle = self._client.get_workflow_handle(self._workflow_id, run_id=run_id) + try: + async for event in handle.fetch_history_events(page_size=1): + attributes = event.workflow_execution_started_event_attributes + if attributes.continued_execution_run_id: + return attributes.continued_execution_run_id + # A reset run's start event is the base run's, copied, and the + # original run id it carries is kept across resets, so it names + # the run the chain of resets began from. + original = attributes.original_execution_run_id + if original and original != run_id: + return original + return None + except RPCError as error: + if error.status != RPCStatusCode.NOT_FOUND: + raise + # The run's History is gone: the chain's retained part starts here. + return None + + async def _successor(self, topic: str, run_id: str) -> tuple[str, int] | None: + """The run that carries on after ``run_id``, and where its stream starts. + + A continue-as-new names its successor in the close event, and the + successor's streams start at zero. A reset does not: the base run is + closed with no word of the reset in its own History, and only describe + names the run reset from it. That run reads on streams of its own that + continue the base run's offset space where it inherited a subscription, + and such a stream refuses any offset below its floor, so the read + resumes at the floor the stream reports. + """ + handle = self._client.get_workflow_handle(self._workflow_id, run_id=run_id) + events = handle.fetch_history_events( + event_filter_type=WorkflowHistoryEventFilterType.CLOSE_EVENT + ) + closed = False + try: + async for event in events: + closed = True + if event.HasField( + "workflow_execution_continued_as_new_event_attributes" + ): + attributes = ( + event.workflow_execution_continued_as_new_event_attributes + ) + if not attributes.new_execution_run_id: + return None + return attributes.new_execution_run_id, 0 + except RPCError as error: + if error.status != RPCStatusCode.NOT_FOUND: + raise + return None + if not closed: + return None + reset_run = await self._reset_run(run_id) + if reset_run is None: + return None + return reset_run, await self._floor(topic, reset_run) + + async def _reset_run(self, run_id: str) -> str | None: + """The run ``run_id`` was reset into, which only describe reports.""" + handle = self._client.get_workflow_handle(self._workflow_id, run_id=run_id) + try: + description = await handle.describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + return None + raise + extended = description.raw_description.workflow_extended_info + return extended.reset_run_id or None + + async def _floor(self, topic: str, run_id: str) -> int: + """The first offset a run's stream holds. + + A stream the run inherited a subscription to exists from the reset on + and starts at the inherited offset. One the base run only published to + is created on the run's first publish, at zero, and does not exist + before that. + """ + try: + return (await self._stream(topic, run_id).describe()).base_offset + except StreamNotFoundError: + return 0 + + +class NativeActivityStreamHandle(NativeStreamHandle): + """The topics one activity owns, from outside, over the stream service. + + An activity's streams belong to one activity execution, not to a chain of + runs: a retry writes to the same stream and a read ends when the activity + reaches a terminal status. So the handle pins the execution on first use + and never follows a successor. A standalone activity is its own owner; an + activity a workflow scheduled is reached through that workflow's run. + """ + + def __init__( + self, + client: Client, + activity_id: str, + workflow_id: str | None, + run_id: str | None, + *, + opened: set[SharedKey] | None = None, + ) -> None: + """Address ``activity_id``'s topics, pinned to ``run_id`` when one is given.""" + super().__init__(client, workflow_id or "", run_id, opened=opened) + self._activity_id = activity_id + + def _stream(self, topic: str, run_id: str) -> WorkflowStreamHandle: + return self._service().activity_stream( + self._activity_id, topic, workflow_id=self._workflow_id, run_id=run_id + ) + + def _store_target(self, run_id: str) -> StorageDriverWorkflowInfo: + # A workflow's activity stores under that workflow, as the worker does + # for its activities; a standalone activity has no workflow to name. + if self._workflow_id: + return super()._store_target(run_id) + return StorageDriverWorkflowInfo(namespace=self._client.namespace) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A ref to ``topic`` of this activity's streams, pinned as this handle is.""" + return StreamRef.for_activity( + self._activity_id, + workflow_id=self._workflow_id or None, + run_id=self._run_id, + topic=topic, + ) + + async def close(self) -> None: + """Refuse: an activity's streams end with the activity. + + Raises: + ValueError: Always; only a standalone stream is closed by hand. + """ + raise ValueError( + "an activity's streams end with the activity; only a standalone stream " + "can be closed" + ) + + async def _current_run(self) -> str: + if self._workflow_id: + return await super()._current_run() + try: + description = await self._client.get_activity_handle( + self._activity_id + ).describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"activity {self._activity_id!r} was not found" + ) from error + raise + assert description.activity_run_id is not None + return description.activity_run_id + + async def _first_run(self) -> str: + return await self._current_run() + + async def _predecessor(self, run_id: str) -> str | None: + return None + + async def _successor(self, topic: str, run_id: str) -> tuple[str, int] | None: + return None + + +class _StandaloneTarget: + """A standalone stream as the producer addresses an owned one. + + The producer pins a run before its first write and names it in every + cursor it hands out. A standalone stream has no run to pin, so its id + stands in that place from the start and nothing is ever resolved. + """ + + def __init__(self, handle: ServiceStreamHandle, stream_id: str) -> None: + self._handle = handle + self.owner_run_id = stream_id + + def pin(self, run_id: str) -> None: + self.owner_run_id = run_id + + async def append( + self, *records: WireRecord, producer_id: str = "", sequence: int = 0 + ) -> Appended: + return await self._handle.append( + *records, producer_id=producer_id, sequence=sequence + ) + + +class NativeStandaloneStreamHandle: + """One standalone stream's topics from outside, over the stream service. + + A standalone stream has an id of its own and no owner, so there is no + chain to follow and no run to pin; a cursor names the stream id where an + owned stream's names a run, and a cursor from another stream is refused. + Its topics share one log, so a topic read is the server's filter over it. + + A read on an id nobody has created yet parks on the server until the + stream appears, so a reader can attach before the producer's first write. + ``latest`` and a producer's append on such an id raise + :class:`temporalio.streams.StreamNotFoundError`: they have nothing to wait + on. The policy is the server's lifecycle: ``retention`` is the age past + which an open stream's records are reclaimed and how long a closed one + stays readable, ``max_records`` drops the oldest records as new ones land, + and ``max_bytes`` refuses an append that would take the held bytes past it + rather than reclaiming anything. + """ + + def __init__( + self, client: Client, stream_id: str, *, opened: set[SharedKey] | None = None + ) -> None: + """Address the standalone stream ``stream_id``, which must exist or be created later.""" + self._client = client + self._stream_id = stream_id + self._opened = set() if opened is None else opened + self._data_converter = client.data_converter + self._converter = client.data_converter.payload_converter + self._streams: StreamClient | None = None + + @property + def stream_id(self) -> str: + """The id of the stream this handle is on.""" + return self._stream_id + + def _service(self) -> StreamClient: + if self._streams is None: + self._streams = shared_client(self._client) + self._opened.add(shared_key(self._client)) + return self._streams + + def _stream(self) -> ServiceStreamHandle: + return self._service().get(self._stream_id) + + def read( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor = BEGINNING, + last: int | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield records on ``topic`` from where the read starts until the stream is closed and drained. + + The server resolves ``BEGINNING``, ``END`` and ``last=`` on the first + poll. On an id that does not exist yet the poll parks until the stream + is created, so the read is open before the first write. + """ + check_read_start(after, last) + topic, result_type = resolve_topic(topic, result_type) + named = None if after == END else _position(after) + if named is not None and named[0] != self._stream_id: + raise StreamCursorError( + f"cursor {after.token!r} names another stream than {self._stream_id!r}" + ) + start: StreamStartPosition | None = None + if last is not None: + start = StreamStartPosition(last_n=last) + elif after == END: + start = StreamStartPosition(tail=True) + elif named is None: + start = StreamStartPosition(earliest=True) + return self._read(topic, named, start, after, result_type) + + async def _read( + self, + topic: str, + named: tuple[str, int] | None, + start: StreamStartPosition | None, + after: Cursor, + result_type: type | None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + stream = self._stream() + decoder: RecordDecoder | None = None + offset = named[1] + 1 if named is not None else 0 + while True: + asked = time.monotonic() + try: + page = await stream.poll( + from_offset=offset, start=start, topics=[topic] + ) + except StreamNotFoundError: + # The server parks a poll on an id nobody has created for its + # wait budget and answers NOT_FOUND when that runs out. The + # stream may still be created, so the read keeps waiting; one + # that has delivered before is gone for good. A server that + # answers at once does not park, so the wait is paced here. + if decoder is not None: + raise + if time.monotonic() - asked < _CREATE_WAIT_PACE: + await asyncio.sleep(_CREATE_WAIT_PACE) + continue + if decoder is None: + decoder = RecordDecoder( + self._converter, + result_type, + after=NativeStreamHandle._previous( + self._stream_id, page, start, after + ), + warn=logger.warning, + ) + start = None + for entry in page.entries: + record = await decode_body(self._data_converter, entry.record) + for out in decoder.decode( + _cursor(self._stream_id, entry.offset), record + ): + yield out + offset = page.next_offset + if page.closed and offset >= page.head_offset: + return + + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: + """The cursor of the newest record on the stream, or ``BEGINNING`` when empty. + + Topics share the stream's offsets, so the newest record may be on + another topic; a read after this cursor still yields only what lands + on ``topic`` later. + + Raises: + StreamNotFoundError: The stream does not exist. + """ + resolve_topic(topic) + head = (await self._stream().describe()).head_offset + return _cursor(self._stream_id, head - 1) if head > 0 else BEGINNING + + def producer( + self, + *, + topic: str | StreamTopic[Any] | None = None, + producer_id: str = "", + attempt: int = 0, + ) -> NativeProducer[Any]: + """A producer on ``topic``; inside an activity its identity is the activity's.""" + topic, _ = resolve_topic(topic) + producer_id, attempt = producer_identity(producer_id, attempt) + return NativeProducer( + _StandaloneTarget(self._stream(), self._stream_id), + self._no_run, + self._data_converter, + self._store_target, + topic, + producer_id, + attempt, + ) + + async def _no_run(self) -> str: + return self._stream_id + + def _store_target(self, _run_id: str) -> StorageDriverWorkflowInfo: + # No workflow owns the stream, so an offloaded body has only the + # namespace to be stored under. + return StorageDriverWorkflowInfo(namespace=self._client.namespace) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A ref to ``topic`` of this stream.""" + return StreamRef.for_standalone(self._stream_id, topic=topic) + + async def close(self) -> None: + """Seal the stream: appends are refused from now on and the tail stays readable. Idempotent.""" + await self._stream().close() + + +class NativeStreams(ProviderPlugin): + """The server-side provider. + + Takes no options: the streams are on the server the client is already + connected to. Construct one, pass it to the worker as a plugin and open + handles from it anywhere else; :meth:`close` releases the channels this + provider opened to the stream service. + """ + + def __init__(self) -> None: + """Create the provider.""" + # What this provider's handles opened, so closing it leaves another + # provider's channels on the same loop alone. + self._opened: set[SharedKey] = set() + + def workflow_provider(self) -> _NativeWorkflowProvider: + """The workflow half, over the server's commands and delivered ranges.""" + return _NativeWorkflowProvider() + + def get_stream_handle( + self, + client: Client, + workflow_id: str | StreamRef, + *, + run_id: str | None = None, + ) -> StreamHandle: + """A handle on ``workflow_id``'s topics; without ``run_id`` it follows the chain. + + A :class:`temporalio.streams.StreamRef` in place of the id opens the + stream it names, whatever its owner kind, with the ref's topic as the + handle's default. + """ + if isinstance(workflow_id, StreamRef): + if run_id is not None: + raise ValueError("a StreamRef names the run itself; pass no run_id") + return open_ref(self, client, workflow_id) + return NativeStreamHandle(client, workflow_id, run_id, opened=self._opened) + + async def create_standalone_stream( + self, + client: Client, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> NativeStandaloneStreamHandle: + """Create the standalone stream ``stream_id`` on the server and return a handle on it. + + The three bounds are the server's lifecycle: ``retention`` reclaims + records older than it on an open stream and times a closed one's + deletion, ``max_records`` drops the oldest records as new ones land, + and ``max_bytes`` refuses an append that would take the held bytes past + it. A create of an id that exists with the same policy returns a handle + on it; with another policy the server refuses it as a ``ValueError``. + """ + if not stream_id: + raise ValueError("stream_id must not be empty") + if retention is not None and retention <= timedelta(0): + raise ValueError("retention must be positive") + if max_records is not None and max_records <= 0: + raise ValueError("max_records must be positive") + if max_bytes is not None and max_bytes <= 0: + raise ValueError("max_bytes must be positive") + streams = shared_client(client) + self._opened.add(shared_key(client)) + try: + await streams.create( + stream_id, + retention=retention, + max_items=max_records, + max_bytes=max_bytes, + ) + except RPCError as error: + # The id exists with this policy; a policy that differs arrives + # typed, as the ValueError the contract names. + if error.status != RPCStatusCode.ALREADY_EXISTS: + raise + return NativeStandaloneStreamHandle(client, stream_id, opened=self._opened) + + def get_standalone_stream_handle( + self, client: Client, stream_id: str + ) -> NativeStandaloneStreamHandle: + """A handle on the standalone stream ``stream_id``. + + Nothing here creates the stream. A read on an id that does not exist + yet parks on the server until it is created; ``latest`` and a + producer's append raise :class:`temporalio.streams.StreamNotFoundError`. + """ + if not stream_id: + raise ValueError("stream_id must not be empty") + return NativeStandaloneStreamHandle(client, stream_id, opened=self._opened) + + def get_activity_stream_handle( + self, + client: Client, + activity_id: str, + *, + workflow_id: str | None = None, + run_id: str | None = None, + ) -> NativeActivityStreamHandle: + """A handle on the topics ``activity_id`` owns, apart from any workflow's. + + Without ``workflow_id`` the activity is a standalone one and ``run_id`` + pins its run; with one it is that workflow's activity and ``run_id`` + pins the workflow's run. + """ + return NativeActivityStreamHandle( + client, activity_id, workflow_id, run_id, opened=self._opened + ) + + async def close(self) -> None: + """Close the channels this provider opened to the stream service. + + The application calls this; no worker or client owns the provider's + lifetime, because one provider serves the workers built from a client + and every handle opened outside them. + """ + await close_shared_clients(*self._opened) + self._opened.clear() diff --git a/temporalio/streams/providers/nexus.py b/temporalio/streams/providers/nexus.py new file mode 100644 index 0000000000..f4ad1f9cea --- /dev/null +++ b/temporalio/streams/providers/nexus.py @@ -0,0 +1,1863 @@ +"""The Nexus front: the outside surface behind one Temporal-authenticated endpoint. + +Two halves in one module. :class:`NexusStreams` is a provider that implements +the outside half only: it hands out handles that append and read through the +endpoint, and it has no workflow half, because a workflow's publishes and +reads ride the Workflow Task and cannot cross an RPC. +:class:`TemporalStreamsHandler` runs in a worker next to any storage provider +and serves two sync operations, ``append`` and ``read``, by fronting that +provider's own handles, so the store behind the endpoint is invisible to +callers and an operator switches it without touching them. + +The wire types and the service definition come from +``temporal_streams.nexusrpc.yaml``, so any language nexgen targets can be +handed the same contract. A record crosses as the serialized +``temporal.api.stream.v1.StreamRecord``, the same bytes every store keeps, so +a caller in another language decodes it with the api protos alone. What stays +hand-written here is what the generator cannot express yet: the handler's +dedupe and long-poll collect loop, and the caller's batching, cursor handling +and codec. + +Reads hand out batches and an append carries one batch per call, because a +Nexus operation per record costs too much for token streams. Record bodies +cross the handler untouched: it reads the store as raw payloads and forwards +them as they are, so a codec that changes the payload encoding survives the +hop and the handler's worker never needs the key. Supersession records are +not transported: the caller's reader re-synthesizes them from the attempts it +observes, which is the policy module's job on every provider. Cursors pass +through opaque, so the caller cannot tell which store produced them; a +foreign one is refused by the store behind the endpoint and reaches the +caller on the first read. + +Both operations address a stream by a reference, ``StreamRef`` on the wire: +the owner (a workflow, an activity, or a standalone stream), the ids that +name it, and the topic. The handler maps the reference onto the store's own +accessor for that owner and refuses an owner the store cannot host with +:class:`temporalio.streams.StreamUnsupportedError`, so a reference is good +behind any endpoint that serves the owner. The same reference is what an +operation of the application's own returns to hand a stream to its caller; +:meth:`NexusStreamHandle.ref` makes one and ``client.get_stream_handle(ref)`` +opens it. + +The handler keeps one parked read per stream reference and serves +consecutive calls from it, so an idle caller does not leave one abandoned +long poll on the store per call. A call whose token does not match the parked +position replaces the subscription, and an idle one is released after a +minute; that is the residual cost. + +Two stated prototype limits. Append deduplication lives in handler memory, by +batch index per producer attempt, so a handler that has no state for a +producer attempt refuses to continue it rather than starting a fresh delegate +whose numbering the store would drop as a repeat; the caller opens a new +attempt, which readers report as a supersession. And any caller the endpoint +admits may touch any owner's streams in the namespace; the endpoint's own +authorization is the boundary. + +A failure from the endpoint reaches the caller as the +:class:`temporalio.streams.StreamError` the store raised, when the handler +named one, and as :class:`temporalio.service.RPCError` otherwise, never as an +HTTP or urllib exception. + +The third piece is the consumer side of a Nexus operation. +:class:`StreamConsumerOperation`, built with :func:`stream_consumer_operation`, +is an asynchronous operation handler whose input is a ``StreamRef`` and whose +result is what a consume function folded the stream's records into. It does +not poll. On start it registers a callback listener on the stream's +notification channel, with a URL the hosting process serves and a header that +names the operation, and reads the stream from its start through the client's +provider, the front when the client carries one. The server posts every +notification on the channel to that URL, the handler reads from its cursor to +the head, and the close completes the operation through the caller's +completion callback. The delivery is what the server's channel library posts: +``POST`` to the listener's URL, the ``Notification`` as protobuf JSON with +``Content-Type: application/json``, the listener's own headers and the +channel's name in ``Temporal-Notification-Channel``. A process that hosts the +handler feeds each such request to :meth:`StreamConsumerOperation.deliver`; +``nexus_consumer_service`` in this package is the standalone one. +""" + +from __future__ import annotations + +import asyncio +import email.utils +import http.client +import inspect +import json +import logging +import time +import urllib.error +import urllib.request +import uuid +import weakref +from collections import OrderedDict +from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any, Generic, NoReturn, TypeVar, cast + +import nexusrpc +import nexusrpc.handler +from google.protobuf import json_format +from google.protobuf.message import DecodeError + +import temporalio.api.notification.v1 +import temporalio.client +import temporalio.common +import temporalio.converter +import temporalio.nexus +from temporalio.api.common.v1 import Payload +from temporalio.api.operatorservice.v1 import ListNexusEndpointsRequest +from temporalio.client import Callback, Client, ClientConfig +from temporalio.common import RawValue +from temporalio.service import ConnectConfig, RPCError, RPCStatusCode, ServiceClient +from temporalio.streams._errors import ( + StreamClosedError, + StreamCursorError, + StreamError, + StreamNotFoundError, + StreamProducerError, + StreamUnsupportedError, +) +from temporalio.streams._provider import StreamHandle, StreamProducer, StreamProvider +from temporalio.streams._record import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamRecord, + check_read_start, +) +from temporalio.streams._ref import StreamOwnerKind, StreamRef +from temporalio.streams._topic import StreamTopic, resolve_topic +from temporalio.streams._wire import ( + RecordDecoder, + WireRecord, + producer_identity, + to_wire, +) +from temporalio.streams.providers._nexus_generated import ( + AppendInput, + AppendOutput, + ReadInput, + ReadOutput, + RecordWire, + TemporalStreams, +) +from temporalio.streams.providers._nexus_generated import StreamRef as WireStreamRef +from temporalio.workflow import Notification + +__all__ = [ + "NOTIFICATION_CHANNEL_HEADER", + "STREAM_CONSUMER_TOKEN_HEADER", + "ConsumerState", + "Delivery", + "NexusProducer", + "NexusStreamHandle", + "NexusStreams", + "StreamConsumerOperation", + "TemporalStreamsHandler", + "WireStreamRef", + "stream_consumer_operation", +] + +T = TypeVar("T") +S = TypeVar("S") + +# The reference's identity on the handler, topic included, so one parked read +# and one producer state key on exactly what the wire names. +_StreamKey = tuple[str, str, str, str, str, str] +_ProducerKey = tuple[_StreamKey, str, int] + +_WORKFLOW_SIDE_ERROR = ( + "the nexus provider is an outside transport; a worker publishes and reads " + "through a storage provider, so give the worker one of those" +) + +# The contract leaves these unset, so the handler is the one place that says +# what an omitted read bound means. The wait is long because the handler +# shortens it to the request deadline anyway, and a shorter default only +# means more round trips on an idle stream. +_DEFAULT_MAX_RECORDS = 100 +_DEFAULT_READ_WAIT = timedelta(seconds=30) +_DEADLINE_MARGIN = timedelta(milliseconds=500) +_DEFAULT_MAX_PRODUCERS = 10_000 +_DEFAULT_SUBSCRIPTION_IDLE = timedelta(seconds=60) +_QUEUE_DEPTH = 1000 +_APPEND_TIMEOUT = timedelta(seconds=30) +_ROUND_TRIP_MARGIN = timedelta(seconds=5) +# What ReadInput accepts in temporal_streams.nexusrpc.yaml. Repeated here so +# a caller's mistake is refused where it was made rather than on the wire. +_MIN_READ_WAIT = timedelta(0) +_MAX_READ_WAIT = timedelta(milliseconds=60_000) +_MIN_MAX_RECORDS = 1 +_MAX_MAX_RECORDS = 1000 + +_definition = nexusrpc.get_service_definition(TemporalStreams) +if _definition is None: # pragma: no cover + raise RuntimeError("the generated stream service carries no nexus definition") +# Both halves read the wire names off the generated definition, so a rename in +# the contract cannot leave the caller posting to a path the handler no longer +# serves. +_SERVICE_NAME = _definition.name +_OPERATION_NAMES = { + operation.method_name: operation.name + for operation in _definition.operation_definitions.values() +} +_APPEND_OPERATION = _OPERATION_NAMES["append"] +_READ_OPERATION = _OPERATION_NAMES["read"] + +# The stream conditions that cross the endpoint under their own name, so the +# caller raises the class the store raised. +_STREAM_ERRORS: dict[str, type[StreamError]] = { + cls.__name__: cls + for cls in ( + StreamError, + StreamNotFoundError, + StreamCursorError, + StreamProducerError, + StreamClosedError, + StreamUnsupportedError, + ) +} +_HTTP_TO_RPC = { + 400: RPCStatusCode.INVALID_ARGUMENT, + 401: RPCStatusCode.UNAUTHENTICATED, + 403: RPCStatusCode.PERMISSION_DENIED, + 404: RPCStatusCode.NOT_FOUND, + 408: RPCStatusCode.DEADLINE_EXCEEDED, + 409: RPCStatusCode.ALREADY_EXISTS, + 412: RPCStatusCode.FAILED_PRECONDITION, + 429: RPCStatusCode.RESOURCE_EXHAUSTED, + 500: RPCStatusCode.INTERNAL, + 501: RPCStatusCode.UNIMPLEMENTED, + 503: RPCStatusCode.UNAVAILABLE, + 504: RPCStatusCode.DEADLINE_EXCEEDED, +} + +_OutputT = TypeVar("_OutputT") + +logger = logging.getLogger(__name__) + + +def _require_topic(topic: str) -> None: + if not topic: + raise ValueError("topic must not be empty") + + +def _check_ref(ref: WireStreamRef) -> None: + """Refuse a reference whose owner lacks the id that names it.""" + _require_topic(ref.topic) + if ref.kind == "workflow" and not ref.workflow_id: + raise ValueError("a stream reference with a workflow owner needs workflow_id") + if ref.kind == "activity" and not ref.activity_id: + raise ValueError("a stream reference with an activity owner needs activity_id") + if ref.kind == "standalone" and not ref.stream_id: + raise ValueError("a stream reference with a standalone owner needs stream_id") + + +def _stream_key(ref: WireStreamRef) -> _StreamKey: + return ( + ref.kind, + ref.workflow_id or "", + ref.run_id or "", + ref.activity_id or "", + ref.stream_id or "", + ref.topic, + ) + + +@dataclass(frozen=True) +class _Address: + """The owner a handle is bound to; a topic completes it into a reference.""" + + kind: StreamOwnerKind + workflow_id: str | None = None + run_id: str | None = None + activity_id: str | None = None + stream_id: str | None = None + + def wire(self, topic: str) -> WireStreamRef: + return WireStreamRef( + kind=self.kind, + workflow_id=self.workflow_id, + run_id=self.run_id, + activity_id=self.activity_id, + stream_id=self.stream_id, + topic=topic, + ) + + def ref(self, topic: str) -> StreamRef: + # The same members under the SDK's own name, so what an operation + # returns is what client.get_stream_handle(ref) opens. + return StreamRef( + self.kind, + topic, + workflow_id=self.workflow_id, + run_id=self.run_id, + activity_id=self.activity_id, + stream_id=self.stream_id, + ) + + +def _handler_error(error: StreamError) -> nexusrpc.HandlerError: + """Carry a stream condition across the endpoint under its own class name.""" + kind = ( + nexusrpc.HandlerErrorType.NOT_FOUND + if isinstance(error, StreamNotFoundError) + else nexusrpc.HandlerErrorType.BAD_REQUEST + ) + return nexusrpc.HandlerError( + f"{type(error).__name__}: {error}", type=kind, retryable_override=False + ) + + +def _bad_request(message: str) -> nexusrpc.HandlerError: + return nexusrpc.HandlerError( + message, type=nexusrpc.HandlerErrorType.BAD_REQUEST, retryable_override=False + ) + + +@dataclass +class _Subscription: + """One parked read on the store, shared by consecutive calls.""" + + records: asyncio.Queue[StreamRecord[Any]] + position: str + last_used: float + last_n: int | None = None + """The newest-N start this subscription was opened with, when it had no token.""" + pump: asyncio.Task[None] | None = None + failure: BaseException | None = None + done: bool = False + + +@dataclass +class _ProducerState: + delegate: StreamProducer + batch_index: int + sequence: int + last_cursor: str | None + finished: bool = False + + +@nexusrpc.handler.service_handler(service=TemporalStreams) +class TemporalStreamsHandler: + """Serves the stream endpoint by fronting one storage provider's handles. + + The provider is an explicit instance the application constructed, so a + caller and a handler can coexist in one process, and so an operator can + run handlers for a new store next to handlers for the old one. + """ + + def __init__( + self, + provider: StreamProvider, + client: Client | None, + *, + max_producers: int = _DEFAULT_MAX_PRODUCERS, + subscription_idle: timedelta = _DEFAULT_SUBSCRIPTION_IDLE, + ) -> None: + """Serve the endpoint out of ``provider``'s handles, opened with ``client``. + + ``max_producers`` bounds the dedupe state kept per producer attempt; + the oldest is evicted, and a producer evicted mid-life gets a clear + refusal on its next batch rather than a silent drop. + ``subscription_idle`` is how long a parked read outlives its last + caller. + """ + self._provider = provider + self._client = client + self._producers: OrderedDict[_ProducerKey, _ProducerState] = OrderedDict() + self._max_producers = max_producers + self._subscriptions: dict[_StreamKey, _Subscription] = {} + # Held weakly, and by the call that is using one for as long as it + # runs. A strong map would keep an entry per address ever read or + # appended to, and nothing would ever reach it again. + self._read_locks: weakref.WeakValueDictionary[_StreamKey, asyncio.Lock] = ( + weakref.WeakValueDictionary() + ) + self._append_locks: weakref.WeakValueDictionary[_ProducerKey, asyncio.Lock] = ( + weakref.WeakValueDictionary() + ) + self._subscription_idle = subscription_idle.total_seconds() + + def _stream(self, ref: WireStreamRef) -> StreamHandle: + """The store's handle for the owner ``ref`` names. + + Raises: + StreamUnsupportedError: The store has no accessor for that owner. + """ + # A storage provider wants a client; the memory provider, which the + # tests front, accepts none, so the cast only lies where it is unread. + client = cast(Client, self._client) + if ref.kind == "workflow": + return self._provider.get_stream_handle( + client, cast(str, ref.workflow_id), run_id=ref.run_id or None + ) + if ref.kind == "activity": + return self._provider.get_activity_stream_handle( + client, + cast(str, ref.activity_id), + workflow_id=ref.workflow_id or None, + run_id=ref.run_id or None, + ) + return self._provider.get_standalone_stream_handle( + client, cast(str, ref.stream_id) + ) + + @nexusrpc.handler.sync_operation + async def append( + self, _ctx: nexusrpc.handler.StartOperationContext, input: AppendInput + ) -> AppendOutput: + """Append on the caller's account, writing a repeated batch once. + + Raises: + nexusrpc.HandlerError: ``BAD_REQUEST`` when the batch index or + sequence does not continue the producer attempt, when the + attempt is one this handler has no state for, when a payload + is not a serialized ``Payload``, or when the store cannot host + the reference's owner; ``NOT_FOUND`` when the store has no + such owner. + """ + try: + return await self._append(input) + except StreamError as error: + raise _handler_error(error) from error + except ValueError as error: + raise _bad_request(str(error)) from error + + async def _append(self, input: AppendInput) -> AppendOutput: + _check_ref(input.stream) + key: _ProducerKey = ( + _stream_key(input.stream), + input.producer_id, + input.attempt, + ) + # The repeat check and the commit have an await between them, so two + # in-flight copies of one batch would both pass the check and both + # reach the store. The read path serialises per address for the same + # reason. + lock = self._append_locks.setdefault(key, asyncio.Lock()) + async with lock: + return await self._append_locked(input, key) + + async def _append_locked( + self, input: AppendInput, key: _ProducerKey + ) -> AppendOutput: + state = self._producers.get(key) + if state is None: + if input.batch_index != 1 or input.sequence != 0: + # A fresh delegate would restart its numbering at zero, and + # the store would drop that as a repeat of the first batch. + # Refusing here turns a silent loss into a failure the caller + # can act on by opening a new attempt. + raise StreamProducerError( + f"producer {input.producer_id!r} attempt {input.attempt} resumed " + f"at batch {input.batch_index} on a handler that has no state for " + "it; open a new attempt" + ) + delegate = self._stream(input.stream).producer( + topic=input.stream.topic, + producer_id=input.producer_id, + attempt=input.attempt, + ) + state = _ProducerState(delegate, 0, 0, None) + elif input.batch_index == state.batch_index: + # The last batch again: the store already holds it, and where it + # landed is the answer the original got. + self._producers.move_to_end(key) + return AppendOutput(cursor=state.last_cursor) + elif input.batch_index < state.batch_index: + raise StreamProducerError( + f"batch {input.batch_index} was written and is no longer the last " + f"for producer {input.producer_id!r} attempt {input.attempt}" + ) + elif input.batch_index != state.batch_index + 1: + raise StreamProducerError( + f"batch {input.batch_index} skips ahead of {state.batch_index} for " + f"producer {input.producer_id!r} attempt {input.attempt}" + ) + elif state.finished: + raise StreamProducerError( + f"producer {input.producer_id!r} attempt {input.attempt} already " + "finished this topic; open a new attempt" + ) + if input.sequence != state.sequence: + raise StreamProducerError( + f"sequence {input.sequence} does not continue at {state.sequence} for " + f"producer {input.producer_id!r} attempt {input.attempt}" + ) + payloads = self._payloads(input.payloads) + cursor: str | None = None + if payloads: + appended = await state.delegate.append(*(RawValue(p) for p in payloads)) + cursor = appended.token if appended is not None else None + if input.finish: + await state.delegate.finish() + # Recorded only once the store accepted the batch, so a failed append + # is not mistaken for a repeat when the caller retries it. A finish is + # recorded the same way rather than dropping the state: a lost + # response would otherwise leave the caller with a batch it can + # neither repeat nor abandon. + state.batch_index = input.batch_index + state.sequence += len(payloads) + state.last_cursor = cursor + state.finished = state.finished or bool(input.finish) + self._producers[key] = state + self._producers.move_to_end(key) + while len(self._producers) > self._max_producers: + self._producers.popitem(last=False) + return AppendOutput(cursor=cursor) + + @staticmethod + def _payloads(encoded: list[bytes] | None) -> list[Payload]: + payloads = [] + for index, raw in enumerate(encoded or []): + try: + payloads.append(Payload.FromString(raw)) + except DecodeError as error: + raise _bad_request( + f"payloads[{index}] is not a serialized Temporal Payload: {error}" + ) from error + return payloads + + @nexusrpc.handler.sync_operation + async def read( + self, ctx: nexusrpc.handler.StartOperationContext, input: ReadInput + ) -> ReadOutput: + """Answer with the records after the caller's token, or time out. + + Raises: + nexusrpc.HandlerError: ``BAD_REQUEST`` when the store refuses the + token or cannot host the reference's owner, ``NOT_FOUND`` when + it has no such owner. + """ + try: + return await self._read(ctx, input) + except StreamError as error: + raise _handler_error(error) from error + except ValueError as error: + raise _bad_request(str(error)) from error + + async def _read( + self, ctx: nexusrpc.handler.StartOperationContext, input: ReadInput + ) -> ReadOutput: + _check_ref(input.stream) + topic = input.stream.topic + stream = self._stream(input.stream) + if input.latest_only: + return ReadOutput(next_token=(await stream.latest(topic=topic)).token) + max_records = ( + _DEFAULT_MAX_RECORDS if input.max_records is None else input.max_records + ) + wait = ( + _DEFAULT_READ_WAIT.total_seconds() + if input.wait_ms is None + else input.wait_ms / 1000 + ) + if ctx.request_deadline is not None: + # The server answers the caller with a timeout at the deadline + # whatever this call does, so collecting past it only holds the + # slot for an answer nobody receives. + deadline = ctx.request_deadline + if deadline.tzinfo is None: + deadline = deadline.replace(tzinfo=timezone.utc) + left = deadline - datetime.now(timezone.utc) - _DEADLINE_MARGIN + wait = max(0.0, min(wait, left.total_seconds())) + after = input.after_token or "" + if after and input.last_n is not None: + raise ValueError( + "pass either after_token or last_n, not both: a token resumes a " + "read and last_n starts one" + ) + key = _stream_key(input.stream) + await self._expire_subscriptions() + lock = self._read_locks.setdefault(key, asyncio.Lock()) + async with lock: + subscription = self._subscriptions.get(key) + if ( + subscription is None + or subscription.position != after + or (not after and subscription.last_n != input.last_n) + ): + if subscription is not None: + await self._drop(key) + subscription = self._subscribe(key, stream, topic, after, input.last_n) + try: + records, next_token = await self._drain(subscription, max_records, wait) + except Exception: + # The pump failed. Keeping the subscription would answer the + # caller's retry from the same token with end of stream, so a + # transient read failure would read as the stream ending. + await self._drop(key) + raise + subscription.position = next_token + subscription.last_used = time.monotonic() + done = ( + subscription.done + and subscription.records.empty() + and subscription.failure is None + ) + if done: + # The store ended the read: the run or chain is closed and the + # tail has been handed over. Nothing more will arrive on it. + await self._drop(key) + return ReadOutput(records=records, next_token=next_token, done=done) + + def _subscribe( + self, + key: _StreamKey, + stream: StreamHandle, + topic: str, + after: str, + last_n: int | None, + ) -> _Subscription: + # Raw payloads: the handler forwards what the store holds without + # decoding it, so an encoding only the caller's codec understands + # passes through untouched. A foreign token is refused right here. + source = ( + stream.read(topic=topic, last=last_n, result_type=RawValue) + if last_n is not None + else stream.read( + topic=topic, + after=Cursor(after) if after else BEGINNING, + result_type=RawValue, + ) + ) + subscription = _Subscription( + records=asyncio.Queue(maxsize=_QUEUE_DEPTH), + position=after, + last_used=time.monotonic(), + last_n=last_n, + ) + + async def pump() -> None: + try: + async for record in source: + if record.kind is RecordKind.SUPERSEDED: + continue + await subscription.records.put(record) + except asyncio.CancelledError: + raise + except Exception as error: + subscription.failure = error + finally: + subscription.done = True + await source.aclose() + + subscription.pump = asyncio.create_task(pump()) + self._subscriptions[key] = subscription + return subscription + + async def _drain( + self, subscription: _Subscription, max_records: int, wait: float + ) -> tuple[list[RecordWire], str]: + records: list[RecordWire] = [] + next_token = subscription.position + queue = subscription.records + deadline = time.monotonic() + wait + while len(records) < max_records: + if queue.empty(): + remaining = deadline - time.monotonic() + if records or remaining <= 0: + break + if subscription.done: + if subscription.failure is None: + break + # Honour the wait even though nothing can arrive, so a + # caller polling a failed stream does not spin on the + # endpoint. + await asyncio.sleep(remaining) + break + try: + record = await asyncio.wait_for(queue.get(), remaining) + except asyncio.TimeoutError: + break + else: + record = queue.get_nowait() + records.append(self._wire(record)) + next_token = record.cursor.token + if not records and subscription.failure is not None: + # Left on the subscription rather than cleared: the caller is + # answered with the failure and the subscription is dropped, so + # there is nothing for a second reader of it to be misled by. + raise subscription.failure + return records, next_token + + @staticmethod + def _wire(record: StreamRecord[Any]) -> RecordWire: + # The record read as RawValue carries the stored body untouched; the + # default converter passes a RawValue through, so no encoding happens + # here. + wire = to_wire( + temporalio.converter.DataConverter.default.payload_converter, + topic=record.topic, + kind=record.kind, + value=record.value, + producer_id=record.producer_id, + attempt=record.attempt, + sequence=record.sequence, + ) + return RecordWire(token=record.cursor.token, record=wire.SerializeToString()) + + async def close(self) -> None: + """Release every parked read. + + Call it when the worker hosting this handler stops, so subscriptions + waiting on the store do not outlive it. + """ + for key in list(self._subscriptions): + await self._drop(key) + + async def _drop(self, key: _StreamKey) -> None: + subscription = self._subscriptions.pop(key, None) + if subscription is None or subscription.pump is None: + return + subscription.pump.cancel() + try: + await subscription.pump + except (asyncio.CancelledError, Exception): + pass + + async def _expire_subscriptions(self) -> None: + cutoff = time.monotonic() - self._subscription_idle + for key in [k for k, s in self._subscriptions.items() if s.last_used < cutoff]: + lock = self._read_locks.get(key) + if lock is not None and lock.locked(): + continue + await self._drop(key) + + +class _EndpointFailure(Exception): + """What the transport saw: the endpoint's answer, or the reason it gave none.""" + + def __init__(self, detail: str, status: int | None) -> None: + super().__init__(detail) + self.detail = detail + self.status = status + + +def _translate(failure: _EndpointFailure) -> Exception: + """The exception a caller raises for an endpoint failure. + + The handler names a stream condition by its class in the failure message, + so the same class is raised here; anything else is an ``RPCError`` with + the status the HTTP answer maps to, or ``UNAVAILABLE`` when there was none. + """ + message = failure.detail + try: + parsed = json.loads(message) + if isinstance(parsed, dict) and isinstance(parsed.get("message"), str): + message = parsed["message"] + except ValueError: + pass + if failure.status is not None: + for name, cls in _STREAM_ERRORS.items(): + prefix = f"{name}: " + if message.startswith(prefix): + return cls(message[len(prefix) :]) + status = _HTTP_TO_RPC.get(failure.status, RPCStatusCode.UNKNOWN) + else: + status = RPCStatusCode.UNAVAILABLE + return RPCError(message, status, b"") + + +def _post( + url: str, body: bytes, headers: Mapping[str, str], timeout: timedelta +) -> bytes: + timeout_ms = int(timeout.total_seconds() * 1000) + # Everything is inside the try, because urllib wraps only the request in + # URLError: a bad url raises from Request(), and a socket timeout or a + # dropped connection raises from getresponse() and read() as + # builtins.TimeoutError or an http.client exception. On 3.11 and later + # TimeoutError is asyncio.TimeoutError, so letting one out would be + # indistinguishable from the caller's own wait_for expiring. + try: + request = urllib.request.Request( + url, + data=body, + headers={ + "Content-Type": "application/json", + # Tells the server how long the handler may park, so it does + # not time the call out ahead of a wait the caller asked for. + "Request-Timeout": f"{timeout_ms}ms", + **headers, + }, + method="POST", + ) + with urllib.request.urlopen( + request, timeout=(timeout + _ROUND_TRIP_MARGIN).total_seconds() + ) as response: + return response.read() + except urllib.error.HTTPError as error: + raise _EndpointFailure( + error.read().decode(errors="replace"), error.code + ) from error + except urllib.error.URLError as error: + raise _EndpointFailure( + f"stream endpoint unreachable at {url}: {error.reason}", None + ) from error + except (TimeoutError, http.client.HTTPException, OSError, ValueError) as error: + raise _EndpointFailure( + f"stream endpoint at {url} did not answer: {error!r}", None + ) from error + + +class _Front: + """Everything a handle needs to reach one endpoint.""" + + def __init__(self, streams: NexusStreams, client: Client | None) -> None: + self._streams = streams + self._client = client + data_converter = streams._data_converter + if data_converter is None: + data_converter = ( + client.data_converter + if client is not None + else temporalio.converter.DataConverter.default + ) + self.converter = data_converter.payload_converter + self.codec = data_converter.payload_codec + self.read_wait = streams._read_wait + self.max_records = streams._max_records + + async def _base_url(self) -> str: + endpoint_id = await self._streams._endpoint_id(self._client) + return ( + f"{self._streams._http_address}/nexus/endpoints/{endpoint_id}" + f"/services/{_SERVICE_NAME}" + ) + + async def invoke( + self, + operation: str, + request: Any, + output: type[_OutputT], + *, + timeout: timedelta, + ) -> _OutputT: + # The contract types carry their own JSON encoding, so the raw caller + # and the worker serving the operation agree on the body without + # either of them spelling the fields out. That encoding is a transfer + # type hook, which only the internal converter applies. + contract = ( + temporalio.converter.DataConverter.default._get_internal_payload_converter() + ) + url = f"{await self._base_url()}/{operation}" + try: + raw = await asyncio.to_thread( + _post, + url, + contract.to_payloads([request])[0].data, + self._streams._headers, + timeout, + ) + except _EndpointFailure as failure: + raise _translate(failure) from failure + payload = Payload(metadata={"encoding": b"json/plain"}, data=raw or b"{}") + return contract.from_payloads([payload], [output])[0] + + +class NexusProducer(Generic[T]): + """Appends through the stream endpoint; the store behind it does the rest. + + Calls are serialized and the batch index is committed only once the + endpoint answered. A batch whose call raised stays pending and is sent + again under the same index, either when the caller retries the same + values or ahead of whatever the caller sends next, so an ambiguous + failure writes the batch once and loses nothing. + """ + + def __init__( + self, + front: _Front, + stream: WireStreamRef, + producer_id: str, + attempt: int, + ) -> None: + """Bind this producer to the stream ``stream`` names behind the endpoint.""" + self._front = front + self._stream = stream + self._producer_id = producer_id + self._attempt = attempt + self._batch_index = 0 + self._sequence = 0 + self._last: Cursor | None = BEGINNING + self._pending: tuple[AppendInput, tuple[bytes, ...], bool] | None = None + self._lock = asyncio.Lock() + + @property + def producer_id(self) -> str: + """Who this producer writes as.""" + return self._producer_id + + @property + def attempt(self) -> int: + """The generation this producer is writing.""" + return self._attempt + + async def append(self, *values: T) -> Cursor | None: + """Append ``values`` through the endpoint and return where the last one landed. + + A repeat answers with the original's position and an empty call with + the last one; ``None`` when the store behind the endpoint learns + positions only at read time. + """ + if not values: + return self._last + payloads = self._front.converter.to_payloads(list(values)) + answer = await self._call(payloads, finish=False) + self._last = Cursor(answer.cursor) if answer.cursor else None + return self._last + + async def finish(self) -> None: + """Write ``FINISH`` for this producer on this topic.""" + await self._call([], finish=True) + + async def _call(self, payloads: list[Payload], *, finish: bool) -> AppendOutput: + # Identity is taken before the codec runs: a codec may encrypt with a + # fresh nonce each time, and the retry has to be recognised anyway. + identity = tuple(payload.SerializeToString() for payload in payloads) + if self._front.codec is not None: + # The endpoint is the edge of this process, so a configured codec + # runs here rather than at the handler: whoever hosts the endpoint + # never holds the plaintext. + payloads = await self._front.codec.encode(payloads) + encoded = [payload.SerializeToString() for payload in payloads] + async with self._lock: + if self._pending is not None and self._pending[1:] != (identity, finish): + # The caller moved on from a batch the endpoint may or may + # not have applied. It goes first under its own index, so the + # handler drops it if it landed and writes it if not, and the + # new batch takes the index after it. + await self._send(*self._pending) + elif self._pending is not None: + return await self._send(*self._pending) + request = AppendInput( + stream=self._stream, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence, + batch_index=self._batch_index + 1, + payloads=encoded, + finish=finish, + ) + return await self._send(request, identity, finish) + + async def _send( + self, request: AppendInput, identity: tuple[bytes, ...], finish: bool + ) -> AppendOutput: + self._pending = (request, identity, finish) + answer = await self._front.invoke( + _APPEND_OPERATION, request, AppendOutput, timeout=_APPEND_TIMEOUT + ) + self._batch_index = request.batch_index + self._sequence = request.sequence + len(request.payloads or []) + int(finish) + self._pending = None + return answer + + +class NexusStreamHandle: + """One owner's stream through the endpoint, re-synthesizing supersession.""" + + def __init__(self, front: _Front, address: _Address) -> None: + """Address the owner ``address`` names; each call completes it with a topic.""" + self._front = front + self._address = address + + def read( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor = BEGINNING, + last: int | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield records on ``topic`` from where the read starts, one endpoint batch at a time. + + ``BEGINNING`` and ``last=`` are resolved by the store behind the + endpoint on the first call. ``END`` is the endpoint's newest position, + asked for with a ``latest_only`` call when the read starts, and the + read resumes after it, so it yields only what is appended from then on. + + The token is opaque here, so a cursor from another store is refused by + the store behind the endpoint and raises + :class:`temporalio.streams.StreamCursorError` on the first iteration. + + ``aclose()`` on the result releases nothing at the endpoint at once. + The contract carries no unsubscribe operation, so the handler cannot + be told; it reclaims the parked read when it has gone idle, which is + a minute by default. Until then the subscription and its long poll on + the store stay, and a caller that stops and starts many reads on one + topic should expect that lag rather than an immediate release. + """ + check_read_start(after, last) + topic, result_type = resolve_topic(topic, result_type) + return self._read(topic, after, last, result_type) + + async def _read( + self, topic: str, after: Cursor, last: int | None, result_type: type | None + ) -> AsyncGenerator[StreamRecord[Any], None]: + if after == END: + after = await self.latest(topic=topic) + # Where last= lands is the store's to say, so BEGINNING stands in for + # the position before it: a resume from it may repeat a record, where + # anything later could skip one. + decoder = RecordDecoder( + self._front.converter, result_type, after=after, warn=logger.warning + ) + token = after.token + last_n = last + stream = self._address.wire(topic) + while True: + answer = await self._front.invoke( + _READ_OPERATION, + ReadInput( + stream=stream, + after_token=token, + last_n=last_n, + max_records=self._front.max_records, + wait_ms=int(self._front.read_wait.total_seconds() * 1000), + ), + ReadOutput, + timeout=self._front.read_wait + _ROUND_TRIP_MARGIN, + ) + for wire in answer.records or []: + cursor = Cursor(wire.token) + try: + record = WireRecord.FromString(wire.record) + except DecodeError as error: + # Same answer as every other reader: skip and say so. + logger.warning("skipping stream record at %s: %s", cursor, error) + continue + if self._front.codec is not None and record.HasField("body"): + record.body.CopyFrom( + (await self._front.codec.decode([record.body]))[0] + ) + for out in decoder.decode(cursor, record): + yield out + token = answer.next_token or token + if token: + last_n = None + if answer.done: + return + + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: + """The newest position on ``topic`` behind the endpoint, for following from now.""" + topic, _ = resolve_topic(topic) + answer = await self._front.invoke( + _READ_OPERATION, + ReadInput(stream=self._address.wire(topic), latest_only=True), + ReadOutput, + timeout=_APPEND_TIMEOUT, + ) + token = answer.next_token or "" + return Cursor(token) if token else BEGINNING + + def producer( + self, + *, + topic: str | StreamTopic[Any] | None = None, + producer_id: str = "", + attempt: int = 0, + ) -> NexusProducer[Any]: + """A producer on ``topic``; inside an activity its identity is the activity's.""" + topic, _ = resolve_topic(topic) + producer_id, attempt = producer_identity(producer_id, attempt) + return NexusProducer( + self._front, self._address.wire(topic), producer_id, attempt + ) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A :class:`temporalio.streams.StreamRef` to ``topic`` of this owner. + + Plain data naming the owner as this handle addresses it and the + topic, with no cursor and no endpoint, so an operation can return it + and whoever receives it opens it with ``client.get_stream_handle(ref)`` + on the front or on the store itself. + """ + name, _ = resolve_topic(topic) + return self._address.ref(name) + + async def close(self) -> None: + """Refused: the contract carries no operation that seals a stream. + + Raises: + ValueError: The handle is on a workflow's or an activity's stream, + which ends with its owner. + StreamUnsupportedError: The handle is on a standalone stream; seal + it on the store's own handle. + """ + if self._address.kind != "standalone": + raise ValueError( + "only a standalone stream can be closed; this handle is on an owner's " + "stream, which ends when the owner does" + ) + raise StreamUnsupportedError( + "the stream endpoint has no operation that seals a standalone stream; " + "close it on the store's own handle" + ) + + +class NexusStreams(StreamProvider, temporalio.client.Plugin): + """The outside half of a provider, over one Nexus endpoint. + + A client plugin but not a worker plugin: a workflow publishes and reads + through the storage provider its worker was given, and this front only + serves code outside a workflow, so ``Client.connect(plugins=[front])`` + makes ``client.get_stream_handle()`` go through the endpoint while a + worker built from that client is left without a provider. The endpoint + hides which store sits behind it. + """ + + def __init__( + self, + *, + endpoint: str, + http_address: str = "http://127.0.0.1:7243", + headers: Mapping[str, str] | None = None, + read_wait: timedelta = _DEFAULT_READ_WAIT, + max_records: int = _DEFAULT_MAX_RECORDS, + data_converter: temporalio.converter.DataConverter | None = None, + ) -> None: + """Point the front at one endpoint. + + Args: + endpoint: The Nexus endpoint's name, resolved to its id through + the operator service of the client a handle is opened with. + When a handle is opened without a client it has to be the id, + because the HTTP ingress dispatches by id. + http_address: The server's Nexus HTTP ingress. + headers: Sent on every request; where an authorization header + belongs. + read_wait: How long the handler may park a read before answering + with what it has. + max_records: The most records one read answer carries. + data_converter: Overrides the client's converter for record + bodies. A payload codec configured here runs on this side of + the endpoint, so records are encoded before they leave the + process and the handler's worker never holds the key. + """ + if not endpoint: + raise ValueError("endpoint must not be empty") + # Checked here rather than on the first read, because the contract + # refuses an out-of-range value with a payload validation error that + # is neither a StreamError nor an RPCError, a long way from the line + # that got it wrong. + if not _MIN_READ_WAIT <= read_wait <= _MAX_READ_WAIT: + raise ValueError( + f"read_wait must be between {_MIN_READ_WAIT} and {_MAX_READ_WAIT}, " + f"got {read_wait}" + ) + if read_wait.microseconds % 1000: + raise ValueError( + f"read_wait is carried in whole milliseconds, got {read_wait}" + ) + if not _MIN_MAX_RECORDS <= max_records <= _MAX_MAX_RECORDS: + raise ValueError( + f"max_records must be between {_MIN_MAX_RECORDS} and " + f"{_MAX_MAX_RECORDS}, got {max_records}" + ) + self._endpoint = endpoint + self._http_address = http_address.rstrip("/") + self._headers = dict(headers or {}) + self._read_wait = read_wait + self._max_records = max_records + self._data_converter = data_converter + self._resolved: str | None = None + self._resolving = asyncio.Lock() + + def workflow_provider(self) -> NoReturn: + """Raise: this front has no workflow half.""" + raise StreamUnsupportedError(_WORKFLOW_SIDE_ERROR) + + def get_stream_handle( + self, client: Client | None, workflow_id: str, *, run_id: str | None = None + ) -> NexusStreamHandle: + """A handle on ``workflow_id``'s stream through the endpoint. + + ``client`` is not used for transport: it resolves the endpoint's name + and supplies the data converter when none was configured. + """ + return NexusStreamHandle( + _Front(self, client), _Address("workflow", workflow_id, run_id) + ) + + def get_activity_stream_handle( + self, + client: Client | None, + activity_id: str, + *, + workflow_id: str | None = None, + run_id: str | None = None, + ) -> NexusStreamHandle: + """A handle on the streams ``activity_id`` owns, through the endpoint. + + The reference carries the activity owner across; whether the store + behind the endpoint can host it is the store's answer, and a refusal + reaches the caller as :class:`temporalio.streams.StreamUnsupportedError` + on the first call. + """ + return NexusStreamHandle( + _Front(self, client), + _Address("activity", workflow_id, run_id, activity_id), + ) + + def get_standalone_stream_handle( + self, client: Client | None, stream_id: str + ) -> NexusStreamHandle: + """A handle on the standalone stream ``stream_id``, through the endpoint. + + The reference carries the standalone owner across; the store behind + the endpoint answers whether it hosts one, and a refusal reaches the + caller as :class:`temporalio.streams.StreamUnsupportedError` on the + first call. + """ + if not stream_id: + raise ValueError("stream_id must not be empty") + return NexusStreamHandle( + _Front(self, client), _Address("standalone", stream_id=stream_id) + ) + + async def create_standalone_stream( + self, + client: Client | None, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> NoReturn: + """Refused: the contract carries no operation that creates a stream. + + Raises: + StreamUnsupportedError: Always. Create the stream on the store + behind the endpoint and hand its reference to callers. + """ + raise StreamUnsupportedError( + "the stream endpoint has no operation that creates a standalone stream; " + "create it on the store behind the endpoint and pass its StreamRef" + ) + + async def close(self) -> None: + """Nothing to release: each call opens and closes its own connection.""" + + def configure_client(self, config: ClientConfig) -> ClientConfig: + """Set this front as the client's ``stream_provider``.""" + config["stream_provider"] = self + return config + + async def connect_service_client( + self, + config: ConnectConfig, + next: Callable[[ConnectConfig], Awaitable[ServiceClient]], + ) -> ServiceClient: + """Connect unchanged.""" + return await next(config) + + async def _endpoint_id(self, client: Client | None) -> str: + if self._resolved is not None: + return self._resolved + async with self._resolving: + if self._resolved is None: + if client is None: + self._resolved = self._endpoint + else: + found = await client.operator_service.list_nexus_endpoints( + ListNexusEndpointsRequest(name=self._endpoint) + ) + if not found.endpoints: + raise ValueError( + f"no nexus endpoint is named {self._endpoint!r}" + ) + self._resolved = found.endpoints[0].id + return self._resolved + + +# --------------------------------------------------------------------------- +# A Nexus operation as a stream consumer. +# --------------------------------------------------------------------------- + +NOTIFICATION_CHANNEL_HEADER = "Temporal-Notification-Channel" +"""The header the server adds to a channel's callback delivery, naming the channel.""" + +STREAM_CONSUMER_TOKEN_HEADER = "Temporal-Stream-Consumer-Token" +"""The header a consumer's listener registration carries, naming the operation. + +The server sends a listener's own headers back on every delivery, so this is +how one delivery URL serves any number of operations. +""" + +_COMPLETION_TIMEOUT = timedelta(seconds=30) +_CLOSED_KEY = "closed" +_FAILURE_CONTENT_TYPE = "application/json" +_CONTENT_TYPE_HEADER = "Content-Type" +_STATE_HEADER = "Nexus-Operation-State" +_TOKEN_HEADER = "Nexus-Operation-Token" +_START_TIME_HEADER = "Nexus-Operation-Start-Time" + + +ConsumeFunction = Callable[[StreamRecord[Any], S], "S | Awaitable[S]"] +ChannelRule = Callable[ + [StreamRef], + "temporalio.client.ChannelAddress | tuple[str, temporalio.common.Execution | None]", +] + + +def _address(where: Any) -> tuple[str, temporalio.common.Execution | None]: + """The channel name and the owner a rule answered with, as an address or a pair.""" + channel = getattr(where, "channel", None) + if isinstance(channel, str): + return channel, getattr(where, "execution", None) + channel, owner = where + return channel, owner + + +@dataclass(frozen=True) +class ConsumerState(Generic[S]): + """What a consumer operation holds for one token, read-only. + + .. warning:: + This API is experimental and unstable. + """ + + ref: StreamRef + """The stream being consumed.""" + + channel: str + """The channel the listener is registered on.""" + + owner: temporalio.common.Execution | None + """The execution a linked channel belongs to, ``None`` for an independent one.""" + + listener_id: str + """The listener id the server assigned.""" + + cursor: Cursor + """Where the next read starts. :data:`BEGINNING` until a record was consumed.""" + + value: S + """What the consume function has folded the records into so far.""" + + reads: int + """How many read passes reached the store, the start's included.""" + + deliveries: int + """How many notifications were delivered for this operation.""" + + records: int + """How many records the consume function has been handed.""" + + +@dataclass(frozen=True) +class Delivery: + """What one delivered notification led to. + + .. warning:: + This API is experimental and unstable. + """ + + token: str | None + """The operation the delivery named, when the headers carried one.""" + + known: bool + """Whether this process holds that operation. A delivery for one it does not is ignored.""" + + read: bool + """Whether the delivery led to a read. One that brought nothing new does not.""" + + records: int + """How many records that read handed to the consume function.""" + + completed: bool + """Whether the delivery closed the operation.""" + + +@dataclass +class _Consumption(Generic[S]): + ref: StreamRef + channel: str + owner: temporalio.common.Execution | None + listener_id: str + callback_url: str | None + callback_headers: dict[str, str] + started: datetime + value: S + cursor: Cursor = BEGINNING + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + reads: int = 0 + deliveries: int = 0 + records: int = 0 + finished: bool = False + done: bool = False + opening: asyncio.Task[None] | None = None + + +def _header(headers: Mapping[str, str], name: str) -> str | None: + wanted = name.lower() + for key, value in headers.items(): + if key.lower() == wanted: + return value + return None + + +def _parse_notification(body: bytes | str | Mapping[str, Any]) -> Notification: + """The notification a delivery carries, from the JSON the server posts.""" + text = json.dumps(body) if isinstance(body, Mapping) else body + if isinstance(text, bytes): + text = text.decode() + proto = json_format.Parse( + text, temporalio.api.notification.v1.Notification(), ignore_unknown_fields=True + ) + return Notification._from_proto(proto) + + +def _payload_content(payload: Payload) -> tuple[dict[str, str], bytes]: + """The HTTP content headers and body that carry ``payload`` to the server. + + The mapping is the server's: a plain JSON payload travels as JSON, a null + one as an empty body with no type, and anything else, a codec's output + included, as the serialized payload itself. + """ + metadata = {key: value.decode() for key, value in payload.metadata.items()} + encoding = metadata.get("encoding") + if set(metadata) == {"encoding"}: + if encoding == "json/plain": + return {_CONTENT_TYPE_HEADER: "application/json"}, payload.data + if encoding == "binary/plain": + return {_CONTENT_TYPE_HEADER: "application/octet-stream"}, payload.data + if encoding == "binary/null": + return {}, b"" + return ( + {_CONTENT_TYPE_HEADER: "application/x-temporal-payload"}, + payload.SerializeToString(), + ) + + +def _post_completion( + url: str, body: bytes, headers: Mapping[str, str], timeout: timedelta +) -> None: + """Post an operation completion to the caller's callback URL. + + Unlike :func:`_post` this sets no content type of its own, because a null + result travels without one. + + Raises: + _EndpointFailure: The callback answered with an error or not at all. + """ + try: + request = urllib.request.Request( + url, data=body or None, headers=dict(headers), method="POST" + ) + with urllib.request.urlopen(request, timeout=timeout.total_seconds()): + return + except urllib.error.HTTPError as error: + raise _EndpointFailure( + error.read().decode(errors="replace"), error.code + ) from error + except urllib.error.URLError as error: + raise _EndpointFailure( + f"completion callback unreachable at {url}: {error.reason}", None + ) from error + except (TimeoutError, http.client.HTTPException, OSError, ValueError) as error: + raise _EndpointFailure( + f"completion callback at {url} did not answer: {error!r}", None + ) from error + + +class StreamConsumerOperation(nexusrpc.handler.OperationHandler[StreamRef, S]): + """An asynchronous operation that consumes the stream its input names. + + The caller starts it with a :class:`temporalio.streams.StreamRef` and + awaits its result. On start the handler registers a callback listener on + the stream's notification channel, with the URL the hosting process + serves and this operation's token in a header, and reads the stream from + its start through the client's provider. Every notification the server + delivers leads to one read from the cursor to the head, however many + writes the channel folded into it, and a delivery that brings nothing new + reads nothing. The operation completes through the caller's completion + callback when the notification says the stream closed, when the read + reaches ``FINISH`` or when the store ends the read, with what the consume + function folded the records into. Cancel unregisters the listener and + reports the operation canceled. + + The consume function is a reducer: it takes a record and the value so far + and returns the new value, awaitable or not. It is handed every record + kind and narrows on ``kind`` itself. An exception from it fails the + operation. The cursor and the value live in this process keyed by the + operation token, so a process that loses them loses the operation; a + durable cursor is a follow-up. + + .. warning:: + This API is experimental and unstable. + """ + + def __init__( + self, + consume: ConsumeFunction[S], + *, + initial: Callable[[], S], + listener_url: str, + client: Client | None = None, + channel_for: ChannelRule = temporalio.client.stream_channel, + result_type: type | None = None, + token_header: str = STREAM_CONSUMER_TOKEN_HEADER, + ) -> None: + """Consume with ``consume``, served at ``listener_url``. + + Args: + consume: The reducer each record is handed with the value so far. + initial: Makes the value an operation starts with. + listener_url: Where the server posts the channel's notifications. + The hosting process serves it and feeds each request to + :meth:`deliver`. + client: Registers the listener and opens the stream. Leave it + unset in a handler hosted by a Temporal worker, where the + operation context's client is used. + channel_for: Names the channel for a ref and the execution a + linked channel belongs to, a workflow or a standalone + activity, as a :class:`temporalio.client.ChannelAddress` or a + ``(channel, execution)`` pair with ``None`` for an + independent channel. The default is + :func:`temporalio.client.stream_channel`, the server's rule + for native streams; a store that names channels its own + way passes its rule. + result_type: What record values are decoded as. + token_header: The header carrying the operation token on the + registration and so on every delivery. + """ + if not listener_url: + raise ValueError("listener_url must not be empty") + self._consume = consume + self._initial = initial + self._listener_url = listener_url + self._given_client = client + self._channel_for = channel_for + self._result_type = result_type + self._token_header = token_header + self._states: dict[str, _Consumption[S]] = {} + + @property + def tokens(self) -> list[str]: + """The operations this process holds, oldest first.""" + return list(self._states) + + def state(self, token: str) -> ConsumerState[S] | None: + """What is held for ``token``, or ``None`` once it completed or never was.""" + held = self._states.get(token) + if held is None: + return None + return ConsumerState( + ref=held.ref, + channel=held.channel, + owner=held.owner, + listener_id=held.listener_id, + cursor=held.cursor, + value=held.value, + reads=held.reads, + deliveries=held.deliveries, + records=held.records, + ) + + def _client(self) -> Client: + if self._given_client is not None: + return self._given_client + return temporalio.nexus.client() + + async def start( + self, ctx: nexusrpc.handler.StartOperationContext, input: StreamRef + ) -> nexusrpc.handler.StartOperationResultAsync: + """Register as the stream's listener and start reading it. + + The read runs after the start answers, so a long stream does not hold + the start request, and the first delivery waits on it. + """ + client = self._client() + token = uuid.uuid4().hex + channel, owner = _address(self._channel_for(input)) + # Registered before the first read: a record appended between the + # read and the registration would otherwise be missed, where one + # appended between the registration and the read is read twice at + # worst, once by the read and once by the delivery it provokes, and + # the cursor makes the second a no-op. + listener_id = await client.register_channel_listener( + channel, + Callback(url=self._listener_url, headers={self._token_header: token}), + execution=owner, + ) + state: _Consumption[S] = _Consumption( + ref=input, + channel=channel, + owner=owner, + listener_id=listener_id, + callback_url=ctx.callback_url, + callback_headers=dict(ctx.callback_headers), + started=datetime.now(timezone.utc), + value=self._initial(), + ) + self._states[token] = state + state.opening = asyncio.create_task(self._open(token, state, client)) + return nexusrpc.handler.StartOperationResultAsync(token) + + async def _open(self, token: str, state: _Consumption[S], client: Client) -> None: + try: + async with state.lock: + if state.done: + return + await self._catch_up(token, state, client) + if state.finished and not state.done: + await self._complete(token, state, client) + except Exception: + # The next delivery reads again from the cursor, so a failure here + # costs nothing but the records it would have consumed early. + logger.exception("the first read of %s failed", state.ref) + + async def deliver( + self, headers: Mapping[str, str], body: bytes | str | Mapping[str, Any] + ) -> Delivery: + """Act on one notification the server posted to the listener URL. + + ``headers`` are the request's, in any case; ``body`` is the + ``Notification`` as protobuf JSON, raw or already parsed. The delivery + is matched to its operation by the token header. Reading from the + cursor to the head happens under the operation's lock, so a delivery + that arrives during the first read waits for it and then finds the + cursor at the head. + + A failure to read raises, so the hosting process answers the server + with an error and the server retries the delivery; a failure in the + consume function fails the operation instead. + """ + token = _header(headers, self._token_header) + state = self._states.get(token) if token is not None else None + if token is None or state is None: + logger.warning( + "ignoring a channel delivery for an operation this process does not hold" + ) + return Delivery(token, known=False, read=False, records=0, completed=False) + client = self._client() + notification = _parse_notification(body) + closed = await self._closed(notification, client) + async with state.lock: + if state.done: + return Delivery(token, True, read=False, records=0, completed=True) + state.deliveries += 1 + reads_before = state.reads + records = await self._catch_up(token, state, client) + completed = False + if (closed or state.finished) and not state.done: + await self._complete(token, state, client) + completed = True + return Delivery( + token, + True, + read=state.reads > reads_before, + records=records, + completed=completed, + ) + + async def _closed(self, notification: Notification, client: Client) -> bool: + payload = notification.metadata.get(_CLOSED_KEY) + if payload is None: + return False + try: + converter = client.data_converter + if converter.payload_codec is not None: + [payload] = await converter.payload_codec.decode([payload]) + return bool(converter.payload_converter.from_payload(payload, bool)) + except Exception: + logger.warning( + "could not decode the notification's closed flag", exc_info=True + ) + return False + + async def _catch_up( + self, token: str, state: _Consumption[S], client: Client + ) -> int: + """Read from the cursor to the head, consuming, and return how many records.""" + handle = client.get_stream_handle(state.ref) + head = await handle.latest() + if head == state.cursor: + return 0 + state.reads += 1 + source = handle.read(after=state.cursor, result_type=self._result_type) + count = 0 + exhausted = True + try: + async for record in source: + try: + value = self._consume(record, state.value) + if inspect.isawaitable(value): + value = await value + except Exception as error: + await self._complete(token, state, client, error=error) + return count + state.value = cast(S, value) + state.cursor = record.cursor + state.records += 1 + count += 1 + if record.kind is RecordKind.FINISH: + state.finished = True + exhausted = False + break + if record.cursor == head: + exhausted = False + break + finally: + await source.aclose() + if exhausted: + # The store ended the read: nothing more will arrive on it. + state.finished = True + return count + + async def _complete( + self, + token: str, + state: _Consumption[S], + client: Client, + *, + error: BaseException | None = None, + ) -> None: + state.done = True + if error is None: + await self._post_result(token, state, client) + else: + await self._post_failure( + token, state, "failed", f"{type(error).__name__}: {error}" + ) + await self._unregister(client, state) + self._states.pop(token, None) + + async def cancel( + self, ctx: nexusrpc.handler.CancelOperationContext, token: str + ) -> None: + """Unregister the listener and report the operation canceled. + + Raises: + nexusrpc.HandlerError: ``NOT_FOUND`` when this process holds no + operation under ``token``. + """ + del ctx + state = self._states.get(token) + if state is None: + raise nexusrpc.HandlerError( + "no stream consumer operation is held under this token", + type=nexusrpc.HandlerErrorType.NOT_FOUND, + retryable_override=False, + ) + client = self._client() + if state.opening is not None and not state.opening.done(): + state.opening.cancel() + async with state.lock: + if state.done: + return + state.done = True + await self._unregister(client, state) + await self._post_failure( + token, state, "canceled", "the caller canceled the stream consumer" + ) + self._states.pop(token, None) + + async def close(self) -> None: + """Unregister every listener this process still holds, completing nothing. + + Call it when the hosting process stops, so the server does not keep + posting to a URL nobody serves. + """ + client = self._given_client + for token, state in list(self._states.items()): + if state.opening is not None and not state.opening.done(): + state.opening.cancel() + state.done = True + if client is not None: + await self._unregister(client, state) + self._states.pop(token, None) + + async def _unregister(self, client: Client, state: _Consumption[S]) -> None: + try: + await client.unregister_channel_listener( + state.channel, state.listener_id, execution=state.owner + ) + except RPCError: + logger.warning( + "could not unregister listener %s from channel %s", + state.listener_id, + state.channel, + exc_info=True, + ) + + async def _post_result( + self, token: str, state: _Consumption[S], client: Client + ) -> None: + converter = client.data_converter + [payload] = converter.payload_converter.to_payloads([state.value]) + if converter.payload_codec is not None: + [payload] = await converter.payload_codec.encode([payload]) + headers, body = _payload_content(payload) + await self._post(token, state, "succeeded", headers, body) + + async def _post_failure( + self, token: str, state: _Consumption[S], outcome: str, message: str + ) -> None: + await self._post( + token, + state, + outcome, + {_CONTENT_TYPE_HEADER: _FAILURE_CONTENT_TYPE}, + json.dumps({"message": message}).encode(), + ) + + async def _post( + self, + token: str, + state: _Consumption[S], + outcome: str, + content: Mapping[str, str], + body: bytes, + ) -> None: + if not state.callback_url: + logger.warning( + "operation %s %s with no completion callback to tell", token, outcome + ) + return + headers = { + **state.callback_headers, + _TOKEN_HEADER: token, + _STATE_HEADER: outcome, + _START_TIME_HEADER: email.utils.format_datetime(state.started, usegmt=True), + **content, + } + try: + await asyncio.to_thread( + _post_completion, state.callback_url, body, headers, _COMPLETION_TIMEOUT + ) + except _EndpointFailure: + logger.exception("could not complete operation %s as %s", token, outcome) + + +def stream_consumer_operation( + consume: ConsumeFunction[S], + *, + initial: Callable[[], S], + listener_url: str, + client: Client | None = None, + channel_for: ChannelRule = temporalio.client.stream_channel, + result_type: type | None = None, + token_header: str = STREAM_CONSUMER_TOKEN_HEADER, +) -> StreamConsumerOperation[S]: + """Turn ``consume`` into an asynchronous operation handler that consumes a stream. + + A service handler returns the result from an operation handler factory, + and the hosting process feeds the deliveries it receives at + ``listener_url`` to the same object's :meth:`StreamConsumerOperation.deliver`. + See :class:`StreamConsumerOperation` for the arguments and the contract. + """ + return StreamConsumerOperation( + consume, + initial=initial, + listener_url=listener_url, + client=client, + channel_for=channel_for, + result_type=result_type, + token_header=token_header, + ) diff --git a/temporalio/streams/providers/nexus_consumer_service.py b/temporalio/streams/providers/nexus_consumer_service.py new file mode 100644 index 0000000000..0aea10d466 --- /dev/null +++ b/temporalio/streams/providers/nexus_consumer_service.py @@ -0,0 +1,509 @@ +r"""A standalone Nexus handler process that consumes a stream through its channel. + +The Temporal frontend forwards a caller's ``StartOperation`` to an endpoint +whose target is an external URL, over the Nexus HTTP protocol. This module +serves that protocol with aiohttp for one :class:`nexusrpc.handler.Handler`, +start and cancel, and one more route the server's notification channels post +to, which it hands to the :class:`StreamConsumerOperation` instances it was +given. It is the shape the PoC demonstrates: a process with a URL of its own, +no worker, consuming a stream the server tells it about. + +Run it with one command, against a server that serves channels and a stream +front endpoint named ``streams``:: + + uv run python -m temporalio.streams.providers.nexus_consumer_service \\ + --address 127.0.0.1:7813 --http http://127.0.0.1:7823 \\ + --endpoint streams --port 8813 --register-endpoint consumers + +``--register-endpoint`` creates a Nexus endpoint whose target is this process, +so a workflow calls ``CollectStream.collect`` on it with a ``StreamRef`` and +gets the stream's values back when the stream closes. The server has to allow +this host in ``callback.allowedAddresses``. + +The wire details are the Nexus SDK's: a start is ``POST /{service}/{operation}`` +with the input as the body, the caller's completion callback in the ``callback`` +query parameter and its headers under the ``Nexus-Callback-`` prefix; an +asynchronous start answers ``201`` with the operation token; a cancel is +``POST /{service}/{operation}/cancel`` with the token in +``Nexus-Operation-Token`` and answers ``202``; a handler error answers the +status its type maps to with a JSON failure body. The body of a start is +turned into a payload the way the server does it, by content type, and a +synchronous result goes back the same way. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import re +import uuid +from collections.abc import Mapping, Sequence +from datetime import datetime, timedelta, timezone +from typing import Any + +import nexusrpc +import nexusrpc.handler +from aiohttp import web + +import temporalio.converter +from temporalio.api.common.v1 import Payload +from temporalio.api.nexus.v1 import EndpointSpec, EndpointTarget +from temporalio.api.operatorservice.v1 import CreateNexusEndpointRequest +from temporalio.client import Client +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams._record import RecordKind, StreamRecord +from temporalio.streams._ref import StreamRef +from temporalio.streams.providers.nexus import ( + NexusStreams, + StreamConsumerOperation, + _payload_content, + stream_consumer_operation, +) + +__all__ = [ + "CollectStream", + "CollectStreamHandler", + "NexusHttpService", + "collect_values", + "main", +] + +logger = logging.getLogger(__name__) + +_CALLBACK_PREFIX = "nexus-callback-" +_CONTENT_PREFIX = "content-" +_REQUEST_ID_HEADER = "nexus-request-id" +_REQUEST_TIMEOUT_HEADER = "request-timeout" +_OPERATION_TOKEN_HEADER = "nexus-operation-token" +_OPERATION_STATE_HEADER = "nexus-operation-state" +_RETRYABLE_HEADER = "nexus-request-retryable" +_CALLBACK_QUERY = "callback" +_TOKEN_QUERY = "token" +_DEFAULT_DELIVERIES_PATH = "/deliveries" + +# The status each handler error type answers with, per the Nexus HTTP spec. +_ERROR_STATUS = { + nexusrpc.HandlerErrorType.BAD_REQUEST: 400, + nexusrpc.HandlerErrorType.UNAUTHENTICATED: 401, + nexusrpc.HandlerErrorType.UNAUTHORIZED: 403, + nexusrpc.HandlerErrorType.NOT_FOUND: 404, + nexusrpc.HandlerErrorType.REQUEST_TIMEOUT: 408, + nexusrpc.HandlerErrorType.CONFLICT: 409, + nexusrpc.HandlerErrorType.RESOURCE_EXHAUSTED: 429, + nexusrpc.HandlerErrorType.INTERNAL: 500, + nexusrpc.HandlerErrorType.NOT_IMPLEMENTED: 501, + nexusrpc.HandlerErrorType.UNAVAILABLE: 503, + nexusrpc.HandlerErrorType.UPSTREAM_TIMEOUT: 520, +} +_OPERATION_FAILED_STATUS = 424 + +_DURATION = re.compile(r"(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)") +_UNIT_SECONDS = { + "ns": 1e-9, + "us": 1e-6, + "µs": 1e-6, + "ms": 1e-3, + "s": 1.0, + "m": 60.0, + "h": 3600.0, +} + + +class _NeverCancelled(nexusrpc.handler.OperationTaskCancellation): + """The task cancellation of a request no worker is going to cancel.""" + + def is_cancelled(self) -> bool: + return False + + def cancellation_reason(self) -> str | None: + return None + + def wait_until_cancelled_sync(self, timeout: float | None = None) -> bool: + del timeout + return False + + async def wait_until_cancelled(self) -> None: + await asyncio.Event().wait() + + +def _deadline(timeout: str | None) -> datetime | None: + """The request deadline a Go duration string in ``Request-Timeout`` names.""" + if not timeout: + return None + seconds = 0.0 + matched = False + for amount, unit in _DURATION.findall(timeout): + seconds += float(amount) * _UNIT_SECONDS[unit] + matched = True + if not matched: + return None + return datetime.now(timezone.utc) + timedelta(seconds=seconds) + + +def _media_type(content_type: str) -> tuple[str, dict[str, str]]: + media, _, rest = content_type.partition(";") + params: dict[str, str] = {} + for part in rest.split(";"): + key, _, value = part.strip().partition("=") + if key: + params[key.strip().lower()] = value.strip().strip('"') + return media.strip().lower(), params + + +def _payload_from_content(headers: Mapping[str, str], body: bytes) -> Payload: + """The payload a start body is, mapped the way the server maps Nexus content.""" + content = { + key[len(_CONTENT_PREFIX) :]: value + for key, value in headers.items() + if key.startswith(_CONTENT_PREFIX) + } + content_type = content.pop("type", "") + content.pop("length", None) + if not content_type: + if not content and not body: + return Payload(metadata={"encoding": b"binary/null"}) + return _unknown(content, body) + media, params = _media_type(content_type) + if media == "application/x-temporal-payload": + return Payload.FromString(body) + if media == "application/json": + if params.get("format") == "protobuf" and params.get("message-type"): + return Payload( + metadata={ + "encoding": b"json/protobuf", + "messageType": params["message-type"].encode(), + }, + data=body, + ) + return Payload(metadata={"encoding": b"json/plain"}, data=body) + if media == "application/x-protobuf" and params.get("message-type"): + return Payload( + metadata={ + "encoding": b"binary/protobuf", + "messageType": params["message-type"].encode(), + }, + data=body, + ) + if media == "application/octet-stream": + return Payload(metadata={"encoding": b"binary/plain"}, data=body) + content["type"] = content_type + return _unknown(content, body) + + +def _unknown(content: Mapping[str, str], body: bytes) -> Payload: + metadata = {key: value.encode() for key, value in content.items()} + metadata["encoding"] = b"unknown/nexus-content" + return Payload(metadata=metadata, data=body) + + +class _PayloadSerializer: + """Hands a start's payload to the handler through the data converter.""" + + def __init__( + self, converter: temporalio.converter.DataConverter, payload: Payload + ) -> None: + self._converter = converter + self._payload = payload + + async def serialize(self, value: Any) -> nexusrpc.Content: + del value + raise NotImplementedError("the service serializes results itself") + + async def deserialize( + self, content: nexusrpc.Content, as_type: type[Any] | None = None + ) -> Any: + del content + payload = self._payload + if self._converter.payload_codec is not None: + [payload] = await self._converter.payload_codec.decode([payload]) + try: + [value] = self._converter.payload_converter.from_payloads( + [payload], [as_type] if as_type is not None else None + ) + except Exception as error: + raise nexusrpc.HandlerError( + f"invalid operation input: {error}", + type=nexusrpc.HandlerErrorType.BAD_REQUEST, + retryable_override=False, + ) from error + return value + + +def _failure(error: nexusrpc.HandlerError) -> web.Response: + headers = {} + if error.retryable_override is not None: + headers[_RETRYABLE_HEADER] = "true" if error.retryable_override else "false" + error_type = ( + error.type + if isinstance(error.type, nexusrpc.HandlerErrorType) + else nexusrpc.HandlerErrorType.UNKNOWN + ) + return web.json_response( + {"message": error.message}, + status=_ERROR_STATUS.get(error_type, 500), + headers=headers, + ) + + +class NexusHttpService: + """Serves one Nexus handler over HTTP, plus the route channel deliveries reach. + + ``handler`` is the Nexus SDK's dispatcher over the service handlers the + process hosts. ``consumers`` are the stream consumer operations among + them; a delivery is offered to each until one holds the operation it + names. ``data_converter`` turns start bodies into inputs and results + into bodies; the caller's converter when a client is at hand. + + .. warning:: + This API is experimental and unstable. + """ + + def __init__( + self, + handler: nexusrpc.handler.Handler, + consumers: Sequence[StreamConsumerOperation[Any]], + *, + data_converter: temporalio.converter.DataConverter | None = None, + deliveries_path: str = _DEFAULT_DELIVERIES_PATH, + ) -> None: + """Serve ``handler`` and route deliveries at ``deliveries_path`` to ``consumers``.""" + if deliveries_path.count("/") != 1 or not deliveries_path.startswith("/"): + raise ValueError( + "deliveries_path must be one path segment, so it cannot be mistaken " + "for a start request" + ) + self._handler = handler + self._consumers = list(consumers) + self._converter = data_converter or temporalio.converter.DataConverter.default + self._deliveries_path = deliveries_path + + @property + def deliveries_path(self) -> str: + """The route the channel's deliveries are served at.""" + return self._deliveries_path + + def application(self) -> web.Application: + """The aiohttp application serving the routes; run it with the usual runner.""" + app = web.Application() + app.add_routes( + [ + web.post(self._deliveries_path, self._deliver), + web.post("/{service}/{operation}", self._start), + web.post("/{service}/{operation}/cancel", self._cancel), + ] + ) + return app + + async def _start(self, request: web.Request) -> web.Response: + headers = {key.lower(): value for key, value in request.headers.items()} + body = await request.read() + ctx = nexusrpc.handler.StartOperationContext( + service=request.match_info["service"], + operation=request.match_info["operation"], + headers={ + key: value + for key, value in headers.items() + if not key.startswith((_CONTENT_PREFIX, _CALLBACK_PREFIX)) + }, + request_id=headers.get(_REQUEST_ID_HEADER) or uuid.uuid4().hex, + callback_url=request.query.get(_CALLBACK_QUERY), + callback_headers={ + key[len(_CALLBACK_PREFIX) :]: value + for key, value in headers.items() + if key.startswith(_CALLBACK_PREFIX) + }, + task_cancellation=_NeverCancelled(), + request_deadline=_deadline(headers.get(_REQUEST_TIMEOUT_HEADER)), + ) + input = nexusrpc.LazyValue( + serializer=_PayloadSerializer( + self._converter, _payload_from_content(headers, body) + ), + headers={}, + stream=None, + ) + try: + result = await self._handler.start_operation(ctx, input) + except nexusrpc.HandlerError as error: + return _failure(error) + except nexusrpc.OperationError as error: + return web.json_response( + {"message": str(error)}, + status=_OPERATION_FAILED_STATUS, + headers={_OPERATION_STATE_HEADER: error.state.value}, + ) + if isinstance(result, nexusrpc.handler.StartOperationResultAsync): + return web.json_response( + {"token": result.token, "state": "running"}, status=201 + ) + [payload] = self._converter.payload_converter.to_payloads([result.value]) + if self._converter.payload_codec is not None: + [payload] = await self._converter.payload_codec.encode([payload]) + content, data = _payload_content(payload) + return web.Response(status=200, body=data, headers=content) + + async def _cancel(self, request: web.Request) -> web.Response: + headers = {key.lower(): value for key, value in request.headers.items()} + token = headers.get(_OPERATION_TOKEN_HEADER) or request.query.get(_TOKEN_QUERY) + if not token: + return _failure( + nexusrpc.HandlerError( + "missing operation token", + type=nexusrpc.HandlerErrorType.BAD_REQUEST, + ) + ) + ctx = nexusrpc.handler.CancelOperationContext( + service=request.match_info["service"], + operation=request.match_info["operation"], + headers={ + key: value + for key, value in headers.items() + if key != _OPERATION_TOKEN_HEADER + }, + task_cancellation=_NeverCancelled(), + ) + try: + await self._handler.cancel_operation(ctx, token) + except nexusrpc.HandlerError as error: + return _failure(error) + return web.Response(status=202) + + async def _deliver(self, request: web.Request) -> web.Response: + body = await request.read() + headers = dict(request.headers) + try: + for consumer in self._consumers: + if (await consumer.deliver(headers, body)).known: + break + except Exception as error: + # A 5xx makes the server retry the delivery, which is what a + # failed read needs. + logger.exception("a channel delivery failed") + return web.json_response({"message": str(error)}, status=500) + return web.Response(status=200) + + +# --------------------------------------------------------------------------- +# The demo service: collect a stream's values into the operation's result. +# --------------------------------------------------------------------------- + + +def collect_values(record: StreamRecord[Any], values: list[Any]) -> list[Any]: + """The demo's consume function: keep every published value, in order.""" + if record.kind is RecordKind.DATA: + values.append(record.value) + return values + + +@nexusrpc.service +class CollectStream: + """Consumes the stream a ref names and answers with its values when it closes.""" + + collect: nexusrpc.Operation[StreamRef, list[Any]] + + +@nexusrpc.handler.service_handler(service=CollectStream) +class CollectStreamHandler: + """Serves :class:`CollectStream` out of one consumer operation.""" + + def __init__(self, consumer: StreamConsumerOperation[list[Any]]) -> None: + """Serve ``collect`` with ``consumer``, which the deliveries also reach.""" + self._consumer = consumer + + @nexusrpc.handler.operation_handler + def collect(self) -> nexusrpc.handler.OperationHandler[StreamRef, list[Any]]: + """The consumer operation, as the operation handler factory hands it out.""" + return self._consumer + + +async def _register_endpoint(client: Client, name: str, url: str) -> None: + try: + await client.operator_service.create_nexus_endpoint( + CreateNexusEndpointRequest( + spec=EndpointSpec( + name=name, + target=EndpointTarget(external=EndpointTarget.External(url=url)), + ) + ) + ) + logger.info("registered nexus endpoint %s -> %s", name, url) + except RPCError as error: + if error.status is not RPCStatusCode.ALREADY_EXISTS: + raise + logger.info("nexus endpoint %s exists; leaving it as it is", name) + + +async def main(argv: Sequence[str] | None = None) -> None: + """Serve the demo consumer until interrupted; see the module docstring.""" + parser = argparse.ArgumentParser(description=(__doc__ or "").split("\n\n")[0]) + parser.add_argument( + "--address", default="127.0.0.1:7233", help="the server's gRPC address" + ) + parser.add_argument("--namespace", default="default") + parser.add_argument( + "--http", + default="http://127.0.0.1:7243", + help="the server's Nexus HTTP ingress, where the stream front is read", + ) + parser.add_argument( + "--endpoint", required=True, help="the stream front's Nexus endpoint name" + ) + parser.add_argument("--host", default="127.0.0.1", help="the interface to serve on") + parser.add_argument("--port", type=int, default=8813, help="the port to serve on") + parser.add_argument( + "--listener-url", + default=None, + help="the URL the server posts deliveries to; defaults to this process's route", + ) + parser.add_argument( + "--register-endpoint", + default=None, + metavar="NAME", + help="create a Nexus endpoint of this name whose target is this process", + ) + args = parser.parse_args(argv) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" + ) + + base_url = f"http://{args.host}:{args.port}" + client = await Client.connect( + args.address, + namespace=args.namespace, + plugins=[NexusStreams(endpoint=args.endpoint, http_address=args.http)], + ) + consumer = stream_consumer_operation( + collect_values, + initial=list, + listener_url=args.listener_url or base_url + _DEFAULT_DELIVERIES_PATH, + client=client, + ) + service = NexusHttpService( + nexusrpc.handler.Handler([CollectStreamHandler(consumer)]), + [consumer], + data_converter=client.data_converter, + ) + runner = web.AppRunner(service.application()) + await runner.setup() + try: + await web.TCPSite(runner, args.host, args.port).start() + if args.register_endpoint: + await _register_endpoint(client, args.register_endpoint, base_url) + logger.info( + "serving %s at %s, deliveries at %s", + CollectStream.__name__, + base_url, + consumer._listener_url, # pyright: ignore[reportPrivateUsage] + ) + await asyncio.Event().wait() + finally: + await consumer.close() + await runner.cleanup() + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + pass diff --git a/temporalio/streams/providers/redis.py b/temporalio/streams/providers/redis.py new file mode 100644 index 0000000000..c537177e30 --- /dev/null +++ b/temporalio/streams/providers/redis.py @@ -0,0 +1,2119 @@ +"""The client-side (Redis) provider. + +Streams live in a store the customer runs, Redis here, behind the external +workflow streams transport. A workflow's publish is buffered by the worker, +staged invisibly under a token when the Workflow Task completes, and promoted +only once a marker in History proves the task was accepted. Consumption is +recorded the same way, as ranges and boundaries in History. + +The mapping, in one place: + +- One Redis stream per topic, the topic's log, read by the workflow and by + outside readers alike. The transport derives an input key and an output key + for a topic, because the direction is part of every key it renders; this + provider's backend renders both onto the log. An outside producer appends a + record once, and it is where the workflow's subscription reads and where an + outside read follows; the workflow's staged batches land in the same log + when its task completes and are promoted there, so an outside reader sees + the producer's records and the workflow's in one order. The workflow does + not read its own records: the backend drops the entries the workflow staged + from every read it serves the transport, the live read and the replay read + alike, so a recorded range and its replay agree. The wake path is the + transport's own: a producer appends, then wakes the subscribed run through + the input key. A wake the server refuses because the consuming run is + closing is sent again after a short wait: a run that continues as new hands + the log to its successor, the records are already where the successor + reads them, and only the wake has to follow. The retries stop when the + chain's current run takes the wake or the chain proves terminal. A run + still refusing when the window passes is inside a Workflow Task that tried + to close it while the wake sat buffered; the wake is dropped, because the + record is in the log and a run that does not close after all rechecks the + log at its next park. +- A reset run inherits the base run's History up to the reset point, and + with it the ranges the base run's task completions recorded. It re-reads + them from the log and replays them against the inherited markers, so the + batches those tasks published are not published again, and live reading + continues from the last inherited boundary. The reset-point task itself is + run again, and what the base run consumed after the reset point is still + in the log, so the reset run reads it again and publishes again: an + outside reader sees that task's batch twice, once from each run. Nothing + in the log marks the reset. +- A record rides as the transport's payload: the serialized ``StreamRecord`` + proto as a ``binary/plain`` value, which the worker's codec encodes and + decodes like any other payload. +- Producer identity dedupes through the transport's idempotency: the session + is ``producer#attempt`` and every record carries a sequence, so a retried + batch is dropped with the original position and a new attempt passes. +- Cursors are ``redis:-`` and name an entry of the topic's log, so a + cursor a workflow reader returned seeds an outside read and the other way + round. A reader opened without a cursor starts where the chain's + predecessor run committed, which is the transport's own rule. ``END`` and + ``last=N`` are positioned against the log when the read starts: outside, + on the first step of the generator, since the call itself cannot reach the + store; inside a workflow, by the worker right after the Workflow Task that + opened the subscription, which records the entry it resolved in the marker + beside the subscription, so replay and a cold start read it from History + and never ask the log again. +- A workflow's streams are keyed by the chain's first run, so a handle + follows continue-as-new by construction and ``run_id`` only decides whose + close ends a read. +- An activity's own streams are one Redis stream per topic, keyed by the + namespace, the workflow id, empty for a standalone activity, the run the + activity execution belongs to, and the activity id. The run is the + workflow's for a workflow's activity and the activity's own for a + standalone one, so a retry writes to the same stream, which a reader sees + as ``SUPERSEDED``, and an id started again, in a new run, starts a new + one, as it does on the server-side provider. Inside the activity the run + is in its info; a handle opened outside without one describes the owner + and takes its current run. The owner is one key component joined with + ``/``, a character the chain keys percent-encode out of every id, so no + chain key and no key derived from one can name an activity's stream. + There is no input stream, because no workflow reads these, and no + staging, because an activity's append is visible as soon as the store + accepts it. A read ends when the owner is terminal and the retained tail + is delivered: a standalone activity is described directly, and a + workflow's activity through its workflow, whose close ends the read, as + does the activity leaving the pending set once its stream exists. An + activity that never wrote has no stream, so a read on it waits for the + workflow. Nothing gates an append after that, so a late attempt still + lands, and a read that has ended does not see it. +- A task's publishes are staged as one batch. The transport's own per-task + batch limits are lifted for this provider, because a synchronous publish + cannot wait for the worker to stage a full batch; a batch it cannot stage + fails the task. +- Retention is trimming, with no consumer floor. By default a record older + than :data:`DEFAULT_RETENTION`, seven days, is trimmed by the next append + the provider makes to its key, an activity's stream included, whatever any + reader has reached; ``retention=None`` turns the age trim off, and + ``max_len`` adds a count cap that is off by default. A replay that reaches + a recorded range the trim removed fails its Workflow Task, an outside + cursor below the trim is refused, and a fully trimmed topic reads as + empty. A run that has to replay cold after seven days of consuming fails, + so a long-lived consumer continues as new inside the window, or is + configured with a longer one. + +- Every record carries the SHA-256 of its converted body under + ``temporal.io/content-hash``, stamped before the payload codec runs, and the + append script compares that hash rather than the stored bytes, so a retry + through a codec that differs on every call is still the same append while a + divergent one is refused. A record without a body is matched by its + plaintext record instead. +- A standalone stream is one more key scheme: ``standalone/`` as the + owner component, a hash under it that holds the policy and the seal, and one + log per topic beside the hash. ``create_stream`` writes the hash once and + refuses a different policy for an id that has one; a handle on an id with no + hash raises ``StreamNotFoundError`` at its first use. Every append applies + the policy to the topic it wrote, ``max_records`` and ``retention`` as on the + other logs and ``max_bytes`` by a byte total the hash keeps per topic, + dropping the oldest entries until the topic fits. ``close()`` sets the seal: + a later append is refused with ``StreamClosedError``, the retained records + stay readable and a read ends once it has delivered them. Nothing in + Temporal knows these keys, so no tooling lists them. + +Keys written by the earlier layout. Before the log, a topic was two keys: the +transport's input key, ``::::`` +with every id percent-encoded, and its output key, the same with an ``output`` +component before the topic. The log is the input key, so the records outside +producers wrote, the ranges a workflow recorded and the cursors its readers +returned, ``redis:in:-``, read as they did. The output key is not read +any more: batches a workflow promoted under the old layout are not delivered to +an outside reader, and an outside cursor minted then names an entry of that +key, so a reader holding one starts over from ``BEGINNING`` or positions itself +with ``latest()``. Nothing moves or deletes an old output key; it ages out under +the operator's own retention. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import re +import time +from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Sequence +from dataclasses import dataclass, replace +from datetime import timedelta +from typing import Any, Final, Generic, TypeVar, get_args +from urllib.parse import quote + +from google.protobuf.message import DecodeError + +from temporalio import workflow +from temporalio.api.common.v1 import Payload +from temporalio.client import ActivityExecutionStatus, Client, WorkflowExecutionStatus +from temporalio.contrib.external_workflow_streams import ( + AFTER, + AppendConflictError, + AppendNotAcknowledgedError, + ChainKeyMismatchError, + ExternalStreamProducer, + Offset, + StartAtTail, + StreamDirection, + WakeNotAcknowledgedError, + WorkflowChainKey, + external_output_stream, + external_stream, +) +from temporalio.contrib.external_workflow_streams import ( + BEGINNING as TRANSPORT_BEGINNING, +) +from temporalio.contrib.external_workflow_streams import ( + RecordKind as TransportRecordKind, +) +from temporalio.contrib.external_workflow_streams import ( + StreamError as TransportStreamError, +) +from temporalio.contrib.external_workflow_streams import ( + StreamRecord as TransportRecord, +) +from temporalio.contrib.external_workflow_streams._backend import ( + DEFAULT_WATCH_BLOCK, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._errors import StreamIntegrityError +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputStage, + OutputStageConflictError, + OutputStageManifest, + OutputStageNotFoundError, + OutputStageResolutionError, + OutputStageStatus, + OutputStreamRecord, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._output_client import ( + _reconcile_output_stage, +) +from temporalio.contrib.external_workflow_streams._redis import ( + _BEGINNING_SENTINEL, + _OUTPUT_STAGE_FIELD, + RedisStreamBackend, + _text, + _to_record, +) +from temporalio.contrib.external_workflow_streams._redis import ( + _parse as _parse_entry_id, +) +from temporalio.contrib.external_workflow_streams._wake import WakeTransport +from temporalio.converter import ( + ActivitySerializationContext, + SerializationContext, + WorkflowSerializationContext, +) +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams._body import CONTENT_HASH_KEY, content_hash +from temporalio.streams._errors import ( + StreamClosedError, + StreamCursorError, + StreamError, + StreamNotFoundError, + StreamProducerError, +) +from temporalio.streams._provider import ReadSource, WriteSink +from temporalio.streams._record import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamRecord, + check_read_start, +) +from temporalio.streams._ref import StreamRef +from temporalio.streams._topic import StreamTopic, resolve_topic +from temporalio.streams._wire import ( + RecordDecoder, + WireRecord, + cursor_position, + mint_cursor, + producer_identity, + to_wire, +) +from temporalio.streams.providers import ProviderPlugin +from temporalio.worker import ReplayerConfig, WorkerConfig + +__all__ = ["DEFAULT_RETENTION", "RedisProducer", "RedisStreamHandle", "RedisStreams"] + +T = TypeVar("T") + +_PROVIDER = "redis" +#: What a workflow reader's cursor carried when a topic was two keys. It named +#: an entry of the input key, which is the log now, so the form is still read. +_LEGACY_INPUT_PREFIX = "in:" +_REDIS_ID = re.compile(r"\d+-\d+") +_READ_BATCH = 256 +_STAGE_FIELD: Final = _OUTPUT_STAGE_FIELD.encode() + +#: How long a record is kept when the constructor is not told otherwise. +#: +#: An age rather than a count, because the count a topic can afford depends on +#: its record size and the age does not, and because a count cap refuses a task +#: whose batch does not fit under it. Seven days is long enough to replay a +#: consumer that was evicted over a weekend and short enough that a chain +#: nobody reads any more does not keep its records for good. +DEFAULT_RETENTION: Final = timedelta(days=7) + +#: How long a wake the server refused is sent again before it is given up, and +#: the pause between attempts. The pause is there because in the instant between +#: two runs of a chain the successor is not yet the run a Signal resolves to. +_WAKE_RETRY_WINDOW: Final = timedelta(seconds=5) +_WAKE_RETRY_BACKOFF: Final = timedelta(milliseconds=200) + +#: Bits of a wake counter given to an entry id's sequence part. Redis assigns +#: sequence numbers from 0 within one millisecond, so a million appends to one +#: log inside the same millisecond would be needed to reach the cap. +_WAKE_SEQUENCE_BITS: Final = 20 + + +def _wake_counter(offset: Offset) -> int: + """A wake counter from a Redis entry id ``-``. + + ``ms * 2**20 + min(seq, 2**20 - 1)``, which increases with the id's own + order and fits the server's signed 64-bit counter for any millisecond + timestamp before the year 2248. Ids past the cap within one millisecond + share a counter, which only lets a later wake fold into a pending one that + reports an earlier id of that same millisecond. + """ + ms, seq = _parse_entry_id(offset) + cap = (1 << _WAKE_SEQUENCE_BITS) - 1 + return (ms << _WAKE_SEQUENCE_BITS) + min(seq, cap) + + +# The transport's per-task batch limits are a backpressure point that awaits +# inside the workflow, and a synchronous publish has nowhere to await. Lifted +# to where they cannot fire, so a task's publishes stage as one batch. +_UNBOUNDED_OUTPUT = external_output_stream.with_options( + max_records=1 << 30, max_logical_bytes=1 << 40 +) + +logger = logging.getLogger(__name__) + + +def _require_topic(topic: str) -> None: + if not topic: + raise ValueError("topic must not be empty") + + +def _position(after: Cursor) -> Offset | None: + """The log entry a cursor names, or ``None`` for BEGINNING. + + Raises: + StreamCursorError: The cursor is another provider's, or does not name + a Redis entry id. + """ + token = cursor_position(after, provider=_PROVIDER) + if token is None: + return None + if token.startswith(_LEGACY_INPUT_PREFIX): + token = token[len(_LEGACY_INPUT_PREFIX) :] + if not _REDIS_ID.fullmatch(token): + raise StreamCursorError( + f"cursor {after.token!r} does not name a Redis stream position" + ) + return Offset(token) + + +def _entry_id(token: str | bytes) -> tuple[int, int]: + """A Redis entry id as the ``(ms, seq)`` pair it orders by.""" + text = token.decode() if isinstance(token, bytes) else token + ms, _, seq = text.partition("-") + return int(ms), int(seq or 0) + + +#: Given one page of log entries, newest first, the ids of the ones that are +#: not records to the reader asking. +_SkipIn = Callable[[list[Any]], Awaitable[set[str]]] + + +async def _tail_after( + store: Any, name: str, before_last: int, *, skip_in: _SkipIn | None = None +) -> Offset | None: + """The entry the newest ``before_last`` records of the log ``name`` come after. + + ``None`` when the log holds no more than ``before_last`` records, which + means a read from the beginning. With ``before_last=0`` it is the newest + record itself, the boundary a read at ``END`` starts after. Walks the log + from its newest entry, leaving out what ``skip_in`` names. + """ + needed = before_last + 1 + end = "+" + seen = 0 + while True: + page: Any = await store.xrevrange(name, end, "-", count=max(needed - seen, 16)) + if not page: + return None + skipped = await skip_in(page) if skip_in is not None else set() + for entry_id, _fields in page: + token = _text(entry_id) + if token in skipped: + continue + seen += 1 + if seen == needed: + return Offset(token) + ms, seq = _entry_id(page[-1][0]) + if seq: + end = f"{ms}-{seq - 1}" + elif ms: + end = f"{ms - 1}-18446744073709551615" + else: + return None + + +def _trim_floor(backend: Any) -> str: + retention = getattr(backend, "_retention", None) + if retention is None: + return "" + # The worker's clock names the floor, so a skewed worker shifts the window by + # its skew, the same as the backend's own trim. + floor = int((time.time() - retention.total_seconds()) * 1000) + return f"{max(floor, 0)}-0" + + +def _trim_maxlen(backend: Any) -> str: + max_len = getattr(backend, "_max_len", None) + return "" if max_len is None else str(max_len) + + +def _fields_args(record: TransportRecord) -> list[Any]: + """The record's stored fields, name then value, as the append scripts take them.""" + args: list[Any] = [] + for name, value in sorted(record.to_fields().items()): + args.append(name.encode()) + args.append(value) + return args + + +def _append_args(backend: Any, record: TransportRecord, digest: str) -> list[Any]: + """What the log append script takes: the trims, the identity and digest, the fields. + + ``digest`` is the plaintext hash of what the record carries, taken before + the payload codec ran, so a retry the codec encoded differently is still + recognized as the same append. + """ + return [ + _trim_floor(backend).encode(), + _trim_maxlen(backend).encode(), + str(record.idempotency_key).encode(), + digest.encode(), + *_fields_args(record), + ] + + +def _plaintext_digest(wire: WireRecord) -> str: + """The digest an append is matched by: the body's hash, or the record's without one.""" + if wire.HasField("body"): + return content_hash(wire.body) + return hashlib.sha256(wire.SerializeToString(deterministic=True)).hexdigest() + + +def _stamp_hash(wire: WireRecord) -> None: + """Put the body's plaintext hash on ``wire`` under the shared metadata key.""" + if wire.HasField("body"): + wire.metadata[CONTENT_HASH_KEY].CopyFrom( + Payload( + metadata={"encoding": b"binary/plain"}, + data=content_hash(wire.body).encode(), + ) + ) + + +#: Append one record to a log, or answer where it already is. +#: +#: The provider's own write rather than the transport's, so the retention trims +#: ride along instead of costing their own round trips; they are exact for the +#: reason the backend's own trims are. The idempotency hash is read before the +#: log is touched, so an identity already used with a different plaintext digest +#: refuses the record rather than writing it, and one used with the same digest +#: answers with the original position, which is what settles a call whose answer +#: was lost. +_LOG_APPEND_LUA: Final = """ +local minid = ARGV[1] +local maxlen = ARGV[2] +local existing = redis.call('HGET', KEYS[2], ARGV[3]) +if existing then + local sep = string.find(existing, '|') + if string.sub(existing, sep + 1) ~= ARGV[4] then + return {'conflict', ''} + end + return {'ok', string.sub(existing, 1, sep - 1)} +end +local id = redis.call('XADD', KEYS[1], '*', unpack(ARGV, 5)) +redis.call('HSET', KEYS[2], ARGV[3], id .. '|' .. ARGV[4]) +if minid ~= '' then + redis.call('XTRIM', KEYS[1], 'MINID', minid) +end +if maxlen ~= '' then + redis.call('XTRIM', KEYS[1], 'MAXLEN', maxlen) +end +return {'ok', id} +""" + + +class _LogAppend: + """One record on one log, a topic's or an activity's, in one Redis call.""" + + def __init__(self, backend: RedisStreamBackend) -> None: + """Bind to ``backend``'s client.""" + self._backend = backend + self._script = backend._client.register_script(_LOG_APPEND_LUA) + + async def write(self, *, name: str, record: TransportRecord, digest: str) -> Offset: + """Append ``record`` to the log ``name`` and return where it landed. + + A repeat of the same ``(session, sequence)`` with the same ``digest`` + returns the original position and writes nothing. + """ + outcome, placed = await self._script( + keys=[name, f"{name}:idem"], + args=_append_args(self._backend, record, digest), + ) + if _text(outcome) == "conflict": + raise AppendConflictError(record.idempotency_key) + return Offset(_text(placed)) + + +@dataclass(frozen=True) +class _StandaloneOwner: + """A stream with an id of its own and no owner, and where Redis keeps it.""" + + namespace: str + stream_id: str + + def meta(self, key_prefix: str) -> str: + """The hash holding the stream's policy, seal and byte totals.""" + namespace = quote(self.namespace, safe="") + return f"{key_prefix}:{namespace}:standalone/{quote(self.stream_id, safe='')}" + + def key(self, key_prefix: str, topic: str) -> str: + """The log holding ``topic`` of this stream; one more component than the hash.""" + return f"{self.meta(key_prefix)}:{quote(topic, safe='')}" + + def __str__(self) -> str: + """The stream, for messages.""" + return f"standalone stream {self.stream_id!r}" + + +def _policy_fields( + retention: timedelta | None, max_records: int | None, max_bytes: int | None +) -> list[bytes]: + """The policy as the hash stores it: milliseconds, counts, and blanks for none.""" + return [ + b"" + if retention is None + else str(int(retention.total_seconds() * 1000)).encode(), + b"" if max_records is None else str(max_records).encode(), + b"" if max_bytes is None else str(max_bytes).encode(), + ] + + +#: Create a standalone stream's hash, or say whether the one there agrees. +_CREATE_STANDALONE_LUA: Final = """ +if redis.call('EXISTS', KEYS[1]) == 1 then + local held = redis.call('HMGET', KEYS[1], 'retention_ms', 'max_records', 'max_bytes') + if held[1] == ARGV[1] and held[2] == ARGV[2] and held[3] == ARGV[3] then + return 'same' + end + return 'conflict' +end +redis.call('HSET', KEYS[1], 'retention_ms', ARGV[1], 'max_records', ARGV[2], + 'max_bytes', ARGV[3], 'sealed', '0') +return 'created' +""" + + +#: Append one record to a standalone stream's topic under the stream's policy. +#: +#: The policy lives in the hash, so the script reads it rather than being told: +#: a sealed stream refuses, a missing one says so, and the trims apply to the +#: topic just written. ``max_bytes`` has no Redis trim of its own, so each entry +#: carries the size of the record it holds, the hash keeps a byte total per +#: topic, and the oldest entries are dropped one at a time until the topic fits. +_STANDALONE_APPEND_LUA: Final = """ +local meta = redis.call('HMGET', KEYS[3], 'sealed', 'retention_ms', 'max_records', + 'max_bytes', ARGV[6]) +if not meta[1] then + return {'missing', ''} +end +if meta[1] == '1' then + return {'closed', ''} +end +local existing = redis.call('HGET', KEYS[2], ARGV[1]) +if existing then + local sep = string.find(existing, '|') + if string.sub(existing, sep + 1) ~= ARGV[2] then + return {'conflict', ''} + end + return {'ok', string.sub(existing, 1, sep - 1)} +end +local id = redis.call('XADD', KEYS[1], '*', unpack(ARGV, 7)) +redis.call('HSET', KEYS[2], ARGV[1], id .. '|' .. ARGV[2]) +local total = tonumber(meta[5] or '0') + tonumber(ARGV[4]) +local floor = nil +if meta[2] ~= '' then + floor = tonumber(ARGV[3]) - tonumber(meta[2]) +end +while true do + local entries = redis.call('XRANGE', KEYS[1], '-', '+', 'COUNT', 1) + if #entries == 0 then break end + local entry = entries[1] + local drop = false + if meta[3] ~= '' and redis.call('XLEN', KEYS[1]) > tonumber(meta[3]) then drop = true end + if meta[4] ~= '' and total > tonumber(meta[4]) then drop = true end + if floor and tonumber(string.match(entry[1], '^(%d+)')) < floor then drop = true end + if not drop then break end + local size = 0 + local fields = entry[2] + for i = 1, #fields, 2 do + if fields[i] == ARGV[5] then size = tonumber(fields[i + 1]) end + end + redis.call('XDEL', KEYS[1], entry[1]) + total = total - size +end +redis.call('HSET', KEYS[3], ARGV[6], total) +return {'ok', id} +""" + + +#: The entry field holding the size a standalone stream's policy counts: the +#: serialized record, as the memory provider measures it, not the stored payload +#: with its envelope. +_SIZE_FIELD: Final = b"__size" + + +class _StandaloneAppend: + """One record on a standalone stream's topic, under its policy, in one call.""" + + def __init__(self, backend: RedisStreamBackend, owner: _StandaloneOwner) -> None: + """Bind to ``backend``'s client and ``owner``'s hash.""" + self._owner = owner + self._meta = owner.meta(_prefix(backend)) + self._script = backend._client.register_script(_STANDALONE_APPEND_LUA) + + async def write( + self, *, name: str, topic: str, record: TransportRecord, digest: str, size: int + ) -> Offset: + """Append ``record`` to the log ``name`` and return where it landed. + + ``size`` is what the stream's ``max_bytes`` counts for this record. + + Raises: + StreamNotFoundError: The stream was never created. + StreamClosedError: The stream is sealed. + AppendConflictError: The identity is held with a different digest. + """ + outcome, placed = await self._script( + keys=[name, f"{name}:idem", self._meta], + args=[ + str(record.idempotency_key).encode(), + digest.encode(), + str(int(time.time() * 1000)).encode(), + str(size).encode(), + _SIZE_FIELD, + f"bytes:{quote(topic, safe='')}".encode(), + *_fields_args(record), + _SIZE_FIELD, + str(size).encode(), + ], + ) + result = _text(outcome) + if result == "missing": + raise StreamNotFoundError(f"{self._owner} was not found") + if result == "closed": + raise StreamClosedError( + f"{self._owner} is closed and takes no more records" + ) + if result == "conflict": + raise AppendConflictError(record.idempotency_key) + return Offset(_text(placed)) + + +def _prefix(backend: Any) -> str: + """The key prefix ``backend`` writes under, for keys the provider derives itself.""" + return backend._key_prefix + + +@dataclass(frozen=True) +class _ActivityOwner: + """The activity whose streams a handle addresses, and where Redis keeps them. + + ``workflow_id`` is ``None`` for a standalone activity. ``run_id`` is the + workflow's run for a workflow's activity and the activity's own run for a + standalone one. It is part of the key, so an activity execution's streams + are its own and an id started again in a new run starts new ones, and it + decides whose close ends a read. ``None`` means the run is not known yet: + a handle opened outside without one asks the server for the current run + before it touches a key. + """ + + namespace: str + workflow_id: str | None + activity_id: str + run_id: str | None + + def key(self, key_prefix: str, topic: str) -> str: + """The Redis stream holding ``topic`` of this activity's streams.""" + if self.run_id is None: + raise RuntimeError(f"the run of {self} was not resolved before its key") + # The chain keys percent-encode every id, so none of their components + # holds a "/", and a component built around one can never equal a + # chain key or a key derived from one. + namespace = quote(self.namespace, safe="") + owner = f"activity/{quote(self.workflow_id or '', safe='')}/" + owner += f"{quote(self.run_id, safe='')}/{quote(self.activity_id, safe='')}" + return f"{key_prefix}:{namespace}:{owner}:{quote(topic, safe='')}" + + def context(self) -> SerializationContext: + """What the owner's payloads are coded under. + + A workflow's activity writes the workflow's data, as it does on the + workflow's own topics; a standalone activity has no workflow, so its + own identity is the context. + """ + if self.workflow_id is not None: + return WorkflowSerializationContext( + namespace=self.namespace, workflow_id=self.workflow_id + ) + return ActivitySerializationContext( + namespace=self.namespace, + activity_id=self.activity_id, + activity_type=None, + activity_task_queue=None, + workflow_id=None, + workflow_type=None, + is_local=False, + ) + + def __str__(self) -> str: + """The owner, for messages.""" + if self.workflow_id is None: + return f"activity {self.activity_id!r}" + return f"activity {self.activity_id!r} of workflow {self.workflow_id!r}" + + +async def _resolve_owner(client: Client, owner: _ActivityOwner) -> _ActivityOwner: + """``owner`` with its run known, or ``StreamNotFoundError`` when the server does not know it. + + One describe: it says whether the owner exists and, for a handle opened + without a run, which run is current. A standalone activity is described + itself; a workflow's activity is described through its workflow, because + the server does not describe it on its own. + """ + try: + if owner.workflow_id is None: + described = await client.get_activity_handle( + owner.activity_id, run_id=owner.run_id + ).describe() + run_id = owner.run_id or described.activity_run_id + else: + description = await client.get_workflow_handle( + owner.workflow_id, run_id=owner.run_id + ).describe() + run_id = owner.run_id or description.run_id + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError(f"{owner} was not found") from error + raise + if not run_id: + raise StreamNotFoundError(f"the server reports no run for {owner}") + return replace(owner, run_id=run_id) + + +async def _retained(client: Any, name: str, offset: Offset) -> bool: + """Whether the record at ``offset`` on the stream ``name`` survived trimming. + + A record at or after the first retained entry is there. On an emptied + stream the last id Redis generated says whether the record ever was. + """ + if not await client.exists(name): + # Nothing was ever written under this key, so nothing was trimmed from it. + return True + info = await client.xinfo_stream(name) + wanted = _entry_id(offset.token) + first = info.get("first-entry") + if first: + return _entry_id(first[0]) <= wanted + return wanted > _entry_id(info["last-generated-id"]) + + +async def _require_standalone(store: Any, meta: str, owner: _StandaloneOwner) -> None: + """Raise ``StreamNotFoundError`` unless ``owner``'s hash exists.""" + if not await store.exists(meta): + raise StreamNotFoundError(f"{owner} was not found") + + +async def _sealed(store: Any, meta: str) -> bool: + """Whether the standalone stream behind ``meta`` is sealed.""" + return _text(await store.hget(meta, "sealed") or b"0") == "1" + + +async def _newest(store: Any, name: str) -> Cursor: + """The cursor of the newest entry of the log ``name``, or ``BEGINNING``.""" + newest: Any = await store.xrevrange(name, "+", "-", count=1) + if not newest: + return BEGINNING + return mint_cursor(_PROVIDER, _text(newest[0][0])) + + +def _is_staged(fields: Any) -> bool: + """Whether a log entry is one the workflow staged, rather than a producer's.""" + return _STAGE_FIELD in fields + + +class _TopicLogBackend(RedisStreamBackend): + """The transport's Redis backend with this provider's layout and trims. + + One key per topic: the transport renders a topic's input key and its + output key apart, because the direction is part of every key it derives, + and this backend renders both onto the log, the input key's rendering. + Every read the transport makes through an input key, the live watch and + the replay range alike, drops the entries the workflow staged itself, so + a workflow never reads its own records and a recorded range replays to + what was delivered. Reads through an output key are the outside reader's + and see the whole log, with the stage protocol deciding what is visible. + + Trims are exact rather than approximate: Redis's approximate trim drops + whole macro nodes only, so a stream shorter than one node, a hundred + entries by default, would never trim and the window would not mean what + it says. Only the logs are trimmed; the idempotency and stage hashes + beside them keep one entry per record and stage. + """ + + def __init__( + self, + *, + client: Any, + key_prefix: str, + retention: timedelta | None, + max_len: int | None, + wake_transport: WakeTransport = "auto", + ) -> None: + super().__init__(client=client, key_prefix=key_prefix) + self._retention = retention + self._max_len = max_len + self.wake_transport = wake_transport + + def wake_counter_for(self, offset: Offset) -> int: + """The entry id's own order, so producers and workers rank wakes alike.""" + return _wake_counter(offset) + + def wake_counter_now(self) -> int: + """The entry id rule applied to the clock, with the largest sequence. + + Above every entry appended before now, so a wake that reports no + position, the worker's shutdown sweep, is not folded under the + channel's latest notification. + """ + now_ms = int(time.time() * 1000) + return (now_ms << _WAKE_SEQUENCE_BITS) + (1 << _WAKE_SEQUENCE_BITS) - 1 + + def stream_key(self, key: StreamKey) -> str: + """The topic's log, whichever direction the transport asks for.""" + return super().stream_key(replace(key, direction=StreamDirection.INPUT)) + + async def read_after( + self, + key: StreamKey, + after: Any, + *, + max_records: int, + block: timedelta | None = DEFAULT_WATCH_BLOCK, + ) -> list[TransportRecord]: + """The live read, without the entries the workflow staged itself. + + A batch that held nothing but the workflow's own entries is read past + without blocking again, so the transport's recheck, which asks for one + record, is not answered "nothing" while a producer's record sits behind + a staged batch. + """ + if key.direction is StreamDirection.OUTPUT: + return await super().read_after( + key, after, max_records=max_records, block=block + ) + name = self.stream_key(key) + start = _BEGINNING_SENTINEL if after.is_beginning else after.offset.serialize() + block_ms = None if block is None else int(block.total_seconds() * 1000) + if block_ms is not None and block_ms <= 0: + block_ms = None + while True: + found: Any = await self._client.xread( + {name: start}, count=max_records, block=block_ms + ) + entries = found[0][1] if found else [] + if not entries: + return [] + records = [ + _to_record(entry_id, fields) + for entry_id, fields in entries + if not _is_staged(fields) + ] + if records: + return records + start = _text(entries[-1][0]) + block_ms = None + + def describe_window(self) -> str: + """The configured window, for messages.""" + parts = [] + if self._retention is not None: + parts.append(f"retention={self._retention}") + if self._max_len is not None: + parts.append(f"max_len={self._max_len}") + return ", ".join(parts) or "no retention" + + async def append(self, key: StreamKey, record: Any) -> Any: + placed = await super().append(key, record) + await self._trim(key) + return placed + + async def stage_output( + self, manifest: OutputStageManifest, records: Sequence[StagedOutputRecord] + ) -> OutputStage: + # A stage is invisible until its task commits, and the trim has no consumer + # floor to hold it: a window at or below the batch takes entries out of the + # stage that was just written, and the commit then fails on a missing record + # for as long as the task retries. Flooring the trim instead would need the + # floor this provider deliberately does not keep, and would not hold anyway, + # because the trim that removes the stage is not the one that staged it. + if self._max_len is not None and manifest.record_count >= self._max_len: + raise ValueError( + f"this task publishes {manifest.record_count} records and max_len is " + f"{self._max_len}: the window has to exceed the largest batch a task " + "publishes, or the batch is trimmed before it commits" + ) + stage = await super().stage_output(manifest, records) + await self._trim(manifest.stream_key) + return stage + + async def abort_output(self, manifest: OutputStageManifest) -> OutputStage: + """Resolve the stage as aborted and take its entries out of the log. + + A batch whose task was not accepted yields nothing to any reader, and + on this provider the workflow's own entries share the log with the + producers' records, so leaving them would count them against every + window and bound the log holds. The stage's hashes stay, status and + offsets included, so a reader that captured the barrier settles it the + same way. + """ + stage = await super().abort_output(manifest) + if stage.records: + await self._client.xdel( + self.stream_key(manifest.stream_key), + *(record.offset.serialize() for record in stage.records), + ) + return stage + + async def _output_stage_from_offsets( + self, + manifest: OutputStageManifest, + status: OutputStageStatus, + encoded_offsets: str, + ) -> OutputStage: + # An aborted stage's entries are gone from the log here, so they are + # stood in for by their positions: the stage still names what it held. + if status is not OutputStageStatus.ABORTED: + return await super()._output_stage_from_offsets( + manifest, status, encoded_offsets + ) + offsets = [Offset(value) for value in encoded_offsets.split(",") if value] + if len(offsets) != manifest.record_count: + raise StreamIntegrityError( + "an output stage's stored offsets do not match its manifest" + ) + return OutputStage( + manifest, + tuple( + OutputStreamRecord(TransportRecordKind.DATA, b"", offset) + for offset in offsets + ), + status, + ) + + async def read_range( + self, key: StreamKey, first: Offset, last: Offset + ) -> list[TransportRecord]: + # The replay read. Said here, where the trim is known, rather than left + # to the range checks, which can only report the record as missing. + if not await self.retains(key, first): + raise StreamIntegrityError( + f"the recorded range [{first}, {last}] on topic " + f"{key.stream_name!r} is past the redis provider's retention " + f"({self.describe_window()}): the records were trimmed, so this " + "run cannot be replayed" + ) + entries: Any = await self._client.xrange( + self.stream_key(key), first.serialize(), last.serialize() + ) + # The same entries the live read dropped, so the range holds the count + # the marker recorded. + return [ + _to_record(entry_id, fields) + for entry_id, fields in entries + if key.direction is StreamDirection.OUTPUT or not _is_staged(fields) + ] + + async def tail_cursor(self, key: StreamKey, *, before_last: int = 0) -> Any: + """The boundary the newest ``before_last`` records begin after. + + Through an input key the workflow's own staged entries are not + records, as on every read the transport makes; through an output key + only an aborted batch's entries are left out, since a pending one may + still commit. + """ + name = self.stream_key(key) + skip_in = ( + self._staged_in + if key.direction is StreamDirection.INPUT + else self._aborted_in(key) + ) + after = await _tail_after(self._client, name, before_last, skip_in=skip_in) + return TRANSPORT_BEGINNING if after is None else AFTER(after) + + @staticmethod + async def _staged_in(page: list[Any]) -> set[str]: + return {_text(entry_id) for entry_id, fields in page if _is_staged(fields)} + + def _aborted_in(self, key: StreamKey) -> _SkipIn: + async def aborted(page: list[Any]) -> set[str]: + staged = { + _text(entry_id): _text(fields[_STAGE_FIELD]) + for entry_id, fields in page + if _is_staged(fields) + } + if not staged: + return set() + stage_ids = sorted(set(staged.values())) + statuses = dict( + zip( + stage_ids, + await self._client.hmget(self._output_status_key(key), stage_ids), + ) + ) + return { + entry_id + for entry_id, stage_id in staged.items() + if _text(statuses.get(stage_id) or "") + == OutputStageStatus.ABORTED.value + } + + return aborted + + async def retains(self, key: StreamKey, offset: Offset) -> bool: + """Whether the record at ``offset`` survived trimming.""" + return await _retained(self._client, self.stream_key(key), offset) + + async def _trim(self, key: StreamKey) -> None: + name = self.stream_key(key) + if self._retention is not None: + # The worker's clock names the floor, so a skewed worker shifts + # the window by its skew. + floor = int((time.time() - self._retention.total_seconds()) * 1000) + await self._client.xtrim( + name, minid=f"{max(floor, 0)}-0", approximate=False + ) + if self._max_len is not None: + await self._client.xtrim(name, maxlen=self._max_len, approximate=False) + + +def _drive(coroutine: Coroutine[Any, Any, None]) -> None: + """Run a transport publish to completion without yielding to the loop. + + The transport's publish only awaits when the task's batch is full. A + synchronous publish has nowhere to wait, so a publish that would have + waited fails the task instead, loudly. + """ + try: + coroutine.send(None) + except StopIteration: + return + coroutine.close() + raise StreamError( + "this Workflow Task's output batch is full, and a synchronous publish " + "cannot wait for the worker to stage it" + ) + + +def _parse(cursor: Cursor, raw: bytes, warn: Any) -> WireRecord | None: + try: + return WireRecord.FromString(raw) + except DecodeError as error: + # Same answer as an undecodable body: skip and say so, so one bad + # record cannot pin a reader. + warn("skipping stream record at %s: %s", cursor, error) + return None + + +class _RedisReadSource: + """One subscription of the running workflow, over the transport's input key.""" + + def __init__(self, subscription: Any) -> None: + self._subscription = subscription + self._records = subscription.records() + + async def next_batch(self) -> list[tuple[Cursor, WireRecord]]: + while True: + # One record per batch: the transport reports readiness per record, + # and a batch here would invent a boundary replay never observed. + offset, body = await self._records.__anext__() + cursor = mint_cursor(_PROVIDER, offset.token) + wire = _parse(cursor, body, workflow.logger.warning) + if wire is not None: + return [(cursor, wire)] + + def close(self) -> None: + self._subscription.close() + + +class _RedisWriteSink: + def __init__(self, topic: str) -> None: + self._topic = _UNBOUNDED_OUTPUT.topic(topic, type=bytes) + + def publish(self, record: WireRecord) -> None: + # Buffered in the transport's per-task batch, staged by the worker when + # the task completes and promoted once History proves the task was + # accepted: rule 1 through the transport's own commit. The hash is a + # digest over bytes already in hand, so it costs no I/O here. + _stamp_hash(record) + _drive(self._topic.publish(record.SerializeToString())) + + +class _RedisWorkflowProvider: + """The workflow half: the transport's subscriptions and staged output.""" + + def __init__(self, idle_timeout: timedelta) -> None: + self._input = external_stream.with_options(idle_timeout=idle_timeout) + + def open_reader( + self, topic: str, *, after: Cursor, last: int | None = None + ) -> ReadSource: + _require_topic(topic) + check_read_start(after, last) + subscribe = self._input.topic(topic, type=bytes).subscribe + if last is not None or after == END: + # The tail is where the log is when the worker looks, which the + # workflow thread cannot see: the transport has the worker resolve + # it after this task and record the entry with the subscription. + return _RedisReadSource(subscribe(start_at_tail=StartAtTail(last or 0))) + position = _position(after) + # Without a position the transport resumes where the chain's + # predecessor run committed; with one, that is where the wait starts + # and what the marker's header records. + start = None if position is None else AFTER(position) + return _RedisReadSource(subscribe(start_cursor=start)) + + def open_writer(self, topic: str) -> WriteSink: + _require_topic(topic) + return _RedisWriteSink(topic) + + def on_workflow_start(self) -> None: + pass + + async def on_workflow_finish(self) -> None: + pass + + +async def _chain(client: Client, workflow_id: str) -> WorkflowChainKey: + try: + description = await client.get_workflow_handle(workflow_id).describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"workflow {workflow_id!r} was not found" + ) from error + raise + return WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + + +#: A chain still has a consumer while a run is in either of these: a run that +#: continued as new hands the stream to its successor rather than ending it. +_STILL_CONSUMING: Final = ( + WorkflowExecutionStatus.RUNNING, + WorkflowExecutionStatus.CONTINUED_AS_NEW, +) + +#: What the server says when a Signal reaches a run whose Workflow Task is +#: closing it: on a continue-as-new handover for an instant, and after a task +#: that tried to close while the Signal sat buffered, until the next one settles. +_CLOSING_REFUSAL: Final = "workflow is closing" + + +def _chain_ended(error: WakeNotAcknowledgedError) -> bool: + """Whether the server refused a wake because the chain has ended.""" + cause = error.__cause__ + return isinstance(cause, RPCError) and cause.status == RPCStatusCode.NOT_FOUND + + +def _refused_as_closing(error: WakeNotAcknowledgedError) -> bool: + """Whether the server refused a wake because the run is closing.""" + return _CLOSING_REFUSAL in str(error) + + +def _storage_error(error: Exception, what: str) -> StreamError: + return StreamError(f"{what}: {error}") + + +def _integrity_error(error: Exception, what: str) -> StreamError: + # Named as a loss rather than a transient read failure, because no retry brings + # a trimmed record back and the caller's next move is different. + return StreamNotFoundError(f"{what}: {error}") + + +class RedisProducer(Generic[T]): + """Appends to a topic from outside workflow code. + + Every append is visible as soon as the store accepts it. Each record goes + to the topic's log, where the workflow's subscription reads it and where + outside readers see it beside the workflow's own records, and a workflow + subscribed to the topic is woken; the cursor returned names the entry. + """ + + def __init__( + self, + streams: RedisStreams, + client: Client, + workflow_id: str | None, + topic: str, + producer_id: str, + attempt: int, + owner: _ActivityOwner | None = None, + standalone: _StandaloneOwner | None = None, + ) -> None: + """Bind this producer to ``topic`` of the chain, of ``owner`` or of ``standalone``.""" + self._streams = streams + self._client = client + self._workflow_id = workflow_id + self._owner = owner + self._standalone = standalone + self._topic = topic + self._producer_id = producer_id + self._attempt = attempt + self._converter = client.data_converter.payload_converter + # One-based, because zero on the wire says the producer does not + # number its records and this one does. + self._sequence = 1 + self._last = BEGINNING + self._input: Any = None + self._append: _LogAppend | _StandaloneAppend | None = None + self._name: str | None = None + self._codec: StreamPayloadCodec[bytes] | None = None + + @property + def producer_id(self) -> str: + """Who this producer writes as.""" + return self._producer_id + + @property + def attempt(self) -> int: + """The generation this producer is writing.""" + return self._attempt + + @property + def _session(self) -> str: + # The transport dedupes on this and the sequence. The attempt is part + # of it so a retried append is dropped while a new generation writing + # different words at the same sequence is not. + return ( + f"{self._producer_id}#{self._attempt}" + if self._attempt + else self._producer_id + ) + + async def append(self, *values: T) -> Cursor: + """Append ``values`` and return the cursor of the last record as stored. + + A repeat returns where the original landed, because the transport + answers a byte-identical repeat with the original offset; an empty + call returns the position of this producer's last record. + """ + if not values: + return self._last + return await self._write( + [ + to_wire( + self._converter, + topic=self._topic, + kind=RecordKind.DATA, + value=value, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence + index, + ) + for index, value in enumerate(values) + ] + ) + + async def finish(self) -> None: + """Write ``FINISH`` for this producer on this topic. + + Written as an ordinary record rather than the transport's own + terminal, which would end every outside read of the topic; a + ``FINISH`` here only says this producer is done. + """ + await self._write( + [ + to_wire( + self._converter, + topic=self._topic, + kind=RecordKind.FINISH, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=self._sequence, + ) + ] + ) + + async def _connect(self) -> None: + if self._codec is not None: + return + backend = self._streams._require_backend() + if self._standalone is not None: + # No owner to code under and no one to wake: a standalone stream + # is read from outside only, and its hash says whether it exists. + self._name = self._standalone.key(_prefix(backend), self._topic) + self._append = _StandaloneAppend(backend, self._standalone) + self._codec = StreamPayloadCodec(self._client.data_converter, bytes) + return + self._append = _LogAppend(backend) + if self._owner is not None: + # No transport producer to bind and no one to wake: an activity's + # stream has no workflow reader and no chain to check the key against. + # Inside the activity the run is known and nothing is described. + if self._owner.run_id is None: + self._owner = await _resolve_owner(self._client, self._owner) + self._name = self._owner.key(_prefix(backend), self._topic) + self._codec = StreamPayloadCodec( + self._client.data_converter.with_context(self._owner.context()), bytes + ) + return + assert self._workflow_id is not None + chain = await _chain(self._client, self._workflow_id) + try: + # The transport's producer is bound for its wake and its key: the + # append itself is the provider's, so the trims ride along with it. + input_ = await ExternalStreamProducer.connect( + backend=backend, + workflow=chain, + client=self._client, + session_id=self._session, + ) + except ChainKeyMismatchError as error: + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} is not the chain this producer " + f"was opened on: {error}" + ) from error + self._input = input_.topic(self._topic, type=bytes) + self._name = backend.stream_key(self._input.stream_key) + self._codec = StreamPayloadCodec( + self._client.data_converter.with_context( + WorkflowSerializationContext( + namespace=chain.namespace, workflow_id=chain.workflow_id + ) + ), + bytes, + ) + + async def _write(self, records: list[WireRecord]) -> Cursor: + await self._connect() + try: + last = await self._place(records) + except AppendConflictError as error: + raise StreamProducerError( + f"producer {self._session!r} already wrote a different record at " + f"sequence {error.key.sequence}" + ) from error + except AppendNotAcknowledgedError as error: + raise _storage_error(error, "an append was not acknowledged") from error + except TransportStreamError as error: + raise _storage_error(error, "the store refused an append") from error + self._sequence += len(records) + assert last is not None + self._last = mint_cursor(_PROVIDER, last.token) + return self._last + + async def _place(self, records: list[WireRecord]) -> Offset | None: + """Append ``records`` to the owner's log and return the last position.""" + assert self._codec is not None and self._append is not None + assert self._name is not None + last: Offset | None = None + for index, record in enumerate(records): + # The digest is taken and stamped before the codec runs, so a retry + # the codec encodes differently still matches its original. + digest = _plaintext_digest(record) + _stamp_hash(record) + # Built here rather than handed to the transport's publish, so the + # trims ride along. The identity is the one the transport would + # derive, so a record the log already holds is reused. + staged = TransportRecord( + kind=TransportRecordKind.DATA, + payload=await self._codec.encode(record.SerializeToString()), + producer_session_id=self._session, + sequence=self._sequence + index, + ) + if isinstance(self._append, _StandaloneAppend): + last = await self._append.write( + name=self._name, + topic=self._topic, + record=staged, + digest=digest, + size=record.ByteSize(), + ) + else: + last = await self._append.write( + name=self._name, record=staged, digest=digest + ) + if self._owner is None and self._standalone is None: + await self._wake(last) + return last + + async def _wake(self, position: Offset | None) -> None: + """Wake the consuming workflow, following the chain past a closing run. + + The records are appended before this is called; what can fail is + telling the consumer. The server refuses a Signal while the run it + resolves to is closing, and a consumer that reads a terminal record + straight from the store and continues as new on it closes in exactly + that way, ahead of the wake for that record. The streams are keyed by + the chain, so the records are already where the successor reads them + and only the wake has to follow: it is sent again, addressed to the + workflow id as every wake is, until the chain's current run takes it. + A chain that has ended instead is the ordinary ending of a terminal + record racing the consumer acting on it, not an error. + + A run that still refuses when the window passes is inside a Workflow + Task that tried to close it while this wake sat buffered, which the + server answers by failing that task and holding the run closed to + Signals until the next one settles. The wake is dropped then: if the + run closes, nothing is owed; if it does not, its next park rechecks + the log and finds the record, the transport's own rule for a record + appended before a park. Any other refusal that outlasts the window is + raised. + + ``NOT_FOUND`` answers the chain question without a describe: the wake + call names the chain's first run, and the server refuses it that way + only once the chain has ended. A Signal, which names the Workflow ID + alone, refuses an ended run with it too. + """ + deadline = time.monotonic() + _WAKE_RETRY_WINDOW.total_seconds() + while True: + try: + await self._input.wake(position=position) + return + except WakeNotAcknowledgedError as error: + if _chain_ended(error) or await self._chain_is_terminal(): + return + if time.monotonic() >= deadline: + if _refused_as_closing(error): + logger.info( + "dropping the wake for %r on topic %r: the run is closing a " + "Workflow Task, and its next park rechecks the log", + self._workflow_id, + self._topic, + ) + return + raise + except TransportStreamError as error: + raise _storage_error(error, "the wake could not be sent") from error + await asyncio.sleep(_WAKE_RETRY_BACKOFF.total_seconds()) + + async def _chain_is_terminal(self) -> bool: + """Whether the chain has ended for good, rather than handing over. + + A run that continued as new is not the end: its successor is the + consumer now, and a closing run still describes as running. A chain + whose History is gone has ended. + """ + assert self._workflow_id is not None + handle = self._client.get_workflow_handle(self._workflow_id) + try: + status = (await handle.describe()).status + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + return True + raise + return status is not None and status not in _STILL_CONSUMING + + +class RedisStreamHandle: + """One owner's topics from outside. + + The owner is ``workflow_id``'s workflow, whose topic logs are read through + the transport's output read so a staged batch is a barrier until its task + settles; with ``activity_id`` an activity, read from the stream the + provider keeps for it, a standalone activity without ``workflow_id`` or an + activity that workflow scheduled; or with ``stream_id`` a standalone + stream, read from the logs under its hash until it is sealed. + """ + + def __init__( + self, + streams: RedisStreams, + client: Client, + workflow_id: str | None, + run_id: str | None, + activity_id: str | None = None, + *, + stream_id: str | None = None, + ) -> None: + """Address the owner's topics; ``run_id`` decides whose close ends a read.""" + self._streams = streams + self._client = client + self._workflow_id = workflow_id + self._run_id = run_id + self._owner: _ActivityOwner | None = None + self._standalone: _StandaloneOwner | None = None + converter = client.data_converter + if stream_id is not None: + # No owner, so nothing to code the bodies under. + self._standalone = _StandaloneOwner(client.namespace, stream_id) + elif activity_id is not None: + self._owner = _ActivityOwner( + client.namespace, workflow_id, activity_id, run_id + ) + converter = converter.with_context(self._owner.context()) + else: + if workflow_id is None: + raise ValueError( + "a stream handle needs a workflow_id, an activity_id or a stream_id" + ) + converter = converter.with_context( + WorkflowSerializationContext( + namespace=client.namespace, workflow_id=workflow_id + ) + ) + self._converter = client.data_converter.payload_converter + self._codec: StreamPayloadCodec[bytes] = StreamPayloadCodec(converter, bytes) + + def read( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor = BEGINNING, + last: int | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield records on ``topic`` from where the read starts until the owner closes. + + For a workflow that is the chain, or the pinned run; for an activity it is + the activity reaching a terminal status, learned as the module docstring + says. ``END`` and ``last=`` are positioned against the log on the first + step of the generator, since this call cannot reach the store. + + Two refusals and they do not land together. A cursor another provider minted, + or one that is not a Redis entry id, is refused by this call: reading the + token needs nothing from the store. A well-formed cursor the retention has + trimmed is refused on the first step of the generator, because answering that + needs a round trip and this call is not a coroutine. Neither yields a record + first. + + Raises: + ValueError: ``last`` is not positive or came with a cursor. + StreamCursorError: The cursor is another provider's, or does not name + a Redis entry. + """ + check_read_start(after, last) + topic, result_type = resolve_topic(topic, result_type) + # A tail start is resolved in the generator; a cursor is parsed here so + # a foreign one fails this call, not the first iteration. + tail = last if last is not None else (0 if after == END else None) + position = None if tail is not None else _position(after) + if self._standalone is not None: + return self._read_standalone( + self._standalone, topic, position, after, result_type, tail=tail + ) + if self._owner is not None: + return self._read_owned( + self._owner, topic, position, after, result_type, tail=tail + ) + return self._read(topic, position, after, result_type, tail=tail) + + async def _read_standalone( + self, + owner: _StandaloneOwner, + topic: str, + position: Offset | None, + after: Cursor, + result_type: type | None, + *, + tail: int | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + backend = self._streams._require_backend() + store = backend._client + meta = owner.meta(_prefix(backend)) + await _require_standalone(store, meta, owner) + name = owner.key(_prefix(backend), topic) + if tail is not None: + position = await _tail_after(store, name, tail) + after = ( + BEGINNING + if position is None + else mint_cursor(_PROVIDER, position.token) + ) + decoder = RecordDecoder( + self._converter, result_type, after=after, warn=logger.warning + ) + if position is not None and not await _retained(store, name, position): + raise StreamCursorError( + f"cursor {after.token!r} names a record on {topic!r} that the " + f"stream's policy has dropped" + ) + start = _BEGINNING_SENTINEL if position is None else position.token + block = int(self._streams._poll.total_seconds() * 1000) or None + closed = False + while True: + found: Any = await store.xread( + {name: start}, count=_READ_BATCH, block=block + ) + entries = found[0][1] if found else [] + for entry_id, fields in entries: + placed = _to_record(entry_id, fields) + assert placed.offset is not None + start = placed.offset.token + if placed.kind is not TransportRecordKind.DATA: + continue + minted = mint_cursor(_PROVIDER, placed.offset.token) + wire = _parse( + minted, await self._codec.decode(placed.payload), logger.warning + ) + if wire is None: + continue + for record in decoder.decode(minted, wire): + yield record + if entries: + continue + if closed: + return + # One more pass after learning of the seal, so a record that landed + # between the read and the seal is not lost. + closed = await _sealed(store, meta) + + async def _read_owned( + self, + owner: _ActivityOwner, + topic: str, + position: Offset | None, + after: Cursor, + result_type: type | None, + *, + tail: int | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + backend = self._streams._require_backend() + # Described on every read, so a handle without a run reads the run that + # is current when the read starts, and ends with it. + owner = await _resolve_owner(self._client, owner) + store = backend._client + name = owner.key(_prefix(backend), topic) + if tail is not None: + position = await _tail_after(store, name, tail) + after = ( + BEGINNING + if position is None + else mint_cursor(_PROVIDER, position.token) + ) + decoder = RecordDecoder( + self._converter, result_type, after=after, warn=logger.warning + ) + if ( + position is not None + and isinstance(backend, _TopicLogBackend) + and not await _retained(store, name, position) + ): + # Refused rather than resumed from the first retained record, + # which would skip whatever the trim took in between. + raise StreamCursorError( + f"cursor {after.token!r} names a record on {topic!r} that the " + f"provider's retention has trimmed ({backend.describe_window()})" + ) + start = _BEGINNING_SENTINEL if position is None else position.token + # XREAD BLOCK 0 waits forever, which is not what a zero poll asks for. + block = int(self._streams._poll.total_seconds() * 1000) or None + closed = False + while True: + found: Any = await store.xread( + {name: start}, count=_READ_BATCH, block=block + ) + entries = found[0][1] if found else [] + for entry_id, fields in entries: + placed = _to_record(entry_id, fields) + assert placed.offset is not None + start = placed.offset.token + if placed.kind is not TransportRecordKind.DATA: + continue + minted = mint_cursor(_PROVIDER, placed.offset.token) + wire = _parse( + minted, await self._codec.decode(placed.payload), logger.warning + ) + if wire is None: + continue + for record in decoder.decode(minted, wire): + yield record + if entries: + continue + if closed: + return + # One more pass after learning the owner is terminal, so a record + # that landed between the read and the describe is not lost. + closed = await self._owner_closed(owner, store, name) + + async def _owner_closed(self, owner: _ActivityOwner, store: Any, name: str) -> bool: + """Whether the owning activity is terminal, as far as this store can tell. + + A standalone activity says so itself. The server does not describe a + workflow's activity, so its workflow is asked: the workflow closing ends + the read, and so does the activity leaving the pending set once its + stream exists. Before the first write there is no stream, so an + activity that never wrote is read until its workflow closes. + """ + try: + if owner.workflow_id is None: + described = await self._client.get_activity_handle( + owner.activity_id, run_id=owner.run_id + ).describe() + return described.status != ActivityExecutionStatus.RUNNING + description = await self._client.get_workflow_handle( + owner.workflow_id, run_id=owner.run_id + ).describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + # The owner's History is gone, so there is nothing left to follow. + return True + raise + status = description.status + if status is not None and status != WorkflowExecutionStatus.RUNNING: + # An activity's streams are not the chain's: a run that continued + # as new took its activities with it. + return True + pending = any( + info.activity_id == owner.activity_id + for info in description.raw_description.pending_activities + ) + if pending: + return False + return bool(await store.exists(name)) + + async def _read( + self, + topic: str, + position: Offset | None, + after: Cursor, + result_type: type | None, + *, + tail: int | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + backend = self._streams._require_backend() + assert self._workflow_id is not None + chain = await _chain(self._client, self._workflow_id) + key = chain.stream_key(topic, direction=StreamDirection.OUTPUT) + if tail is not None: + resolved = await backend.tail_cursor(key, before_last=tail) + position = None if resolved.is_beginning else resolved.offset + after = ( + BEGINNING + if position is None + else mint_cursor(_PROVIDER, position.token) + ) + decoder = RecordDecoder( + self._converter, result_type, after=after, warn=logger.warning + ) + if ( + position is not None + and isinstance(backend, _TopicLogBackend) + and not await backend.retains(key, position) + ): + # Refused rather than resumed from the first retained record, + # which would skip whatever the trim took in between. + raise StreamCursorError( + f"cursor {after.token!r} names a record on {topic!r} that the " + f"provider's retention has trimmed ({backend.describe_window()})" + ) + cursor = TRANSPORT_BEGINNING if position is None else AFTER(position) + closed = False + while True: + try: + result = await backend.read_output_after( + key, + cursor, + max_records=_READ_BATCH, + block=self._streams._poll, + ) + except StreamIntegrityError as error: + # Integrity loss is permanent, and the taxonomy the transport builds + # on it is the difference between a retry that clears and one that + # never will. Filed as a store read, an operator retries forever. + raise _integrity_error(error, "the store lost records") from error + except TransportStreamError as error: + raise _storage_error(error, "the store could not be read") from error + for placed in result.records: + cursor = AFTER(placed.offset) + if placed.kind is not TransportRecordKind.DATA: + continue + minted = mint_cursor(_PROVIDER, placed.offset.token) + wire = _parse( + minted, await self._codec.decode(placed.payload), logger.warning + ) + if wire is None: + continue + for record in decoder.decode(minted, wire): + yield record + if result.pending is not None: + # A staged batch whose task History has not settled yet is a + # barrier: nothing past it is read until History says whether + # the task was accepted. + try: + resolved = await _reconcile_output_stage( + backend=backend, + client=self._client, + workflow_id=self._workflow_id, + manifest=result.pending.manifest, + ) + except StreamIntegrityError as error: + raise _integrity_error( + error, "a staged batch lost records" + ) from error + except ( + OutputStageConflictError, + OutputStageNotFoundError, + OutputStageResolutionError, + ) as error: + # The transport's own stage vocabulary does not cross the public + # surface: to a reader this is a batch that cannot be settled. + raise _storage_error( + error, "a staged batch could not be settled" + ) from error + except TransportStreamError as error: + raise _storage_error( + error, "a staged batch could not be settled" + ) from error + if not resolved: + await asyncio.sleep(self._streams._poll.total_seconds()) + continue + if result.records: + continue + if closed: + return + # One more pass after learning the workflow closed, so a batch + # promoted between the read and the describe is not lost. + closed = await self._closed() + + async def _closed(self) -> bool: + assert self._workflow_id is not None + handle = self._client.get_workflow_handle( + self._workflow_id, run_id=self._run_id + ) + try: + status = (await handle.describe()).status + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + # The run's History is gone; nothing more can be promoted. + return True + raise + if status is None or status == WorkflowExecutionStatus.RUNNING: + return False + # The streams are keyed by the chain, so a run that continued as new + # is not the end unless the handle was pinned to it. + return not (self._run_id is None and status in _STILL_CONSUMING) + + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: + """The cursor of the newest committed record on ``topic``, for following from now. + + ``BEGINNING`` when the topic holds no committed record, which a topic whose + records retention has all trimmed answers too: the two are the same state to + a reader, and a read from it starts at the first record retained after it + rather than at the tail the caller asked to follow from. + """ + topic, _ = resolve_topic(topic) + backend = self._streams._require_backend() + if self._standalone is not None: + await _require_standalone( + backend._client, + self._standalone.meta(_prefix(backend)), + self._standalone, + ) + return await _newest( + backend._client, self._standalone.key(_prefix(backend), topic) + ) + if self._owner is not None: + owner = await _resolve_owner(self._client, self._owner) + return await _newest(backend._client, owner.key(_prefix(backend), topic)) + assert self._workflow_id is not None + chain = await _chain(self._client, self._workflow_id) + try: + tail = await backend.output_tail( + chain.stream_key(topic, direction=StreamDirection.OUTPUT) + ) + except TransportStreamError as error: + raise _storage_error(error, "the store's tail could not be read") from error + if tail.is_beginning: + return BEGINNING + assert tail.offset is not None + return mint_cursor(_PROVIDER, tail.offset.token) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A ref to ``topic`` of this owner's stream, pinned as this handle is.""" + if self._standalone is not None: + return StreamRef.for_standalone(self._standalone.stream_id, topic=topic) + if self._owner is not None: + return StreamRef.for_activity( + self._owner.activity_id, + workflow_id=self._owner.workflow_id, + run_id=self._owner.run_id, + topic=topic, + ) + assert self._workflow_id is not None + return StreamRef.for_workflow( + self._workflow_id, run_id=self._run_id, topic=topic + ) + + async def close(self) -> None: + """Seal a standalone stream. An owned stream ends with its owner, not by a caller. + + The seal is a flag in the stream's hash that every append reads, so a + later append is refused with ``StreamClosedError`` and a read ends once + it has delivered the retained tail. Idempotent. + + Raises: + ValueError: This handle is on a workflow's or an activity's stream. + StreamNotFoundError: The stream was never created. + """ + if self._standalone is None: + raise ValueError( + "only a standalone stream can be closed; this handle is on an owned " + "stream, which ends when its workflow or activity does" + ) + backend = self._streams._require_backend() + meta = self._standalone.meta(_prefix(backend)) + await _require_standalone(backend._client, meta, self._standalone) + await backend._client.hset(meta, "sealed", "1") + + def producer( + self, + *, + topic: str | StreamTopic[Any] | None = None, + producer_id: str = "", + attempt: int = 0, + ) -> RedisProducer[Any]: + """A producer on ``topic``; inside an activity its identity is the activity's.""" + topic, _ = resolve_topic(topic) + producer_id, attempt = producer_identity(producer_id, attempt) + return RedisProducer( + self._streams, + self._client, + self._workflow_id, + topic, + producer_id, + attempt, + owner=self._owner, + standalone=self._standalone, + ) + + +class RedisStreams(ProviderPlugin): + """The client-side provider over Redis streams. + + Construct one, pass it to the worker as a plugin and open handles from + it anywhere else. The Redis client is opened on first use, so it belongs + to the loop that uses it, and released by :meth:`close`; a ``backend`` + the caller hands in stays the caller's to close. + """ + + def __init__( + self, + *, + url: str = "redis://127.0.0.1:6379", + key_prefix: str = "temporal-streams", + idle_timeout: timedelta = timedelta(seconds=1), + client: Any | None = None, + poll_interval: timedelta = timedelta(milliseconds=500), + retention: timedelta | None = DEFAULT_RETENTION, + max_len: int | None = None, + wake_transport: WakeTransport = "auto", + ) -> None: + """Create the provider. + + Args: + url: The Redis to connect to when no ``client`` is given. + key_prefix: Prepended to every key, so one Redis serves several + deployments. + idle_timeout: How long a workflow reader with nothing to read + holds its Workflow Task open before the worker parks it. + client: A ``redis.asyncio.Redis`` the caller opened, with + ``decode_responses=False``, and closes itself; the provider + puts its own key layout and trims on top of it. + poll_interval: How long an outside reader that is caught up waits + for a record before asking whether the workflow closed. + retention: Trim records older than this from a topic's log on + every append the provider makes to it. + :data:`DEFAULT_RETENTION`, seven days, unless the caller says + otherwise; ``None`` keeps every record until ``max_len`` + trims it, or for good when that is unset too. This is + retention without a consumer floor: nothing holds a record + for a reader that has not reached it. A workflow whose replay + reaches a recorded range past the window fails its Workflow + Task with the transport's ``StreamIntegrityError`` until the + window is raised, an outside ``read(after=)`` below the window + raises ``StreamCursorError``, and a live reader that falls + behind the window misses records. The floor the server-side + provider keeps would need a consumer registry in Redis. + max_len: Keep at most this many entries per key, trimmed on the + same appends and with the same consequences. Off unless set. + It must exceed the largest batch a task publishes, or a stage + is trimmed before its commit; a batch at or above it is + refused where it is staged. + wake_transport: How a producer and a worker wake a workflow after + an append. ``"channel"`` notifies the stream's channel with + the entry id as the position: on a server with channels + linked to a workflow the channel lives in the reading + workflow's own state and nothing subscribes, on one with + independent channels every subscribed reader is woken. + ``"signal"`` uses the reserved Signal, which writes a + History event. ``"auto"`` notifies the channel and steps + down to the Signal on a server without channels. + """ + # Checked against the alias so an untyped caller still gets a ValueError. + if wake_transport not in get_args(WakeTransport): + raise ValueError(f"unknown wake transport {wake_transport!r}") + if retention is not None and retention <= timedelta(0): + raise ValueError("retention must be positive") + if max_len is not None and max_len < 1: + raise ValueError("max_len must be positive") + self._url = url + self._key_prefix = key_prefix + self._idle_timeout = idle_timeout + self._client = client + self._backend: _TopicLogBackend | None = None + self._owned_client: Any = None + self._poll = poll_interval + self._retention = retention + self._max_len = max_len + self._wake_transport: WakeTransport = wake_transport + + def _require_backend(self) -> _TopicLogBackend: + if self._backend is None: + client = self._client + if client is None: + import redis.asyncio + + # A dead peer would otherwise hold a blocking read open forever. + # Several block periods, so a healthy socket that is merely idle + # inside one read window is never abandoned. + client = self._owned_client = redis.asyncio.from_url( + self._url, + decode_responses=False, + socket_timeout=DEFAULT_WATCH_BLOCK.total_seconds() * 6, + ) + self._backend = _TopicLogBackend( + client=client, + key_prefix=self._key_prefix, + retention=self._retention, + max_len=self._max_len, + wake_transport=self._wake_transport, + ) + return self._backend + + def configure_worker(self, config: WorkerConfig) -> WorkerConfig: + """Set this provider and its transport backend on the worker.""" + config = super().configure_worker(config) + config["external_stream_backend"] = self._require_backend() + return config + + def configure_replayer(self, config: ReplayerConfig) -> ReplayerConfig: + """Set this provider and its transport backend on the replayer.""" + config = super().configure_replayer(config) + config["external_stream_backend"] = self._require_backend() + return config + + def workflow_provider(self) -> _RedisWorkflowProvider: + """The workflow half, over the transport's subscriptions and staged output.""" + return _RedisWorkflowProvider(self._idle_timeout) + + def get_stream_handle( + self, client: Client, workflow_id: str, *, run_id: str | None = None + ) -> RedisStreamHandle: + """A handle on ``workflow_id``'s topics; it follows the chain by construction.""" + return RedisStreamHandle(self, client, workflow_id, run_id) + + def get_activity_stream_handle( + self, + client: Client, + activity_id: str, + *, + workflow_id: str | None = None, + run_id: str | None = None, + ) -> RedisStreamHandle: + """A handle on the topics ``activity_id`` owns, apart from any workflow's. + + Without ``workflow_id`` the activity is a standalone one and ``run_id`` + names its run; with one it is that workflow's activity and ``run_id`` + names the workflow's run. The streams are keyed by that run, so a + retry writes to the same ones and a reader sees the attempt change as + ``SUPERSEDED``, while an id started again in a new run starts new + ones. Without ``run_id`` each read, ``latest()`` and the first append + of a producer describe the owner and take the run current at that + moment. A read ends when the activity is terminal and the retained + tail is delivered; the store has no gate on an append after that, so + a late attempt still lands, and a read that has ended does not see it. + """ + return RedisStreamHandle(self, client, workflow_id, run_id, activity_id) + + async def create_standalone_stream( + self, + client: Client, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> RedisStreamHandle: + """Create the standalone stream ``stream_id``, or find it with the same policy. + + The policy is written once into the stream's hash and applied on every + append to any of its topics. ``retention`` left unset takes this + provider's default window; the other two bounds are off unless set. + + Raises: + ValueError: ``stream_id`` is empty, a bound is not positive, or + the stream exists with a different policy. + """ + if not stream_id: + raise ValueError("stream_id must not be empty") + if retention is not None and retention <= timedelta(0): + raise ValueError("retention must be positive") + if max_records is not None and max_records <= 0: + raise ValueError("max_records must be positive") + if max_bytes is not None and max_bytes <= 0: + raise ValueError("max_bytes must be positive") + backend = self._require_backend() + owner = _StandaloneOwner(client.namespace, stream_id) + policy = _policy_fields( + self._retention if retention is None else retention, max_records, max_bytes + ) + outcome = await backend._client.register_script(_CREATE_STANDALONE_LUA)( + keys=[owner.meta(_prefix(backend))], args=policy + ) + if _text(outcome) == "conflict": + raise ValueError( + f"{owner} exists with another policy; a policy is set when the " + "stream is created and does not change" + ) + return RedisStreamHandle(self, client, None, None, stream_id=stream_id) + + def get_standalone_stream_handle( + self, client: Client, stream_id: str + ) -> RedisStreamHandle: + """A handle on the standalone stream ``stream_id``. + + Nothing is checked here: a ``read``, ``latest``, ``producer`` or + ``close`` on a stream that was never created raises + :class:`temporalio.streams.StreamNotFoundError` when it is used. + """ + return RedisStreamHandle(self, client, None, None, stream_id=stream_id) + + async def close(self) -> None: + """Release the Redis connection this provider opened; a caller's stays open.""" + self._backend = None + client, self._owned_client = self._owned_client, None + if client is not None: + await client.aclose() diff --git a/temporalio/streams/providers/temporal_streams.nexusrpc.yaml b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml new file mode 100644 index 0000000000..1b027cfebc --- /dev/null +++ b/temporalio/streams/providers/temporal_streams.nexusrpc.yaml @@ -0,0 +1,234 @@ +# The stream endpoint's wire contract. Inputs reject unknown fields so a typo +# fails loudly at the handler; outputs accept them so a handler can add a +# field before every caller has been rebuilt from the newer contract. +nexusrpc: "1.0.0" + +services: + TemporalStreams: + fqn: TemporalStreams + description: >- + The stream endpoint's two operations. One Temporal-authenticated endpoint + hides the store behind it, so an operator switches storage without + touching callers. Reads hand out batches and an append carries one batch + per call, because a Nexus operation per record costs too much for token + streams. A record crosses as the serialized + temporal.api.stream.v1.StreamRecord, the same bytes every store keeps, so + a caller in any language decodes it with the api protos alone. + operations: + append: + fqn: append + description: >- + Append one batch on the caller's account. The handler writes a batch + once: a repeat of the last batch_index answers with where the + original landed, and an index that skips ahead, one already behind + the last, a sequence that does not continue, or a producer attempt + the handler has no state for is refused. Supersession records are not + transported: a reader re-synthesizes them from the attempts it + observes. + input: { $ref: "#/$defs/AppendInput" } + output: { $ref: "#/$defs/AppendOutput" } + read: + fqn: read + description: >- + Answer with the records after the caller's token, or time out. The + call parks until it has max_records records or wait_ms elapses, + whichever comes first, and returns whatever it collected. It says + when the store has ended the read, so the caller can stop. + input: { $ref: "#/$defs/ReadInput" } + output: { $ref: "#/$defs/ReadOutput" } + +$defs: + StreamRef: + type: object + description: >- + A stream, named by its owner and a topic: what an operation returns to + hand a stream to its caller, and what read and append take in place of + an owner spelled out. It names no cursor and no store, so the same + reference is good behind any endpoint that serves the owner. A member + the owner kind does not use is absent or null; null is how the SDK + writes an unset member of its own StreamRef. + properties: + kind: + type: string + enum: [workflow, activity, standalone] + description: >- + What owns the stream. A workflow's streams are keyed by workflow_id + and, when pinned, run_id. An activity's own streams are keyed by + activity_id and, when a workflow scheduled it, that workflow's ids. + A standalone stream has its own stream_id and no execution behind + it. + workflow_id: + oneOf: [{ type: string }, { type: "null" }] + description: >- + The owning workflow, or the workflow that scheduled the owning + activity. Required for a workflow owner. + run_id: + oneOf: [{ type: string }, { type: "null" }] + description: >- + Pin the owner to this run. Absent follows the execution chain + across continue-as-new, which is what a producer normally wants. + activity_id: + oneOf: [{ type: string }, { type: "null" }] + description: "The owning activity. Required for an activity owner." + stream_id: + oneOf: [{ type: string }, { type: "null" }] + description: "The stream's own id. Required for a standalone owner." + topic: + type: string + description: "The topic on the owner's stream." + required: + - kind + - topic + additionalProperties: false + + AppendInput: + type: object + description: "One append call: who is writing, where, and what." + properties: + stream: { $ref: "#/$defs/StreamRef" } + producer_id: + type: string + description: >- + Identifies the writer across its retries, so its attempts can be + ordered. Never empty here: the caller resolved it, from the activity + context when it was not given. + attempt: + type: integer + description: >- + The generation this producer is writing. A later attempt supersedes + an earlier one. + sequence: + type: integer + minimum: 0 + description: >- + The producer's sequence of the first record in this batch; the batch + is numbered from it and a finish takes the next number. It has to + continue where the previous batch ended, so the handler and the + store agree on every record's position within the attempt. + batch_index: + type: integer + minimum: 1 + description: >- + Counts this producer's batches from 1. The handler answers a repeat + of the last index with the original's position, and refuses one that + skips ahead, one already behind the last, or one that resumes an + attempt it never saw start. + payloads: + type: array + items: + type: string + contentEncoding: base64 + description: >- + The record bodies to write, in order, each a serialized + temporal.api.common.v1.Payload. A payload codec configured on the + caller has already run on them. Empty on a call that only finishes. + finish: + type: boolean + description: >- + Write FINISH for this producer after the batch: it will write nothing + more on the topic. Says nothing about the producer's outcome. + required: + - stream + - producer_id + - attempt + - sequence + - batch_index + additionalProperties: false + + AppendOutput: + type: object + description: "Where the append landed, when the store can say." + properties: + cursor: + type: string + description: >- + Opaque token naming the last record written by this call, or by the + original when the call repeated the last batch. Absent when the + store learns positions only at read time; such a caller positions + itself with a latest_only read. + additionalProperties: true + + ReadInput: + type: object + description: "One read call: where to resume from and how long to wait." + properties: + stream: { $ref: "#/$defs/StreamRef" } + after_token: + type: string + description: >- + Opaque cursor from an earlier record or append. The read resumes + strictly after the record it names, so a caller never sees that + record twice. Empty starts at the beginning. The token is produced by + whichever store sits behind the endpoint, so a caller cannot tell + which one that is, and a token from another store is refused. + max_records: + type: integer + minimum: 1 + maximum: 1000 + description: "Return at most this many records. Defaults to 100 when omitted." + wait_ms: + type: integer + minimum: 0 + maximum: 60000 + description: >- + Wait at most this long for records to arrive before answering. The + handler shortens it to fit the request deadline. A call that collects + nothing answers with no records and the caller's own token. + latest_only: + type: boolean + description: >- + Answer with the newest position and no records, for a reader that + wants to follow from now. + last_n: + type: integer + minimum: 1 + description: >- + With no after_token, start at the newest this many records, or at + all of them when the stream holds fewer. Records of every kind + count. Refused alongside an after_token, which is how a read + resumes. + required: + - stream + additionalProperties: false + + RecordWire: + type: object + description: "One record on the wire: its cursor and the record itself." + properties: + token: + type: string + description: >- + Opaque cursor naming this record. Pass it as after_token to resume + just past it. + record: + type: string + contentEncoding: base64 + description: >- + The serialized temporal.api.stream.v1.StreamRecord: topic, kind, + producer_id, attempt, sequence and body, as the store holds it. + required: + - token + - record + additionalProperties: true + + ReadOutput: + type: object + description: "What one read call answered, and where to resume." + properties: + records: + type: array + items: + $ref: "#/$defs/RecordWire" + description: "The records after the caller's token, in stream order." + next_token: + type: string + description: >- + Opaque cursor to pass as after_token on the following call. Echoes + the caller's own token when the call collected nothing. + done: + type: boolean + description: >- + The store ended the read: the owning execution, or its chain, is + closed and every retained record after the caller's token has been + delivered. Nothing more will arrive, so the caller stops. + additionalProperties: true diff --git a/temporalio/streams/providers/workflow_streams.py b/temporalio/streams/providers/workflow_streams.py new file mode 100644 index 0000000000..8a5ed5daad --- /dev/null +++ b/temporalio/streams/providers/workflow_streams.py @@ -0,0 +1,1598 @@ +"""The provider over the shipped Workflow Streams transport (Option 0). + +Speaks the shipped contrib feature's wire format, the +``__temporal_workflow_stream_*`` Signal, Update and Query, so interface code +and existing Workflow Streams code share one log, and old histories replay. +Records live in the owning workflow's History, which is also this provider's +limit: the shipped caps (payloads in History, the Signal cap, bounded +subscribers) are transport properties and remain. + +The mapping, in one place: + +- A topic is the shipped topic of the same name in the workflow's one log. + A record rides as the item's ``Payload``: its data is the serialized + ``StreamRecord`` proto and its encoding is ``binary/plain``. The shipped + code stores and returns that ``Payload`` untouched, so the body's own + encoding never meets the transport. +- An outside publish is an Update, ``__temporal_streams_publish``, carrying + the shipped publish Signal's input. Producer identity dedupes through the + shipped publisher state, the publisher id being ``producer#attempt`` and + the sequence where the batch's records end, and the Update adds what a + Signal has no room for: a response. The workflow answers with the run and + offset the batch landed at, so ``append()`` returns a cursor, and it + refuses a repeat that carries *different* content at a sequence it holds, + or one behind its most recent, before accepting the Update, so the caller + gets :class:`temporalio.streams.StreamProducerError` and the log takes + nothing. Content is compared by a hash the workflow keeps per producer. + The Update's id is derived from producer, sequence and content, so a + retry after a lost reply is answered by the server from the first + outcome. The Action cost is unchanged: one Update per append batch in + place of one Signal. + + The Signal stays as the transport a worker that predates the Update + serves. A producer that is told twice, across a task boundary, that the + workflow has no publish Update falls back to it for the rest of its life + and returns ``None`` from ``append()``, since a Signal learns positions at + read time; ``publish_transport="signal"`` on the provider picks it from + the start. Records land in the same log either way, so a log written by + Signals reads the same. The shipped Signal handler's dedupe is one table + per publisher across topics, so on that transport one identity writing + two topics at the same sequence has its second batch dropped; the Update + keeps one per producer and topic, as the other providers do. +- A log belongs to one run and is not carried across continue-as-new, so a + cursor names the run as well as the offset. A handle without a run id + reads run after run: each log through the poll Update while its run is + open and through the tail Query once it has closed, then the successor's + from its first record. A run that was reset is followed too, into the run + describe names, at the position the read had reached: the reset run + rebuilt the base run's log up to the reset point by replay, so the items + before it sit at the same offsets. The reset itself is not reported to + the reader as a record; that is a wire change for a later round. +- The workflow-side stream object belongs to the workflow instance, found + through the handler the shipped class registers on it. An evicted and + rebuilt workflow gets its own, so a task that failed leaks nothing into + the next attempt's log and a replayed run does not see records twice. + The provider's handlers are registered as the instance is initialised, + before the first task's Signals and Updates are applied, so a publish or + poll that arrives with that task finds them; the stream object is bound + on first use, adopting one the workflow built in its ``__init__`` or + constructing the provider's own. +- An activity a workflow scheduled keeps its own streams inside that + workflow's log, under the reserved topic ``activity//`` + with ``%`` and ``/`` in the id percent-encoded, the way the native + provider reserves ``activity/`` in the owner's map. The record itself + carries the plain name. A standalone activity has no workflow to host a + log, so its streams are refused, and a workflow's own topic may not start + with the reserved prefix. A standalone stream, one with no owner at all, + is refused too: there is no workflow whose state could hold it. A housing + workflow per stream id is the design option for that, not built here. +- A record's body meets the client's data converter at the transport's + envelope rather than one record at a time. A batch travels as a Signal or + Update argument and comes back as an Update or Query result, and the SDK + runs the payload codec and external storage over those the way it does + over every payload it sends, off the workflow thread, so a batch above the + storage threshold is offloaded as a claim and a codec protects it in + History. The bodies inside are left as the payload converter produced + them, because the workflow thread reads them straight out of its state + and could not decode a codec's output or redeem a claim there. So the + worker's converter has to match the clients', as it does for every other + payload, and a client with a converter of its own cannot read another's + records. What each record does carry is the plaintext hash of its body + under ``temporal.io/content-hash``, stamped by the producer before the + envelope is encoded, and the workflow matches a repeated batch by those + hashes rather than by the bytes. +- A handle names its stream as a :class:`temporalio.streams.StreamRef` with + ``ref()``, a workflow's or an activity's, and ``close()`` refuses, since + an owned stream ends with its owner. +""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import logging +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Generic, Literal, TypeVar + +from google.protobuf.message import DecodeError + +from temporalio import workflow +from temporalio.api.common.v1 import Payload +from temporalio.client import ( + Client, + WorkflowExecutionStatus, + WorkflowHandle, + WorkflowHistoryEventFilterType, + WorkflowQueryFailedError, + WorkflowUpdateFailedError, + WorkflowUpdateRPCTimeoutOrCancelledError, + WorkflowUpdateStage, +) +from temporalio.contrib.workflow_streams import ( + POLL_UPDATE_NAME, + PUBLISH_SIGNAL_NAME, + STREAM_DRAINING_ERROR_TYPE, + TRUNCATED_OFFSET_ERROR_TYPE, + PollInput, + PollResult, + PublishEntry, + PublishInput, + WorkflowStream, +) +from temporalio.converter import PayloadConverter +from temporalio.exceptions import ApplicationError +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams._body import CONTENT_HASH_KEY, content_hash +from temporalio.streams._errors import ( + StreamCursorError, + StreamError, + StreamNotFoundError, + StreamProducerError, + StreamUnsupportedError, +) +from temporalio.streams._provider import ReadSource, WriteSink +from temporalio.streams._record import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamRecord, + check_read_start, +) +from temporalio.streams._ref import StreamRef +from temporalio.streams._topic import StreamTopic, resolve_topic +from temporalio.streams._wire import ( + RecordDecoder, + WireRecord, + cursor_position, + mint_cursor, + producer_identity, + to_wire, +) +from temporalio.streams.providers import ProviderPlugin +from temporalio.worker._workflow_instance import QUERY_HANDLER_NOT_FOUND + +__all__ = [ + "PRODUCER_CONFLICT_ERROR_TYPE", + "PublishTransport", + "WorkflowStreamsActivityHandle", + "WorkflowStreamsHandle", + "WorkflowStreamsProducer", + "WorkflowStreamsProvider", +] + +T = TypeVar("T") + +PublishTransport = Literal["update", "signal"] +"""How an outside producer's batches reach the workflow.""" + +PRODUCER_CONFLICT_ERROR_TYPE = "StreamProducerConflict" +"""The ``ApplicationError.type`` the publish Update refuses a producer conflict with. + +Public so a caller driving the Update itself can tell the refusal from any +other failure. +""" + +_PROVIDER = "workflow_streams" +_PUBLISH_UPDATE = "__temporal_streams_publish" +_TAIL_QUERY = "__temporal_streams_tail" +_LATEST_QUERY = "__temporal_streams_latest" +_START_QUERY = "__temporal_streams_start" +_ENCODING = b"binary/plain" +# The topics an activity owns live in its workflow's log under this prefix, +# so no workflow topic may start with it. +_ACTIVITY_PREFIX = "activity/" +# The same cap the shipped poll path answers under, because both are one +# response through the same server. +_MAX_TAIL_RESPONSE_BYTES = 1_000_000 +# The server's failure type for an accepted Update whose run closed before +# answering it: the poll's way of saying the run is over. +_UPDATE_OUTLIVED_RUN = "AcceptedUpdateCompletedWorkflow" +# The SDK's rejection of an Update no handler is registered for. It shares +# its wording with the Query one. +_HANDLER_NOT_FOUND = QUERY_HANDLER_NOT_FOUND + +logger = logging.getLogger(__name__) + + +def _run_is_closing(error: RPCError) -> bool: + """Whether the server refused an Update because the run is completing. + + The window between a run deciding to close, or continue as new, and its + close being recorded; the next attempt learns how it closed. + """ + return ( + error.status == RPCStatusCode.FAILED_PRECONDITION and "closing" in error.message + ) + + +def _require_topic(topic: str) -> None: + if not topic: + raise ValueError("topic must not be empty") + if topic.startswith(_ACTIVITY_PREFIX): + raise ValueError( + f"topic {topic!r} is reserved: names under {_ACTIVITY_PREFIX!r} hold the " + "streams of the workflow's activities on the workflow_streams provider" + ) + + +def _activity_topic(activity_id: str, topic: str) -> str: + """The reserved name ``topic`` of ``activity_id``'s streams takes in the log. + + The id is percent-encoded so an id holding ``/`` cannot be read as two + components; the name comes last, so it may hold anything. + """ + escaped = activity_id.replace("%", "%25").replace("/", "%2F") + return f"{_ACTIVITY_PREFIX}{escaped}/{topic}" + + +def _cursor(run_id: str, offset: int) -> Cursor: + return mint_cursor(_PROVIDER, f"{run_id}:{offset}") + + +def _position(after: Cursor) -> tuple[str, int] | None: + """The ``(run id, offset)`` a cursor of this provider names, or ``None`` for BEGINNING.""" + token = cursor_position(after, provider=_PROVIDER) + if token is None: + return None + run_id, _, offset = token.rpartition(":") + try: + if not run_id: + raise ValueError + return run_id, int(offset) + except ValueError: + raise StreamCursorError( + f"cursor {after.token!r} does not name a run and an offset on the " + "workflow_streams provider" + ) from None + + +def _wrap(record: WireRecord) -> Payload: + return Payload(metadata={"encoding": _ENCODING}, data=record.SerializeToString()) + + +def _unwrap(cursor: Cursor, payload: Payload, warn: Any) -> WireRecord | None: + try: + return WireRecord.FromString(payload.data) + except DecodeError as error: + # An item another publisher put on this topic, or a corrupt one: + # skip and say so, so one bad record cannot pin a reader. + warn("skipping stream record at %s: %s", cursor, error) + return None + + +def _entry_data(payload: Payload) -> str: + # The documented wire form of PublishEntry.data. + return base64.b64encode(payload.SerializeToString()).decode("ascii") + + +def _content(publish: PublishInput) -> str: + """A digest of one batch's content, length-delimited so a resplit cannot collide. + + Each record counts by the plaintext hash stamped on it under + ``CONTENT_HASH_KEY``, so a body's encoding never enters the identity a + repeat is matched by; a record without a body, such as ``FINISH``, counts + by its bytes. + """ + digest = hashlib.sha256() + for entry in publish.items: + record = WireRecord.FromString(_decode_payload(entry.data).data) + stamped = record.metadata.get(CONTENT_HASH_KEY) + identity = ( + stamped.data + if stamped is not None + else record.SerializeToString(deterministic=True) + ) + for part in (entry.topic.encode(), identity): + digest.update(len(part).to_bytes(8, "big")) + digest.update(part) + return digest.hexdigest() + + +def _producer_key(publish: PublishInput) -> tuple[str, str]: + # A batch is one topic's, so its first entry names the stream. + topic = publish.items[0].topic if publish.items else "" + return publish.publisher_id, topic + + +def _decode_payload(data: str) -> Payload: + return Payload.FromString(base64.b64decode(data)) + + +def _publish_id(publish: PublishInput, content: str) -> str: + # One id per (producer, sequence, content): a retry after a lost reply + # is answered from the first outcome without reaching the workflow, and a + # divergent repeat is a new Update the workflow gets to refuse. + key = f"{publish.publisher_id}\0{publish.sequence}\0{content}".encode() + return f"streams-publish-{hashlib.sha256(key).hexdigest()[:32]}" + + +@dataclass +class _PublishResult: + """The publish Update's answer: where the batch's last record sits.""" + + run_id: str + last_offset: int + + +@dataclass +class _Held: + """What the workflow keeps of a producer's most recent accepted batch.""" + + sequence: int + content: str + last_offset: int + + +class _Shipped: + """Constructs the shipped stream object, which insists on a caller named ``__init__``.""" + + def __init__(self) -> None: + self.stream = WorkflowStream() + + +class _InstanceStream: + """The provider's handlers on one workflow instance, over the shipped stream object. + + Built when the instance is initialised, before the workflow's own + ``__init__`` and before any Signal or Update of the first task is + applied, so the handlers are found by whatever arrives with that task. + The shipped stream object is bound on first use rather than here: a + workflow migrating from the contrib feature constructs its own in + ``__init__``, which the shipped class refuses to do twice, so this + adopts that one when it exists and constructs the provider's own when + nothing has, from a handler that may write or from the workflow half. + """ + + def __init__(self) -> None: + self._stream: WorkflowStream | None = None + # Per producer and topic, the most recent batch taken by the publish + # Update, so a repeat is answered and a divergent one refused. A + # producer's identity is one per stream, as on every provider, so + # the same identity on two topics is two producers. Workflow state, + # rebuilt on replay like the log itself. + self._producers: dict[tuple[str, str], _Held] = {} + if workflow.get_query_handler(_TAIL_QUERY) is None: + # The poll Update stops answering once the workflow is closing, + # and a reader between polls at that moment would lose what the + # final task published. The log is workflow state, so a Query + # still serves it after completion. + workflow.set_query_handler(_TAIL_QUERY, self._tail) + if workflow.get_query_handler(_LATEST_QUERY) is None: + workflow.set_query_handler(_LATEST_QUERY, self._latest) + if workflow.get_query_handler(_START_QUERY) is None: + workflow.set_query_handler(_START_QUERY, self._start) + if workflow.get_update_handler(_PUBLISH_UPDATE) is None: + workflow.set_update_handler( + _PUBLISH_UPDATE, self._publish, validator=self._validate_publish + ) + if workflow.get_update_handler(POLL_UPDATE_NAME) is None: + # Stands in for the shipped poll handler until a stream object is + # bound, whose constructor then registers the real one over it. + workflow.set_update_handler( + POLL_UPDATE_NAME, self._poll, validator=self._validate_poll + ) + + @property + def held(self) -> WorkflowStream | None: + """The shipped stream object, if the workflow or a handler has bound one. + + Looks one up and never constructs, so a Query may ask. + """ + if self._stream is None: + self._stream = _registered_stream() + return self._stream + + @property + def stream(self) -> WorkflowStream: + """The shipped stream object, adopting the workflow's own or constructing the provider's.""" + held = self.held + if held is None: + held = self._stream = _Shipped().stream + return held + + def _validate_poll(self, payload: PollInput) -> None: + held = self.held + if held is not None: + held._validate_poll(payload) # pyright: ignore[reportPrivateUsage] + + async def _poll(self, payload: PollInput) -> PollResult: + return await self.stream._on_poll(payload) # pyright: ignore[reportPrivateUsage] + + def _validate_publish(self, publish: PublishInput) -> None: + """Refuse a conflicting batch before the Update is accepted. + + Refused here rather than in the handler so the refusal writes no + event: the caller learns of it and the log is untouched. + + Raises: + ApplicationError: Typed ``PRODUCER_CONFLICT_ERROR_TYPE``. The + batch repeats this producer's most recent sequence with + other content, or names a sequence behind it. + """ + held = self._producers.get(_producer_key(publish)) + if held is None: + return + if publish.sequence < held.sequence: + raise ApplicationError( + f"producer {publish.publisher_id!r} already wrote past sequence " + f"{publish.sequence}; only its most recent append can be repeated", + type=PRODUCER_CONFLICT_ERROR_TYPE, + ) + if publish.sequence == held.sequence and _content(publish) != held.content: + raise ApplicationError( + f"producer {publish.publisher_id!r} repeated sequence " + f"{publish.sequence} with different content", + type=PRODUCER_CONFLICT_ERROR_TYPE, + ) + + def _publish(self, publish: PublishInput) -> _PublishResult: + """Take one outside batch into the log and answer where it landed. + + A repeat of the producer's most recent batch, which the validator + let through because its content matches, is answered with the + original position and writes nothing. The Update keeps its own + table rather than the shipped Signal handler's, which is one per + publisher across topics; a producer stays on one transport for its + life, so the two never judge the same batch. + """ + run_id = workflow.info().run_id + key = _producer_key(publish) + held = self._producers.get(key) + if held is not None and publish.sequence == held.sequence: + return _PublishResult(run_id=run_id, last_offset=held.last_offset) + stream = self.stream + for entry in publish.items: + stream.topic(entry.topic).publish(_decode_payload(entry.data)) + last = stream.next_offset - 1 + self._producers[key] = _Held(publish.sequence, _content(publish), last) + return _PublishResult(run_id=run_id, last_offset=last) + + def _tail(self, from_offset: int, topic: str) -> dict[str, Any]: + """One page of ``topic``'s items at or past ``from_offset``. + + Filtered and capped here rather than at the caller, because a Query + response has to fit the server's blob limit and a log the reader only + wants one topic of can be much larger than that. + """ + stream = self.held + if stream is None: + # Nothing has been bound, so nothing has been published. + return { + "items": [], + "next_offset": 0, + "more_ready": False, + "base_offset": 0, + } + items: list[dict[str, Any]] = [] + size = 0 + next_offset = stream.next_offset + more_ready = False + held = stream.items_from(from_offset) + for offset, item_topic, payload in held: + if item_topic != topic: + continue + data = base64.b64encode(payload.SerializeToString()).decode("ascii") + if items and size + len(data) > _MAX_TAIL_RESPONSE_BYTES: + next_offset, more_ready = offset, True + break + size += len(data) + items.append({"offset": offset, "topic": item_topic, "data": data}) + return { + "items": items, + "next_offset": next_offset, + "more_ready": more_ready, + # Where the retained log starts at or past ``from_offset``: a + # reader whose position is below it has fallen behind truncation. + "base_offset": held[0][0] if held else next_offset, + } + + def _latest(self, topic: str) -> int: + """The newest offset holding a record on ``topic``, or -1 when it has none. + + The log orders every topic together, so the head of the log is not an + answer about one topic. Scanning here costs one Query rather than + shipping the log to the caller to find the same thing. + """ + stream = self.held + if stream is None: + return -1 + for offset, item_topic, _ in reversed(stream.items_from(0)): + if item_topic == topic: + return offset + return -1 + + def _start(self, topic: str, last_n: int) -> int: + """Where a read starts: the log's head, or ``topic``'s newest ``last_n`` records. + + Zero ``last_n`` asks for the head, which is where ``END`` starts. + """ + stream = self.held + return 0 if stream is None else start_offset(stream, topic, last_n) + + +def start_offset(stream: WorkflowStream, topic: str, last_n: int) -> int: + """The log offset a read of ``topic`` starts at. + + ``last_n`` of zero is the head of the log, so only what is published next + is read. Otherwise it is the offset of ``topic``'s ``last_n``-th newest + item, or the oldest one the log holds when there are fewer. The log is + workflow state, so the workflow half answers the same on every replay. + """ + if last_n <= 0: + return stream.next_offset + items = stream.items_from(0) + seen = 0 + for offset, item_topic, _ in reversed(items): + if item_topic == topic: + seen += 1 + if seen == last_n: + return offset + return items[0][0] if items else stream.next_offset + + +def _registered_stream() -> WorkflowStream | None: + handler = workflow.get_signal_handler(PUBLISH_SIGNAL_NAME) + if handler is None: + return None + stream = getattr(handler, "__self__", None) + if not isinstance(stream, WorkflowStream): + raise RuntimeError( + f"the {PUBLISH_SIGNAL_NAME!r} signal on this workflow is handled by " + "something other than a WorkflowStream, so the workflow_streams " + "provider cannot share its log" + ) + return stream + + +def _instance() -> _InstanceStream: + # Found on the instance rather than in a process-level map keyed by run + # id: the SDK rebuilds an evicted workflow from history as a new object, + # and a map would hand that object the stale log with its unregistered + # handlers and the records of a task that failed. + handler = workflow.get_update_handler(_PUBLISH_UPDATE) + registered = getattr(handler, "__self__", None) + if isinstance(registered, _InstanceStream): + return registered + return _InstanceStream() + + +class _WSReadSource: + """Reads the signal-fed log the shipped feature keeps in workflow state.""" + + def __init__( + self, stream: WorkflowStream, topic: str, start: int, run_id: str + ) -> None: + self._stream = stream + self._topic = topic + self._offset = start + self._run_id = run_id + self._closed = False + + async def next_batch(self) -> list[tuple[Cursor, WireRecord]]: + while True: + if self._closed: + raise StopAsyncIteration + await workflow.wait_condition( + lambda: self._closed or self._stream.next_offset > self._offset + ) + if self._closed: + raise StopAsyncIteration + batch: list[tuple[Cursor, WireRecord]] = [] + for offset, topic, payload in self._stream.items_from(self._offset): + if topic != self._topic: + continue + cursor = _cursor(self._run_id, offset) + wire = _unwrap(cursor, payload, workflow.logger.warning) + if wire is not None: + batch.append((cursor, wire)) + self._offset = self._stream.next_offset + if batch: + return batch + + def close(self) -> None: + self._closed = True + + +class _WSWriteSink: + def __init__(self, stream: WorkflowStream, topic: str) -> None: + self._handle = stream.topic(topic) + + def publish(self, record: WireRecord) -> None: + # Appending to workflow state commits with the task, and a poll + # Update's result rides the same task completion, so a failed task + # leaks nothing: rule 1 through the shipped mechanics. + self._handle.publish(_wrap(record)) + + +class _WSWorkflowProvider: + """The workflow half: the shipped stream object of one workflow instance. + + Made per instance by the worker, so the stream object it captures on + first use is this instance's, and the finish hook lets go of that one + rather than of whatever the thread's handler lookup answers at the time. + """ + + def __init__(self) -> None: + self._stream: WorkflowStream | None = None + + def _own_stream(self) -> WorkflowStream: + if self._stream is None: + self._stream = _instance().stream + return self._stream + + def open_reader( + self, topic: str, *, after: Cursor, last: int | None = None + ) -> ReadSource: + check_read_start(after, last) + _require_topic(topic) + run_id = workflow.info().run_id + start = 0 + # The log is workflow state, so END and last= resolve against it here + # and land on the same offset on every replay. + if last is not None: + return _WSReadSource( + self._own_stream(), + topic, + start_offset(self._own_stream(), topic, last), + run_id, + ) + if after == END: + stream = self._own_stream() + return _WSReadSource(stream, topic, stream.next_offset, run_id) + named = _position(after) + if named is not None: + if named[0] != run_id: + raise StreamCursorError( + f"cursor {after.token!r} names another run; a run's log is its own" + ) + start = named[1] + 1 + return _WSReadSource(self._own_stream(), topic, start, run_id) + + def open_writer(self, topic: str) -> WriteSink: + _require_topic(topic) + return _WSWriteSink(self._own_stream(), topic) + + def on_workflow_start(self) -> None: + # Called as the instance is initialised, so the handlers exist before + # the first task's Updates are evaluated: an outside publish or poll + # that arrives with that task is served rather than rejected. The + # stream object itself is bound later, once the workflow's own + # __init__ has had its chance to construct one. + _instance() + + async def on_workflow_finish(self) -> None: + # An Option 0 stream dies with its run, and a parked long-poll Update + # would otherwise hold completion open. Same recipe the shipped + # feature documents before a return or a continue-as-new. Asked of + # the instance rather than this object's cache, because a poll can + # bind the stream before workflow code touches it. + held = _instance().held + if held is None: + return + held.detach_pollers() + await workflow.wait_condition(workflow.all_handlers_finished) + + +class WorkflowStreamsProducer(Generic[T]): + """Appends through the publish Update, or the shipped publish Signal. + + Direct rather than through ``WorkflowStreamClient`` because the interface + owns the publisher identity: it must be ``producer#attempt`` for the + shipped dedupe to drop a retry and pass a new generation, and the client + would use its own random id. + + Sequences are committed only after the server accepted the batch. A + batch whose send raised stays pending and goes out again under the same + sequence, either when the caller retries the same values or ahead of + whatever the caller sends next, so an ambiguous failure writes the batch + once and loses nothing. A batch the workflow refused is dropped from the + pending slot: the refusal is the answer. + """ + + def __init__( + self, + handle: Any, + converter: PayloadConverter, + topic: str, + producer_id: str, + attempt: int, + *, + item_topic: str | None = None, + transport: PublishTransport = "update", + retry_cooldown: timedelta = timedelta(milliseconds=100), + ) -> None: + """Bind this producer to ``topic`` on the workflow behind ``handle``. + + ``item_topic`` is the name the log files the records under when it + differs from the name the records carry, as an activity's reserved + topics do. ``transport`` is how batches travel; ``"update"`` falls + back to ``"signal"`` on a workflow that has no publish Update, after + one retry ``retry_cooldown`` later. + """ + self._handle = handle + self._converter = converter + self._topic = topic + self._item_topic = topic if item_topic is None else item_topic + self._producer_id = producer_id + self._attempt = attempt + self._transport: PublishTransport = transport + self._retry_cooldown = retry_cooldown + # One-based, because zero on the wire says the producer does not + # number its records and this one does. + self._sequence = 1 + self._pending: tuple[list[PublishEntry], int] | None = None + self._last: Cursor | None = None + + @property + def transport(self) -> PublishTransport: + """How this producer's batches travel now: the Signal once it fell back.""" + return self._transport + + @property + def producer_id(self) -> str: + """Who this producer writes as.""" + return self._producer_id + + @property + def attempt(self) -> int: + """The generation this producer is writing.""" + return self._attempt + + @property + def _publisher_id(self) -> str: + return ( + f"{self._producer_id}#{self._attempt}" + if self._attempt + else self._producer_id + ) + + async def append(self, *values: T) -> Cursor | None: + """Append ``values`` and return the cursor of the batch's last record. + + A repeat of this producer's most recent append with the same content + is answered with the position the original landed at; an empty call + returns the position of the last record this producer wrote, or + ``None`` before its first. ``None`` is also the answer on the Signal + transport, which learns positions at read time, so a caller that + wants to follow from now asks :meth:`WorkflowStreamsHandle.latest`. + + Raises: + StreamProducerError: The workflow holds this sequence with other + content, or the producer already wrote past it. + StreamNotFoundError: The workflow does not exist or has closed. + """ + if not values: + return self._last + return await self._send([(RecordKind.DATA, value) for value in values]) + + async def finish(self) -> None: + """Write ``FINISH`` for this producer on this topic.""" + await self._send([(RecordKind.FINISH, None)]) + + def _entries( + self, batch: list[tuple[RecordKind, Any]] + ) -> tuple[list[PublishEntry], int]: + sequence = self._sequence + entries = [] + for kind, value in batch: + wire = to_wire( + self._converter, + topic=self._topic, + kind=kind, + value=value, + producer_id=self._producer_id, + attempt=self._attempt, + sequence=sequence, + ) + sequence += 1 + if wire.HasField("body"): + # The plaintext hash the workflow matches a repeat by, taken + # as the converter produced the body, before the transport's + # codec meets the envelope. + wire.metadata[CONTENT_HASH_KEY].CopyFrom( + Payload( + metadata={"encoding": _ENCODING}, + data=content_hash(wire.body).encode(), + ) + ) + entries.append( + PublishEntry(topic=self._item_topic, data=_entry_data(_wrap(wire))) + ) + return entries, sequence + + async def _send(self, batch: list[tuple[RecordKind, Any]]) -> Cursor | None: + entries, next_sequence = self._entries(batch) + if self._pending is not None and self._pending[0] != entries: + # The caller moved on from a batch whose send raised. It goes + # first, under the sequence it already had, so a copy the server + # did accept is answered or dropped and one it never saw lands. + # The new batch is then renumbered behind it. + await self._deliver(*self._pending) + entries, next_sequence = self._entries(batch) + return await self._deliver(entries, next_sequence) + + async def _deliver( + self, entries: list[PublishEntry], next_sequence: int + ) -> Cursor | None: + # The dedupe sequence is where this producer's records end, not how + # many batches it has sent. The two differ once a retry batches its + # records differently from the send it is repeating, and a counter of + # batches then either drops a batch of new records or lets records + # that are already there through a second time. + self._pending = (entries, next_sequence) + publish = PublishInput( + items=entries, publisher_id=self._publisher_id, sequence=next_sequence + ) + if self._transport == "signal": + landed = await self._signal(publish) + else: + landed = await self._update(publish) + self._sequence = next_sequence + self._pending = None + if landed is not None: + self._last = landed + return landed + + async def _update(self, publish: PublishInput) -> Cursor | None: + content = _content(publish) + retried = False + while True: + try: + answer = await self._handle.execute_update( + _PUBLISH_UPDATE, + publish, + id=_publish_id(publish, content), + result_type=_PublishResult, + ) + except WorkflowUpdateFailedError as error: + cause = error.cause + cause_type = getattr(cause, "type", None) + if cause_type == PRODUCER_CONFLICT_ERROR_TYPE: + # Final for this batch: there is nothing to send again. + self._pending = None + raise StreamProducerError(str(cause)) from error + if cause_type == _UPDATE_OUTLIVED_RUN: + # Accepted, then the run closed before the handler ran: + # the same answer a Signal to a closed run gets. + raise StreamNotFoundError( + f"workflow {self._handle.id!r} closed before taking the " + "batch, so its stream cannot be appended to" + ) from error + if _HANDLER_NOT_FOUND not in str(cause): + raise StreamError( + f"the publish update on workflow {self._handle.id!r} " + f"failed: {error}" + ) from error + if not retried: + # A rejection comes back with the completion of the task + # that made it, so a retry reaches a later task, and the + # start hook registers the handler on the first one. + retried = True + await asyncio.sleep(self._retry_cooldown.total_seconds()) + continue + # Rejected across a task boundary: the workflow's worker + # predates the publish Update. Its Signal takes the batch, + # and every later one from this producer. + logger.info( + "workflow %r has no publish update; appending by Signal", + self._handle.id, + ) + self._transport = "signal" + return await self._signal(publish) + except RPCError as error: + if _run_is_closing(error): + # Where a Signal would be carried to the successor, the + # Update is refused; the retry reaches the run that + # takes over, since the handle follows the chain. + await asyncio.sleep(self._retry_cooldown.total_seconds()) + continue + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"workflow {self._handle.id!r} was not found, so its stream " + "cannot be appended to" + ) from error + raise + return _cursor(answer.run_id, answer.last_offset) + + async def _signal(self, publish: PublishInput) -> Cursor | None: + # Always None: a Signal has no response to carry the position. + try: + await self._handle.signal(PUBLISH_SIGNAL_NAME, publish) + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"workflow {self._handle.id!r} was not found, so its stream " + "cannot be appended to" + ) from error + raise + return None + + +class WorkflowStreamsHandle: + """One workflow's log from outside, through the shipped poll Update and a tail Query.""" + + def __init__( + self, + client: Client, + workflow_id: str, + run_id: str | None, + poll_cooldown: timedelta, + publish_transport: PublishTransport = "update", + ) -> None: + """Address ``workflow_id``'s log, pinned to ``run_id`` when one is given.""" + self._client = client + self._workflow_id = workflow_id + self._run_id = run_id + self._poll_cooldown = poll_cooldown + self._publish_transport: PublishTransport = publish_transport + self._converter = client.data_converter.payload_converter + + def read( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor = BEGINNING, + last: int | None = None, + result_type: type | None = None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + """Yield records on ``topic`` from where the read starts until the chain, or the pinned run, closes. + + ``BEGINNING`` is the oldest item the first retained run's log still + holds: the poll Update reads offset zero as the log's base. ``END`` + and ``last=`` start on the current run, or the pinned one, at an + offset its workflow answers by Query when the read starts. + + The chain is followed across continue-as-new, into the successor's + log from its first record, and across a reset, into the run describe + names at the position the read had reached. A handle pinned to a run + ends with that run either way. + """ + check_read_start(after, last) + topic, result_type = resolve_topic(topic, result_type) + wire_topic = self._wire_topic(topic) + # Parsed here so a foreign cursor fails this call, not the first + # iteration of the generator. + named = None if after == END else _position(after) + if named is not None and self._run_id is not None and named[0] != self._run_id: + raise StreamCursorError( + f"cursor {after.token!r} names another run than this handle is pinned to" + ) + return self._read(wire_topic, named, after, last, result_type) + + def _wire_topic(self, topic: str) -> str: + """The name the log files ``topic`` under: the topic itself for a workflow's.""" + _require_topic(topic) + return topic + + async def _read( + self, + topic: str, + named: tuple[str, int] | None, + after: Cursor, + last: int | None, + result_type: type | None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + if named is not None: + run_id, offset = named[0], named[1] + 1 + elif last is not None or after == END: + run_id, offset = await self._start_on_current_run(topic, last or 0) + # A synthesized record is positioned before the first one read. + after = _cursor(run_id, offset - 1) + else: + run_id, offset = self._run_id or await self._first_run(), 0 + decoder = RecordDecoder( + self._converter, result_type, after=after, warn=logger.warning + ) + while True: + handle = self._handle(run_id) + next_offset = offset + polls = self._poll(handle, topic, offset) + try: + async for item_offset, payload in polls: + next_offset = item_offset + 1 + for record in self._records(decoder, run_id, item_offset, payload): + yield record + finally: + await polls.aclose() + status = await self._status(handle) + if status is None: + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} run {run_id!r} was not found" + ) + if status == WorkflowExecutionStatus.RUNNING: + # The subscription ended early, on an RPC timeout for + # instance; the run is still open, so pick up where it left. + offset = next_offset + continue + # What landed after the last poll is still in workflow state, so + # the tail comes back by Query rather than being lost with the run. + tail_offset = next_offset + while True: + page, tail_offset, more = await self._tail(handle, topic, tail_offset) + for offset_, payload in page: + for record in self._records(decoder, run_id, offset_, payload): + yield record + if not more: + break + if self._run_id is not None: + return + following = await self._following(handle, status, tail_offset) + if following is None: + return + run_id, offset = following + + async def _following( + self, + handle: WorkflowHandle[Any, Any], + status: WorkflowExecutionStatus, + position: int, + ) -> tuple[str, int] | None: + """The run that carries on after ``handle``'s, and where to read it from. + + A continue-as-new names its successor in the close event, and the + successor's log starts over at zero. A reset does not: the base run is + closed with no word of it in its own History, and only describe names + the run reset from it. That run rebuilt its log by replaying the base + run's History up to the reset point, so the items before that point + sit at the same offsets in both logs, and the read carries on at the + position it reached. Past the reset point the two logs differ, and a + reader already there is not told: reporting the reset to consumers as + a record is a wire change for a later round. + """ + if status == WorkflowExecutionStatus.CONTINUED_AS_NEW: + successor = await self._successor(handle) + return None if successor is None else (successor, 0) + reset_run = await self._reset_run(handle) + return None if reset_run is None else (reset_run, position) + + async def _reset_run(self, handle: WorkflowHandle[Any, Any]) -> str | None: + """The run ``handle``'s run was reset into, which only describe reports.""" + description = await self._describe(handle) + if description is None: + return None + extended = description.raw_description.workflow_extended_info + return extended.reset_run_id or None + + async def _poll( + self, handle: WorkflowHandle[Any, Any], topic: str, offset: int + ) -> AsyncGenerator[tuple[int, Payload], None]: + """Drive the shipped poll Update on one run until that run closes. + + Written here rather than through ``WorkflowStreamClient.subscribe`` + for two reasons. That loop swallows a cancellation of the caller's + task, so a consumer's ``asyncio.timeout`` or task cancel around a read + would end the subscription and let the read resubscribe forever; here + the cancellation leaves ``read()`` as what it was. And it follows + continue-as-new with offsets this provider does not carry across + runs, which is the outer loop's job. + """ + cooldown = self._poll_cooldown.total_seconds() + unhandled = False + while True: + try: + update = await handle.start_update( + POLL_UPDATE_NAME, + PollInput(topics=[topic], from_offset=offset), + wait_for_stage=WorkflowUpdateStage.ACCEPTED, + result_type=PollResult, + ) + result = await update.result() + except WorkflowUpdateRPCTimeoutOrCancelledError as error: + if isinstance(error.__cause__, asyncio.CancelledError): + # The SDK wraps a cancelled await in this error; the + # consumer cancelled us, so that is what comes out. + raise asyncio.CancelledError() from error + # The RPC itself timed out while the run is still open. + continue + except WorkflowUpdateFailedError as error: + cause = getattr(error.cause, "type", None) + if cause == TRUNCATED_OFFSET_ERROR_TYPE: + # Restarting from the beginning would hand the caller + # records it already handled, and only the caller can + # decide to do that. + raise StreamCursorError( + f"offset {offset} of workflow {self._workflow_id!r} run " + f"{handle.run_id!r} is no longer retained" + ) from error + if cause == STREAM_DRAINING_ERROR_TYPE: + # Pollers are detached because the run is closing; the + # next attempt learns how it closed. + await asyncio.sleep(cooldown) + continue + if cause == _UPDATE_OUTLIVED_RUN: + return + if _HANDLER_NOT_FOUND in str(error.cause): + if await self._status(handle) != WorkflowExecutionStatus.RUNNING: + # The run closed with the rejecting task, as a run + # that continues as new on its first task does; the + # caller describes it and follows the chain. + return + if unhandled: + raise StreamError( + f"workflow {self._workflow_id!r} run {handle.run_id!r} " + "does not serve the poll update: the workflow_streams " + "provider is not installed on its worker, and no " + "WorkflowStream was constructed" + ) from error + # A rejection comes back with the completion of the task + # that made it, so a retry reaches a later task, and the + # start hook registers the handler on the first one. Only + # a second rejection means there is no handler to wait for. + unhandled = True + await asyncio.sleep(cooldown) + continue + raise StreamError( + f"the poll update on workflow {self._workflow_id!r} failed: {error}" + ) from error + except RPCError as error: + if _run_is_closing(error): + # Continuing as new, or completing: the next attempt + # learns how it closed, the way a draining poll does. + await asyncio.sleep(cooldown) + continue + # The run closed and its poll Update went with it, or the + # workflow does not exist; the caller describes to tell which. + if error.status != RPCStatusCode.NOT_FOUND: + raise + return + for item in result.items: + if item.topic == topic: + yield item.offset, Payload.FromString(base64.b64decode(item.data)) + offset = result.next_offset + if not result.more_ready and cooldown > 0: + await asyncio.sleep(cooldown) + + def _records( + self, decoder: RecordDecoder, run_id: str, offset: int, payload: Payload + ) -> list[StreamRecord[Any]]: + cursor = _cursor(run_id, offset) + wire = _unwrap(cursor, payload, logger.warning) + if wire is None: + return [] + return decoder.decode(cursor, wire) + + def _handle(self, run_id: str | None) -> WorkflowHandle[Any, Any]: + return self._client.get_workflow_handle(self._workflow_id, run_id=run_id) + + async def _describe(self, handle: WorkflowHandle[Any, Any]) -> Any | None: + """The description of ``handle``'s run, or ``None`` when it is gone.""" + try: + return await handle.describe() + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + return None + raise + + async def _status( + self, handle: WorkflowHandle[Any, Any] + ) -> WorkflowExecutionStatus | None: + description = await self._describe(handle) + return None if description is None else description.status + + async def _first_run(self) -> str: + """The oldest retained run of the chain, walking back from the latest.""" + try: + run_id = (await self._handle(None).describe()).run_id + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} was not found" + ) from error + raise + assert run_id is not None + while True: + previous = await self._predecessor(run_id) + if previous is None: + return run_id + run_id = previous + + async def _predecessor(self, run_id: str) -> str | None: + try: + async for event in self._handle(run_id).fetch_history_events(page_size=1): + attributes = event.workflow_execution_started_event_attributes + if attributes.continued_execution_run_id: + return attributes.continued_execution_run_id + # A reset run's start event is the base run's, copied, and the + # original run id it carries is kept across resets, so it names + # the run the chain of resets began from. + original = attributes.original_execution_run_id + return original if original and original != run_id else None + except RPCError as error: + if error.status != RPCStatusCode.NOT_FOUND: + raise + # The run's History is gone: the chain's retained part starts here. + return None + + async def _successor(self, handle: WorkflowHandle[Any, Any]) -> str | None: + events = handle.fetch_history_events( + event_filter_type=WorkflowHistoryEventFilterType.CLOSE_EVENT + ) + try: + async for event in events: + if event.HasField( + "workflow_execution_continued_as_new_event_attributes" + ): + attributes = ( + event.workflow_execution_continued_as_new_event_attributes + ) + return attributes.new_execution_run_id or None + except RPCError as error: + if error.status != RPCStatusCode.NOT_FOUND: + raise + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} run {handle.run_id!r} was not found, " + "so its successor cannot be followed" + ) from error + return None + + async def _tail( + self, handle: WorkflowHandle[Any, Any], topic: str, from_offset: int + ) -> tuple[list[tuple[int, Payload]], int, bool]: + """One page of ``topic``'s tail as ``(items, next offset, more to come)``.""" + try: + wire = await handle.query( + _TAIL_QUERY, args=[from_offset, topic], result_type=dict + ) + except WorkflowQueryFailedError as error: + if QUERY_HANDLER_NOT_FOUND not in str(error): + raise StreamError( + f"the tail query on workflow {self._workflow_id!r} failed: {error}" + ) from error + # The workflow never opened a stream through this provider, so + # there is no tail to serve. + return [], from_offset, False + except RPCError as error: + if error.status != RPCStatusCode.NOT_FOUND: + raise + # The History is gone; nothing is left to serve. + return [], from_offset, False + if from_offset and wire.get("base_offset", from_offset) > from_offset: + # Restarting from the base would hand the caller records it + # already handled, and only the caller can decide to do that. + raise StreamCursorError( + f"offset {from_offset} of workflow {self._workflow_id!r} run " + f"{handle.run_id!r} is no longer retained" + ) + items = [ + (item["offset"], Payload.FromString(base64.b64decode(item["data"]))) + for item in wire["items"] + ] + return items, wire["next_offset"], bool(wire["more_ready"]) + + async def _start_on_current_run(self, topic: str, last_n: int) -> tuple[str, int]: + """The run a read at ``END`` or of the newest records starts on, and the offset. + + Raises: + StreamUnsupportedError: The workflow's worker runs a provider that + predates these reads and cannot answer where they start. + """ + handle = self._handle(self._run_id) + description = await self._describe(handle) + if description is None: + raise StreamNotFoundError(f"workflow {self._workflow_id!r} was not found") + run_id = description.run_id + assert run_id is not None + try: + offset = await self._handle(run_id).query( + _START_QUERY, args=[topic, last_n], result_type=int + ) + except WorkflowQueryFailedError as error: + if QUERY_HANDLER_NOT_FOUND not in str(error): + raise StreamError( + f"the start query on workflow {self._workflow_id!r} failed: {error}" + ) from error + raise StreamUnsupportedError( + f"workflow {self._workflow_id!r} does not answer where a read at END " + "or of the last records starts; its worker's provider predates them" + ) from error + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} was not found" + ) from error + raise + return run_id, offset + + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: + """The newest position holding a record on ``topic``. + + The log is one per run, so the cursor names the run it was read from: + the pinned run, or the latest run of the chain. One log orders every + topic, so the answer comes from a Query that scans it for this topic + rather than from the head of the log, which usually names some other + topic's record. + """ + topic, _ = resolve_topic(topic) + wire_topic = self._wire_topic(topic) + handle = self._handle(self._run_id) + description = await self._describe(handle) + if description is None: + raise StreamNotFoundError(f"workflow {self._workflow_id!r} was not found") + try: + head = await handle.query(_LATEST_QUERY, wire_topic, result_type=int) + except WorkflowQueryFailedError as error: + if QUERY_HANDLER_NOT_FOUND not in str(error): + raise StreamError( + f"the latest query on workflow {self._workflow_id!r} failed: " + f"{error}" + ) from error + # The workflow never opened a stream through this provider, so + # the topic holds nothing. + head = -1 + except RPCError as error: + if error.status == RPCStatusCode.NOT_FOUND: + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} was not found" + ) from error + raise + run_id = description.run_id + assert run_id is not None + if head >= 0: + return _cursor(run_id, head) + # An empty topic on the chain's first run is the beginning of the + # stream; on a successor it is a position of its own, because + # BEGINNING would send a chain-following read back to the first run. + if await self._predecessor(run_id) is None: + return BEGINNING + return _cursor(run_id, -1) + + def producer( + self, + *, + topic: str | StreamTopic[Any] | None = None, + producer_id: str = "", + attempt: int = 0, + ) -> WorkflowStreamsProducer[Any]: + """A producer on ``topic``; inside an activity its identity is the activity's.""" + topic, _ = resolve_topic(topic) + wire_topic = self._wire_topic(topic) + producer_id, attempt = producer_identity(producer_id, attempt) + return WorkflowStreamsProducer( + self._handle(self._run_id), + self._converter, + topic, + producer_id, + attempt, + item_topic=wire_topic, + transport=self._publish_transport, + retry_cooldown=self._poll_cooldown, + ) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A ref to ``topic`` of this workflow's stream, pinned as this handle is.""" + return StreamRef.for_workflow( + self._workflow_id, run_id=self._run_id, topic=topic + ) + + async def close(self) -> None: + """Refuse: an owned stream ends with its owner, not by a caller.""" + raise ValueError( + "only a standalone stream can be closed; this handle is on an owned " + "stream, which ends when its workflow or activity does" + ) + + +class WorkflowStreamsActivityHandle(WorkflowStreamsHandle): + """The streams one activity of a workflow owns, kept in that workflow's log. + + Each topic is the reserved topic ``activity//`` of the + workflow's log, so an activity's ``tokens`` and its workflow's ``tokens`` + are two streams. The activity belongs to one run, so the handle pins the + run on first use and never follows a successor. A read ends when the + workflow closes, or once the activity has been seen pending, is pending + no longer and the read delivered a record: a stream the activity never + wrote has nothing to close, so a read on it waits for the workflow. + + A read here polls the tail Query and describes the workflow between + polls, every ``poll_cooldown``, rather than parking on the poll Update: + an Update parked in the workflow returns only when the log grows, so it + could not notice the activity ending, and each abandoned one would stay + parked against the run's Update caps. + """ + + def __init__( + self, + client: Client, + workflow_id: str, + run_id: str | None, + activity_id: str, + poll_cooldown: timedelta, + publish_transport: PublishTransport = "update", + ) -> None: + """Address ``activity_id``'s streams inside ``workflow_id``'s log.""" + super().__init__(client, workflow_id, run_id, poll_cooldown, publish_transport) + self._activity_id = activity_id + + def _wire_topic(self, topic: str) -> str: + return _activity_topic(self._activity_id, topic) + + def ref(self, *, topic: str | StreamTopic[Any] | None = None) -> StreamRef: + """A ref to ``topic`` of this activity's streams, through its workflow.""" + return StreamRef.for_activity( + self._activity_id, + workflow_id=self._workflow_id, + run_id=self._run_id, + topic=topic, + ) + + async def _read( + self, + topic: str, + named: tuple[str, int] | None, + after: Cursor, + last: int | None, + result_type: type | None, + ) -> AsyncGenerator[StreamRecord[Any], None]: + if named is not None: + run_id, offset = named[0], named[1] + 1 + elif last is not None or after == END: + run_id, offset = await self._start_on_current_run(topic, last or 0) + after = _cursor(run_id, offset - 1) + else: + run_id, offset = await self._current_run(), 0 + handle = self._handle(run_id) + decoder = RecordDecoder( + self._converter, result_type, after=after, warn=logger.warning + ) + cooldown = self._poll_cooldown.total_seconds() + seen_pending = False + delivered = False + ended = False + while True: + more = True + while more: + page, offset, more = await self._tail(handle, topic, offset) + for offset_, payload in page: + for record in self._records(decoder, run_id, offset_, payload): + delivered = True + yield record + if ended: + return + description = await self._describe(handle) + if description is None: + raise StreamNotFoundError( + f"workflow {self._workflow_id!r} run {run_id!r} was not found" + ) + pending = any( + info.activity_id == self._activity_id + for info in description.raw_description.pending_activities + ) + seen_pending = seen_pending or pending + # One more pass after learning the stream ended, so a record that + # landed between the tail and the describe is not lost. + ended = description.status != WorkflowExecutionStatus.RUNNING or ( + seen_pending and not pending and delivered + ) + if not ended: + await asyncio.sleep(cooldown) + + async def _current_run(self) -> str: + description = await self._describe(self._handle(self._run_id)) + if description is None: + raise StreamNotFoundError(f"workflow {self._workflow_id!r} was not found") + assert description.run_id is not None + return description.run_id + + +class WorkflowStreamsProvider(ProviderPlugin): + """The provider over the shipped Workflow Streams transport.""" + + def __init__( + self, + *, + poll_cooldown: timedelta = timedelta(milliseconds=100), + publish_transport: PublishTransport = "update", + ) -> None: + """Create the provider. + + Args: + poll_cooldown: How long an outside reader that is caught up waits + between polls. Backlogs drain at full speed regardless. Also + how long a producer waits before retrying a publish Update + the workflow's first task rejected. + publish_transport: How an outside producer's batches reach the + workflow. ``"update"``, the default, answers each batch with + its position and refuses a conflicting repeat; on a workflow + whose worker predates the publish Update the producer falls + back to the Signal by itself. ``"signal"`` is the transport + of the first release and skips the detection. Both cost one + Action per batch. + """ + self._poll_cooldown = poll_cooldown + self._publish_transport: PublishTransport = publish_transport + + def workflow_provider(self) -> _WSWorkflowProvider: + """The workflow half, over the running instance's shipped stream object.""" + return _WSWorkflowProvider() + + def get_stream_handle( + self, client: Client, workflow_id: str, *, run_id: str | None = None + ) -> WorkflowStreamsHandle: + """A handle on ``workflow_id``'s log; without ``run_id`` it follows the chain.""" + return WorkflowStreamsHandle( + client, workflow_id, run_id, self._poll_cooldown, self._publish_transport + ) + + def get_activity_stream_handle( + self, + client: Client, + activity_id: str, + *, + workflow_id: str | None = None, + run_id: str | None = None, + ) -> WorkflowStreamsActivityHandle: + """A handle on the streams an activity of ``workflow_id`` owns. + + They live in the workflow's log under the reserved topics + ``activity//``, and ``run_id`` pins the workflow's + run. Without ``workflow_id`` the activity is a standalone one, which + has no workflow to host a log, so this provider refuses it; such an + activity addresses a workflow's stream by ``workflow_id`` instead. + + Raises: + StreamUnsupportedError: ``workflow_id`` was not given. + """ + if workflow_id is None: + raise StreamUnsupportedError( + "the workflow_streams provider cannot hold a stream a standalone " + "activity owns: its log lives inside a running workflow, and no " + "workflow hosts this activity's" + ) + return WorkflowStreamsActivityHandle( + client, + workflow_id, + run_id, + activity_id, + self._poll_cooldown, + self._publish_transport, + ) + + async def create_standalone_stream( + self, + client: Client, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> WorkflowStreamsHandle: + """Refused: this provider has no store for a stream without an owner. + + Every log here is a running workflow's state, served by that + workflow's handlers, and a standalone stream has no workflow. A + housing workflow started for the stream id, with the retention policy + as its state and ``close()`` as a Signal, would be one way to offer + it on this transport; it is a design option for a later round, not + built here. + + Raises: + StreamUnsupportedError: Always. + """ + raise StreamUnsupportedError( + "the workflow_streams provider does not host standalone streams: every " + "log is a running workflow's state, and a stream without an owner has " + "no workflow" + ) + + def get_standalone_stream_handle( + self, client: Client, stream_id: str + ) -> WorkflowStreamsHandle: + """Refused: this provider has no store for a stream without an owner. + + See :meth:`create_standalone_stream`. + + Raises: + StreamUnsupportedError: Always. + """ + raise StreamUnsupportedError( + "the workflow_streams provider does not host standalone streams: every " + "log is a running workflow's state, and a stream without an owner has " + "no workflow" + ) + + async def close(self) -> None: + """Nothing to release: the provider holds no connection of its own.""" diff --git a/temporalio/testing/_workflow.py b/temporalio/testing/_workflow.py index 3ab07e5aa0..c0c4291719 100644 --- a/temporalio/testing/_workflow.py +++ b/temporalio/testing/_workflow.py @@ -500,6 +500,7 @@ def __init__( super().__init__(_client_with_interceptors(client, *interceptors)) self._server = server self._auto_time_skipping = True + self._time_skipping_waiters = 0 async def shutdown(self) -> None: await self._server.shutdown() @@ -544,25 +545,42 @@ async def time_skipping_unlocked(self) -> AsyncIterator[None]: if not self._supports_time_skipping or not self._auto_time_skipping: yield None return - # Unlock to start time skipping, lock again to stop it - await self.client.test_service.unlock_time_skipping( - temporalio.api.testservice.v1.UnlockTimeSkippingRequest() - ) + # The test server holds one time-skipping lock per in-flight task and + # counts one unlock per request, and it skips as soon as the count is + # zero. Concurrent waiters therefore share a single unlock; a second + # one would let the clock jump while another workflow still has a task + # in flight. The waiter is counted before the await so a second waiter + # cannot slip in and unlock again. + self._time_skipping_waiters += 1 + if self._time_skipping_waiters == 1: + try: + await self.client.test_service.unlock_time_skipping( + temporalio.api.testservice.v1.UnlockTimeSkippingRequest() + ) + except BaseException: + self._time_skipping_waiters -= 1 + raise try: yield None - # Lock it back, throwing on error - await self.client.test_service.lock_time_skipping( - temporalio.api.testservice.v1.LockTimeSkippingRequest() - ) - except: + except BaseException: # Lock it back, swallowing error try: - await self.client.test_service.lock_time_skipping( - temporalio.api.testservice.v1.LockTimeSkippingRequest() - ) - except: + await self._lock_time_skipping_after_waiter() + except Exception: logger.exception("Failed locking time skipping after error") raise + # Lock it back, throwing on error + await self._lock_time_skipping_after_waiter() + + async def _lock_time_skipping_after_waiter(self) -> None: + # Only the last waiter locks. The count drops first so a failed lock + # call does not leave a phantom waiter behind. + self._time_skipping_waiters -= 1 + if self._time_skipping_waiters: + return + await self.client.test_service.lock_time_skipping( + temporalio.api.testservice.v1.LockTimeSkippingRequest() + ) class _AssertionErrorInterceptor( diff --git a/temporalio/worker/_activity.py b/temporalio/worker/_activity.py index f72df9cf4b..0f289931ba 100644 --- a/temporalio/worker/_activity.py +++ b/temporalio/worker/_activity.py @@ -33,6 +33,7 @@ import temporalio.common import temporalio.converter import temporalio.exceptions +import temporalio.streams from temporalio.converter import ( StorageDriverActivityInfo, StorageDriverStoreContext, @@ -63,9 +64,11 @@ def __init__( metric_meter: temporalio.common.MetricMeter, client: temporalio.client.Client, encode_headers: bool, + stream_provider: temporalio.streams.StreamProvider | None = None, ) -> None: self._bridge_worker = bridge_worker self._task_queue = task_queue + self._stream_provider = stream_provider self._activity_executor = activity_executor self._shared_state_manager = shared_state_manager self._running_activities: dict[bytes, _RunningActivity] = {} @@ -666,6 +669,10 @@ async def _execute_activity( runtime_metric_meter=None if sync_non_threaded else self._metric_meter, client=self._client if not running_activity.sync else None, cancellation_details=running_activity.cancellation_details, + stream_provider=( + self._stream_provider if not running_activity.sync else None + ), + sync=running_activity.sync, ) ) temporalio.activity.logger.debug("Starting activity") @@ -942,6 +949,7 @@ def heartbeat_fn(*details: Any) -> None: runtime_metric_meter=runtime_metric_meter, client=None, cancellation_details=cancellation_details, + sync=True, ) ) if not cancel_thread_raiser: diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index 7c03c2cd43..fa6d5ba18c 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from temporalio.api.enums.v1.command_type_pb2 import CommandType +from temporalio.api.stream.v1 import StreamRecord from temporalio.bridge._visitor import PayloadVisitor from temporalio.bridge._visitor_functions import VisitorFunctions from temporalio.bridge.proto.workflow_activation.workflow_activation_pb2 import ( @@ -26,6 +27,7 @@ StartChildWorkflowExecution, WorkflowCommand, ) +from temporalio.streams._body import CONTENT_HASH_KEY @dataclass(frozen=True) @@ -116,6 +118,18 @@ async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): await super()._visit_coresdk_workflow_commands_ScheduleNexusOperation(fs, o) + async def _visit_temporal_api_stream_v1_StreamRecord( + self, fs: VisitorFunctions, o: StreamRecord + ) -> None: + if o.HasField("body"): + await self._visit_temporal_api_common_v1_Payload(fs, o.body) + for key, value in o.metadata.items(): + # The declared content hash is the server's dedupe identity and is + # read as sent, so neither the codec nor the store may rewrite it. + if key == CONTENT_HASH_KEY: + continue + await self._visit_temporal_api_common_v1_Payload(fs, value) + async def _visit_coresdk_workflow_commands_WorkflowCommand( self, fs: VisitorFunctions, o: WorkflowCommand ) -> None: diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index b3eb1a4d12..bfc86f026c 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -8,20 +8,25 @@ from collections.abc import AsyncIterator, Mapping, Sequence from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass +from typing import Any from typing_extensions import TypedDict +import temporalio.api.enums.v1 import temporalio.api.history.v1 +import temporalio.api.stream.v1 import temporalio.bridge.proto.workflow_activation import temporalio.bridge.worker import temporalio.client import temporalio.converter import temporalio.runtime +import temporalio.streams import temporalio.worker import temporalio.workflow from ..common import HeaderCodecBehavior from ._interceptor import Interceptor +from ._stream_ranges import fetch_range from ._worker import load_default_build_id from ._workflow import _WorkflowWorker from ._workflow_instance import ( @@ -56,18 +61,52 @@ def __init__( runtime: temporalio.runtime.Runtime | None = None, disable_safe_workflow_eviction: bool = False, header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + external_stream_backend: Any | None = None, + stream_provider: temporalio.streams.StreamProvider | None = None, + stream_client: temporalio.client.Client | None = None, ) -> None: """Create a replayer to replay workflows from history. See :py:meth:`temporalio.worker.Worker.__init__` for a description of most of the arguments. Most of the same arguments need to be passed to the replayer that were passed to the worker when the workflow originally - ran. + ran, ``stream_provider`` included when the workflow used streams. + + A workflow that read a server-side stream cannot be replayed from its + history alone. History records the offsets each Workflow Task consumed + and never the records; on a live task the server re-supplies them from + the stream. Three cases: + + * The history carries the records in + :py:attr:`temporalio.client.WorkflowHistory.stream_slices`, put + there by :py:meth:`fetch_stream_slices` while the stream was + retained and carried by ``to_json()`` and ``from_json()``. They are + handed to the replay and no server is contacted. + * The history carries none and ``stream_client`` is given: a client + connected to the server that still holds the streams. The replayer + fetches every range the completed tasks recorded from the stream + service and hands the records to the replay, so the workflow sees + what it saw the first time. A range the stream no longer holds fails + that replay with :py:class:`temporalio.streams.StreamNotFoundError`. + The stream service is reached on a channel of its own, opened with + the client's connection settings (target, TLS, API key, headers, + retries), as :py:class:`temporalio.client_stream.Connection` + describes. The replayer closes the channel it opened when the + replay is finished. + * The history carries none and there is no client: replaying it fails + and the message names both remedies. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that will be shared across all replay calls and never explicitly shut down. Users are encouraged to provide their own if needing more control. + + A history containing external workflow stream markers can only be + replayed with the same backend configured, because replay re-reads the + recorded ranges from the provider. Without them the replay fails the + same way the original Workflow would have with no backends configured -- + which is correct, but is a configuration error rather than a finding + about the history. """ self._config = ReplayerConfig( workflows=list(workflows), @@ -86,6 +125,9 @@ def __init__( runtime=runtime, disable_safe_workflow_eviction=disable_safe_workflow_eviction, header_codec_behavior=header_codec_behavior, + external_stream_backend=external_stream_backend, + stream_provider=stream_provider, + stream_client=stream_client, ) self._initial_config = self._config.copy() self._default_workflow_logic_flags = set(_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS) @@ -107,6 +149,31 @@ def _set_default_workflow_logic_flag( else: self._default_workflow_logic_flags.discard(flag) + @staticmethod + async def fetch_stream_slices( + client: temporalio.client.Client, history: temporalio.client.WorkflowHistory + ) -> temporalio.client.WorkflowHistory: + """Return ``history`` with the stream records its tasks consumed attached. + + History records only the offsets each Workflow Task consumed from a + server-side stream, so an export cannot be replayed without the + server that still holds the stream. This fetches every recorded range + from the stream service through ``client`` while the stream is + retained and returns a copy of ``history`` with the records in + :py:attr:`temporalio.client.WorkflowHistory.stream_slices`. Its + ``to_json()`` then carries them, and a replayer given the result, or a + ``from_json()`` of it, needs no server. Ranges recorded before a reset + point are fetched from the run the workflow was reset from. + + Raises :py:class:`temporalio.streams.StreamNotFoundError` for a range + the stream no longer holds. + """ + return temporalio.client.WorkflowHistory( + history.workflow_id, + history.events, + await _stream_slices(client, history), + ) + def config(self, *, active_config: bool = False) -> ReplayerConfig: """Config, as a dictionary, used to create this replayer. @@ -210,6 +277,7 @@ async def _workflow_replay_iterator( pusher = None workflow_worker_task = None bridge_worker_scope = None + stream_client = self._config.get("stream_client") try: last_replay_failure: Exception | None @@ -279,6 +347,7 @@ def on_eviction_hook( ), should_enforce_versioning_behavior=False, assert_local_activity_valid=lambda a: None, + external_stream_backend=self._config.get("external_stream_backend"), encode_headers=self._config.get( "header_codec_behavior", HeaderCodecBehavior.NO_CODEC ) @@ -287,6 +356,7 @@ def on_eviction_hook( default_workflow_logic_flags=frozenset( self._default_workflow_logic_flags ), + stream_provider=self._config.get("stream_provider"), ) external_storage = data_converter.external_storage storage_driver_types = ( @@ -364,6 +434,33 @@ def on_eviction_hook( # Yield iterator async def replay_iterator() -> AsyncIterator[WorkflowReplayResult]: async for history in histories: + # The records a consuming workflow read are not in its + # history unless they were captured into it. Otherwise + # fetch them from the stream service, or report here what + # is missing, rather than let Core fail the first task on + # input it was never given. + stream_slices: list[bytes] = [] + if history.stream_slices: + stream_slices = [ + s.SerializeToString() for s in history.stream_slices + ] + elif stream_client is not None: + try: + fetched = await _stream_slices(stream_client, history) + except temporalio.streams.StreamNotFoundError as err: + yield WorkflowReplayResult( + history=history, replay_failure=err + ) + continue + stream_slices = [s.SerializeToString() for s in fetched] + else: + missing = _stream_records_missing(history) + if missing is not None: + yield WorkflowReplayResult( + history=history, replay_failure=RuntimeError(missing) + ) + continue + # Clear last complete and push history last_replay_complete.clear() await pusher.push_history( @@ -371,6 +468,7 @@ async def replay_iterator() -> AsyncIterator[WorkflowReplayResult]: temporalio.api.history.v1.History( events=history.events ).SerializeToString(), + stream_slices, ) # Wait for worker error or last replay to complete. This @@ -397,6 +495,13 @@ async def replay_iterator() -> AsyncIterator[WorkflowReplayResult]: yield replay_iterator() finally: + # The stream channel is opened by the fetch, on this loop, and is + # the replayer's to close: nothing else in the process asked for + # it, and leaving it open outlives the replay it served. + if stream_client is not None: + from temporalio.client_stream import close_shared_clients, shared_key + + await close_shared_clients(shared_key(stream_client)) # Close the pusher if pusher is not None: pusher.close() @@ -416,6 +521,118 @@ async def replay_iterator() -> AsyncIterator[WorkflowReplayResult]: logger.warning("Failed to finalize shutdown", exc_info=True) +def _consumed_ranges( + history: temporalio.client.WorkflowHistory, +) -> list[tuple[int, temporalio.api.stream.v1.StreamRange]]: + """Every range a completed task recorded, with the id of the event that recorded it.""" + out: list[tuple[int, temporalio.api.stream.v1.StreamRange]] = [] + for event in history.events: + if not event.HasField("workflow_task_completed_event_attributes"): + continue + attributes = event.workflow_task_completed_event_attributes + for consumed in attributes.consumed_stream_ranges: + out.append((event.event_id, consumed)) + return out + + +def _eras( + history: temporalio.client.WorkflowHistory, +) -> list[tuple[str, list[tuple[int, temporalio.api.stream.v1.StreamRange]]]]: + """The recorded ranges grouped by the run whose streams hold them. + + A reset copies the base run's history into the new run, so the ranges + before the reset point were consumed from the base run's streams, and the + run they belong to is only named by the ``WorkflowTaskFailed`` event that + marks the reset point, after them in the history. Ranges after the last + reset point belong to the run itself, which that event names as well; a + history with no reset point belongs to the run its start event names. This + is the split the server makes when it re-supplies a cache miss. + """ + reset = temporalio.api.enums.v1.WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_RESET_WORKFLOW + eras: list[tuple[str, list[tuple[int, temporalio.api.stream.v1.StreamRange]]]] = [] + pending: list[tuple[int, temporalio.api.stream.v1.StreamRange]] = [] + own_run_id = history.run_id + for event in history.events: + if event.HasField("workflow_task_failed_event_attributes"): + failed = event.workflow_task_failed_event_attributes + if failed.cause == reset and failed.base_run_id: + eras.append((failed.base_run_id, pending)) + pending = [] + own_run_id = failed.new_run_id or own_run_id + continue + if not event.HasField("workflow_task_completed_event_attributes"): + continue + attributes = event.workflow_task_completed_event_attributes + for consumed in attributes.consumed_stream_ranges: + pending.append((event.event_id, consumed)) + eras.append((own_run_id, pending)) + return eras + + +def _stream_records_missing(history: temporalio.client.WorkflowHistory) -> str | None: + """Why this history cannot be replayed without a stream client, or ``None``.""" + ranges = [r for _, r in _consumed_ranges(history) if r.to_offset > r.from_offset] + if not ranges: + return None + streams = sorted({r.stream_id for r in ranges}) + return ( + f"workflow {history.workflow_id!r} consumed records from stream(s) " + f"{', '.join(repr(s) for s in streams)} in {len(ranges)} task(s), and History " + "records only the offsets. Either create the Replayer with stream_client= (a " + "Client connected to the server that still holds the streams) so the records " + "can be fetched from the stream service, or replay a history exported with " + "them: Replayer.fetch_stream_slices(client, history) attaches the records " + "while the stream is retained and WorkflowHistory.to_json() carries them." + ) + + +async def _stream_slices( + client: temporalio.client.Client, history: temporalio.client.WorkflowHistory +) -> list[temporalio.api.stream.v1.StreamSlice]: + """Fetch the records the history's tasks consumed, as ``StreamSlice`` messages. + + The shape is the one the server puts on a poll response when it re-supplies + a cache miss: one slice per recorded range, tagged with the completion that + recorded it, an empty range included, fetched from the run whose stream + holds it (the run reset from, for a range recorded before a reset point). + Raises :py:class:`temporalio.streams.StreamNotFoundError` for a range the + stream no longer holds. + """ + if not _consumed_ranges(history): + return [] + # Imported here: the stream client needs the grpc extra, which a replayer + # without a stream client never touches. + from temporalio.client_stream import shared_client + + streams = shared_client(client) + # The handle that served each run's stream, so later ranges of the same + # stream go straight to it. + served_by: dict[tuple[str, str], Any] = {} + slices: list[temporalio.api.stream.v1.StreamSlice] = [] + for run_id, ranges in _eras(history): + for event_id, consumed in ranges: + stream_slice = temporalio.api.stream.v1.StreamSlice( + stream_id=consumed.stream_id, + run_id=run_id, + from_offset=consumed.from_offset, + to_offset=consumed.to_offset, + workflow_task_completed_event_id=event_id, + ) + if consumed.to_offset > consumed.from_offset: + handle, records, owner_run_id = await fetch_range( + streams, + history.workflow_id, + run_id, + consumed, + served_by.get((run_id, consumed.stream_id)), + ) + served_by[(run_id, consumed.stream_id)] = handle + stream_slice.run_id = owner_run_id or run_id + stream_slice.records.extend(records) + slices.append(stream_slice) + return slices + + class ReplayerConfig(TypedDict, total=False): """TypedDict of config originally passed to :py:class:`Replayer`.""" @@ -433,6 +650,9 @@ class ReplayerConfig(TypedDict, total=False): runtime: temporalio.runtime.Runtime | None disable_safe_workflow_eviction: bool header_codec_behavior: HeaderCodecBehavior + external_stream_backend: Any | None + stream_provider: temporalio.streams.StreamProvider | None + stream_client: temporalio.client.Client | None @dataclass(frozen=True) @@ -450,6 +670,12 @@ class WorkflowReplayResult: :py:class:`temporalio.workflow.NondeterminismError` was encountered during replay - likely indicating your workflow code is incompatible with the history. + + A workflow that read a server-side stream reports here, before any task + runs, when its records could not be fetched: a + :py:class:`temporalio.streams.StreamNotFoundError` when the stream no + longer holds a range a task consumed, or a ``RuntimeError`` when the + replayer was given no ``stream_client`` to fetch them with. """ diff --git a/temporalio/worker/_stream_ranges.py b/temporalio/worker/_stream_ranges.py new file mode 100644 index 0000000000..6bf35dbef3 --- /dev/null +++ b/temporalio/worker/_stream_ranges.py @@ -0,0 +1,173 @@ +"""Reading a recorded stream range back from the stream service. + +History records the offsets each Workflow Task consumed from a server-side +stream and never the records. Two paths have to read those offsets back: the +replayer, for a history it was handed without its records, and a live worker +handed a task whose re-supplied ranges stop short of what History recorded. +Both resolve a subscribed name the way the server does and read exactly the +recorded offsets, so the workflow sees what it saw the first time. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import temporalio.api.stream.v1 +import temporalio.bridge.proto.workflow_activation +import temporalio.service +import temporalio.streams + +if TYPE_CHECKING: + import temporalio.client + +__all__ = ["fetch_range", "fill_short_stream_ranges", "read_range"] + + +async def fill_short_stream_ranges( + activation: temporalio.bridge.proto.workflow_activation.WorkflowActivation, + workflow_id: str, + client: temporalio.client.Client, +) -> None: + """Fetch the records a short ``DeliverStreamRecords`` job lacks, before the activation runs. + + The server re-supplies a replaying workflow's recorded ranges on the task + that carries them, within a budget. A job whose records stop short of its + range is that budget's remainder: the offsets are recorded in History and + the stream still holds them, so they are read from the stream service the + way the replayer reads them for an exported history, and the job is made + whole before the workflow runs on it. A job that covers its range, or that + records an empty observation, is left as it is. The fetch reads from the + activation's own run; a range recorded before a reset point lives on the + base run, which the job does not name yet. + + Raises: + temporalio.streams.StreamNotFoundError: The stream no longer holds the + missing offsets. The task fails rather than run on less input. + """ + short = [ + job.deliver_stream_records + for job in activation.jobs + if job.HasField("deliver_stream_records") + and len(job.deliver_stream_records.records) + < job.deliver_stream_records.to_offset - job.deliver_stream_records.from_offset + ] + if not short: + return + # Imported here: the stream client needs the grpc extra, which a worker + # that never sees a short range never touches. + from temporalio.client_stream import shared_client + + streams = shared_client(client) + served_by: dict[str, Any] = {} + for job in short: + missing = temporalio.api.stream.v1.StreamRange( + stream_id=job.stream_id, + from_offset=job.from_offset + len(job.records), + to_offset=job.to_offset, + ) + handle, records, _ = await fetch_range( + streams, + workflow_id, + activation.run_id, + missing, + served_by.get(job.stream_id), + ) + served_by[job.stream_id] = handle + job.records.extend(records) + + +async def fetch_range( + streams: Any, + workflow_id: str, + run_id: str, + consumed: temporalio.api.stream.v1.StreamRange, + known: Any, +) -> tuple[Any, list[temporalio.api.stream.v1.StreamRecord], str]: + """The records at exactly the recorded range, with the handle that served them. + + A subscribed name is resolved as the server resolves it: a stream the + workflow owns by that name first, else a standalone stream by that id. The + owned stream cannot be asked whether it exists, since a name nobody wrote + reads as empty, so the owned stream is probed for the range's first record + and the standalone one is tried when it has nothing there. ``known`` is the + handle that served this stream before, when there was one. + """ + gone = ( + f"stream {consumed.stream_id!r} no longer holds offsets " + f"[{consumed.from_offset}, {consumed.to_offset}) that a completed task of " + f"workflow {workflow_id!r} run {run_id!r} consumed; its records cannot be " + "replayed" + ) + candidates = ( + [known] + if known is not None + else [ + streams.workflow_stream( + workflow_id, consumed.stream_id, owner_run_id=run_id + ), + streams.get(consumed.stream_id), + ] + ) + last: Exception | None = None + for handle in candidates: + try: + records, owner_run_id = await read_range(handle, consumed, gone) + except temporalio.streams.StreamNotFoundError as error: + last = error + continue + if records is not None: + return handle, records, owner_run_id + raise temporalio.streams.StreamNotFoundError(gone) from last + + +async def read_range( + handle: Any, consumed: temporalio.api.stream.v1.StreamRange, gone: str +) -> tuple[list[temporalio.api.stream.v1.StreamRecord] | None, str]: + """Read ``[from_offset, to_offset)`` from one stream. + + ``None`` when the stream has nothing at the range's first offset, which is + how a name that is not this stream's reads; a stream that has the start but + not the rest, or refuses the offset as truncated or past its head, raises + :class:`temporalio.streams.StreamNotFoundError` with ``gone`` as the reason. + """ + records: list[temporalio.api.stream.v1.StreamRecord] = [] + owner_run_id = "" + offset = consumed.from_offset + while offset < consumed.to_offset: + try: + page = await handle.poll( + from_offset=offset, + max_records=consumed.to_offset - offset, + wait=False, + ) + except temporalio.streams.StreamNotFoundError: + if not records: + return None, "" + raise + except temporalio.streams.StreamCursorError as error: + # Below the truncation floor: the records a task consumed are gone. + raise temporalio.streams.StreamNotFoundError(gone) from error + except temporalio.service.RPCError as error: + # Past the head, or a refusal the server does not type: it refuses + # the offset rather than answering short. + if error.status in ( + temporalio.service.RPCStatusCode.FAILED_PRECONDITION, + temporalio.service.RPCStatusCode.INVALID_ARGUMENT, + temporalio.service.RPCStatusCode.OUT_OF_RANGE, + ): + raise temporalio.streams.StreamNotFoundError(gone) from error + raise + owner_run_id = page.run_id or owner_run_id + if not page.entries: + if not records: + return None, "" + raise temporalio.streams.StreamNotFoundError(gone) + for entry in page.entries: + if entry.offset != offset or offset >= consumed.to_offset: + raise temporalio.streams.StreamNotFoundError( + f"{gone}: the stream answered offset {entry.offset} where " + f"{offset} was due" + ) + records.append(entry.record) + offset += 1 + return records, owner_run_id diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 60f824c4d7..4a81f19e10 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -24,6 +24,7 @@ import temporalio.common import temporalio.runtime import temporalio.service +import temporalio.streams from temporalio.common import ( HeaderCodecBehavior, VersioningBehavior, @@ -152,6 +153,8 @@ def __init__( ), disable_payload_error_limit: bool = False, max_workflow_task_external_storage_concurrency: int = _DEFAULT_WORKFLOW_TASK_EXTERNAL_STORAGE_CONCURRENCY, + external_stream_backend: Any | None = None, + stream_provider: temporalio.streams.StreamProvider | None = None, ) -> None: """Create a worker to process workflows and/or activities. @@ -277,6 +280,16 @@ def __init__( graceful_shutdown_timeout: Amount of time after shutdown is called that activities are given to complete before their tasks are cancelled. + external_stream_backend: External Workflow Stream provider instance. + The provider must declare ``guarantees_immutability = True``; + Worker construction fails otherwise. Input subscriptions use + ``StreamBackend``. Workflow-originated output additionally + requires the provider to implement ``OutputStreamBackend``; + input-only providers remain supported. + + .. warning:: + This option and the External Workflow Streams feature are + experimental. workflow_failure_exception_types: The types of exceptions that, if a workflow-thrown exception extends, will cause the workflow/update to fail instead of suspending the workflow via @@ -343,6 +356,10 @@ def __init__( Defaults to 3. Adjust this value based on your workload's needs. Please report any issues you encounter with this setting or if you feel the default should be changed. + stream_provider: Experimental. The stream provider that workflows + on this worker read and publish through, see + :py:mod:`temporalio.streams`. A provider that is also a + :py:class:`Plugin` sets this itself when passed in ``plugins``. WARNING: This setting is experimental. """ @@ -392,6 +409,8 @@ def __init__( nexus_task_poller_behavior=nexus_task_poller_behavior, disable_payload_error_limit=disable_payload_error_limit, max_workflow_task_external_storage_concurrency=max_workflow_task_external_storage_concurrency, + external_stream_backend=external_stream_backend, + stream_provider=stream_provider, ) plugins_from_client = cast( @@ -417,6 +436,20 @@ def _init_from_config(self, client: temporalio.client.Client, config: WorkerConf Client is safe to take separately since it can't be modified by worker plugins. """ self._config = config + + # The external stream backend is validated here rather than lazily so + # that a provider which cannot guarantee record immutability fails + # Worker construction, before any Workflow can consume through it. + self._external_stream_backend = config.get("external_stream_backend") + if self._external_stream_backend is not None: + from temporalio.contrib.external_workflow_streams._backend import ( + _validate_backend, + ) + + self._external_stream_backend = _validate_backend( + self._external_stream_backend + ) + if not ( config.get("activities") or config.get("nexus_service_handlers") @@ -461,6 +494,11 @@ def _init_from_config(self, client: temporalio.client.Client, config: WorkerConf # Prepend applicable client interceptors to the given ones client_config = config["client"].config(active_config=True) # type: ignore[reportTypedDictNotRequiredAccess] + # A provider registered on the client serves its workers too, so one + # registration covers every context that asks for a stream. + stream_provider = config.get("stream_provider") or client_config.get( + "stream_provider" + ) interceptors_from_client = cast( list[Interceptor], [i for i in client_config["interceptors"] if isinstance(i, Interceptor)], @@ -498,6 +536,7 @@ def _init_from_config(self, client: temporalio.client.Client, config: WorkerConf interceptors=interceptors, metric_meter=self._runtime.metric_meter, client=client, + stream_provider=stream_provider, encode_headers=( client_config["header_codec_behavior"] == HeaderCodecBehavior.CODEC ), @@ -538,6 +577,7 @@ def check_activity(activity: str): self._workflow_worker = _WorkflowWorker( bridge_worker=lambda: self._bridge_worker, + client=client, namespace=config["client"].namespace, # type: ignore[reportTypedDictNotRequiredAccess] task_queue=config["task_queue"], # type: ignore[reportTypedDictNotRequiredAccess] workflows=workflows, @@ -565,6 +605,11 @@ def check_activity(activity: str): encode_headers=client_config["header_codec_behavior"] != HeaderCodecBehavior.NO_CODEC, max_workflow_task_external_storage_concurrency=max_workflow_task_external_storage_concurrency, + external_stream_backend=self._external_stream_backend, + stream_provider=stream_provider, + stream_client=( + config["client"] if stream_provider is not None else None # type: ignore[reportTypedDictNotRequiredAccess] + ), ) tuner = config.get("tuner") @@ -868,6 +913,20 @@ async def raise_on_shutdown(): f"Beginning worker shutdown, will wait {graceful_timeout} before cancelling activities" ) + # Ask Core what state each streaming Run is in, before shutdown is + # initiated. This is the first half of P20's sweep and it has to be here: + # an idle cached Run has no pending work, so Core's workflow-state lane + # finishes on the first input after the shutdown token is cancelled, and + # every Run-status probe from then on answers RunNotFound. That answer + # owes a wake just as NoOpenWorkflowTask does, so a sweep that probes + # later still sends its wake and still looks correct -- while Parked (no + # wake needed) and WftOpen (C15b's, not ours to race) stop being + # reachable answers at all. Only the asking happens here; the wakes go + # out below, once the pollers have stopped, because offering a Run to a + # task queue this Worker is still polling is not a hand-off. + if self._workflow_worker: + await self._workflow_worker.probe_external_stream_runs() + # Initiate core worker shutdown self._bridge_worker.initiate_shutdown() @@ -901,6 +960,23 @@ async def raise_on_shutdown(): if self._nexus_worker: await self._nexus_worker.wait_all_completed() + # Send the wakes the probe above found owed, then tear down external + # stream subscriptions. This is the second half of P20's obligation and + # it happens here rather than inside the workflow worker's poll loop + # because both shutdown paths have to reach it: the poll drain above only + # replaces a worker task whose run() raised, so a clean shutdown would + # otherwise leave every Run registered, its watchers running, and its + # owed wake Signals unsent. + # + # After the wait above, so per-Run teardown stays driven by + # RemoveFromCache: every activation, eviction included, has been + # answered by now, so a FinalizeExternalStreams in flight was answered + # before the manager's state for that Run disappears. Before + # finalize_shutdown, because a Run the probe phase never reached is + # probed here and that still needs the bridge worker. + if self._workflow_worker: + await self._workflow_worker.shutdown_external_streams() + # Do final shutdown try: await self._bridge_worker.finalize_shutdown() @@ -1020,6 +1096,8 @@ class WorkerConfig(TypedDict, total=False): nexus_task_poller_behavior: PollerBehavior disable_payload_error_limit: bool max_workflow_task_external_storage_concurrency: int + external_stream_backend: Any | None + stream_provider: temporalio.streams.StreamProvider | None def _warn_if_activity_executor_max_workers_is_inconsistent( diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 01304d0144..5d7e040058 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -13,10 +13,13 @@ from dataclasses import dataclass from datetime import timezone from types import TracebackType +from typing import TYPE_CHECKING, Any, cast import temporalio.api.common.v1 +import temporalio.api.enums.v1 import temporalio.bridge.proto.common import temporalio.bridge.proto.workflow_activation +import temporalio.bridge.proto.workflow_commands import temporalio.bridge.proto.workflow_completion import temporalio.bridge.runtime import temporalio.bridge.worker @@ -24,6 +27,7 @@ import temporalio.converter import temporalio.converter._extstore import temporalio.exceptions +import temporalio.streams import temporalio.workflow from temporalio.bridge.worker import PollShutdownError from temporalio.converter import StorageDriverStoreContext, StorageDriverWorkflowInfo @@ -35,26 +39,82 @@ _relax_sandbox_for_debugger, ) from ._interceptor import ( + ExecuteWorkflowInput, Interceptor, WorkflowInboundInterceptor, WorkflowInterceptorClassInput, + WorkflowOutboundInterceptor, ) +from ._stream_ranges import fill_short_stream_ranges from ._workflow_instance import ( _DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS, PatchActivationInput, WorkflowInstance, WorkflowInstanceDetails, WorkflowRunner, + _is_workflow_terminal_command, _WorkflowExternFunctions, _WorkflowLogicFlag, ) +if TYPE_CHECKING: + import temporalio.client + logger = logging.getLogger(__name__) # Set to true to log all activations and completions LOG_PROTOS = False +class _StreamHooksInterceptor(WorkflowInboundInterceptor): + """Brackets the workflow function with the stream provider's lifecycle hooks. + + Installed by the worker when it has a stream provider, so no workflow + code has to call anything before it runs or before it returns. The start + hook runs when the instance's loop first turns, after the workflow's own + ``__init__`` and before the first task's Signals and Updates are handled, + because the SDK handles those ahead of the workflow function and a handler + registered any later would be missed by an Update that arrives with that + task. The finish hook runs when the function returns, raises or continues as new, + because a provider that parked a reader against the run has to let go + either way. + It does not run when the run is being evicted from the cache or when the + abandoned coroutine is collected: neither is the workflow ending, the + instance's state is not to be touched during eviction, and at collection + time the runtime on the thread belongs to whichever workflow happens to + be running, so the hook would act on that one. + """ + + def init(self, outbound: WorkflowOutboundInterceptor) -> None: + super().init(outbound) + # The hook has to run after the workflow's own __init__, which may + # register handlers the provider adopts, and before the first task's + # Signals and Updates are handled, which the SDK does ahead of the + # workflow function. The instance is its own event loop and nothing + # is queued on it yet, so a callback queued now runs first when that + # loop first turns, which is after every job of the activation has + # been applied and before any task they created takes a step. + runtime = temporalio.workflow._Runtime.current() + loop = cast(asyncio.AbstractEventLoop, cast(object, runtime)) + loop.call_soon(lambda: runtime.workflow_streams().provider.on_workflow_start()) + + async def execute_workflow(self, input: ExecuteWorkflowInput) -> Any: + runtime = temporalio.workflow._Runtime.current() + provider = runtime.workflow_streams().provider + try: + result = await self.next.execute_workflow(input) + except GeneratorExit: + raise + except BaseException: + # Eviction cancels the primary task the same way a workflow + # cancellation does, and only the cancellation is a run ending. + if not runtime.workflow_is_evicting(): + await provider.on_workflow_finish() + raise + await provider.on_workflow_finish() + return result + + # Value was chosen abitrarily as a small number that allows some concurrency and prevents # large numbers of concurrent external storage operations causing resource contention. # This default limit is per workflow task activation and does not limit the total number @@ -75,6 +135,61 @@ def _set_external_storage_metrics( target.driver_names.extend(sorted(metrics.driver_names)) +def _try_buffer_external_output( + completion: temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion, + runtime: Any, +) -> bool: + """Tell Core to retain a live output batch when this completion can wait. + + A quiescent input snapshot is the ordinary proof that Core can keep this + Workflow Task open. A park recheck that became ready is the other safe + retained transition: the attempted park lost to readiness and Core resumes + the same task. Any user/server command, a confirmed park, or capacity + backpressure takes the ordinary staging path instead. + """ + if runtime.output_rollover_requested: + return False + max_publish_latency = runtime.output_max_publish_latency + if max_publish_latency is None: + return False + + commands = completion.successful.commands + variants = [command.WhichOneof("variant") for command in commands] + became_ready = ( + len(commands) == 1 + and commands[0].HasField("external_stream_park_result") + and commands[0].external_stream_park_result.HasField("became_ready") + ) + # The channel report rides every completion and asks nothing of the + # server, so it does not make a quiescent completion anything else. + quiescent = "workflow_stream_quiescent" in variants and all( + variant + in ( + "workflow_stream_progress", + "workflow_stream_quiescent", + "workflow_stream_channels", + ) + for variant in variants + ) + if not became_ready and not quiescent: + return False + + command = temporalio.bridge.proto.workflow_commands.WorkflowCommand() + command.workflow_output_stream_buffered.max_publish_latency.FromTimedelta( + max_publish_latency + ) + # Keep input progress first and the quiescent snapshot last. This mirrors + # output commits and lets Core observe the cursor delta before retaining the + # task whose output deadline it is about to arm. + insert_at = 0 + while insert_at < len(commands) and commands[insert_at].HasField( + "workflow_stream_progress" + ): + insert_at += 1 + commands.insert(insert_at, command) + return True + + class _WorkflowWorker: # type:ignore[reportUnusedClass] def __init__( self, @@ -104,6 +219,10 @@ def __init__( encode_headers: bool, max_workflow_task_external_storage_concurrency: int, default_workflow_logic_flags: frozenset[_WorkflowLogicFlag] | None = None, + external_stream_backend: Any | None = None, + client: Any = None, + stream_provider: temporalio.streams.StreamProvider | None = None, + stream_client: temporalio.client.Client | None = None, ) -> None: # Debug mode is enabled if specified or if the TEMPORAL_DEBUG env var is truthy debug_mode = debug_mode or bool(os.environ.get("TEMPORAL_DEBUG")) @@ -162,10 +281,58 @@ def __init__( __temporal_assert_local_activity_valid=assert_local_activity_valid, ) ) + self._stream_provider = stream_provider + if stream_provider is not None: + # Innermost, so the lifecycle hooks bracket the workflow function + # itself, after every user interceptor has done its own setup. + self._interceptor_classes.append(_StreamHooksInterceptor) + # For the records a task's re-supplied ranges leave out, which the + # stream service still holds. + self._stream_client = stream_client + + # External Workflow Streams. The manager is per-Worker and owns the + # backend connection and watcher tasks; it is created lazily on the + # Worker's own event loop because that is the loop the watchers must run + # on, and __init__ is not necessarily called from it. + # The runtime exists even when no backend is configured. Recorded stream + # state comes from History, so replay annotations and continuation + # headers must be decoded and validated independently of whether this + # Worker can create a new live subscription. Workflow code still sees + # the feature as unconfigured through `external_streams_configured` in + # its instance details below. + self._external_stream_backend = external_stream_backend + self._external_streams_configured = external_stream_backend is not None + #: Held only to send the reserved wake Signal, which is a raw service + #: call rather than anything the bridge can do -- Core cannot signal a + #: Workflow on this Worker's behalf. + self._client = client + # The taxonomy's counters, created from its own module rather than + # here: P18 names them, documents them, and knows which error class + # belongs to which one. A counter created ad hoc here is a second + # definition of a name an operator alerts on. + from temporalio.contrib.external_workflow_streams._errors import StreamMetrics + + self._stream_metrics = StreamMetrics.create(metric_meter) + self._external_stream_manager: Any = None + #: How the Runs this Worker builds listen for their streams: by the + #: linked channel, the independent channel or the Signal. ``None`` + #: until the server has been asked. + self._channel_support: Any = None self._workflow_failure_exception_types = workflow_failure_exception_types self._patch_activation_callback = patch_activation_callback self._running_workflows: dict[str, _RunningWorkflow] = {} + #: Per-Run stream runtimes, held here rather than read off the instance. + #: A sandboxed Workflow's `instance` is a proxy that exposes only the + #: `WorkflowInstance` protocol, so reaching through it for the runtime + #: silently found nothing -- and the jobs that must never reach + #: `activate()` quietly went to ``_apply`` instead. + self._external_stream_runtimes: dict[str, Any] = {} + # Stages survive activations within the cached Run because Core may + # defer the server completion behind a local activity. A marker can be + # committed by that later activation even though it staged no new + # output of its own. + self._pending_external_output_stages: dict[str, list[Any]] = {} self._disable_eager_activity_execution = disable_eager_activity_execution self._on_eviction_hook = on_eviction_hook self._disable_safe_eviction = disable_safe_eviction @@ -279,6 +446,58 @@ async def drain_poll_queue(self) -> None: except PollShutdownError: return + async def probe_external_stream_runs(self) -> None: + """Asks Core what state each streaming Run is in. P20's sweep, first half. + + Called by the Worker immediately *before* Core's shutdown is initiated, + because that is the last moment the answer exists. An idle cached Run has + no pending work, so Core's ``shutdown_done`` is satisfied by the first + input after the shutdown token is cancelled and the workflow-state lane + ends there; every probe afterwards answers ``RunNotFound``. Since + ``RunNotFound`` owes a wake just as ``NoOpenWorkflowTask`` does, a sweep + that probes too late still sends its wake and still looks right, while + the two answers that mean *don't* send one -- ``Parked``, which needs no + wake, and ``WftOpen``, which belongs to C15b -- can no longer occur. + + Only the asking happens here. The wakes are sent from + :py:meth:`shutdown_external_streams`, after the pollers have stopped: + offering the Run to a task queue this Worker is still polling would be + the opposite of a hand-off. + + The manager bounds this with its own short grace period, so a wedged + Core cannot delay the stop-polling step. + """ + if self._external_stream_manager is not None: + await self._external_stream_manager.probe_runs() + + async def shutdown_external_streams(self) -> None: + """Sends the owed wakes and tears the manager down. P20's second half. + + Called by the Worker once every activation has been dealt with, on + *both* shutdown paths. It deliberately does not live in + :py:meth:`drain_poll_queue`, which the Worker substitutes only for a + worker task whose ``run()`` raised: wiring it there means a clean + ``Worker.shutdown()`` never sweeps at all, leaving Runs registered, + watchers running, and buffers and backend connections open in a process + that is about to exit. + + Here, and not with the probe, because per-Run teardown is driven by + ``RemoveFromCache`` and nothing else: a ``FinalizeExternalStreams`` in + flight has to be answered before the manager's state for that Run + disappears. + + It is not folded into eviction either, because an *idle cached Run + receives no eviction activation at shutdown at all* -- ``shutdown_done`` + treats a Run with no pending work as finished -- and that is exactly the + Run that most needs a wake: its records are buffered here and nothing + else will ever tell the Workflow they arrived. + + The manager bounds the sweep with its own grace period, so this cannot + hold shutdown open indefinitely. + """ + if self._external_stream_manager is not None: + await self._external_stream_manager.shutdown() + async def _activate_inline_for_debug( self, loop: asyncio.AbstractEventLoop, @@ -329,12 +548,18 @@ async def _handle_activation( await self._handle_cache_eviction(act, cache_remove_job) return + if self._external_stream_manager is not None: + self._external_stream_manager.note_workflow_task_started(act.run_id) + if self._external_streams_configured and self._channel_support is None: + await self._decide_channel_subscriptions(act) + # Build default success completion (e.g. remove-job-only activations) completion = ( temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() ) completion.successful.SetInParent() workflow = None + workflow_id: str | None = None data_converter = self._data_converter download_metrics = temporalio.converter._extstore.StorageOperationMetrics() try: @@ -356,6 +581,11 @@ async def _handle_activation( "Cache already exists for activation with initialize job" ) + # Before the bodies are decoded, so what is fetched is decoded with + # the rest, and before the workflow runs on the range. + if self._stream_client is not None: + await fill_short_stream_ranges(act, workflow_id, self._stream_client) + workflow_context = temporalio.converter.WorkflowSerializationContext( namespace=self._namespace, workflow_id=workflow_id, @@ -395,7 +625,19 @@ async def _handle_activation( ) self._running_workflows[act.run_id] = workflow - if self._debug_mode: + # Two of the four stream jobs are *themselves* backend operations, + # and a third has to be prepared by one. Routing them through the + # synchronous `activate()` would put a multi-second transaction + # inside a call running under a 2-second deadlock timeout -- failing + # the Workflow Task for a perfectly healthy backend, and getting + # worse the more records replay must validate. + # + # So they are partitioned here, in the async layer that already + # awaits `decode_activation` before handing anything to the executor. + handled = await self._handle_external_stream_jobs(act, workflow) + if handled is not None: + completion = handled + elif self._debug_mode: # Inline on the main thread so pdb / breakpoint() can read # stdin. The loop blocks during the activation — that's the # intended single-stepping semantic. @@ -433,6 +675,24 @@ async def _handle_activation( workflow.deadlocked_activation_task = activate_task raise deadlock_exc from None + output_runtime = self._external_stream_runtimes.get(act.run_id) + if output_runtime is not None and completion.HasField("successful"): + # A subscription opened at a stream's tail is positioned here, + # off the Workflow thread and before the completion goes out, so + # the marker records the boundary the watcher starts from. + await output_runtime.resolve_pending_starts() + if ( + output_runtime is not None + and completion.HasField("successful") + and not act.is_replaying + and output_runtime.has_output + ): + await self._stage_or_buffer_external_output( + act, + completion, + output_runtime, + ) + except Exception as err: if isinstance(err, _DeadlockError): err.swap_traceback() @@ -480,6 +740,26 @@ async def _handle_activation( f"Failed converting activation exception: {inner_err}" ) + # One place for every failed completion, however it was reached. An + # external stream failure arrives two ways -- raised out here by a + # runtime-only job, or raised on the Workflow thread by a delivery and + # already turned into a failure by `activate()` -- and both must carry + # the same cause and increment the same counter. + if self._external_stream_backend is not None and completion.HasField("failed"): + try: + self._note_external_stream_failure(completion) + except Exception: + # Reporting a failure may not *become* one. This runs outside + # the block that turns an exception into a failed completion, + # so anything raised here would escape with the completion + # unsent -- turning a Workflow Task failure the server can see + # into a Workflow Task timeout it cannot explain. + logger.exception( + "Failed classifying an external stream failure on workflow " + "with run ID %s", + act.run_id, + ) + completion.run_id = act.run_id # Encode completion @@ -531,6 +811,29 @@ async def _handle_activation( logger.exception( "Failed completing activation on workflow with run ID %s", act.run_id ) + else: + if workflow_id is not None: + await self._promote_external_output( + run_id=act.run_id, + workflow_id=workflow_id, + ) + # A wake accepted by the service stays outstanding until the task it + # caused completes successfully. Failed/rejected tasks replay, and + # readiness rebuilt during that replay must remain coalesced behind + # the original wake. Only Core accepting this completion proves the + # cycle ended and permits another buffered generation to wake. + if self._external_stream_manager is not None and completion.HasField( + "successful" + ): + self._external_stream_manager.note_workflow_task_completed( + act.run_id, + terminal=any( + map( + _is_workflow_terminal_command, + completion.successful.commands, + ) + ), + ) async def _handle_cache_eviction( self, @@ -634,6 +937,17 @@ async def _handle_cache_eviction( # Remove from map and send completion if act.run_id in self._running_workflows: del self._running_workflows[act.run_id] + # Per-Run stream teardown is driven by `RemoveFromCache` and nothing + # else, so a `FinalizeExternalStreams` in flight is always answered + # before the manager's state for the Run disappears. It is done here + # rather than inside the instance because `disable_safe_eviction` + # skips the instance's eviction job entirely, and a Run whose + # watchers outlived it would keep a backend connection open forever. + self._external_stream_runtimes.pop(act.run_id, None) + await self._abort_dead_external_output(run_id=act.run_id) + self._pending_external_output_stages.pop(act.run_id, None) + if self._external_stream_manager is not None: + await self._external_stream_manager.evict_run(act.run_id) try: await self._bridge_worker().complete_workflow_activation( temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion( @@ -727,6 +1041,8 @@ def _create_workflow_instance( ) # Create instance from details + runtime = self._create_external_stream_runtime(act, init) + self._external_stream_runtimes[act.run_id] = runtime det = WorkflowInstanceDetails( # Sharing the underlying converter could let another workflow's state # affect conversion and replay, so each instance needs a fresh one. @@ -743,12 +1059,724 @@ def _create_workflow_instance( last_completion_result=init.last_completion_result, last_failure=last_failure, default_workflow_logic_flags=frozenset(self._default_workflow_logic_flags), + external_stream_runtime=runtime, + external_streams_configured=self._external_streams_configured, + stream_provider=self._stream_provider, ) if defn.sandboxed: return self._workflow_runner.create_instance(det) else: return self._unsandboxed_workflow_runner.create_instance(det) + async def _handle_external_stream_jobs( + self, + act: temporalio.bridge.proto.workflow_activation.WorkflowActivation, + _workflow: _RunningWorkflow, + ) -> ( + temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion | None + ): + """Answers the runtime-only stream jobs without calling ``activate()``. + + Returns the synthesized completion when this activation was entirely + one of those jobs, or ``None`` to let the normal path run. + + ``PrepareExternalStreamPark`` and ``FinalizeExternalStreams`` run no + user Workflow code at all -- they cannot resolve futures and their only + legal answers are their own result command or an activation failure. + ``ReplayExternalStreams`` is *prepared* here rather than handled: its + recorded ranges are read and validated into the buffers, then the job + passes through so ``_apply`` delivers from memory exactly as it does for + a live resolve. + """ + runtime = self._external_stream_runtimes.get(act.run_id) + if runtime is None: + if any(j.HasField("replay_external_streams") for j in act.jobs): + raise RuntimeError( + "received an external stream replay job without the per-Run " + "validation runtime" + ) + return None + + if any(j.HasField("resolve_external_stream_waits") for j in act.jobs): + # Core is telling this Worker the wait set has moved on, which is + # also the only notice that a *confirmed* park is over: a wake Signal + # and a fresh quiescent snapshot both clear Core's `park_generation`, + # and neither is visible in the backend. The intent installed for + # that park comes out here, before the Workflow resumes, so nothing + # can read a generation that no longer exists -- not a producer + # choosing what its wake names, and not this Worker's own shutdown + # sweep. The job itself passes through to `_apply` unchanged. + await self._stream_manager().resolve_park(act.run_id) + + park = next( + ( + j.prepare_external_stream_park + for j in act.jobs + if j.HasField("prepare_external_stream_park") + ), + None, + ) + finalize = next( + ( + j.finalize_external_streams + for j in act.jobs + if j.HasField("finalize_external_streams") + ), + None, + ) + replay = next( + ( + j.replay_external_streams + for j in act.jobs + if j.HasField("replay_external_streams") + ), + None, + ) + + if replay is not None: + # Prepared, not handled: fill and validate every recorded range + # before the job reaches the Workflow thread. A transient backend + # error or an integrity violation surfaces from here through the + # activation-failure path rather than as a deadlock timeout, which + # would misattribute a storage problem to the Workflow's own code. + plan = None + if replay.replay_annotation: + plan = await self._stream_manager().prepare_replay( + act.run_id, replay.replay_annotation + ) + # The other half of the same record's decoding. `prepare_replay` + # bound the codec to the stream key the *marker* recorded; the + # converter that runs inside `activate()` would otherwise stay bound + # to this Run's identity, and an offline `Replayer` supplies its own + # namespace for that -- one record, two Workflow identities. Bound + # out here for the reason `_create_external_stream_runtime` gives: + # `with_context` clones the user's component converters, and the + # runtime crosses into the Workflow sandbox. + if plan is not None: + runtime.install_replay_converters(self._replay_stream_converters(plan)) + return None + + if park is None and finalize is None: + return None + + completion = ( + temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() + ) + completion.successful.SetInParent() + + if finalize is not None: + # **No backend work at all** (ADR-010). The terminal is read from + # the runtime's own in-memory blocked snapshot: the boundary is not + # "wherever the stream is now", it is where this Workflow Task's + # deliveries stopped, which was fixed the moment the last activation + # returned. Refreshing it against the backend would be actively + # wrong -- it could name a position replay must not reproduce. + command = completion.successful.commands.add() + command.external_stream_finalized.quiescence_generation = ( + finalize.quiescence_generation + ) + command.external_stream_finalized.final_observation_delta = ( + runtime.add_terminal() + ) + return completion + + assert park is not None + # The set to park is **Core's**, not this Worker's registration list. + # `park.waits` is the complete blocked snapshot Core is holding the + # Workflow Task for; `blocked_snapshot()` is every subscription the + # runtime has registered, which is a superset -- a subscription that + # delivered a record and was not awaited again is registered and not + # blocked. Parking the superset rechecks a wait nothing is waiting on, + # so its records abort a park that was entirely legitimate and the + # handshake repeats on every idle timeout; and it installs an intent for + # a wait Core is not parking, which is an intent with no park behind it. + # The runtime supplies only the cursor boundary for each of Core's + # waits. + snapshot = runtime.blocked_snapshot() + became_ready = await self._stream_manager().prepare_park( + act.run_id, + park.quiescence_generation, + { + wait.wait_id: snapshot[wait.wait_id] + for wait in park.waits + if wait.wait_id in snapshot + }, + ) + command = completion.successful.commands.add() + command.external_stream_park_result.quiescence_generation = ( + park.quiescence_generation + ) + if became_ready: + # A recheck found records, so this parking generation is abandoned + # and Core issues a normal resolve activation next -- rather than + # running user code from inside the park path. + command.external_stream_park_result.became_ready.SetInParent() + else: + command.external_stream_park_result.confirmed.SetInParent() + command.external_stream_park_result.final_observation_delta = ( + runtime.add_terminal() + ) + return completion + + async def _stage_or_buffer_external_output( + self, + act: temporalio.bridge.proto.workflow_activation.WorkflowActivation, + completion: temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion, + runtime: Any, + ) -> None: + """Retain a safe batch, or stage it before reporting this completion.""" + if _try_buffer_external_output(completion, runtime): + return + staged_output = await self._stage_external_output(act, completion, runtime) + pending = self._pending_external_output_stages.setdefault(act.run_id, []) + for topic in staged_output.topics: + if topic.manifest not in pending: + pending.append(topic.manifest) + + async def _stage_external_output( + self, + act: temporalio.bridge.proto.workflow_activation.WorkflowActivation, + completion: temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion, + runtime: Any, + ) -> Any: + """Stage Workflow output before its compact marker command reaches Core.""" + staged = await runtime.stage_output(act.history_floor_event_id) + command = temporalio.bridge.proto.workflow_commands.WorkflowCommand() + output = command.workflow_output_stream_commit + output.request_rollover = runtime.output_rollover_requested + manifest = output.manifest + manifest.schema_version = staged.schema_version + manifest.fingerprint_version = staged.fingerprint_version + manifest.stage_token = staged.stage_token + manifest.history_floor_event_id = staged.history_floor_event_id + manifest.run_id = staged.run_id + manifest.provider_id = staged.provider_id + manifest.provider_format_version = staged.provider_format_version + for staged_topic in staged.topics: + source = staged_topic.manifest + topic = manifest.topics.add() + topic.topic = source.stream_key.stream_name + topic.record_count = source.record_count + topic.logical_byte_count = source.logical_byte_count + topic.logical_fingerprint = source.fingerprint + topic.finished = staged_topic.finished + for counts in staged.segment_record_counts: + segment = manifest.segments.add() + segment.record_counts_by_topic.extend(counts) + + # Input progress remains first. It validates consumed input before any + # command that could depend on it, while this internal commit sits ahead + # of the user's server-bound commands. + commands = completion.successful.commands + insert_at = 0 + while insert_at < len(commands) and commands[insert_at].HasField( + "workflow_stream_progress" + ): + insert_at += 1 + commands.insert(insert_at, command) + runtime.output_stage_recorded() + return staged + + async def _promote_external_output( + self, + *, + run_id: str, + workflow_id: str, + ) -> None: + """Promote a staged batch only after History proves its marker. + + Core's completion call has already made its server RPC by this point, + but it deliberately absorbs report errors in order to drive eviction + and task retry. A successful bridge await therefore is not itself a + commit proof. Reusing the cold-client History predicate keeps definite + rejection and ambiguous transport outcomes safely pending/aborted while + making the healthy path visible without waiting for a reader. + """ + backend = self._external_stream_backend + client = self._client + if backend is None or client is None: + return + + from temporalio.contrib.external_workflow_streams._output_client import ( + _reconcile_output_stage, + ) + + pending = self._pending_external_output_stages.get(run_id) + if not pending: + return + unresolved: list[Any] = [] + for manifest in pending: + try: + resolved = await _reconcile_output_stage( + backend=backend, + client=client, + workflow_id=workflow_id, + manifest=manifest, + ) + if not resolved: + unresolved.append(manifest) + except asyncio.CancelledError: + raise + except Exception as err: + # Promotion is an optimization over the durable pending + # barrier. A cold client repeats this exact reconciliation, so + # a transient History/backend failure must not retroactively + # turn an already-reported Workflow Task into an SDK failure. + self._stream_metrics.record(err) + logger.warning( + "Could not opportunistically reconcile external output stage " + "%s for Workflow %s; it remains pending for a client", + manifest.stage_token, + workflow_id, + exc_info=True, + ) + unresolved.append(manifest) + if unresolved: + self._pending_external_output_stages[run_id] = unresolved + else: + self._pending_external_output_stages.pop(run_id, None) + + async def _abort_dead_external_output(self, *, run_id: str) -> None: + """Abort the staged batches of an evicted Run that History has rejected. + + A Run evicted because its completion was not accepted may have staged + a batch that no marker will ever name. History already says so when + the task was failed or timed out, so such a stage is aborted here + instead of waiting for a reader to find the barrier. A stage whose + marker is authoritative is never re-promoted from here: promotion is + attempted once after the completion, and a stage it left pending is + owed to a cold client's repair, as the contract tests pin down. + """ + backend = self._external_stream_backend + client = self._client + pending = self._pending_external_output_stages.get(run_id) + if backend is None or client is None or not pending: + return + + from temporalio.contrib.external_workflow_streams._output_client import ( + _apply_output_stage_decision, + _history_decision, + _HistoryDecision, + ) + + for manifest in pending: + workflow_id = manifest.stream_key.workflow_id + try: + decision = await _history_decision( + client=client, + workflow_id=workflow_id, + manifest=manifest, + ) + if decision is not _HistoryDecision.ABORT: + continue + await _apply_output_stage_decision( + backend=backend, + manifest=manifest, + decision=decision, + ) + except asyncio.CancelledError: + raise + except Exception as err: + self._stream_metrics.record(err) + logger.warning( + "Could not abort the dead external output stage %s for " + "Workflow %s at eviction; it remains pending for a client", + manifest.stage_token, + workflow_id, + exc_info=True, + ) + + def _replay_stream_converters( + self, plan: Any + ) -> dict[int, temporalio.converter.DataConverter]: + """One converter per wait the marker binds, in the recorded context. + + From the annotation's header, which is the only place a replayed + record's stream is written down -- the Workflow has not run far enough + to have re-created the subscription that would otherwise carry it. + + Bound from the Worker's own converter rather than from the runtime's, + which is already bound to this Run: a second ``with_context`` over the + first would ask a user's component converter to rebind itself, and + nothing in the protocol promises that composes. + + Memoized per ``(namespace, workflow_id)`` exactly as the manager's own + preparation is, so a marker binding several waits of one Workflow clones + the component converters once rather than once per wait. + """ + bound: dict[tuple[str, str], temporalio.converter.DataConverter] = {} + converters: dict[int, temporalio.converter.DataConverter] = {} + for wait_id, binding in plan.annotation.header.streams.items(): + key = binding.stream_key + cached = bound.get((key.namespace, key.workflow_id)) + if cached is None: + cached = self._data_converter.with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=key.namespace, + workflow_id=key.workflow_id, + ) + ) + bound[(key.namespace, key.workflow_id)] = cached + converters[wait_id] = cached + return converters + + def _stream_manager(self) -> Any: + """The Worker's subscription manager, created on first use. + + Lazily, because it captures the running event loop -- the one its + watcher tasks must run on -- and ``__init__`` is not necessarily called + from that loop. + """ + if self._external_stream_manager is None: + from temporalio.contrib.external_workflow_streams._manager import ( + StreamSubscriptionManager, + ) + + self._external_stream_manager = StreamSubscriptionManager( + backend=self._external_stream_backend, + # The Worker's converter, for the **asynchronous half** of + # decoding a record: external-payload retrieval and the user's + # PayloadCodec. Both are arbitrary asynchronous work and neither + # needs the topic's declared type, so both belong out here on + # this loop -- the same place `decode_activation` awaits them + # for every other payload an activation carries. Without this + # the Workflow thread awaits them inside `activate()`, which + # performs I/O in a deterministic event loop and puts a user + # codec under the 2-second deadlock timeout. + data_converter=self._data_converter, + notify_ready=self._bridge_worker().notify_external_stream_ready, + # A Worker whose Run cannot take local readiness owes the same + # Signal a producer owes. Without this the record sits buffered + # while the Workflow waits for a task that nothing will create: + # parked, cached-with-no-open-task, and evicted all look the + # same from here, and all three are answered the same way. + send_wake=self._send_external_stream_wake, + # Read-only, and deliberately not the readiness call: readiness + # asserts a buffered record, so probing with it on the way out + # would manufacture a Workflow Task for a Run that had nothing + # waiting. + run_status=self._bridge_worker().external_stream_run_status, + shutdown_wake_failed_metric=self._record_shutdown_wake_failed, + # Only a prefix on this Worker's sender identity, which the + # manager makes unique per instance. Passed so a wake request ID + # stays traceable to a client in server-side logs; it is + # deliberately not the identity itself, since two Workers + # sharing one Client share this string. + client_identity=( + self._client.service_client.config.identity + if self._client is not None + else "" + ), + ) + return self._external_stream_manager + + def _note_external_stream_failure( + self, + completion: temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion, + ) -> None: + """Applies the external stream failure taxonomy to a failed completion. + + Three of the taxonomy's four rows are Workflow Task failures that differ + from every other Workflow Task failure -- and from each other -- only in + **the error type and the metric**. The server retries a failed Workflow + Task regardless of cause, so nothing about the retry distinguishes a + backend outage that will clear on its own from integrity loss that needs + an operator, or from a converter mismatch that needs a code change. + This is what makes those rows tellable apart: + + - ``force_cause`` is set to the external-storage cause, which is what + separates all three from an ordinary Workflow bug in server-side + Workflow Task failure reporting; + - the matching counter, and only the matching counter, is incremented, + so an alert on integrity loss is not diluted by a backend outage. + + Row four -- the annotation not matching the subscriptions Workflow code + creates -- is deliberately absent: it is ordinary nondeterminism, gets + no stream cause and no stream counter, and is fixed by versioning the + Workflow rather than by touching the backend. + + The failure is inspected rather than the exception, because half of + these failures never exist as an exception out here: a decode that + raises on the Workflow thread is converted inside ``activate()``, and + what comes back is a completion. The application failure type is the + exception's class name, and the chain is walked because the raising + frame may have wrapped it. + """ + failure = completion.failed.failure + counter = None + while True: + counter = self._stream_metrics.counter_for( + failure.application_failure_info.type + ) + if counter is not None or not failure.HasField("cause"): + break + failure = failure.cause + if counter is None: + return + + completion.failed.force_cause = temporalio.api.enums.v1.WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_EXTERNAL_STORAGE_FAILURE + counter.add(1) + + def _record_shutdown_wake_failed(self, subscription: Any) -> None: + """Counts a shutdown wake that could not be acknowledged. + + A dropped wake is silent by nature -- the Workflow simply waits, and + nothing distinguishes that from a producer having nothing to say -- so it + gets a metric rather than only a log line. + """ + self._stream_metrics.shutdown_wake_failed.add( + 1, + { + "namespace": self._namespace, + "task_queue": self._task_queue, + "stream_name": subscription.stream_key.stream_name, + }, + ) + + async def _send_external_stream_wake(self, subscription: Any) -> None: + """Sends the wake a subscription owes, over the backend's wake transport. + + Addressed to the chain rather than to a Run, so it lands on the current + Run of the chain -- which may already be a successor by the time this + runs. + + A failure is logged **and re-raised to the manager that called this**. + There is no Workflow Task to fail out here, so the exception is not a + way of reporting anything to a Workflow -- it is how the caller learns + the wake was not acknowledged. The shutdown sweep is the caller that + acts on it: it retries within its grace period and counts the wake on + ``external_stream_shutdown_wake_failed`` when the retries run out. + Returning normally instead makes an unacknowledged wake + indistinguishable from a delivered one, which ends that retry loop after + one attempt and reports a clean shutdown that lost a record. + + The request ID is derived from the wake's identity, so a retry is the + same wake rather than a second one -- which is what makes re-sending an + attempt that may in fact have arrived safe. + + The manager's live watcher path is the other caller, and it must guard + this call itself: an exception escaping ``_report_ready`` ends the + watcher task for good. + + ``subscription`` is whatever the manager composes a wake from, which is + not always a subscription: a stale park intent retired after its wait was + closed or its Run evicted still owes the wake it silenced, and the + manager carries that obligation on an object of its own. + """ + from temporalio.contrib.external_workflow_streams._wake import ( + WakeRequest, + channel_for, + send_wake, + wake_position, + wake_transport_of, + ) + + if self._client is None: + # Raised for the same reason a failed send is: the wake is owed and + # will not be sent, and a caller told nothing counts it as + # delivered. + raise RuntimeError( + "an external stream wake Signal is owed but this Worker has no " + "client to send it with; the Workflow would wait out its idle " + "timeout instead" + ) + + key = subscription.stream_key + # Asked of the manager rather than read straight from the backend. The + # park this wake must name is whatever is installed *now* -- a generation + # cached when the watcher started would name a park since abandoned and + # resolved -- but "installed" is not "live", and only the manager's + # owed-removal ledger can tell an intent Core is still parked on from one + # this Worker has already decided to remove. Naming the latter sends a + # Signal Core discards while reporting success, which is how the shutdown + # sweep came to count an obsolete generation as a handoff it had made. + generation = ( + await self._stream_manager().wake_park_generation(subscription) or 0 + ) + # The furthest record this Worker has read, which is the record the + # Run has not consumed yet. A cleanup wake has no subscription left to + # have read anything. + read = getattr(subscription, "prefetch_cursor", None) + position, position_counter = wake_position( + subscription.backend, read.offset if read is not None else None + ) + address = channel_for(key) + try: + await send_wake( + self._client, + WakeRequest( + namespace=key.namespace, + workflow_id=key.workflow_id, + first_execution_run_id=key.first_execution_run_id, + stream_name=key.stream_name, + wait_id=subscription.wait_id, + park_generation=generation, + # This Worker's own identity, not the client's: two Workers + # in one process share a Client, and a shared identity would + # give their unparked wakes the same request ID for the + # server to deduplicate -- losing the second Worker's wake. + sender_identity=self._stream_manager().wake_sender_identity, + # Each wake cycle is a separate ask, not a retry of the last + # completed one. The manager coalesces reports until Core + # accepts that cycle's successful task completion. + wake_counter=subscription.wake_counter, + position=position, + position_counter=position_counter, + channel=address.channel, + channel_execution=address.execution, + ), + transport=wake_transport_of(subscription.backend), + ) + except Exception: + logger.exception( + "Failed sending external stream wake Signal for %s wait %s", + key, + subscription.wait_id, + ) + raise + + async def _decide_channel_subscriptions( + self, act: temporalio.bridge.proto.workflow_activation.WorkflowActivation + ) -> None: + """Decides how the readers listen, from what the server offers. + + Decided once per Worker and before the Run is built, because the + decision is a pair of lang flags the Run reads at construction: one + for the independent channel and its subscribe command, one for the + channel linked to the stream's owner, which needs no command. The + linked kind shows only on a running workflow, so the probe asks about + the workflow whose first task this is. A probe that cannot tell, or an + activation without a start to name a workflow, leaves the question + open for the next activation; the Runs built meanwhile do not + subscribe, and their records still arrive by the Signal. + """ + from temporalio.contrib.external_workflow_streams._wake import ( + ChannelSupport, + channel_support, + ) + + if self._client is None: + self._channel_support = ChannelSupport.NONE + return + workflow_id = next( + ( + job.initialize_workflow.workflow_id + for job in act.jobs + if job.HasField("initialize_workflow") + ), + None, + ) + if not workflow_id: + return + try: + answer = await channel_support(self._client, workflow_id) + except Exception: + logger.warning( + "Could not ask the server whether it has notification channels", + exc_info=True, + ) + return + if answer is None: + return + self._channel_support = answer + self._set_default_workflow_logic_flag( + _WorkflowLogicFlag.SUBSCRIBE_NOTIFICATION_CHANNELS, + enabled=answer is not ChannelSupport.NONE, + ) + self._set_default_workflow_logic_flag( + _WorkflowLogicFlag.LINKED_NOTIFICATION_CHANNELS, + enabled=answer is ChannelSupport.LINKED, + ) + + def _create_external_stream_runtime( + self, + act: temporalio.bridge.proto.workflow_activation.WorkflowActivation, + init: temporalio.bridge.proto.workflow_activation.InitializeWorkflow, + ) -> Any: + """The per-Run handle Workflow code reaches the manager through. + + Created even when no backends are registered because recorded state is + authoritative: continuation headers and replay annotations still need + to be decoded, and quiet marker bindings still need their reverse + nondeterminism check. The instance installs this handle into Workflow + code only when backends were configured, preserving ``subscribe()``'s + explicit error naming the Worker option. + """ + from temporalio.contrib.external_workflow_streams._api import ( + DEFAULT_IDLE_TIMEOUT, + ) + from temporalio.contrib.external_workflow_streams._continuation import ( + read_continuation_header, + ) + from temporalio.contrib.external_workflow_streams._output_continuation import ( + read_output_continuation_header, + ) + from temporalio.contrib.external_workflow_streams._runtime import ( + WorkflowStreamRuntime, + ) + + # Decode before consulting configuration. The reserved header is + # must-understand History state, so an unsupported encoding cannot turn + # into an ordinary first execution on a Worker whose current Workflow + # code no longer calls subscribe(). A valid non-empty continuation also + # needs its recorded backend configuration even if current code removed + # every stream call; otherwise the successor silently discards the + # cursor and binding its predecessor committed. + continuation = read_continuation_header(dict(init.headers)) + output_continuation = read_output_continuation_header(dict(init.headers)) + if ( + continuation is not None + and continuation.cursors + or output_continuation is not None + ) and not self._external_streams_configured: + raise RuntimeError( + "Workflow History contains external stream continuation state, " + "but external streams are not configured on this Worker; pass " + "external_stream_backend=... to the Worker" + ) + + return WorkflowStreamRuntime( + manager=self._stream_manager(), + backend=self._external_stream_backend, + run_id=act.run_id, + namespace=self._namespace, + workflow_id=init.workflow_id, + # The *chain* key, not this Run: the stream spans the whole + # Continue-As-New chain, so a new Run continues the same stream + # rather than starting a fresh one. + first_execution_run_id=init.first_execution_run_id, + # Bound to the consuming Workflow, the same way `decode_activation` + # binds every other payload this activation carries. A converter + # that derives a key from the Workflow it serves would otherwise get + # the right context for the Workflow's own argument and no context + # at all for a stream record delivered in the very same activation. + # + # Bound *here* rather than inside the runtime's `codec_for`: the + # runtime crosses into the Workflow sandbox, and `with_context` runs + # user code to clone the component converters -- work that belongs + # on this side of the boundary, and that a per-Run handle need do + # only once. + # + # `with_context` and not `_with_contexts`: the store context names + # the Workflow as the payload's *storer*, and a stream record is + # stored by its producer, not by this Run. `with_context` also + # returns the converter unchanged unless a component implements + # `WithSerializationContext`, so the default converter is untouched. + data_converter=self._data_converter.with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._namespace, + workflow_id=init.workflow_id, + ) + ), + default_idle_timeout=DEFAULT_IDLE_TIMEOUT, + # Read here, before the Workflow object exists and therefore before + # any subscribe() call: a start cursor restored after a subscription + # was established would already have been overwritten by BEGINNING + # (ADR-022). + continuation=continuation, + output_continuation=output_continuation, + ) + def nondeterminism_as_workflow_fail(self) -> bool: return any( issubclass(temporalio.workflow.NondeterminismError, typ) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 5dcc4aa966..1bd5aeae07 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -18,6 +18,7 @@ from collections.abc import ( Awaitable, Callable, + Collection, Coroutine, Generator, Iterable, @@ -47,6 +48,7 @@ import temporalio.api.common.v1 import temporalio.api.enums.v1 import temporalio.api.sdk.v1 +import temporalio.api.stream.v1 import temporalio.bridge.proto.activity_result import temporalio.bridge.proto.child_workflow import temporalio.bridge.proto.common @@ -58,7 +60,9 @@ import temporalio.converter import temporalio.exceptions import temporalio.nexus.system +import temporalio.streams import temporalio.workflow +import temporalio.workflow._streams from temporalio.converter import StorageDriverStoreContext, StorageDriverWorkflowInfo from temporalio.converter._payload_converter import ( _TemporalTransferTypePayloadConverter, @@ -94,9 +98,32 @@ logger = logging.getLogger(__name__) +QUERY_HANDLER_NOT_FOUND = "expected but not found" +"""The phrase a query for an unregistered handler comes back with. + +A caller that has to recognise the condition has only the failure message to +go on, so it matches this constant rather than a copy of the sentence. +""" + # Set to true to log all cases where we're ignoring things during delete LOG_IGNORE_DURING_DELETE = False +# Core answers a query carrying this id on the query's own task, alone. Held in +# step with LEGACY_QUERY_ID in sdk-core, crates/sdk-core/src/worker/workflow/mod.rs. +_LEGACY_QUERY_ID = "legacy_query" + + +def _is_workflow_terminal_command( + command: temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.WorkflowCommand, +) -> bool: + """Whether a command ends the current Workflow Run.""" + return ( + command.HasField("complete_workflow_execution") + or command.HasField("continue_as_new_workflow_execution") + or command.HasField("fail_workflow_execution") + or command.HasField("cancel_workflow_execution") + ) + async def _shield_await(fut: asyncio.Future[Any]) -> Any: """Await a future without cancelling it if the awaiting task is cancelled. @@ -184,6 +211,21 @@ class WorkflowInstanceDetails: default_workflow_logic_flags: frozenset[_WorkflowLogicFlag] = field( default_factory=lambda: _DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS ) + external_stream_runtime: Any = None + """Opaque per-Run handle to the Worker's External Workflow Stream manager. + + Crosses into the sandbox by reference, like the rest of these details, which + is the whole point: the manager owns the Worker's backend connections and + watcher tasks, and a copy re-created inside the sandbox would watch nothing. + Workflow code only ever sees this handle -- never a provider instance. + + Present for validation even when no ``external_stream_backend`` was + configured on the Worker, because recorded state must be handled based on + History rather than current Worker configuration. + """ + external_streams_configured: bool = False + """Whether Workflow code may use the runtime for new subscriptions.""" + stream_provider: temporalio.streams.StreamProvider | None = None class WorkflowInstance(ABC): @@ -274,6 +316,168 @@ def create_instance(self, det: WorkflowInstanceDetails) -> WorkflowInstance: _ExceptionHandler: TypeAlias = Callable[[asyncio.AbstractEventLoop, _Context], Any] +# Match the server's per-batch limits. A record over its limit is refused where +# it is published, because a rejected command would be reissued on every +# replay; a task's records are split into commands that fit the batch limits. +# +# Copied rather than learned: the activation does not carry them and the +# server does not report them, so this is a second copy of a number somebody +# else owns. If the server lowers one, or makes it per namespace, the split +# here stops fitting and the command is rejected on every replay, which is +# the failure the split exists to avoid. Carrying them on the activation is +# what would fix that, and it needs a Core and server change. +_STREAM_CONTINUITY_REMEDY = ( + "This fails the Workflow Task and will keep failing it, because the range " + "is recorded as consumed and will not be sent again. Reset the workflow to " + "before the subscription to start its stream reading over, or terminate it " + "if its output is no longer wanted." +) + +_MAX_STREAM_RECORDS_PER_BATCH = 1000 +_MAX_STREAM_RECORD_BYTES = 1 << 20 +_MAX_STREAM_BATCH_BYTES = 2 << 20 + + +def _stream_batches( + records: Sequence[temporalio.api.stream.v1.StreamRecord], +) -> Iterator[list[temporalio.api.stream.v1.StreamRecord]]: + """Split one task's records for a stream into batches the server accepts.""" + batch: list[temporalio.api.stream.v1.StreamRecord] = [] + size = 0 + for record in records: + record_size = record.ByteSize() + if batch and ( + len(batch) >= _MAX_STREAM_RECORDS_PER_BATCH + or size + record_size > _MAX_STREAM_BATCH_BYTES + ): + yield batch + batch, size = [], 0 + batch.append(record) + size += record_size + if batch: + yield batch + + +def _is_completion_command( + command: temporalio.bridge.proto.workflow_commands.WorkflowCommand, +) -> bool: + return ( + command.HasField("complete_workflow_execution") + or command.HasField("continue_as_new_workflow_execution") + or command.HasField("fail_workflow_execution") + or command.HasField("cancel_workflow_execution") + ) + + +class _StreamBuffer: + """Holds the stream ranges delivered to a workflow so far. + + Delivery is driven by the server, not by whether workflow code happens to be + reading. A range arrives once, is recorded in History as consumed, and is + never sent again, so anything not yet read has to be kept here rather than + dropped. + """ + + def __init__(self, stream_id: str = "") -> None: + self._stream_id = stream_id + self._records: list[temporalio.workflow._DeliveredStreamRecord] = [] + self._waiters: list[asyncio.Future] = [] + # Where the next range has to start. Unknown until the first one + # arrives, because a subscription may start wherever the stream is and + # the server is the one that resolves that. + self._next_offset: int | None = None + self._closed = False + + @property + def closed(self) -> bool: + """Whether workflow code has said it wants no more of this stream.""" + return self._closed + + def close(self) -> None: + """Stop keeping what arrives, and let go of what is held. Idempotent. + + There is no unsubscribe command, so the server keeps delivering for + the life of the run. Holding those records would grow the instance + without bound for a reader nobody will read again. Dropping them is + replay-safe because the close happens at the same point of the same + workflow code every time, so the same ranges are dropped; continuity + is still tracked, so a range that repeats or skips is still caught. + """ + self._closed = True + self._records = [] + waiters, self._waiters = self._waiters, [] + for waiter in waiters: + if not waiter.done(): + waiter.set_result(None) + + def extend( + self, + records: Sequence[temporalio.api.stream.v1.StreamRecord], + from_offset: int = 0, + to_offset: int | None = None, + ) -> None: + if to_offset is None: + to_offset = from_offset + len(records) + # A range is recorded as consumed once and never resent, so one that + # repeats, skips or mis-sizes would hand the workflow duplicate or + # shifted records with nothing to say so. Failing the task is what + # makes the fault visible. + if to_offset - from_offset != len(records): + raise RuntimeError( + f"stream {self._stream_id!r} delivered {len(records)} records " + f"for offsets [{from_offset}, {to_offset}). {_STREAM_CONTINUITY_REMEDY}" + ) + if self._next_offset is not None and from_offset != self._next_offset: + raise RuntimeError( + f"stream {self._stream_id!r} delivered offsets [{from_offset}, " + f"{to_offset}) but the last range ended at {self._next_offset}. " + f"{_STREAM_CONTINUITY_REMEDY}" + ) + self._next_offset = to_offset + # An empty range still counts as a delivery, but there is nothing to + # hand a reader, so only a non-empty one wakes anyone. A closed buffer + # counts the range and keeps nothing: continuity is still checked + # above, and nobody is left to read what it held. + if not records or self._closed: + return + # Offsets are dense inside a delivered range and the range arrives in + # order, so counting from its start is the position rather than an + # estimate of it. The per-record field is not on the activation, and a + # reader that has to resume elsewhere needs a position it can name. + for index, record in enumerate(records): + # Copied so the buffer outlives the activation that carried it. + kept = temporalio.api.stream.v1.StreamRecord() + kept.CopyFrom(record) + self._records.append( + temporalio.workflow._DeliveredStreamRecord( + record=kept, offset=from_offset + index + ) + ) + waiters, self._waiters = self._waiters, [] + for waiter in waiters: + if not waiter.done(): + waiter.set_result(None) + + def take(self) -> list[temporalio.workflow._DeliveredStreamRecord]: + taken, self._records = self._records, [] + return taken + + def put_back( + self, records: Sequence[temporalio.workflow._DeliveredStreamRecord] + ) -> None: + """Return an unread tail to the front of the buffer.""" + self._records[:0] = records + + def wait_future(self) -> asyncio.Future: + loop = asyncio.get_event_loop() + fut = loop.create_future() + self._waiters.append(fut) + return fut + + def __len__(self) -> int: + return len(self._records) + + class _WorkflowInstanceImpl( # type: ignore[reportImplicitAbstractClass] WorkflowInstance, temporalio.workflow._Runtime, asyncio.AbstractEventLoop ): @@ -298,12 +502,63 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: ) self._extern_functions = det.extern_functions + self._external_stream_runtime = det.external_stream_runtime + self._external_streams_configured = det.external_streams_configured + # Installed on this object, before the Workflow's constructor runs, so a + # ``@workflow.init`` constructor can publish and subscribe. Per-Run state + # must share the Run's lifetime, which this object has and a module global + # would not. + if ( + self._external_stream_runtime is not None + and self._external_streams_configured + ): + from temporalio.contrib.external_workflow_streams._api import ( + _install_runtime, + ) + + _install_runtime(self, self._external_stream_runtime) + self._pending_replay_finish: Any = None + """A marker replay whose last segment the activation's own drain serves. + + Set by ``_apply_replay_external_streams`` and consumed by + ``_finish_replay_external_streams``, which the activation calls once that + drain has run. Held here rather than on the runtime because what it + defers is a step of the *activation*, not of the stream runtime. + """ + self._pending_output_replay_finish = False self._disable_eager_activity_execution = det.disable_eager_activity_execution self._worker_level_failure_exception_types = ( det.worker_level_failure_exception_types ) self._patch_activation_callback = det.patch_activation_callback + self._stream_provider = det.stream_provider + self._streams: temporalio.workflow._streams._WorkflowStreams | None = None + # Keyed by channel name; one subscription per channel per run + self._channel_subscriptions: dict[ + str, temporalio.workflow.ChannelSubscription + ] = {} self._default_workflow_logic_flags = det.default_workflow_logic_flags + self._subscribed_channels: set[str] = set() + #: The channels linked to this workflow that the run listens on. No + #: command: the owner is the listener by construction, so the set only + #: tells a notification carrying ``linked_to`` from one nobody asked for. + self._linked_channels: set[str] = set() + self._linked_channel_subscriptions: dict[ + str, temporalio.workflow.ChannelSubscription + ] = {} + #: The gate's answers for this run, taken on the first stream + #: subscription and kept until continue-as-new; see + #: ``subscribe_stream_channel``. + self._stream_channels_enabled: bool | None = None + self._stream_channels_linked = False + #: Whether a stream reader has been opened on a server with channels. + #: From then on every completion reports the channels the run + #: listens on, the empty set included, and Core keeps the server's + #: subscriptions in step with the report. + self._stream_channels_reporting = False + #: The last report, so a notification on a channel the readers + #: listen on is told from one nobody asked for. + self._stream_channels: list[str] = [] self._primary_task: asyncio.Task[None] | None = None self._cancel_primary_task_pending = False self._time_ns = 0 @@ -401,6 +656,16 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: str, list[temporalio.bridge.proto.workflow_activation.SignalWorkflow] ] = {} + # Stream ranges delivered to this workflow, keyed by stream id. Ranges + # arrive whether or not anything is reading yet, because the server has + # already recorded them as consumed and will not send them again. + self._stream_buffers: dict[str, _StreamBuffer] = {} + # Records this task's publishes append, by stream, until the task + # completes and they become commands. + self._stream_appends: dict[ + str, list[temporalio.api.stream.v1.StreamRecord] + ] = {} + # When we evict, we have to mark the workflow as deleting so we don't # add any commands and we swallow exceptions on tear down self._deleting = False @@ -461,6 +726,9 @@ def activate( temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() ) self._current_completion.successful.SetInParent() + # A failed task's publishes never became commands, so nothing carries + # over into this one. + self._stream_appends = {} self._current_activation_error: Exception | None = None self._deployment_version_for_current_task = ( @@ -474,6 +742,22 @@ def activate( ) self._time_ns = act.timestamp.ToNanoseconds() self._is_replaying = act.is_replaying + if self._external_stream_runtime is not None and not any( + job.HasField("remove_from_cache") for job in act.jobs + ): + # Re-arms the per-activation delivery budget. It has to be reset here + # rather than anywhere later: a producer that keeps a subscription's + # buffer non-empty would otherwise keep the iterator fed for the + # whole of this call, and this call runs on a thread-pool executor + # under a 2-second deadlock timeout that every retry would hit again. + # + # A cache-removal activation is not a Workflow Task. In particular, + # it must still be able to tear down a runtime whose output staging + # failed: treating the eviction's (empty) history floor as a new task + # would reject the still-uncommitted batch and poison eviction. + self._external_stream_runtime.begin_activation( + getattr(act, "history_floor_event_id", None) + ) self._current_thread_id = threading.get_ident() self._current_internal_flags = act.available_internal_flags self._single_batch_activation = self._workflow_logic_flag_enabled( @@ -487,11 +771,25 @@ def activate( job_sets: list[ list[temporalio.bridge.proto.workflow_activation.WorkflowActivationJob] ] = [[], [], [], []] + # An external stream replay marker is what this Workflow Task recorded, + # so it has to be installed before any of the task's code runs and + # publishes, or the install wipes those publishes and the manifest + # check fails them. It is applied right before the first drain of the + # activation, after every job that drain will act on. + replay_jobs: list[ + temporalio.bridge.proto.workflow_activation.WorkflowActivationJob + ] = [] for job in act.jobs: if job.HasField("notify_has_patch"): job_sets[0].append(job) elif job.HasField("signal_workflow") or job.HasField("do_update"): job_sets[1].append(job) + elif job.HasField("notifications_received"): + # Ordered with the Signals, where Core puts them: what a + # task was woken for is known before anything it resolves. + job_sets[1].append(job) + elif job.HasField("replay_external_streams"): + replay_jobs.append(job) elif not job.HasField("query_workflow"): if job.HasField("initialize_workflow"): start_job = job.initialize_workflow @@ -508,30 +806,38 @@ def activate( if start_job: self._workflow_input = self._make_workflow_input(start_job) - if self._single_batch_activation: - # Applying every job before giving workflow tasks a chance to - # run prevents their order in the activation from hiding state - # that arrived in the same workflow task. - for job_set in job_sets: - for job in job_set: - # Let errors bubble out of these to the caller to fail the task - self._apply(job) - if any(job_sets): - self._run_once(check_conditions=bool(job_sets[1] or job_sets[2])) + try: + self._apply_activation_jobs(job_sets, replay_jobs) + except BaseException: + # An error is already on its way out, so the replay is *abandoned* + # rather than closed. Closing runs `verify_replay_consumed`, which + # raises whenever a recorded delivery is still armed -- which it is, + # since the drain that would have taken it is the one that just + # failed -- and an exception raised while another is propagating + # **replaces** it. A user's failure, or an unrelated nondeterminism + # error, would reach the completion path as a nondeterminism error + # blaming a `subscribe()` call nobody touched: the wrong diagnosis, + # and the wrong classification with it, since a workflow-failing + # error would be retried as a task failure instead. + # + # Replay mode is still left, because a Run stuck in it has every + # later drain return nothing at all. Only the checks and the + # cursor move are skipped -- and the reposition must be skipped + # here in any case: an activation that failed committed nothing. + self._abandon_replay_external_streams() + raise else: - # Preserve the legacy scheduling order for histories which do - # not contain the single-batch workflow logic flag. - for index, job_set in enumerate(job_sets): - if not job_set: - continue - for job in job_set: - # Let errors bubble out of these to the caller to fail the task - self._apply(job) - - # Run one iteration of the loop. We do not allow conditions to - # be checked in patch jobs (first index) or query jobs (last - # index). - self._run_once(check_conditions=index == 1 or index == 2) + # A replay marker's last recorded segment is drained by the + # activation's *own* drain above, not by the replay driver, so + # what closes the replay -- the consumed check, the cursor + # reposition, and leaving replay mode -- can only happen here. + self._finish_replay_external_streams() + + # Detected *after* the drain, not during it: `_run_once` already + # drains `self._ready` until empty, so "no coroutine is runnable" is + # its post-condition and quiescence is a registry check rather than + # an event-loop change. + self._emit_external_stream_commands() except Exception as err: # We want some errors during activation, like those that can happen # during payload conversion, to be able to fail the workflow not the @@ -552,6 +858,9 @@ def activate( ) activation_err = None + if activation_err is None and not self._deleting: + self._flush_stream_appends() + # Apply versioning behavior if one was established if self._versioning_behavior: self._current_completion.successful.versioning_behavior = ( @@ -596,26 +905,83 @@ def activate( ) self._current_completion.failed.failure.application_failure_info.SetInParent() - def is_completion( - command: temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.WorkflowCommand, - ): - return ( - command.HasField("complete_workflow_execution") - or command.HasField("continue_as_new_workflow_execution") - or command.HasField("fail_workflow_execution") - or command.HasField("cancel_workflow_execution") + if any( + map( + _is_workflow_terminal_command, + self._current_completion.successful.commands, ) - - if any(map(is_completion, self._current_completion.successful.commands)): + ): self._warn_if_unfinished_handlers() return self._current_completion + def _apply_activation_jobs( + self, + job_sets: list[ + list[temporalio.bridge.proto.workflow_activation.WorkflowActivationJob] + ], + replay_jobs: list[ + temporalio.bridge.proto.workflow_activation.WorkflowActivationJob + ], + ) -> None: + """Apply one activation's jobs and run the drains they earn. + + An external stream replay marker is installed in front of the first + drain that can publish. A drain that publishes ahead of the install has + its records wiped by it and fails the manifest check, and an install + that earns a drain of its own makes the activation run one more drain + than the task the marker was written for. + """ + if self._single_batch_activation: + # Applying every job before giving workflow tasks a chance to + # run prevents their order in the activation from hiding state + # that arrived in the same workflow task. + for job_set in job_sets: + for job in job_set: + # Let errors bubble out of these to the caller to fail the task + self._apply(job) + for job in replay_jobs: + self._apply(job) + if any(job_sets) or replay_jobs: + self._run_once(check_conditions=bool(job_sets[1] or job_sets[2])) + return + + # Preserve the legacy scheduling order for histories which do + # not contain the single-batch workflow logic flag. + replay_pending = bool(replay_jobs) + # When nothing at index 1 or above will drain, the patch set's drain is + # the first one that can publish, so the install joins that set instead + # of adding a drain behind it. + first_draining_index = 1 if any(job_sets[1:]) else 0 + for index, job_set in enumerate(job_sets): + if not job_set: + continue + for job in job_set: + # Let errors bubble out of these to the caller to fail the task + self._apply(job) + if replay_pending and index >= first_draining_index: + for job in replay_jobs: + self._apply(job) + replay_pending = False + + # Run one iteration of the loop. We do not allow conditions to + # be checked in patch jobs (first index) or query jobs (last + # index). + self._run_once(check_conditions=index == 1 or index == 2) + if replay_pending: + # No job set drained, so the marker's drain is this one, under the + # conditions rule the single-batch branch gives the same activation. + for job in replay_jobs: + self._apply(job) + self._run_once(check_conditions=False) + def _apply( self, job: temporalio.bridge.proto.workflow_activation.WorkflowActivationJob ) -> None: if job.HasField("cancel_workflow"): self._apply_cancel_workflow(job.cancel_workflow) + elif job.HasField("deliver_stream_records"): + self._apply_deliver_stream_records(job.deliver_stream_records) elif job.HasField("do_update"): self._apply_do_update(job.do_update) elif job.HasField("fire_timer"): @@ -624,10 +990,16 @@ def _apply( self._apply_query_workflow(job.query_workflow) elif job.HasField("notify_has_patch"): self._apply_notify_has_patch(job.notify_has_patch) + elif job.HasField("notifications_received"): + self._apply_notifications_received(job.notifications_received) elif job.HasField("remove_from_cache"): self._apply_remove_from_cache(job.remove_from_cache) elif job.HasField("resolve_activity"): self._apply_resolve_activity(job.resolve_activity) + elif job.HasField("resolve_external_stream_waits"): + self._apply_resolve_external_stream_waits(job.resolve_external_stream_waits) + elif job.HasField("replay_external_streams"): + self._apply_replay_external_streams(job.replay_external_streams) elif job.HasField("resolve_child_workflow_execution"): self._apply_resolve_child_workflow_execution( job.resolve_child_workflow_execution @@ -807,6 +1179,412 @@ def _apply_fire_timer( if handle: self._ready.append(handle) + def _apply_resolve_external_stream_waits( + self, + job: temporalio.bridge.proto.workflow_activation.ResolveExternalStreamWaits, + ) -> None: + """Resumes every wait that has something buffered. + + **Performs no I/O.** Everything this touches is already in memory: the + manager reported readiness only once a record was buffered, so the drain + that follows is bounded and cannot block. Reading the backend here would + put a multi-second transaction inside a synchronous activation running + under a 2-second deadlock timeout. + + The job's ``ready_hints`` are hints, not an exhaustive availability + claim, so *every* active wait is resolved and left to find out for + itself whether anything arrived. Resolving only the hinted ones would + strand a record whose readiness notification was coalesced away. + """ + del job # The hints are deliberately unused; see above. + if self._external_stream_runtime is not None: + self._external_stream_runtime.resolve_all_pending() + + def _apply_notifications_received( + self, + job: temporalio.bridge.proto.workflow_activation.NotificationsReceived, + ) -> None: + """Hands the notifications this Workflow Task was woken with to their takers. + + A notification names the channel it came over and, for a channel + linked to this workflow, the owner in ``linked_to``; a public + subscription of the matching kind gets it in arrival order. The + external-stream runtime only notes the latest position per channel, + whichever kind carried it: Core resolves the parked waits itself when + it hands over this job, live and on replay alike, and the resolve job + follows this one in the same activation, so resuming here as well + would wake every wait twice for one task. + """ + for proto in job.notifications: + if proto.HasField("linked_to"): + subscriptions = self._linked_channel_subscriptions + listened: Collection[str] = self._linked_channels + else: + subscriptions = self._channel_subscriptions + listened = self._subscribed_channels.union(self._stream_channels) + subscription = subscriptions.get(proto.channel) + if subscription is not None: + subscription._deliver( + temporalio.workflow.Notification._from_proto(proto) + ) + elif proto.channel not in listened: + # The server fans out to whatever listened at the time, so a + # channel this run never asked for is not the workflow's + # concern. + logger.debug( + "Dropping a notification on channel %r, which this run does " + "not listen on", + proto.channel, + ) + if self._external_stream_runtime is not None: + self._external_stream_runtime.notifications_received(job.notifications) + + def workflow_subscribe_channel( + self, channel: str + ) -> temporalio.workflow.ChannelSubscription: + existing = self._channel_subscriptions.get(channel) + if existing is not None: + return existing + self._issue_channel_subscription(channel) + subscription = temporalio.workflow.ChannelSubscription(channel) + self._channel_subscriptions[channel] = subscription + return subscription + + def workflow_unsubscribe_channel(self, channel: str) -> None: + command = self._add_command() + command.unsubscribe_notification_channel.channel = channel + # Out of the map before the next activation: the server may still hand + # this run a notification it folded onto a task ahead of the command. + self._channel_subscriptions.pop(channel, None) + # Out of the once-per-run set too, so a later subscribe reaches the + # server again instead of being folded into the one it left. + self._subscribed_channels.discard(channel) + + def workflow_linked_channel( + self, channel: str + ) -> temporalio.workflow.ChannelSubscription: + existing = self._linked_channel_subscriptions.get(channel) + if existing is not None: + return existing + self._linked_channels.add(channel) + subscription = temporalio.workflow.ChannelSubscription(channel, linked=True) + self._linked_channel_subscriptions[channel] = subscription + return subscription + + def subscribe_stream_channel(self, address: Any) -> bool: + """Makes this run a listener of an external stream's channel, when it can. + + The external-stream runtime's entry, gated where the public + :py:func:`temporalio.workflow.subscribe_channel` is not: a reader opens + on whatever server the Worker talks to, and a server without channels + fails a Workflow Task that carries a subscription. The Worker asks the + server once and the answer becomes two lang flags, read here on the + run's first stream subscription and kept until continue-as-new, so a + replay takes the path the live run took. + + No command is issued here. The subscribe command is server-bound, so + a completion carrying it cannot be retained, and the task that opens + a reader is the one that should stay open and park. Instead every + completion from now on reports the channels of the open readers in + ``WorkflowStreamChannels``, and Core subscribes the new ones on the + completion that ends the task, after the progress marker, and + unsubscribes the ones whose readers have gone; see + :meth:`_report_stream_channels`. + + With the linked kind a stream owned by this workflow is not reported + at all: its channel lives in this run's state and the run is its + listener by construction. A stream with another owner, or no owner, + listens on the independent channel of its name. + + Returns whether the run listens on the channel after this call. + """ + channel: str = address.channel + if channel in self._linked_channels: + return True + if self._read_only: + return False + if self._stream_channels_enabled is None: + self._stream_channels_enabled = self._workflow_logic_flag_enabled( + _WorkflowLogicFlag.SUBSCRIBE_NOTIFICATION_CHANNELS + ) + if self._stream_channels_enabled: + self._stream_channels_linked = self._workflow_logic_flag_enabled( + _WorkflowLogicFlag.LINKED_NOTIFICATION_CHANNELS + ) + if not self._stream_channels_enabled: + return False + self._stream_channels_reporting = True + if ( + self._stream_channels_linked + and address.workflow_id == self._info.workflow_id + ): + self._linked_channels.add(channel) + return True + + def _stream_channel_name(self, stream_key: Any) -> str | None: + """The channel an open reader's stream is listened on. + + ``None`` for a stream this run is the listener of by construction. + """ + from temporalio.contrib.external_workflow_streams._wake import channel_for + + address = channel_for(stream_key) + if ( + self._stream_channels_linked + and address.workflow_id == self._info.workflow_id + ): + return None + return address.channel + + def _report_stream_channels(self, runtime: Any) -> None: + """Puts the complete set of channels the open readers listen on in the + completion. + + Emitted on every completion once a reader has been opened on a server + with channels, the empty set included: Core treats an absent report as + unchanged, and only the latest report counts, so a reader opened and + closed inside one task is never subscribed. The order is the readers' + ``wait_id`` order with repeats dropped, which replay reproduces, and + Core issues the subscribes in it. + """ + channels: list[str] = [] + for stream_key in runtime.listened_stream_keys(): + channel = self._stream_channel_name(stream_key) + if channel is not None and channel not in channels: + channels.append(channel) + self._stream_channels = channels + self._add_command().workflow_stream_channels.channels.extend(channels) + + def _issue_channel_subscription(self, channel: str) -> None: + """Emits the subscribe command once per channel per run. + + A second command for one channel would only record a second event, so + the public subscription and the stream runtime share this set. + """ + if channel in self._subscribed_channels: + return + self._subscribed_channels.add(channel) + self._add_command().subscribe_notification_channel.channel = channel + + def _apply_replay_external_streams( + self, + job: temporalio.bridge.proto.workflow_activation.ReplayExternalStreams, + ) -> None: + """Delivers a recorded marker's observations, from memory only. + + The recorded ranges were read and validated before this activation was + handed to the Workflow thread, so this performs no I/O at all. + + The live run's *k* activations become *k* drains inside this one + activation. Segments are walked in order with one event-loop drain each, + rather than delivered all at once, because reproducing the record order + while changing how many drains occurred would make ``wait_condition`` + predicates fire a different number of times than they did live + (ADR-018). + + *k*, not *k + 1*: the activation runs a drain of its own for the job set + this job arrived in, so only the first *k - 1* segments are drained here + and the last is left to that one. Closing the replay therefore also + moves to after it, in :meth:`_finish_replay_external_streams`. + + This is safe with respect to Workflow time: every segment of a marker + belongs to one Workflow Task, so ``workflow.now()`` is constant across + them in both directions. + """ + runtime = self._external_stream_runtime + if runtime is None: + raise RuntimeError( + "received an external stream replay job without the per-Run " + "validation runtime" + ) + has_field = getattr(job, "HasField", None) + try: + has_output = bool(has_field and has_field("output")) + except ValueError: + # Compatible with a generated binding from before the additive + # output field (and with input-only replay driver test doubles). + has_output = False + output = cast(Any, job).output if has_output else None + output_segments = output.segments if output is not None else () + if has_output: + runtime.begin_output_replay(output) + self._pending_output_replay_finish = True + plan = runtime.take_replay_plan() + if ( + has_output + and plan is not None + and len(plan.segments) != len(output_segments) + ): + raise temporalio.workflow.NondeterminismError( + "External stream History has incompatible input and output " + "activation schedules. This prerelease marker omitted empty " + "input activations, whose positions cannot be recovered safely." + ) + if plan is None: + if has_output: + # An output-only marker deliberately has no input ReplayPlan. + # Its segment list is still the shared activation schedule: run + # the first k - 1 drains here and leave the last to activate()'s + # ordinary trailing drain, exactly as the input-driven path + # below does. Treating all output-only markers as one segment + # collapsed retained multi-activation Workflow Tasks on replay. + for _segment in output_segments[:-1]: + runtime.begin_output_replay_segment() + runtime.resolve_all_pending() + self._run_once(check_conditions=True) + if output_segments: + runtime.begin_output_replay_segment() + runtime.resolve_all_pending() + return + + # Nothing was prepared -- the input replay job reached the Workflow + # thread without its ranges being read, which would mean delivering + # from a buffer that live watching filled. Better to deliver nothing + # than to deliver something replay did not record. + runtime.resolve_all_pending() + return + + deferred = False + closed = False + try: + # The bindings first, and before any delivery. A recorded run is + # joined to a subscription by `wait_id`, which is an integer and + # therefore says nothing about what that wait was subscribed to; + # without checking the binding, wait 1 moved from one stream to + # another delivers the first stream's recorded bytes through the + # second's subscription rather than failing as nondeterminism. + runtime.begin_replay(plan.annotation.header.streams) + # Every segment but the **last** is drained here. The last one is + # armed and left to the drain the activation runs for the job set + # this job arrived in, because that drain happens whatever this + # method does: a driver that drained all *k* segments itself + # produced *k + 1* drains for the *k* the marker records, and + # ADR-018 requires exactly *k*. Leaving the last one to the + # activation is also what the live run did -- there each + # activation's single trailing drain served the records that + # activation had just been handed. + for segment in plan.segments[:-1]: + if has_output: + runtime.begin_output_replay_segment() + runtime.begin_replay_segment(list(segment.deliveries)) + runtime.resolve_all_pending() + self._run_once(check_conditions=True) + # Reached only once every segment but the last has been drained. + if plan.segments: + if has_output: + runtime.begin_output_replay_segment() + runtime.begin_replay_segment(list(plan.segments[-1].deliveries)) + runtime.resolve_all_pending() + # Closing the replay here would end replay mode before the drain + # that delivers this final segment, so it is handed to + # `_finish_replay_external_streams` -- which the activation calls + # once that drain has run, and which is what leaves replay mode + # from then on. + self._pending_replay_finish = plan + deferred = True + else: + if has_output: + runtime.begin_output_replay_segment() + # A marker that recorded no activation of its own -- a Workflow + # Task that bound a wait and blocked with the stream never + # delivering. **Nothing is deferred, and that is not asymmetry + # for its own sake.** With no recorded segment for the + # activation's drain to serve, that drain is a *live* one: + # records that arrived while this Run was evicted are already in + # the buffer and it would hand them over. Repositioning after it + # would then retract exactly what it had just delivered -- cursor + # back to the marker's boundary, buffer cleared -- and the + # watcher would re-read and re-deliver records Workflow code + # already had. Closing first retracts the buffer *before* the + # drain, so the drain finds nothing and the watcher re-reads from + # the boundary the marker committed. The records are not lost; + # they arrive on the activation the re-read announces. + self._pending_replay_finish = plan + closed = True + if has_output: + # Input has no recorded drain and must reposition before the + # activation's live drain. Output, however, is produced by + # that drain and is validated after it. + self._pending_output_replay_finish = False + self._finish_replay_external_streams() + self._pending_output_replay_finish = True + else: + self._finish_replay_external_streams() + finally: + if not deferred and not closed: + # The walk raised part-way. Leaving replay mode set would make + # every later drain on this Run return nothing at all, turning + # one marker's failure into a Workflow that silently never + # receives again -- and the checks the close performs must not + # run here, where they would replace the error that got us here + # with one of their own. + self._pending_replay_finish = None + self._pending_output_replay_finish = False + if has_output: + runtime.abandon_output_replay() + runtime.end_replay() + + def _abandon_replay_external_streams(self) -> None: + """Leaves replay mode without running any of the checks a close runs. + + For the path where an activation is already failing. See the caller. + """ + self._pending_replay_finish = None + self._pending_output_replay_finish = False + runtime = self._external_stream_runtime + if runtime is not None: + runtime.abandon_output_replay() + runtime.end_replay() + + def _finish_replay_external_streams(self) -> None: + """Closes the replay whose last segment the activation's drain served. + + Called from the activation's ``finally`` so that a drain which raised + still leaves replay mode, for the reason + :meth:`_apply_replay_external_streams` gives. + """ + plan = self._pending_replay_finish + output_pending = getattr(self, "_pending_output_replay_finish", False) + if plan is None and not output_pending: + return + self._pending_replay_finish = None + self._pending_output_replay_finish = False + runtime = self._external_stream_runtime + if runtime is None: + return + try: + # Nothing recorded may be left over. Each of these records was handed + # to Workflow code in the activation its run was recorded in, so a + # replay that ends holding one has run code that consumes less than + # History says was consumed -- what a removed `subscribe()` call + # looks like from here. + if plan is not None: + runtime.verify_replay_consumed() + # The manager knew nothing about this marker while replay was + # running: its watcher has been reading the very same records from + # the subscription's start cursor into the live buffer. The next live + # drain would hand them over again -- observed end-to-end as one + # marker's record delivered twice. Moving the cursors to what the + # marker committed is what makes live delivery resume *after* those + # records rather than in front of them. + # + # After every segment's drain, not inside the walk: a replay that + # raised part-way committed nothing, and advancing a committed cursor + # for records this Workflow may never have received would lose them + # outright. + if plan is not None: + runtime.reposition_after_replay(plan.committed_boundaries) + if output_pending: + runtime.verify_output_replay() + finally: + if output_pending: + # Successful verification already cleared the manifest, so + # this is a no-op on success and cleanup of partial output on + # every validation/input-reposition failure path. + runtime.abandon_output_replay() + if plan is not None: + runtime.end_replay() + def _apply_query_workflow( self, job: temporalio.bridge.proto.workflow_activation.QueryWorkflow ) -> None: @@ -819,7 +1597,8 @@ async def run_query() -> None: if not defn: known_queries = sorted([k for k in self._queries.keys() if k]) raise RuntimeError( - f"Query handler for '{job.query_type}' expected but not found, " + f"Query handler for '{job.query_type}' " + f"{QUERY_HANDLER_NOT_FOUND}, " f"known queries: [{' '.join(known_queries)}]" ) @@ -877,6 +1656,12 @@ def _apply_remove_from_cache( # We consider eviction to be under replay so that certain code like # logging that avoids replaying doesn't run during eviction either self._is_replaying = True + # Drop the stream readiness futures before cancelling tasks. Nothing + # will ever resolve them now, and a coroutine cancelled while awaiting + # one would otherwise be woken by the cancellation and then find its + # future still registered. + if self._external_stream_runtime is not None: + self._external_stream_runtime.clear_pending() # Cancel everything for task in self._tasks: task.cancel() @@ -1144,6 +1929,19 @@ def _apply_resolve_signal_external_workflow( else: fut.set_result(None) + def _apply_deliver_stream_records( + self, + job: temporalio.bridge.proto.workflow_activation.DeliverStreamRecords, + ) -> None: + buffer = self._stream_buffers.get(job.stream_id) + if buffer is None: + # Nothing subscribed. The range is already recorded as consumed and + # will not be sent again, so buffering it is the only way a + # subscription made later in the same task still sees it. + buffer = _StreamBuffer(job.stream_id) + self._stream_buffers[job.stream_id] = buffer + buffer.extend(job.records, job.from_offset, job.to_offset) + def _apply_signal_workflow( self, job: temporalio.bridge.proto.workflow_activation.SignalWorkflow ) -> None: @@ -1310,6 +2108,85 @@ def workflow_get_current_deployment_version( def get_info(self) -> temporalio.workflow.Info: return self._info + def workflow_subscribe_stream( + self, + stream_name_or_id: str, + start: temporalio.api.stream.v1.StreamStartPosition, + ) -> None: + # Reissued on every replay, so the buffer has to exist before the first + # range arrives and the command has to be harmless the second time. A + # repeat subscription leaves the server-side cursor where it is. + self._stream_buffers.setdefault( + stream_name_or_id, _StreamBuffer(stream_name_or_id) + ) + command = self._add_command() + command.subscribe_stream.stream_name_or_id = stream_name_or_id + command.subscribe_stream.start_position.CopyFrom(start) + + def workflow_append_stream_records( + self, + stream_name: str, + records: Sequence[temporalio.api.stream.v1.StreamRecord], + ) -> None: + self._assert_not_read_only("append stream records") + if not records: + raise ValueError("append_stream_records needs at least one record") + kept: list[temporalio.api.stream.v1.StreamRecord] = [] + for record in records: + if record.ByteSize() > _MAX_STREAM_RECORD_BYTES: + raise ValueError( + f"a stream record is limited to {_MAX_STREAM_RECORD_BYTES} " + f"bytes, got {record.ByteSize()}" + ) + copy = temporalio.api.stream.v1.StreamRecord() + copy.CopyFrom(record) + # The workflow is the producer here, whatever the caller set. + copy.producer_id = "" + kept.append(copy) + # Held until the task completes, so a task's publishes on one stream + # become one command and one History event however many there were. + self._stream_appends.setdefault(stream_name, []).extend(kept) + + def _flush_stream_appends(self) -> None: + appends, self._stream_appends = self._stream_appends, {} + if not appends: + return + commands = self._current_completion.successful.commands + # Ahead of any command that ends the run, because the server accepts + # nothing after one of those. + insert_at = len(commands) + for index, command in enumerate(commands): + if _is_completion_command(command): + insert_at = index + break + for stream_name, records in appends.items(): + for batch in _stream_batches(records): + command = temporalio.bridge.proto.workflow_commands.WorkflowCommand() + command.append_stream_records.stream_name = stream_name + command.append_stream_records.records.extend(batch) + commands.insert(insert_at, command) + insert_at += 1 + + def workflow_close_stream_records(self, stream_id: str) -> None: + self._stream_buffers.setdefault(stream_id, _StreamBuffer(stream_id)).close() + + async def workflow_read_stream_records( + self, stream_id: str, max_records: int + ) -> list[temporalio.workflow._DeliveredStreamRecord]: + # Ranges arrive on Workflow Tasks, and a query activation carries none, + # so without this the read waits on a future nothing can resolve and the + # query times out with nothing to say why. + self._assert_not_read_only("read stream") + buffer = self._stream_buffers.setdefault(stream_id, _StreamBuffer(stream_id)) + while not len(buffer) and not buffer.closed: + await buffer.wait_future() + taken = buffer.take() + if max_records and len(taken) > max_records: + # Put the tail back rather than dropping it: nothing will resend it. + buffer.put_back(taken[max_records:]) + taken = taken[:max_records] + return taken + def workflow_get_current_history_length(self) -> int: return self._current_history_length @@ -1358,6 +2235,9 @@ def workflow_instance(self) -> Any: def workflow_is_continue_as_new_suggested(self) -> bool: return self._continue_as_new_suggested + def workflow_is_evicting(self) -> bool: + return self._deleting + def workflow_is_target_worker_deployment_version_changed(self) -> bool: return self._target_worker_deployment_version_changed @@ -1810,6 +2690,20 @@ async def workflow_start_nexus_operation( ) ) + def workflow_streams(self) -> temporalio.workflow._streams._WorkflowStreams: + if self._streams is None: + if self._stream_provider is None: + raise RuntimeError( + "no stream provider is configured on this worker; pass one with " + "Worker(plugins=[provider]) or stream_provider=" + ) + # The workflow half is made per instance, so whatever it keeps + # dies with the instance the way handlers do. + self._streams = temporalio.workflow._streams._WorkflowStreams( + self._stream_provider.workflow_provider() + ) + return self._streams + def workflow_time_ns(self) -> int: return self._time_ns @@ -2310,6 +3204,295 @@ def _add_command(self) -> temporalio.bridge.proto.workflow_commands.WorkflowComm self._assert_not_read_only("add command") return self._current_completion.successful.commands.add() + def _attach_external_stream_continuation(self, command: Any) -> None: + """Puts each subscription's position on the Continue-As-New command. + + The successor Run restores it from its own ``WorkflowExecutionStarted`` + rather than reading the backend, so replay sees the boundary the Run + actually started from instead of wherever the stream has since got to + (ADR-022). + + Written when the command is created and again once the activation has + quiesced -- see :meth:`_refresh_external_stream_continuations`, which is + what makes the second write the one that counts. + + Nothing is attached when the Run held no subscriptions -- an empty header + on every Continue-As-New in every Workflow would be pure overhead. + """ + runtime = self._external_stream_runtime + if runtime is None: + return + continuation = runtime.continuation() + if continuation.cursors: + from temporalio.contrib.external_workflow_streams._continuation import ( + CONTINUATION_HEADER, + write_continuation_header, + ) + + command.headers[CONTINUATION_HEADER].CopyFrom( + write_continuation_header(continuation) + ) + output_continuation = runtime.output_continuation() + if output_continuation is not None: + from temporalio.contrib.external_workflow_streams._output_continuation import ( + OUTPUT_CONTINUATION_HEADER, + write_output_continuation_header, + ) + + command.headers[OUTPUT_CONTINUATION_HEADER].CopyFrom( + write_output_continuation_header(output_continuation) + ) + + def _refresh_external_stream_continuations(self) -> None: + """Re-takes the continuation snapshot now the activation has quiesced. + + Creating the Continue-As-New command does not end the activation. + :meth:`_run_once` drains ``self._ready`` until it is empty and then + re-checks conditions, so a stream consumer scheduled -- or unblocked -- + before the terminal command still gets its turn afterwards, and every + record it takes calls ``record_consumption`` after the header was + serialized. Those consumptions do reach the predecessor's final stream + marker, which is closed on the way out; a header left at the earlier + boundary would therefore describe a different boundary than History + does, and the successor would be handed the record a second time. + + No Workflow code runs from here on, so this is the first point at which + the consumption boundary is stable for the activation. + + Every Continue-As-New command is refreshed rather than the last one, + because two top-level functions can each raise one and only Core decides + which terminal survives. + + **Not gated behind an internal flag.** Core matches a Continue-As-New + command to its ``WorkflowExecutionContinuedAsNew`` event by command type + alone and never compares the command's headers against the recorded ones + (``continue_as_new_workflow_state_machine.rs``), so a replay that + regenerates the header at the later boundary cannot disagree with a + History written at the earlier one. What the successor resumes from is + the copy in its own ``WorkflowExecutionStarted``, which is durable and + which replaying the predecessor does not rewrite -- so a chain already in + flight keeps the boundary it started with. + """ + for command in self._current_completion.successful.commands: + if command.HasField("continue_as_new_workflow_execution"): + self._attach_external_stream_continuation( + command.continue_as_new_workflow_execution + ) + + def _emit_external_stream_commands(self) -> None: + """Answers the two independent questions every activation return poses. + + 1. **Did replay-visible stream state change?** If so a + ``WorkflowStreamProgress`` carries the observation delta, on *every* + completion path. This is what commits the cursor boundary, and it is + not conditional on why the Workflow Task ended: if a consumed record + influenced a command that lands in History while the consumption + itself is never marked, replay re-delivers that record while the + command it produced is already durable. + 2. **Should the Workflow Task be retained?** If so a + ``WorkflowStreamQuiescent`` asks Core to hold it open. It carries no + annotation data; the two questions are genuinely separate. + + The progress command is emitted **first**, because it must precede every + command whose value could depend on the consumed data -- on replay that + is what guarantees a record is validated before the command derived from + it is matched. + + The two are separate in *time* as well, which is why replay answers only + the second: the annotation is already in History, but the wait set the + quiescent command registers is per-Worker runtime state that a replayed + Run has to rebuild. + + Quiescence is also what a Continue-As-New header has been waiting for, so + :meth:`_refresh_external_stream_continuations` runs from here too -- the + boundary it records and the boundary the delta below commits have to be + the same one. + """ + runtime = self._external_stream_runtime + if runtime is None or self._deleting: + return + + # A legacy query is answered on a task of its own, and Core refuses any + # other command beside that answer. The activation ran no Workflow code, + # so the wait set it would report is the one the retained task already + # holds, and there is no observation delta to commit. + if any( + command.HasField("respond_to_query") + and command.respond_to_query.query_id == _LEGACY_QUERY_ID + for command in self._current_completion.successful.commands + ): + return + + # First, because the boundary a Continue-As-New header has to carry is + # the one this activation is about to commit, and it only stops moving + # here. + self._refresh_external_stream_continuations() + + # Read before anything is added below, so "what the Workflow itself + # produced this activation" stays answerable. A terminal command also + # determines whether buffered readiness can ever be useful again. + commands = self._current_completion.successful.commands + produced_commands = len(commands) > 0 + terminal = any(map(_is_workflow_terminal_command, commands)) + + # Records still buffered when a non-terminal activation ends have no readiness + # notification coming: the watcher moved its prefetch cursor past them + # when it buffered them, and it only reports again after a *new* non-empty + # read. Re-reporting is what brings the next activation in, and it is the + # only thing that can. + # + # **Asked of every completion, not only of one a delivery budget + # stopped.** The budget is the obvious way to end an activation with a + # full buffer and it is not the only one. Readiness accepted while an + # activation was open is kept by Core only if the quiescent snapshot that + # follows reports the *same* wait generation: same generation means the + # Workflow never saw the record, a bumped one means it drained and + # re-blocked. So a record that arrives after this activation's last drain + # is accepted at generation G, the wait re-blocks to G+1 on the way out, + # and the snapshot legitimately drops a readiness that now refers to a + # resolved block. Nothing else announces it, and the Workflow waits + # forever on data it is already holding -- until an unrelated later event. + # Gating this on the budget left that hole open, and it is a race, so it + # showed up as an occasional stall rather than a reproducible one. + # + # Cheap where it is not needed: the manager skips every subscription whose + # buffer is empty, which is the ordinary case, and a redundant report is + # answered `Accepted` or `Stale` and retried against the current + # generation. + # + # A terminal completion is the exception. This Run cannot consume a + # later activation, and re-arming after its terminal command reports the + # buffer into the Workflow's closing window. Core answers + # `NoOpenWorkflowTask`, which turns that report into a follow-up Signal + # that can race the terminal report and make the same task replay. + # + # Before every early return below: the waits involved are marked blocked, + # so they are in the quiescent snapshot, and a snapshot is what lets Core + # start the idle timer and eventually park. Parking a Workflow Task whose + # records are already in the local buffer would be wrong, and this is what + # makes Core resolve instead of park. + if not terminal: + runtime.rearm_readiness() + + # The snapshot goes out whether or not this completion retains the task. + # A completion carrying a timer, activity, child workflow, or signal must + # be reported so the server can act on it, so it cannot ask for + # retention -- but the subscriptions stay registered, and the wake Signal + # covers the window that leaves. + # + # "Stay registered" is the load-bearing phrase, and it is Core that + # decides how to read this: it registers the waits and, when the + # completion cannot be retained for, arms no timer. Withholding the + # snapshot here instead is what left a Workflow whose very first block + # rode such a completion registered nowhere, unresumable by any wake -- + # a deadlock in ordinary user code. + snapshot = runtime.quiescent_snapshot() + + retaining = bool(snapshot) and not produced_commands + + if self._is_replaying: + # Every marker for a replayed Workflow Task is already in History, + # so there is nothing here for Core to write. Re-deriving the + # annotation is not merely redundant: the command would be matched + # against the very event it was read from, and Core would see a + # marker arriving for a machine already resolved from lookahead. + # + # What lang accumulated while replaying is dropped rather than + # carried, because otherwise it rides the *next* completion -- which + # is live, and would write a second marker for observations the + # first one already recorded. A registration alone is enough to + # trigger that, so it happens even for a stream that never delivered + # anything. + # + # The *quiescent* command below is emitted all the same, because it + # is not about the annotation at all: it is the only thing that + # registers a wait set with Core, and Core's wait set is runtime + # state that a replayed Run rebuilds from nothing. A Run handed over + # and replayed that reported no snapshot would finish replay with an + # empty wait set -- every later readiness answered as though the Run + # had no subscriptions, every wake Signal marking nothing ready, and + # every Workflow Task it created completing with no activation in + # it. That Run is unresumable, with its records sitting in the + # stream. + # + # The cost is that Core starts its idle timer here too, and on + # replay it should not run one at all: a replay slower than the idle + # timeout can therefore queue a park handshake between two replay + # activations. That is a rare timing hazard against a certain and + # total failure, so it is the right trade to make from this side -- + # but removing it needs Core either to hold its timers while + # `replaying` or to register the wait set from the marker lookahead + # it already reads. + runtime.start_new_annotation() + else: + parts: list[bytes] = [] + # The delta first. `take_observation_delta` is what *closes* this + # activation's segment, so a rollover decision read before it is a + # decision about the annotation as it stood one activation ago: the + # segment that crossed the high-water mark went out with + # `request_rollover = false`, and the following activation was then + # free to add another frame and overflow before Core had ever been + # asked to roll over. Reading it after also picks up the runtime + # having stopped delivering because the annotation could afford no + # more, which is decided while this same segment is being closed. + delta = runtime.take_observation_delta() + if delta is not None: + parts.append(delta) + # And before the terminal is added: closing the annotation starts a + # fresh one, whose accumulator is nowhere near the high-water mark. + request_rollover = runtime.request_rollover + # Both of these end the Workflow Task. Retention is refused outright + # for the first; for the second Core takes the rollover as + # authoritative *over* a retention request, because the annotation + # is the thing that has to stop growing. + if runtime.annotation_started and (not retaining or request_rollover): + # This completion ends the Workflow Task, so Core writes the + # marker for everything accumulated and clears it. Two things + # follow, and neither is optional. + # + # The terminal has to ride *this* delta. Core is + # annotation-blind and never manufactures one, and it asks for + # one only on the boundaries it decides itself -- so on a + # completion Python decided, a marker whose annotation has no + # terminal is what gets written, and that is durable and wrong + # (ADR-008). + # + # And the next annotation has to begin from a fresh header. Core + # accumulates by byte append, so an accumulator that kept the + # header it already emitted would start the *next* marker at + # whatever frame came first -- the observed failure is an + # annotation beginning with a terminal frame, read back as + # "schema version 3". + parts.append(runtime.add_terminal()) + + if parts: + progress = temporalio.bridge.proto.workflow_commands.WorkflowCommand() + progress.workflow_stream_progress.observation_delta = b"".join(parts) + progress.workflow_stream_progress.request_rollover = request_rollover + # Inserted at the front rather than appended: the progress + # command must precede every command whose value could depend on + # the consumed data. + commands.insert(0, progress) + + # Whether or not anything is blocked, and on replay too: the report + # is what Core derives the server's subscriptions from, and a task + # that read a record and then scheduled an activity leaves with no + # snapshot but still with readers to keep listening for. + if self._stream_channels_reporting: + self._report_stream_channels(runtime) + + if not snapshot: + return + + quiescent = self._add_command().workflow_stream_quiescent + quiescent.quiescence_generation = 0 + quiescent.idle_timeout.FromTimedelta(runtime.effective_idle_timeout()) + for wait in snapshot: + entry = quiescent.waits.add() + entry.wait_id = wait.wait_id + entry.generation = wait.generation + entry.immediately_parkable = wait.immediately_parkable + def _workflow_logic_flag_enabled(self, flag: _WorkflowLogicFlag) -> bool: if flag in self._current_internal_flags: return True @@ -3902,6 +5085,7 @@ def _apply_command(self) -> None: v.backoff_start_interval.FromTimedelta(self._input.backoff_start_interval) if self._input.headers: temporalio.common._apply_headers(self._input.headers, v.headers) + self._instance._attach_external_stream_continuation(v) if self._input.retry_policy: self._input.retry_policy.apply_to_proto(v.retry_policy) if memo_payloads: @@ -4162,6 +5346,17 @@ class _WorkflowLogicFlag(IntEnum): RAISE_ON_CANCELLING_COMPLETED_ACTIVITY = 1 PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH = 2 + # Whether the run reports the channels its external-stream readers listen + # on, for Core to subscribe it to. Set by the Worker from one probe of the + # server; a run reads it on its first stream subscription and keeps that + # answer until continue-as-new. Numbered away from the values upstream + # hands out in sequence, so a flag the main SDK adds later cannot read a + # History written with this one. + SUBSCRIBE_NOTIFICATION_CHANNELS = 100 + # Whether a stream this workflow owns listens on its linked channel, which + # needs no subscription, rather than on the independent one. Set with the + # flag above from the same probe and read with it. + LINKED_NOTIFICATION_CHANNELS = 101 # TODO: Enable PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH by default after diff --git a/temporalio/worker/workflow_sandbox/_restrictions.py b/temporalio/worker/workflow_sandbox/_restrictions.py index 3e6e0d6ba6..147c8c7e8e 100644 --- a/temporalio/worker/workflow_sandbox/_restrictions.py +++ b/temporalio/worker/workflow_sandbox/_restrictions.py @@ -464,6 +464,13 @@ def with_child_unrestricted(self, *child_path: str) -> SandboxMatcher: SandboxRestrictions.passthrough_modules_minimum = { "grpc", + # The External Workflow Streams subscription manager. It owns the Worker's + # backend connections, its watcher tasks, and the buffers the Workflow + # thread pops from -- all of which live on the Worker's own asyncio loop. + # Re-importing it inside the sandbox would give the Workflow a second, + # empty manager watching nothing, so only an opaque handle to the real one + # may cross the boundary. + "temporalio.contrib.external_workflow_streams._manager", # Due to some side-effecting calls made on import, these need to be # allowed "pathlib", @@ -779,6 +786,34 @@ def _public_callables(parent: Any, *, exclude: set[str] = set()) -> set[str]: "zipfile": SandboxMatcher( children={"ZipFile": SandboxMatcher(use={"extract", "extractall"})} ), + "temporalio": SandboxMatcher( + children={ + "contrib": SandboxMatcher( + children={ + # External stream provider instances hold connections + # and credentials and live on the Worker, outside the + # sandbox. Workflow code uses the backend configured on + # its Worker; it never constructs or imports one, and a + # provider reached from in here would be a second + # instance that no watcher owns. + "external_workflow_streams": SandboxMatcher( + children={ + "_redis": SandboxMatcher( + match_self=True, + leaf_message=( + "External stream providers may not be " + "imported from Workflow code. Register " + "the provider on the Worker with " + "external_stream_backend=... instead." + ), + ), + }, + access={"RedisStreamBackend"}, + ), + } + ), + } + ), "zoneinfo": SandboxMatcher( children={ "ZoneInfo": SandboxMatcher( diff --git a/temporalio/workflow/__init__.py b/temporalio/workflow/__init__.py index c631fd88e8..98a125dcb0 100644 --- a/temporalio/workflow/__init__.py +++ b/temporalio/workflow/__init__.py @@ -57,7 +57,13 @@ as_completed, wait, ) -from ._context import ( +from ._channels import ( + ChannelSubscription, + Notification, + linked_channel, + subscribe_channel, +) +from ._context import ( # noqa: F401 Info, ParentInfo, RootInfo, @@ -97,6 +103,21 @@ uuid7, wait_condition, ) +from ._context import ( + _append_stream_records as _append_stream_records, +) +from ._context import ( + _close_stream_records as _close_stream_records, +) +from ._context import ( + _DeliveredStreamRecord as _DeliveredStreamRecord, +) +from ._context import ( + _read_stream_records as _read_stream_records, +) +from ._context import ( + _subscribe_stream as _subscribe_stream, +) from ._definition import ( DynamicWorkflowConfig, _Definition, @@ -151,6 +172,12 @@ logger, unsafe, ) +from ._streams import ( + StreamReader, + StreamWriter, + stream_reader, + stream_writer, +) from ._workflow_ops import ( ChildWorkflowCancellationType, ChildWorkflowConfig, @@ -258,6 +285,14 @@ "SandboxImportNotificationPolicy", "logger", "unsafe", + "ChannelSubscription", + "Notification", + "linked_channel", + "subscribe_channel", + "StreamReader", + "StreamWriter", + "stream_reader", + "stream_writer", "ChildWorkflowCancellationType", "ChildWorkflowConfig", "ChildWorkflowHandle", diff --git a/temporalio/workflow/_channels.py b/temporalio/workflow/_channels.py new file mode 100644 index 0000000000..07c3bde7af --- /dev/null +++ b/temporalio/workflow/_channels.py @@ -0,0 +1,252 @@ +"""Notification channels from inside workflow code. + +.. warning:: + This module is experimental and may change in future versions. + +A channel carries notifications, not data. A writer tells the channel's +listeners that a source they consume has moved, and each listener reads the +source itself. A workflow that subscribes gets the notifications the server +folded for it with each Workflow Task. They travel in History, so a replay +sees the same ones at the same points. +""" + +from __future__ import annotations + +import asyncio +from collections import deque +from collections.abc import Mapping +from dataclasses import dataclass, field + +import temporalio.api.common.v1 +import temporalio.api.notification.v1 +import temporalio.common +from temporalio.workflow._context import _Runtime + +__all__ = [ + "ChannelSubscription", + "Notification", + "linked_channel", + "subscribe_channel", +] + + +@dataclass(frozen=True) +class Notification: + """One notification from a channel. + + The server folds notifications per listener while one is pending and no + task has been scheduled for it, keeping the one with the highest counter. + A listener therefore sees where a burst of writes ended, not every write. + """ + + channel: str + """The channel the writer notified.""" + + position: bytes + """Where the source stands after the write, in the writer's terms. + + Opaque to the server and handed over as sent. + """ + + counter: int + """Orders notifications from one channel's writers. Higher is later.""" + + metadata: Mapping[str, temporalio.api.common.v1.Payload] = field( + default_factory=dict + ) + """Details for the listener, such as which topic moved, as payloads. + + A codec has been applied. Convert a value with + :py:meth:`temporalio.converter.PayloadConverter.from_payload` on the + converter :py:func:`temporalio.workflow.payload_converter` returns. + """ + + linked_to: temporalio.common.Execution | None = None + """The execution a linked channel belongs to, and the run that received this. + + ``None`` for a notification from an independent channel. A workflow that + holds both kinds of handle on one name gets a notification on the handle + its kind names: :func:`temporalio.workflow.linked_channel` when set, + :func:`temporalio.workflow.subscribe_channel` otherwise. + """ + + @staticmethod + def _from_proto( + proto: temporalio.api.notification.v1.Notification, + ) -> Notification: + return Notification( + channel=proto.channel, + position=proto.position, + counter=proto.counter, + metadata=dict(proto.metadata.items()), + linked_to=( + temporalio.common.Execution.from_proto(proto.linked_to) + if proto.HasField("linked_to") + else None + ), + ) + + +class ChannelSubscription: + """A workflow's handle on one channel, of either kind. + + Prefer :func:`temporalio.workflow.subscribe_channel` for an independent + channel and :func:`temporalio.workflow.linked_channel` for one linked to + this workflow. The handle is an async iterator over the notifications as + they arrive, and :meth:`receive` takes them one at a time. Notifications + wait in arrival order until taken. Two loops on one handle share its + buffer and interleave. :meth:`unsubscribe` ends an independent + subscription; a linked channel lasts as long as the run. + """ + + def __init__(self, channel: str, *, linked: bool = False) -> None: + """Prefer the two module functions named above.""" + self._channel = channel + self._linked = linked + self._closed = False + self._pending: deque[Notification] = deque() + self._waiters: deque[asyncio.Future[None]] = deque() + + @property + def channel(self) -> str: + """The channel this handle is on.""" + return self._channel + + @property + def linked(self) -> bool: + """Whether the channel is the one linked to this workflow. + + A linked handle gets the notifications that carry + :attr:`Notification.linked_to`; an independent one gets the rest. + """ + return self._linked + + @property + def closed(self) -> bool: + """Whether :meth:`unsubscribe` has ended this subscription. + + A closed handle still hands out the notifications it had queued, then + :meth:`receive` raises and iteration ends. + """ + return self._closed + + def unsubscribe(self) -> None: + """End this workflow's subscription to the channel. + + Issues the unsubscribe command once; a second call changes nothing. + Notifications already queued on this handle can still be read, and + one the server put on a scheduled Workflow Task before the command + landed is dropped on arrival. A later + :func:`temporalio.workflow.subscribe_channel` for the same name opens + a new subscription with a new command. + + Raises: + ValueError: The handle is from + :func:`temporalio.workflow.linked_channel`. A linked channel is + part of the run and has no subscription to end. + """ + if self._linked: + raise ValueError("a linked channel has no subscription") + if self._closed: + return + _Runtime.current().workflow_unsubscribe_channel(self._channel) + self._closed = True + # The waiters wake to find the handle closed with nothing queued. + self._wake() + + async def receive(self) -> Notification: + """The next notification on this channel, waiting for one to arrive. + + The wait is a future the delivery resolves, so it adds no command and + replays the same way. + + Raises: + RuntimeError: The subscription is closed and nothing is queued. + """ + notification = await self._next() + if notification is None: + raise RuntimeError("channel subscription closed") + return notification + + def __aiter__(self) -> ChannelSubscription: + """The subscription is its own iterator.""" + return self + + async def __anext__(self) -> Notification: + """The next notification. The iteration ends once the handle is closed and drained.""" + notification = await self._next() + if notification is None: + raise StopAsyncIteration + return notification + + async def _next(self) -> Notification | None: + while not self._pending: + if self._closed: + return None + waiter: asyncio.Future[None] = asyncio.Future() + self._waiters.append(waiter) + try: + await waiter + finally: + if waiter in self._waiters: + self._waiters.remove(waiter) + return self._pending.popleft() + + def _deliver(self, notification: Notification) -> None: + self._pending.append(notification) + self._wake() + + def _wake(self) -> None: + # Every waiter wakes; the ones that find the buffer empty again wait + # once more. + while self._waiters: + waiter = self._waiters.popleft() + if not waiter.done(): + waiter.set_result(None) + + +def subscribe_channel(channel: str) -> ChannelSubscription: + """Subscribe this workflow to ``channel``. + + The first call for a channel in a run issues a command, so gate a new + channel with :func:`temporalio.workflow.patched` as you would a timer. A + second call for the same channel returns the subscription already open, + and the two share its buffer. From then on each Workflow Task carries the + notifications the server folded for this workflow on the channel, and + they arrive here. A successor run after continue-as-new starts with no + subscriptions. + + Args: + channel: Name of the channel, scoped to the namespace. + + Raises: + ValueError: ``channel`` is empty. + """ + if not channel: + raise ValueError("channel must not be empty") + return _Runtime.current().workflow_subscribe_channel(channel) + + +def linked_channel(channel: str) -> ChannelSubscription: + """Listen on the channel named ``channel`` that is linked to this workflow. + + A linked channel lives in this workflow's own state, so the workflow is + its listener by construction: no command, no event, and no gate needed + for a new name. A writer reaches it by naming this workflow as the + execution, as in :py:meth:`temporalio.client.Client.notify_channel` with + ``workflow_id``, and a successor run after continue-as-new is reached by + the same calls. + A second call for the same name returns the handle already open, and the + two share its buffer. The name does not collide with an independent + channel's: a notification carrying :attr:`Notification.linked_to` comes + here, one without it goes to :func:`subscribe_channel`. + + Args: + channel: Name of the channel, scoped to this workflow. + + Raises: + ValueError: ``channel`` is empty. + """ + if not channel: + raise ValueError("channel must not be empty") + return _Runtime.current().workflow_linked_channel(channel) diff --git a/temporalio/workflow/_context.py b/temporalio/workflow/_context.py index 37928afb99..8b688db198 100644 --- a/temporalio/workflow/_context.py +++ b/temporalio/workflow/_context.py @@ -14,6 +14,7 @@ from nexusrpc import InputT, OutputT import temporalio.api.common.v1 +import temporalio.api.stream.v1 import temporalio.common import temporalio.converter @@ -22,9 +23,11 @@ if TYPE_CHECKING: from ._activities import ActivityCancellationType, ActivityHandle + from ._channels import ChannelSubscription from ._event_groups import EventGroup from ._exceptions import ContinueAsNewVersioningBehavior, VersioningIntent from ._nexus import NexusOperationCancellationType, NexusOperationHandle + from ._streams import _WorkflowStreams from ._workflow_ops import ( ChildWorkflowCancellationType, ChildWorkflowHandle, @@ -321,6 +324,28 @@ def workflow_get_current_deployment_version( self, ) -> temporalio.common.WorkerDeploymentVersion | None: ... + @abstractmethod + def workflow_subscribe_stream( + self, + stream_name_or_id: str, + start: temporalio.api.stream.v1.StreamStartPosition, + ) -> None: ... + + @abstractmethod + def workflow_append_stream_records( + self, + stream_name: str, + records: Sequence[temporalio.api.stream.v1.StreamRecord], + ) -> None: ... + + @abstractmethod + def workflow_close_stream_records(self, stream_id: str) -> None: ... + + @abstractmethod + async def workflow_read_stream_records( + self, stream_id: str, max_records: int + ) -> list[_DeliveredStreamRecord]: ... + @abstractmethod def workflow_get_current_history_length(self) -> int: ... @@ -353,6 +378,16 @@ def workflow_instance(self) -> Any: ... @abstractmethod def workflow_is_continue_as_new_suggested(self) -> bool: ... + @abstractmethod + def workflow_is_evicting(self) -> bool: + """Whether this instance is being dropped from the cache rather than ending. + + Eviction cancels the primary task the way a workflow cancellation + does, so anything that runs on the way out has to be able to tell the + two apart. Instance state must not be touched while this is true. + """ + ... + @abstractmethod def workflow_is_target_worker_deployment_version_changed(self) -> bool: ... @@ -499,6 +534,24 @@ async def workflow_start_nexus_operation( event_groups: Sequence[EventGroup] | None = None, ) -> NexusOperationHandle[OutputT]: ... + @abstractmethod + def workflow_streams(self) -> _WorkflowStreams: ... + + @abstractmethod + def workflow_subscribe_channel(self, channel: str) -> ChannelSubscription: ... + + @abstractmethod + def workflow_unsubscribe_channel(self, channel: str) -> None: + """Record the unsubscribe command for ``channel`` and forget its handle. + + Called once per subscription, by the handle that owns it, so a late + notification for the channel finds no handle and is dropped. + """ + ... + + @abstractmethod + def workflow_linked_channel(self, channel: str) -> ChannelSubscription: ... + @abstractmethod def workflow_time_ns(self) -> int: ... @@ -1013,6 +1066,110 @@ async def sleep( ) +def _subscribe_stream( # type: ignore[reportUnusedFunction] + stream_name_or_id: str, + *, + start: temporalio.api.stream.v1.StreamStartPosition | None = None, +) -> None: + """Subscribe this workflow to a server-side stream. + + From here on its Workflow Tasks carry the ranges it has not consumed yet, + and :func:`_read_stream_records` returns them. Safe to call again: a second + subscription to a stream this run already consumes does not move its + cursor, though it does write one event. Calling it on every replay is + harmless because replay matches the command to the event already recorded. + + Only the name or id and the start position go to the server. The rest of the + stream's addressing is resolved there, because a workflow cannot look it up + without doing I/O and a value it carried would be a reading rather than a + fact. A name this workflow has not written yet names a stream it owns, and + subscribing creates it. + + Args: + stream_name_or_id: Stream to consume: the name of one this workflow + owns, or the id of a standalone stream. The server tries them in + that order. + start: Where to start: an absolute offset, the oldest record the + stream holds, the tail as of registration, or the last N records. + Omitted, it is the oldest record held. The server resolves it + once and records the offset, so replay does not resolve it again. + """ + if start is None: + start = temporalio.api.stream.v1.StreamStartPosition(earliest=True) + _Runtime.current().workflow_subscribe_stream(stream_name_or_id, start) + + +def _append_stream_records( # type: ignore[reportUnusedFunction] + records: Sequence[temporalio.api.stream.v1.StreamRecord], + *, + stream_name: str = "", +) -> None: + """Publish records to a server-side stream this workflow owns. + + Returns at once. The records become one command when this Workflow Task + completes, so they are visible when the task is accepted and never if it + fails. Their bodies go to the stream's own log rather than into History, + which gets one fixed-size event naming the offset range, so a task that + publishes a thousand records costs History the same as one that publishes + one. Readers do not have to exist yet, and adding one costs the writer + nothing. + + Args: + records: Records to append, in order. The server stores each with an + empty ``producer_id``, because the workflow is the producer. + stream_name: Name of a stream this workflow owns, created on first + use. Empty means the workflow's default output stream. A workflow + cannot append to a stream another execution owns. + + Raises: + ValueError: ``records`` is empty or one of them is over the server's + per-record size limit. + """ + _Runtime.current().workflow_append_stream_records(stream_name, records) + + +@dataclass(frozen=True) +class _DeliveredStreamRecord: + """One record a consuming workflow was given, with where it sat.""" + + record: temporalio.api.stream.v1.StreamRecord + offset: int + """Its position in the whole stream, which is what a reader resumes from.""" + + +def _close_stream_records(stream_id: str) -> None: # type: ignore[reportUnusedFunction] + """Say this workflow wants no more of ``stream_id``. + + There is no unsubscribe command, so the server keeps delivering for the + life of the run; this drops what arrives instead of holding it for a + reader that has gone. Deterministic on replay, because the same workflow + code closes at the same point and the same ranges are dropped. + + Args: + stream_id: Stream to stop keeping records for. + """ + _Runtime.current().workflow_close_stream_records(stream_id) + + +async def _read_stream_records( # type: ignore[reportUnusedFunction] + stream_id: str, *, max_records: int = 0 +) -> list[_DeliveredStreamRecord]: + """Read the next records of a server-side stream this workflow consumes. + + Waits until at least one record is available. Ranges arrive on Workflow + Tasks, and only the offsets they covered are written to History, so this is + deterministic on replay: the server re-supplies the same ranges by reading + the stream again. Subscribe first with :func:`_subscribe_stream`; this only + reads what has already been delivered to this workflow. + + Args: + stream_id: Stream to read from. + max_records: Most records to return at once, or 0 for everything + available. + """ + return await _Runtime.current().workflow_read_stream_records(stream_id, max_records) + + async def wait_condition( fn: Callable[[], bool], *, diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py new file mode 100644 index 0000000000..bc3bfa8a55 --- /dev/null +++ b/temporalio/workflow/_streams.py @@ -0,0 +1,346 @@ +"""Streams from inside workflow code. + +.. warning:: + This module is experimental and may change in future versions. + +The reader and writer here are the same on every provider. They convert +values, synthesize supersession and buffer nothing the provider did not hand +them; everything provider-specific sits behind the ``ReadSource`` and +``WriteSink`` that the provider's workflow half opens. The provider itself +comes from the worker, the way the payload converter does. +""" + +from __future__ import annotations + +import asyncio +from collections import deque +from collections.abc import AsyncIterator, Callable +from typing import Any, Generic, TypeVar, cast, overload + +from temporalio.streams._provider import ReadSource, WorkflowStreamProvider, WriteSink +from temporalio.streams._record import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamRecord, + check_read_start, +) +from temporalio.streams._topic import StreamTopic, resolve_topic +from temporalio.streams._wire import RecordDecoder, to_wire +from temporalio.workflow._context import _Runtime, payload_converter +from temporalio.workflow._exceptions import ReadOnlyContextError +from temporalio.workflow._sandbox import logger + +__all__ = ["StreamReader", "StreamWriter", "stream_reader", "stream_writer"] + +T = TypeVar("T") + + +class _WorkflowStreams: + """The stream state a workflow instance carries: its provider half and open readers.""" + + def __init__(self, provider: WorkflowStreamProvider) -> None: + self.provider = provider + self.readers: dict[str, StreamReader[Any]] = {} + # Finishing is a statement about the topic, not about the writer + # object that made it, and stream_writer() hands out a new object on + # every call. Rebuilt in order on replay, so it stays deterministic. + self.finished: set[str] = set() + + +class StreamReader(Generic[T]): + """Reads one topic of this workflow's stream from inside workflow code. + + The reader is the async iterator: ``async for record in reader`` yields + every kind of record, including the supersession the reader synthesizes + when a producer's newer attempt appears. Check ``record.kind``, or + iterate :meth:`values` when the workflow only wants data. Iteration ends + when the reader is closed or the provider ends the subscription. One loop + per reader: two loops on one reader share its buffer and interleave. + """ + + def __init__( + self, + source: ReadSource, + *, + topic: str, + result_type: type | None, + after: Cursor, + on_close: Callable[[], None], + ) -> None: + """Prefer :func:`temporalio.workflow.stream_reader`.""" + self._source = source + self._topic = topic + self._result_type = result_type + self._decoder = RecordDecoder( + payload_converter(), result_type, after=after, warn=logger.warning + ) + self._pending: deque[StreamRecord[T]] = deque() + self._lock = asyncio.Lock() + self._closed = False + self._ended = False + self._on_close = on_close + + @property + def topic(self) -> str: + """The name of the topic this reader is subscribed to.""" + return self._topic + + def __aiter__(self) -> StreamReader[T]: + """The reader is its own iterator.""" + return self + + async def __anext__(self) -> StreamRecord[T]: + """The next record, waiting for one to arrive.""" + while True: + if self._pending: + return self._pending.popleft() + if self._closed or self._ended: + raise StopAsyncIteration + await self._fill() + + async def _fill(self) -> None: + # Two loops on one reader must not race the source, so one batch + # fetch is in flight at a time and the second loop takes what the + # first one buffered. + async with self._lock: + if self._pending or self._closed or self._ended: + return + try: + batch = await self._source.next_batch() + except StopAsyncIteration: + # The provider ended the subscription. Iteration stops rather + # than raising, so a workflow that reads to the end of a + # finished stream leaves the loop instead of failing its task. + self._ended = True + return + for cursor, wire in batch: + self._pending.extend(self._decoder.decode(cursor, wire)) + + async def values(self) -> AsyncIterator[T]: + """Iterate the data values, dropping control records.""" + async for record in self: + if record.kind is RecordKind.DATA: + yield cast("T", record.value) + + def close(self) -> None: + """End the subscription. Idempotent. + + A later :func:`temporalio.workflow.stream_reader` on the same topic + opens a new subscription, which is a new command. + """ + if self._closed: + return + self._closed = True + self._source.close() + self._on_close() + + +class StreamWriter(Generic[T]): + """Publishes to one topic of this workflow's stream. + + A workflow can only publish transactionally to its own stream, on every + provider. Writing to somebody else's stream is an activity's job, and it + gets the weaker guarantee that goes with doing I/O. The type parameter is + the topic definition's value type; a writer on a string-named topic takes + any value. + """ + + def __init__(self, sink: WriteSink, topic: str, finished: set[str]) -> None: + """Prefer :func:`temporalio.workflow.stream_writer`.""" + self._sink = sink + self._topic = topic + self._finished = finished + + @property + def topic(self) -> str: + """The name of the topic this writer is bound to.""" + return self._topic + + def publish(self, value: T) -> None: + """Append ``value`` to this topic. + + Synchronous, because there is nothing to wait for inside a task: the + record becomes visible when this Workflow Task is accepted, and never + at all if the task fails, so a reader cannot see a decision the + workflow did not commit. A :class:`temporalio.common.RawValue` passes + through pre-encoded. + + Raises: + ValueError: The topic was already finished in this run, by this + writer or by another one on the same topic. + temporalio.workflow.ReadOnlyContextError: Called from a query or an + update validator, which commit nothing. + """ + if self._topic in self._finished: + raise ValueError(f"topic {self._topic!r} was already finished") + # A query is answered on a task that carries the answer and nothing else, so a + # record published here could only ever be dropped. Said at the call rather than + # thrown away later without a word. + if _Runtime.current().workflow_is_read_only(): + raise ReadOnlyContextError( + "While in read-only function, action attempted: publish to a stream" + ) + self._sink.publish( + to_wire( + payload_converter(), + topic=self._topic, + kind=RecordKind.DATA, + value=value, + ) + ) + + def finish(self) -> None: + """Write ``FINISH`` for this workflow on this topic. Idempotent. + + Says this workflow has nothing more to send on the topic. It does not + say the workflow succeeded, and it does not end anyone's read. The + marker belongs to the topic, so a second writer on the same topic in + the same run finds it already written. + """ + if self._topic in self._finished: + return + self._finished.add(self._topic) + self._sink.publish( + to_wire(payload_converter(), topic=self._topic, kind=RecordKind.FINISH) + ) + + +@overload +def stream_reader( + topic: StreamTopic[T], *, after: Cursor = ..., last: int | None = None +) -> StreamReader[T]: ... + + +@overload +def stream_reader( + topic: str | None = None, + *, + result_type: type[T], + after: Cursor = ..., + last: int | None = None, +) -> StreamReader[T]: ... + + +@overload +def stream_reader( + topic: str | None = None, + *, + result_type: None = None, + after: Cursor = ..., + last: int | None = None, +) -> StreamReader[Any]: ... + + +def stream_reader( + topic: str | StreamTopic[Any] | None = None, + *, + result_type: type | None = None, + after: Cursor = BEGINNING, + last: int | None = None, +) -> StreamReader[Any]: + """Subscribe this workflow to ``topic`` of its own stream. + + ``topic`` is a :func:`temporalio.streams.topic` definition, which carries + the record type, or a plain string with ``result_type=`` for a name + decided at runtime, and without one the reader is on + :data:`temporalio.streams.DEFAULT_TOPIC`, decoded as a string-named topic + is. One subscription per topic per run. A second call + for the same topic returns the reader already open on it, so records go + to whichever loop pulls first; such a call may pass no ``after``, no + ``last`` and no different type. Adding a reader on a new topic is a new command, so + gate it with :func:`temporalio.workflow.patched` as you would a timer. A + reader in a successor run starts a new subscription: nothing crosses + continue-as-new implicitly. + + Args: + topic: The topic, relative to this workflow's stream. Omit it for + the default topic. + result_type: The value type for a string-named or default topic, used as the + decode hint. :class:`temporalio.common.RawValue` returns the + payload untouched. + after: Resume strictly after this record. ``BEGINNING`` starts at + the oldest record the topic still holds and + :data:`temporalio.streams.END` at whatever is appended after the + subscription is registered. Honoured on the first subscription + of a run, because after that the recorded observations decide. + last: Start at the newest ``last`` records instead, or at all of + them when there are fewer. Records of every kind count. Exclusive + with a cursor in ``after``. Where it lands is resolved once, + outside the workflow, and replay reproduces it. + + Raises: + ValueError: ``topic`` is empty, ``result_type`` was passed with a + definition, ``last`` is not positive or came with a cursor, or a + reader on the topic is already open and this call asked for a + different position or type. + temporalio.streams.StreamCursorError: ``after`` was minted by another + provider. + temporalio.streams.StreamUnsupportedError: The provider cannot start + where ``END`` or ``last`` asks. + """ + check_read_start(after, last) + name, result_type = resolve_topic(topic, result_type) + state: _WorkflowStreams = _Runtime.current().workflow_streams() + existing = state.readers.get(name) + if existing is not None: + if ( + after != BEGINNING + or last is not None + or result_type is not existing._result_type + ): + raise ValueError( + f"topic {name!r} already has a reader in this run; a second " + "stream_reader shares it and takes no after=, last= or other type" + ) + return existing + # Passed only when given, so a provider written before last= existed + # still serves every read it can. + source = ( + state.provider.open_reader(name, after=after) + if last is None + else state.provider.open_reader(name, after=after, last=last) + ) + + def forget() -> None: + state.readers.pop(name, None) + + # The decoder positions a synthesized record at the one before it. Where + # END or last= lands is not known here, so BEGINNING stands in: resuming + # from it may deliver a record twice, where END would skip one. + previous = BEGINNING if after == END or last is not None else after + reader: StreamReader[Any] = StreamReader( + source, topic=name, result_type=result_type, after=previous, on_close=forget + ) + state.readers[name] = reader + return reader + + +@overload +def stream_writer(topic: StreamTopic[T]) -> StreamWriter[T]: ... + + +@overload +def stream_writer(topic: str | None = None) -> StreamWriter[Any]: ... + + +def stream_writer(topic: str | StreamTopic[Any] | None = None) -> StreamWriter[Any]: + """Publish to ``topic`` of this workflow's stream, or to its default topic. + + Every call returns a new writer, and they all share the run's record of + which topics were finished, so ``finish()`` on one is seen by the next. + + Args: + topic: A :func:`temporalio.streams.topic` definition, whose value type + the writer's ``publish`` takes, or a plain string for a name + decided at runtime. Omitted, the writer is on + :data:`temporalio.streams.DEFAULT_TOPIC`. Encoding follows each + published value. + + Raises: + ValueError: ``topic`` is empty. + """ + name, _ = resolve_topic(topic) + state = _Runtime.current().workflow_streams() + return StreamWriter(state.provider.open_writer(name), name, state.finished) diff --git a/tests/contrib/external_workflow_streams/__init__.py b/tests/contrib/external_workflow_streams/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/contrib/external_workflow_streams/conformance.py b/tests/contrib/external_workflow_streams/conformance.py new file mode 100644 index 0000000000..92b4489106 --- /dev/null +++ b/tests/contrib/external_workflow_streams/conformance.py @@ -0,0 +1,853 @@ +"""The backend conformance suite (P2). + +**The suite is the deliverable, not the interface.** Each check below encodes +one obligation from the backend contract and fails with a message naming the +obligation, so a provider author reads what they broke rather than a diff of +two lists. + +Checks are plain async functions taking ``(backend, key)`` and raising +``AssertionError``. That shape is what lets two different callers use them: a +provider's own test parametrizes over :data:`CONFORMANCE_CHECKS` and expects +every one to pass, while the suite's own tests call a single check against a +deliberately-broken stub and assert it fails *for the right reason*. A suite +that cannot be shown to fail is not evidence of anything. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from datetime import timedelta + +from temporalio.contrib.external_workflow_streams._backend import ( + AppendConflictError, + ParkIntent, + ParkIntentRemoval, + StreamBackend, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Offset, + RecordKind, + StreamRecord, +) + +Check = Callable[[StreamBackend, StreamKey], Awaitable[None]] + +NO_BLOCK = timedelta(0) + + +def _data(session: str, seq: int, payload: bytes) -> StreamRecord: + return StreamRecord(RecordKind.DATA, payload, session, seq) + + +def _fence(session: str, seq: int) -> StreamRecord: + return StreamRecord(RecordKind.WRITE_FENCE, b"", session, seq) + + +async def _append_all( + backend: StreamBackend, key: StreamKey, payloads: list[bytes], session: str = "s" +) -> list[StreamRecord]: + return [ + await backend.append(key, _data(session, i, p)) for i, p in enumerate(payloads) + ] + + +# --- ordering --------------------------------------------------------------- + + +async def check_offsets_are_totally_ordered( + backend: StreamBackend, key: StreamKey +) -> None: + """Appended records must ascend under the provider's own comparator.""" + placed = await _append_all(backend, key, [b"a", b"b", b"c"]) + offsets = [r.offset for r in placed] + + assert all(o is not None for o in offsets), ( + "append must return the record with its assigned offset" + ) + assert backend.strictly_increasing(offsets), ( # type: ignore[arg-type] + f"appended offsets must strictly increase under compare_offsets, got {offsets}" + ) + + +async def check_offsets_are_not_compared_lexically( + backend: StreamBackend, key: StreamKey +) -> None: + """Ordering must survive a width change in the offset's leading component. + + This is the check that catches string comparison, and it is invisible + without it: ``"9-0" < "10-0"`` is true numerically and false lexically, so + a lexical comparator is correct for nine appends out of ten and then + silently reorders a range. + """ + early, late = await _append_at_widening_boundary(backend, key) + + assert backend.compare_offsets(early, late) < 0, ( + f"offset {early} was appended before {late}, so compare_offsets must " + "order it first; comparing offsets lexically fails here" + ) + assert backend.compare_offsets(late, early) > 0, ( + "compare_offsets must be antisymmetric" + ) + assert backend.compare_offsets(early, early) == 0, ( + "compare_offsets must report an offset as equal to itself" + ) + + +async def _append_at_widening_boundary( + backend: StreamBackend, key: StreamKey +) -> tuple[Offset, Offset]: + """Two offsets whose leading components differ in digit width. + + A provider that assigns offsets from a clock it exposes (the in-memory + reference) is driven across the boundary directly. Anything else -- a real + Redis -- gets whatever it assigns, and the check still holds because the + comparator must order *its own* offsets correctly either way. + """ + now = getattr(backend, "now_ms", None) + if now is not None: + backend.now_ms = 9 # type: ignore[attr-defined] + early = await backend.append(key, _data("widen", 0, b"early")) + backend.now_ms = 10 # type: ignore[attr-defined] + late = await backend.append(key, _data("widen", 1, b"late")) + else: + early = await backend.append(key, _data("widen", 0, b"early")) + late = await backend.append(key, _data("widen", 1, b"late")) + assert early.offset is not None and late.offset is not None + return early.offset, late.offset + + +# --- the inclusive range read ----------------------------------------------- + + +async def check_range_read_includes_both_endpoints( + backend: StreamBackend, key: StreamKey +) -> None: + """``read_range(first, last)`` is closed on both ends. + + A provider implementing it with exclusive semantics passes every live test + and drops the boundary records on the first replay, which is why this is + checked directly rather than inferred from a round trip. + """ + placed = await _append_all(backend, key, [b"a", b"b", b"c"]) + first, last = placed[0].offset, placed[-1].offset + assert first is not None and last is not None + + got = await backend.read_range(key, first, last) + + assert [r.payload for r in got] == [b"a", b"b", b"c"], ( + "read_range must be inclusive of both endpoints; it returned " + f"{[r.payload for r in got]} for the full range" + ) + + +async def check_range_read_of_one_record_returns_it( + backend: StreamBackend, key: StreamKey +) -> None: + """The degenerate range ``[x, x]`` contains exactly the record at ``x``.""" + placed = await _append_all(backend, key, [b"a", b"b"]) + only = placed[0].offset + assert only is not None + + got = await backend.read_range(key, only, only) + + assert [r.payload for r in got] == [b"a"], ( + f"read_range({only}, {only}) must return exactly that record, got " + f"{[r.payload for r in got]}" + ) + + +async def check_range_read_reports_a_missing_record_as_missing( + backend: StreamBackend, key: StreamKey +) -> None: + """A deleted record must be absent, not substituted for. + + Integrity loss must never resolve to an alternate stream result. The bug + this catches is a range read implemented as "N records starting at + ``first``" rather than as the closed interval ``[first, last]``: it returns + the right *count* by reaching past ``last``, so a trimmed stream reads back + as intact and replay delivers a record the original run never saw. + + A fourth record is appended beyond the range precisely so there is + something for such a provider to reach for. + """ + placed = await _append_all(backend, key, [b"a", b"b", b"c", b"d"]) + first, middle, last = placed[0].offset, placed[1].offset, placed[2].offset + assert first is not None and middle is not None and last is not None + + delete = getattr(backend, "delete_for_test", None) + if delete is None: + return # This provider cannot be made to lose a record on demand. + await delete(key, middle) + + got = await backend.read_range(key, first, last) + + assert [r.payload for r in got] == [b"a", b"c"], ( + "read_range must report a deleted record as absent rather than " + f"substituting another from beyond the range, got {[r.payload for r in got]}" + ) + + +# --- the exclusive watch ---------------------------------------------------- + + +async def check_watch_is_exclusive_of_its_boundary( + backend: StreamBackend, key: StreamKey +) -> None: + """``read_after(AFTER(x))`` must not re-deliver the record at ``x``. + + An inclusive watch re-delivers the last consumed record after every + resumption, which a Workflow sees as a duplicate. + """ + placed = await _append_all(backend, key, [b"a", b"b"]) + first = placed[0].offset + assert first is not None + + got = await backend.read_after(key, AFTER(first), max_records=10, block=NO_BLOCK) + + assert [r.payload for r in got] == [b"b"], ( + "read_after must return records strictly after its boundary; it " + f"returned {[r.payload for r in got]} after the first record" + ) + + +async def check_beginning_reads_from_the_start( + backend: StreamBackend, key: StreamKey +) -> None: + """``BEGINNING`` is a boundary before every record, not an offset.""" + await _append_all(backend, key, [b"a", b"b"]) + + got = await backend.read_after(key, BEGINNING, max_records=10, block=NO_BLOCK) + + assert [r.payload for r in got] == [b"a", b"b"], ( + "read_after(BEGINNING) must return every record from the start, got " + f"{[r.payload for r in got]}" + ) + + +async def check_a_consumer_parked_at_the_tail_resumes( + backend: StreamBackend, key: StreamKey +) -> None: + """The core cursor-semantics check. + + A consumer that has drained the stream holds ``AFTER(last)`` -- a boundary + past the tail. It must be able to resume from there once a record whose id + it could not have predicted arrives. A provider that requires the cursor to + name an existing record cannot express this state at all. + """ + placed = await _append_all(backend, key, [b"a"]) + tail = placed[0].offset + assert tail is not None + + # Parked: nothing beyond the boundary yet, and the boundary names a record + # that is the last one in existence. + empty = await backend.read_after(key, AFTER(tail), max_records=10, block=NO_BLOCK) + assert empty == [], ( + f"read_after(AFTER({tail})) must be empty while {tail} is the tail, got {empty}" + ) + + unpredictable = await backend.append(key, _data("later", 0, b"surprise")) + + resumed = await backend.read_after(key, AFTER(tail), max_records=10, block=NO_BLOCK) + + assert [r.payload for r in resumed] == [b"surprise"], ( + "a consumer parked at the tail must resume once a record arrives, " + f"without having to name its id in advance; got {[r.payload for r in resumed]}" + ) + assert resumed[0].offset == unpredictable.offset + + +async def check_watch_respects_max_records( + backend: StreamBackend, key: StreamKey +) -> None: + """Backpressure is the buffer bound, so the read must honour it.""" + await _append_all(backend, key, [b"a", b"b", b"c", b"d"]) + + got = await backend.read_after(key, BEGINNING, max_records=2, block=NO_BLOCK) + + assert len(got) == 2, f"read_after must return at most max_records, got {len(got)}" + + +async def check_watch_returns_empty_rather_than_hanging( + backend: StreamBackend, key: StreamKey +) -> None: + """An elapsed block returns empty; it does not raise and does not hang.""" + got = await backend.read_after( + key, BEGINNING, max_records=10, block=timedelta(milliseconds=50) + ) + + assert got == [], f"an elapsed block must return empty, got {got}" + + +# --- idempotent append ------------------------------------------------------ + + +async def check_identical_reappend_is_a_no_op( + backend: StreamBackend, key: StreamKey +) -> None: + """A retried Activity attempt must not duplicate its record.""" + first = await backend.append(key, _data("retry", 0, b"payload")) + again = await backend.append(key, _data("retry", 0, b"payload")) + + assert again.offset == first.offset, ( + "re-appending byte-identical content under the same idempotency key " + f"must return the original offset {first.offset}, got {again.offset}" + ) + + assert first.offset is not None + in_range = await backend.read_range(key, first.offset, first.offset) + assert len(in_range) == 1, ( + f"an idempotent re-append must write nothing, but {len(in_range)} " + "records occupy the original offset" + ) + + +async def check_reappend_with_different_bytes_is_rejected( + backend: StreamBackend, key: StreamKey +) -> None: + """Idempotency is on identity, not on the key alone. + + A provider that accepts this lets a retried producer attempt with changed + input overwrite what a consumer may already have delivered -- and the + divergence surfaces much later, as an unrelated nondeterminism error. + """ + await backend.append(key, _data("conflict", 0, b"original")) + + try: + await backend.append(key, _data("conflict", 0, b"different")) + except AppendConflictError: + return + raise AssertionError( + "re-appending different bytes under an existing idempotency key must " + "raise AppendConflictError; the append was accepted instead" + ) + + +async def check_different_sessions_do_not_collide( + backend: StreamBackend, key: StreamKey +) -> None: + """The key is ``(session_id, sequence)``, so sequence alone is not enough.""" + one = await backend.append(key, _data("session-a", 0, b"a")) + two = await backend.append(key, _data("session-b", 0, b"b")) + + assert one.offset != two.offset, ( + "two producers each writing their own sequence 0 must get distinct " + "offsets; the idempotency key includes the session id" + ) + + +# --- control records -------------------------------------------------------- + + +async def check_control_records_share_the_data_offset_sequence( + backend: StreamBackend, key: StreamKey +) -> None: + """A fence occupies an offset and advances the cursor like any record.""" + before = await backend.append(key, _data("s", 0, b"a")) + fence = await backend.append(key, _fence("s", 1)) + after = await backend.append(key, _data("s", 2, b"b")) + + offsets = [before.offset, fence.offset, after.offset] + assert backend.strictly_increasing(offsets), ( # type: ignore[arg-type] + f"a write fence must take its place in the offset sequence, got {offsets}" + ) + + assert before.offset is not None and after.offset is not None + got = await backend.read_range(key, before.offset, after.offset) + kinds = [r.kind for r in got] + assert kinds == [RecordKind.DATA, RecordKind.WRITE_FENCE, RecordKind.DATA], ( + f"a range read must return control records in place, got {kinds}" + ) + + +async def check_records_round_trip_unchanged( + backend: StreamBackend, key: StreamKey +) -> None: + """Every field a producer wrote must read back exactly.""" + written = StreamRecord(RecordKind.DATA, b"\x00\xff bytes", "session-x", 42) + placed = await backend.append(key, written) + + assert placed.offset is not None + (got,) = await backend.read_range(key, placed.offset, placed.offset) + + assert got.payload == written.payload, "payload bytes must round-trip exactly" + assert got.producer_session_id == written.producer_session_id + assert got.sequence == written.sequence + assert got.kind == written.kind + + +# --- registration precondition ---------------------------------------------- + + +async def check_immutability_is_declared( + backend: StreamBackend, + key: StreamKey, # pyright: ignore[reportUnusedParameter] +) -> None: + """Declared, not merely true. Undeclared is a registration failure. + + The registry is what enforces this at Worker construction; the check is + here so a provider author finds out while writing the provider rather than + when a Worker refuses to start. + """ + assert type(backend).guarantees_immutability is True, ( + f"{type(backend).__name__} must declare guarantees_immutability = True; " + f"it is {type(backend).guarantees_immutability!r}. The guarantee is a " + "precondition for registration, not a runtime mode." + ) + assert type(backend).provider_id, ( + f"{type(backend).__name__} must declare a provider_id; every annotation " + "header records it" + ) + + +# --- parking (P2b) ---------------------------------------------------------- + +SHORT_LEASE = timedelta(milliseconds=80) + + +async def check_intents_are_keyed_by_stream_and_wait_id( + backend: StreamBackend, key: StreamKey +) -> None: + """Two subscriptions to one stream must each keep their own intent. + + A stream-keyed intent looks correct until a Workflow subscribes to the same + stream twice: the second install overwrites the first, and from then on only + one of the two can ever be woken. Cursors are per subscription, so the + surviving intent also names the wrong position for the lost one. + """ + first = ParkIntent(wait_id=1, cursor=BEGINNING, park_generation=4, run_id="run-a") + second = ParkIntent( + wait_id=2, cursor=AFTER(Offset("100-0")), park_generation=4, run_id="run-a" + ) + + await backend.install_park_intent(key, first) + await backend.install_park_intent(key, second) + + got_first = await backend.park_intent(key, 1) + got_second = await backend.park_intent(key, 2) + + assert got_first == first, ( + "installing a second subscription's intent overwrote the first; park " + f"intents must be keyed (stream key, wait_id), got {got_first!r}" + ) + assert got_second == second + assert got_first.cursor != got_second.cursor # type: ignore[union-attr] + + +async def check_every_parked_subscription_is_enumerable( + backend: StreamBackend, key: StreamKey +) -> None: + """A producer must be able to find every wait it has to wake. + + ``wait_id`` is allocated by a per-Run counter inside ``subscribe()``, which no + producer can see, and the other five parking operations all take one. A + provider that enumerates only the most recent intent, or only distinct + streams, leaves the subscriptions it omits parked on records already sitting + in the stream. + """ + other = StreamKey( + key.namespace, key.workflow_id, key.first_execution_run_id, "another-stream" + ) + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=4, run_id="run-a") + ) + await backend.install_park_intent( + key, ParkIntent(2, AFTER(Offset("100-0")), park_generation=4, run_id="run-a") + ) + await backend.install_park_intent( + other, ParkIntent(3, BEGINNING, park_generation=4, run_id="run-a") + ) + + found = await backend.parked_wait_ids(key) + + assert sorted(found) == [1, 2], ( + f"expected both parked subscriptions on this stream, got {found!r}" + ) + assert 3 not in found, ( + "enumeration must be per stream: waking a subscription parked on a " + "different stream produces a Workflow Task that finds nothing there" + ) + + # And a removed intent stops being enumerable, or a producer keeps signaling + # a wait that has already been resolved. + await backend.remove_park_intent(key, 1) + assert await backend.parked_wait_ids(key) == [2] + + +async def check_an_intent_is_removable_and_removal_is_idempotent( + backend: StreamBackend, key: StreamKey +) -> None: + """An aborted park removes every intent it installed.""" + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=1, run_id="run-a") + ) + + await backend.remove_park_intent(key, 1) + assert await backend.park_intent(key, 1) is None + + # Idempotent: an abort may race a removal that already happened. + await backend.remove_park_intent(key, 1) + assert await backend.park_intent(key, 1) is None + + +async def check_conditional_removal_never_deletes_a_replacement( + backend: StreamBackend, key: StreamKey +) -> None: + """A predecessor's delayed cleanup must not remove a successor's park. + + And it must say which of the two "nothing was removed" cases it met. A key + that is clear is not a key someone else's park is holding: the consumer + treats the first as cleanup that happened -- possibly its own, on a call + whose reply was lost -- and the second as a park it must leave alone. + """ + successor = ParkIntent(1, BEGINNING, park_generation=2, run_id="run-b") + await backend.install_park_intent(key, successor) + + assert ( + await backend.remove_park_intent_if_matches( + key, 1, run_id="run-a", park_generation=2 + ) + is ParkIntentRemoval.MISMATCH + ) + assert await backend.park_intent(key, 1) == successor + + assert ( + await backend.remove_park_intent_if_matches( + key, 1, run_id="run-b", park_generation=1 + ) + is ParkIntentRemoval.MISMATCH + ) + assert await backend.park_intent(key, 1) == successor + + assert ( + await backend.remove_park_intent_if_matches( + key, 1, run_id="run-b", park_generation=2 + ) + is ParkIntentRemoval.REMOVED + ) + assert await backend.park_intent(key, 1) is None + + assert ( + await backend.remove_park_intent_if_matches( + key, 1, run_id="run-b", park_generation=2 + ) + is ParkIntentRemoval.ABSENT + ), "a repeat of a removal that succeeded is an absent key, not a mismatch" + assert ( + await backend.remove_park_intent_if_matches( + key, 1, run_id="run-c", park_generation=9 + ) + is ParkIntentRemoval.ABSENT + ), "an absent key is absent whoever asks about it" + + +async def check_a_new_runs_intent_replaces_its_predecessors( + backend: StreamBackend, key: StreamKey +) -> None: + """The Run ID is the intent's value, not part of its key. + + ``wait_id`` is stable across a Continue-As-New chain and the stream key + already carries the first execution Run ID, so the key is unique within the + chain. Putting the Run ID in the key instead would leave a dead intent + behind on every continuation. + """ + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=1, run_id="run-a") + ) + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=2, run_id="run-b") + ) + + got = await backend.park_intent(key, 1) + assert got is not None and got.run_id == "run-b", ( + f"a new Run's intent must replace its predecessor's, got {got!r}" + ) + + +async def check_recheck_sees_an_append_past_the_cursor( + backend: StreamBackend, key: StreamKey +) -> None: + """The recheck is what closes the append/park race. + + A producer appends *before* it observes the park generation, so an append is + either seen by this recheck or paired with a wake Signal. A recheck that + could not see a record appended after the intent was installed would lose + exactly the appends the ordering was designed to catch. + """ + placed = await _append_all(backend, key, [b"a"]) + tail = placed[0].offset + assert tail is not None + + await backend.install_park_intent( + key, ParkIntent(1, AFTER(tail), park_generation=1, run_id="run-a") + ) + assert not await backend.recheck(key, 1), ( + "recheck reported records with nothing past the intent's cursor" + ) + + await backend.append(key, _data("racer", 0, b"late")) + + assert await backend.recheck(key, 1), ( + "recheck missed a record appended after the intent was installed; the " + "append/park race is not closed" + ) + + +async def check_recheck_of_a_removed_intent_is_false( + backend: StreamBackend, key: StreamKey +) -> None: + await _append_all(backend, key, [b"a"]) + assert not await backend.recheck(key, 99) + + +async def check_the_current_generation_is_readable( + backend: StreamBackend, key: StreamKey +) -> None: + """A producer needs the generation to name in its wake Signal.""" + assert await backend.current_park_generation(key, 1) is None, ( + "no intent is installed, so there is no confirmed park to report; a " + "producer here must send an unparked wake rather than a stale one" + ) + + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=7, run_id="run-a") + ) + + assert await backend.current_park_generation(key, 1) == 7 + + +async def check_a_removed_intent_reports_no_generation( + backend: StreamBackend, key: StreamKey +) -> None: + """Removing an intent is what ends a park, so it must end what readers see. + + The rule the consumer relies on is *an intent exists only while that park is + outstanding*, and the consumer can only hold up its half of it -- installing + one when a park is confirmed and removing it when the park is over. The + provider's half is that removal is visible through + ``current_park_generation``, which is the call every wake path asks and the + only thing that distinguishes a live park from none. + + A provider that answered from a remembered "last generation" beside the + intent would pass every other check here and still break both readers, in + the same direction and invisibly. A producer would name a generation Core + has already discarded, and Core ignores exactly that as stale -- so the + record is appended, the Signal is sent, and the Workflow is never woken. The + consumer's own shutdown sweep would send a parked wake where it owes the + unparked one, and a parked wake's request ID ignores sender identity by + design, so it arrives byte-identical to the wake that ended the park and the + server deduplicates it away. + """ + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=4, run_id="run-a") + ) + assert await backend.current_park_generation(key, 1) == 4 + + await backend.remove_park_intent(key, 1) + + assert await backend.current_park_generation(key, 1) is None, ( + "a removed park intent still reports a generation; a wake naming it is " + "a claim of a park that is over, which Core discards as stale" + ) + + +async def check_a_claim_excludes_a_second_producer( + backend: StreamBackend, key: StreamKey +) -> None: + """Only meaningful for a provider that declares leased claims.""" + if not type(backend).supports_leased_claims: + return + + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=3, run_id="run-a") + ) + + assert await backend.claim_park_generation( + key, 1, 3, claimant="producer-a", lease=SHORT_LEASE + ) + assert not await backend.claim_park_generation( + key, 1, 3, claimant="producer-b", lease=SHORT_LEASE + ), "two producers both believe they own the same park generation's wake" + + +async def check_a_claim_is_renewable_by_its_holder( + backend: StreamBackend, key: StreamKey +) -> None: + """Renewal must not look like contention. + + A producer whose append and Signal are separated by a slow call has to be + able to extend its own claim; if renewal were refused it would have to drop + the claim and race for it again. + """ + if not type(backend).supports_leased_claims: + return + + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=3, run_id="run-a") + ) + + assert await backend.claim_park_generation( + key, 1, 3, claimant="producer-a", lease=SHORT_LEASE + ) + assert await backend.claim_park_generation( + key, 1, 3, claimant="producer-a", lease=SHORT_LEASE + ), "a claim holder could not renew its own claim" + + +async def check_an_expired_claim_is_taken_over( + backend: StreamBackend, key: StreamKey +) -> None: + """The case an unleased claim cannot recover from. + + A producer that crashes between claiming a generation and sending its wake + Signal strands the generation: every other producer sees it claimed and + concludes the wake is handled, so the parked Workflow waits forever with + data sitting in the stream. A lease turns that into a bounded delay. + """ + if not type(backend).supports_leased_claims: + return + + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=3, run_id="run-a") + ) + assert await backend.claim_park_generation( + key, 1, 3, claimant="crashed-producer", lease=SHORT_LEASE + ) + + await asyncio.sleep(SHORT_LEASE.total_seconds() * 2.5) + + assert await backend.claim_park_generation( + key, 1, 3, claimant="producer-b", lease=SHORT_LEASE + ), ( + "an expired claim was not taken over; a producer crashing between its " + "claim and its Signal would strand this generation permanently" + ) + + +async def check_observe_only_providers_always_grant( + backend: StreamBackend, key: StreamKey +) -> None: + """A provider that cannot lease must let every producer signal. + + Duplicate Signals are harmless -- the runtime rechecks every subscription on + wakeup regardless -- and lost ones are not, so refusing a claim it cannot + police would be strictly worse than granting every one. + """ + if type(backend).supports_leased_claims: + return + + await backend.install_park_intent( + key, ParkIntent(1, BEGINNING, park_generation=3, run_id="run-a") + ) + + assert await backend.claim_park_generation( + key, 1, 3, claimant="producer-a", lease=SHORT_LEASE + ) + assert await backend.claim_park_generation( + key, 1, 3, claimant="producer-b", lease=SHORT_LEASE + ), "an observe-only provider must grant every claim so no wake is lost" + + +async def check_distinct_identities_never_share_storage( + backend: StreamBackend, key: StreamKey +) -> None: + """Two different `StreamKey` values must never address the same storage. + + Every component of the identity is a user-chosen string -- a namespace, a + Workflow ID, a Run ID, a topic name -- so a provider that renders the tuple + by joining the fields with a delimiter is not injective. These two are + different Workflows: + + ("ns", "wf", r1, f"{r2}:tokens") and ("ns", f"wf:{r1}", r2, "tokens") + + and a provider that joins on ``:`` gives them one physical location. They + then share records, idempotency state, park intents and claims: each reads + the other's data, and -- the quiet one -- each concludes the other's claim + has already taken responsibility for its wake, so neither signals and both + Runs wait forever on records that are already durable. + + The delimiter here is the one this suite happens to use. A provider using a + different separator has the same obligation against its own, which is why + this check asserts *isolation of behaviour* rather than any property of the + key strings a provider builds -- those are private to it. + """ + left = StreamKey(key.namespace, "wf", "run-a", "run-b:tokens") + right = StreamKey(key.namespace, "wf:run-a", "run-b", "tokens") + + placed_left = await backend.append(left, _data("left", 0, b"left-only")) + placed_right = await backend.append(right, _data("right", 0, b"right-only")) + + assert [ + r.payload + for r in await backend.read_after( + left, BEGINNING, max_records=10, block=NO_BLOCK + ) + ] == [b"left-only"], ( + "two distinct stream identities share one physical stream; joining the " + "identity's fields without escaping is not injective, and these two " + "unrelated Workflows now read each other's records" + ) + assert [ + r.payload + for r in await backend.read_after( + right, BEGINNING, max_records=10, block=NO_BLOCK + ) + ] == [b"right-only"] + + # Park state keyed off the same identity must be isolated for the same + # reason: an intent read across the boundary answers a generation Core has + # never heard of, and the producer's wake naming it is discarded as stale. + await backend.install_park_intent( + left, + ParkIntent(1, AFTER(placed_left.offset), park_generation=4, run_id="a"), # type: ignore[arg-type] + ) + assert await backend.park_intent(right, 1) is None, ( + "a park intent installed on one stream identity is visible on another" + ) + assert await backend.current_park_generation(right, 1) is None + assert await backend.parked_wait_ids(right) == [] + del placed_right + + +#: The parking checks, kept as their own list so a provider can adopt the core +#: contract before the parking extension. +PARKING_CONFORMANCE_CHECKS: list[Check] = [ + check_intents_are_keyed_by_stream_and_wait_id, + check_every_parked_subscription_is_enumerable, + check_an_intent_is_removable_and_removal_is_idempotent, + check_conditional_removal_never_deletes_a_replacement, + check_a_new_runs_intent_replaces_its_predecessors, + check_recheck_sees_an_append_past_the_cursor, + check_recheck_of_a_removed_intent_is_false, + check_the_current_generation_is_readable, + check_a_removed_intent_reports_no_generation, + check_distinct_identities_never_share_storage, + check_a_claim_excludes_a_second_producer, + check_a_claim_is_renewable_by_its_holder, + check_an_expired_claim_is_taken_over, + check_observe_only_providers_always_grant, +] + + +#: Every check, in the order a provider author most usefully reads them. +CONFORMANCE_CHECKS: list[Check] = [ + check_immutability_is_declared, + check_offsets_are_totally_ordered, + check_offsets_are_not_compared_lexically, + check_range_read_includes_both_endpoints, + check_range_read_of_one_record_returns_it, + check_range_read_reports_a_missing_record_as_missing, + check_watch_is_exclusive_of_its_boundary, + check_beginning_reads_from_the_start, + check_a_consumer_parked_at_the_tail_resumes, + check_watch_respects_max_records, + check_watch_returns_empty_rather_than_hanging, + check_identical_reappend_is_a_no_op, + check_reappend_with_different_bytes_is_rejected, + check_different_sessions_do_not_collide, + check_control_records_share_the_data_offset_sequence, + check_records_round_trip_unchanged, +] diff --git a/tests/contrib/external_workflow_streams/conftest.py b/tests/contrib/external_workflow_streams/conftest.py new file mode 100644 index 0000000000..6b35d9bf2c --- /dev/null +++ b/tests/contrib/external_workflow_streams/conftest.py @@ -0,0 +1,191 @@ +"""Redis fixtures for External Workflow Streams tests (X2). + +One Redis server is shared by the whole suite, so every test gets its own key +prefix rather than its own server. The prefix embeds the ``pytest-xdist`` +worker id, so parallel workers cannot collide even when two of them run the +same test module. +""" + +from __future__ import annotations + +import os +import uuid +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from typing import Any + +import pytest +import pytest_asyncio + +#: Asking for either of these is what gives a case a running server. A case +#: that asks for neither never observes a clock, so no environment can fail it. +_SERVER_FIXTURES = frozenset({"client", "env"}) + +#: The environments whose server the suite starts for itself. None of them +#: accepts the subscribe-notification-channel command. +_ENVIRONMENTS_WITHOUT_CHANNELS = ("local", "time-skipping", "envconfig") + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "needs_channel_server: the case needs a server that serves notification " + "channels, named with -E host:port", + ) + config.addinivalue_line( + "markers", + "needs_linked_server: the case needs a server that serves channels linked " + "to a workflow, named with -E host:port; the case skips itself on one " + "with only independent channels", + ) + config.addinivalue_line( + "markers", + "needs_unsubscribe_server: the case needs a server that accepts the " + "unsubscribe-notification-channel command, named with -E host:port; an " + "older channel server fails the Workflow Task that carries it", + ) + + +#: The markers naming a server capability the suite's own servers lack. +_CHANNEL_SERVER_MARKERS = ( + "needs_channel_server", + "needs_linked_server", + "needs_unsubscribe_server", +) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + if config.getoption("--workflow-environment") not in _ENVIRONMENTS_WITHOUT_CHANNELS: + return + skip = pytest.mark.skip( + reason="needs a server that serves notification channels; name one with -E" + ) + for item in items: + if any(item.get_closest_marker(marker) for marker in _CHANNEL_SERVER_MARKERS): + item.add_marker(skip) + + +async def server_channel_support(client: Any) -> Any: + """What the server offers a stream's readers, as the Worker would find it. + + The linked kind shows only on a running workflow, so one is started on a + task queue nobody polls and asked about; the probe is the Worker's own. + """ + from temporalio.contrib.external_workflow_streams._wake import ( + ChannelSupport, + channel_support, + server_has_channels, + ) + + if not await server_has_channels(client): + return ChannelSupport.NONE + probe = uuid.uuid4().hex + handle = await client.start_workflow( + "ChannelSupportProbe", + id=f"channel-support-probe-{probe}", + task_queue=f"nobody-polls-{probe}", + ) + try: + return await channel_support(client, handle.id) + finally: + await handle.terminate() + + +@pytest.fixture(autouse=True) +def skip_under_time_skipping(request: pytest.FixtureRequest) -> None: + """Hold the server-backed cases to a clock the tests can reason about. + + Those cases measure real-server timing: how long a Workflow Task was held + open, the interval a wake sweep runs on, the deadline a shutdown waits out. + The time-skipping server advances the clock whenever workers go idle, which + removes exactly the quantities being measured, so the failures it produces + say nothing about the feature. Which of them fail drifts run to run, so the + server-backed cases are held as a group rather than by name. Everything + else here is offline and keeps running on both environments. + """ + if _SERVER_FIXTURES.isdisjoint(request.fixturenames): + return + if request.getfixturevalue("env").supports_time_skipping: + pytest.skip("this case measures real-server timing; see conftest") + + +DEFAULT_REDIS_URL = "redis://127.0.0.1:6379" + +#: Every key this suite creates starts with this, so a leaked key is +#: attributable and a global cleanup is possible without guessing. +KEY_NAMESPACE = "temporal-extstream-test" + + +def redis_url() -> str: + return os.getenv("TEMPORAL_TEST_REDIS_URL", DEFAULT_REDIS_URL) + + +@dataclass +class RedisKeyspace: + """An isolated slice of the shared Redis server. + + ``client`` is a live ``redis.asyncio.Redis``. Every key a test touches must + be built with :meth:`key`, which is what makes the teardown complete. + """ + + client: "object" # redis.asyncio.Redis, untyped to keep redis an optional import + prefix: str + + def key(self, name: str) -> str: + return f"{self.prefix}{name}" + + async def cleanup(self) -> int: + """Delete every key under this keyspace. Returns the number removed.""" + removed = 0 + batch: list[str] = [] + async for found in self.client.scan_iter(match=f"{self.prefix}*", count=500): # type: ignore[attr-defined] + batch.append(found) + if len(batch) >= 500: + removed += await self.client.delete(*batch) # type: ignore[attr-defined] + batch = [] + if batch: + removed += await self.client.delete(*batch) # type: ignore[attr-defined] + return removed + + +@pytest.fixture(scope="session") +def redis_worker_id(request: pytest.FixtureRequest) -> str: + """The ``pytest-xdist`` worker id, or ``master`` when running serially.""" + return getattr(request.config, "workerinput", {}).get("workerid", "master") + + +@pytest_asyncio.fixture(scope="session") # type: ignore[reportUntypedFunctionDecorator] +async def redis_client() -> AsyncGenerator[object, None]: + """A session-wide connection, skipping the suite if no server is reachable. + + Reachability is checked once per session rather than per test so a missing + Redis produces one clear skip reason instead of one per case. + """ + redis = pytest.importorskip("redis.asyncio", reason="redis is not installed") + + client = redis.from_url(redis_url(), decode_responses=True) + try: + await client.ping() + except Exception as err: + await client.aclose() + pytest.skip(f"Redis is not reachable at {redis_url()}: {err}") + + try: + yield client + finally: + await client.aclose() + + +@pytest_asyncio.fixture # type: ignore[reportUntypedFunctionDecorator] +async def redis_keyspace( + redis_client: object, redis_worker_id: str +) -> AsyncGenerator[RedisKeyspace, None]: + """An isolated key prefix, cleaned up whether or not the test passed.""" + prefix = f"{KEY_NAMESPACE}:{redis_worker_id}:{uuid.uuid4().hex}:" + keyspace = RedisKeyspace(client=redis_client, prefix=prefix) + try: + yield keyspace + finally: + await keyspace.cleanup() diff --git a/tests/contrib/external_workflow_streams/crash_worker.py b/tests/contrib/external_workflow_streams/crash_worker.py new file mode 100644 index 0000000000..60a164190e --- /dev/null +++ b/tests/contrib/external_workflow_streams/crash_worker.py @@ -0,0 +1,220 @@ +"""A Worker in a process of its own, so a test can kill it outright. + +A crash is not a shutdown. Every in-process way of stopping a Worker runs the +graceful path -- which, for External Workflow Streams, is exactly the path that +*finalizes the annotation and writes the marker*. Testing "the Worker went away +before its marker committed" against a Worker that politely committed its marker +on the way out would assert the opposite of the case. + +So the Worker under test runs here, in a child process that is sent ``SIGKILL`` +with a Workflow Task retained. Nothing gets a chance to run. + +Two things live in this module rather than in the test: + +- the Workflow definition, because both processes must register the *same* one; +- a provider that records every offset it reads into Redis, because a crashed + process cannot be asked afterwards what it had read. The read log is the only + evidence that survives the kill, and comparing it with the replacement + Worker's is what turns "the records arrived eventually" into "the same offsets + were read again". + +The provider import is deliberately **inside** a function: this module is +re-imported by the Workflow sandbox, and importing a provider from Workflow code +is refused outright. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from datetime import timedelta +from typing import Any + +from temporalio import workflow +from temporalio.client import Client +from temporalio.worker import Worker + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams._api import external_stream + from temporalio.contrib.external_workflow_streams._output_api import ( + external_output_stream, + ) + +STREAM_NAME = "tokens" + +READ_LOG = "reads" +"""Key suffix under which a recording provider logs the offsets it read.""" + +OUTPUT_STAGE_LOG = "output-stages" +"""Key suffix used to prove an output stage survived a process kill.""" + + +@workflow.defn +class CrashConsumeWorkflow: + """Consumes a fixed number of records and returns them, in order. + + Returning the values is what makes loss and duplication visible: a record + re-delivered after the crash appears twice, and one lost never appears. + """ + + @workflow.run + async def run(self, expected: int) -> list[str]: + tokens = external_stream.topic(STREAM_NAME, type=str) + seen: list[str] = [] + async for token in tokens.subscribe(): + seen.append(token) + if len(seen) >= expected: + break + return seen + + +@workflow.defn +class CrashPublishWorkflow: + """Publishes output on a Workflow Task whose Worker will be killed.""" + + @workflow.run + async def run(self) -> str: + topic = external_output_stream.topic("events", type=str) + await topic.publish("accepted-after-crash") + await topic.finish() + return "done" + + +@workflow.defn +class CrashUpdatePublishWorkflow: + """Waits for an Update that publishes while its report is withheld.""" + + def __init__(self) -> None: + self._finished = False + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: self._finished) + + @workflow.update + async def publish(self, value: str) -> str: + await external_output_stream.topic("events", type=str).publish(value) + return value + + @workflow.signal + def finish(self) -> None: + self._finished = True + + +def read_log_key(prefix: str, reader: str) -> str: + return f"{prefix}:{READ_LOG}:{reader}" + + +def output_stage_log_key(prefix: str) -> str: + return f"{prefix}:{OUTPUT_STAGE_LOG}" + + +def recording_backend(*, url: str, prefix: str, reader: str) -> Any: + """A Redis provider that appends every offset it reads to a Redis list. + + In Redis rather than in memory because the Worker doing the reading may be + killed: whatever it recorded has to outlive the process. + """ + from temporalio.contrib.external_workflow_streams._redis import RedisStreamBackend + + class ReadRecordingBackend(RedisStreamBackend): + async def read_after( # type: ignore[override] + self, + key: Any, + after: Any, + *, + max_records: int, + block: timedelta | None = None, + ) -> Any: + records = await super().read_after( + key, after, max_records=max_records, block=block + ) + await self._log(records) + return records + + async def read_range(self, key: Any, first: Any, last: Any) -> Any: # type: ignore[override] + records = await super().read_range(key, first, last) + await self._log(records) + return records + + async def _log(self, records: Any) -> None: + offsets = [str(r.offset) for r in records if r.offset is not None] + if offsets: + await self._client.rpush(read_log_key(prefix, reader), *offsets) + + return ReadRecordingBackend(url=url, key_prefix=prefix) + + +def blocking_output_backend(*, url: str, prefix: str) -> Any: + """A Redis provider that stops only after output is durably staged. + + Blocking inside ``stage_output`` places the process at the exact boundary + the crash tests need: Redis has atomically stored the pending stage, while + the Worker has not returned the activation completion to Core/server. + """ + from temporalio.contrib.external_workflow_streams._redis import RedisStreamBackend + + class DurablyStageThenBlockBackend(RedisStreamBackend): + async def stage_output(self, manifest: Any, records: Any) -> Any: + stage = await super().stage_output(manifest, records) + await self._client.rpush( + output_stage_log_key(prefix), + json.dumps( + { + "run_id": manifest.run_id, + "stage_token": manifest.stage_token, + "sub_batch_id": manifest.sub_batch_id, + }, + sort_keys=True, + ), + ) + # There is deliberately no release path. The parent sends SIGKILL, + # so neither this call nor Worker shutdown can report completion. + await asyncio.Event().wait() + return stage + + return DurablyStageThenBlockBackend(url=url, key_prefix=prefix) + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--address", required=True) + parser.add_argument("--namespace", required=True) + parser.add_argument("--task-queue", required=True) + parser.add_argument("--redis-url", required=True) + parser.add_argument("--prefix", required=True) + parser.add_argument("--reader", default="crashed") + parser.add_argument( + "--mode", + choices=("consume", "publish", "update"), + default="consume", + ) + args = parser.parse_args() + + client = await Client.connect(args.address, namespace=args.namespace) + workflows: list[type[Any]] + if args.mode == "consume": + backend = recording_backend( + url=args.redis_url, prefix=args.prefix, reader=args.reader + ) + workflows = [CrashConsumeWorkflow] + else: + backend = blocking_output_backend(url=args.redis_url, prefix=args.prefix) + workflows = ( + [CrashPublishWorkflow] + if args.mode == "publish" + else [CrashUpdatePublishWorkflow] + ) + async with Worker( + client, + task_queue=args.task_queue, + workflows=workflows, + external_stream_backend=backend, + ): + # Until killed. There is deliberately no shutdown path here. + await asyncio.Event().wait() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/contrib/external_workflow_streams/m1_gate.py b/tests/contrib/external_workflow_streams/m1_gate.py new file mode 100644 index 0000000000..0f23117b3a --- /dev/null +++ b/tests/contrib/external_workflow_streams/m1_gate.py @@ -0,0 +1,146 @@ +"""The machinery behind the required-test gates (P16a, P16b). + +The required-test lists are the milestone gates. A gate that is only a claim in +a document is not a gate at all: someone renames a test, the case it covered +quietly stops existing, and the list still says the milestone is met. So the +list is parsed from the plan itself and every case is mapped to a test that has +to actually exist. + +Three things this checks, each catching a different way the gate rots: + +- the case **count** still matches what the plan's heading declares, so a case + added to the plan cannot be silently unmapped; +- every case is either covered or recorded as open, so a new case fails the gate + rather than passing by omission -- and a partially covered one is never + counted as covered, which is how a gate stops meaning anything; +- every mapped test **exists**, so a rename or deletion fails here rather than + leaving the list pointing at nothing. + +What this deliberately does *not* do is assert that a mapped test passes. That +is the test suite's job, and duplicating it here would only mean running +everything twice. +""" + +from __future__ import annotations + +import ast +import re +from dataclasses import dataclass +from pathlib import Path + +#: The required-test lists live in the vendored Core checkout, which is the same +#: branch the Core-side work is on. Reading them from there rather than from a +#: copy is what makes the count check meaningful -- a copy would drift silently. +PLAN_DIR = ( + Path(__file__).resolve().parents[3] + / "temporalio" + / "bridge" + / "sdk-core" + / "arch_docs" + / "streaming-poc-docs" + / "required-tests" +) + +TESTS_DIR = Path(__file__).resolve().parent + +CORE_TESTS = ( + Path(__file__).resolve().parents[3] + / "temporalio" + / "bridge" + / "sdk-core" + / "crates" + / "sdk-core" + / "src" + / "core_tests" + / "external_streams.rs" +) + + +@dataclass(frozen=True) +class RequiredCase: + """One bullet of a required-test list.""" + + number: int + text: str + + +def declared_count(list_name: str) -> int: + """The count the plan's own heading declares. + + Read rather than hardcoded: the plan says to update the heading whenever a + case is added or removed, and reading it is what turns that instruction into + something enforced. + """ + heading = (PLAN_DIR / list_name).read_text().splitlines()[0] + match = re.search(r"(\d+)\s+cases", heading) + assert match, f"{list_name} heading does not declare a case count: {heading!r}" + return int(match.group(1)) + + +def required_cases(list_name: str) -> list[RequiredCase]: + """Every bullet, in document order, with continuation lines joined. + + Bullets wrap across lines in the plan, and a parser that treated each line as + a case would count the same case several times -- inflating the total to + something that happened to match by accident. + """ + cases: list[RequiredCase] = [] + current: list[str] | None = None + for line in (PLAN_DIR / list_name).read_text().splitlines(): + if line.startswith("- "): + if current is not None: + cases.append(RequiredCase(len(cases) + 1, " ".join(current))) + current = [line[2:].strip()] + elif current is not None and line.startswith(" ") and line.strip(): + current.append(line.strip()) + elif current is not None and not line.strip(): + cases.append(RequiredCase(len(cases) + 1, " ".join(current))) + current = None + if current is not None: + cases.append(RequiredCase(len(cases) + 1, " ".join(current))) + return cases + + +def _python_test_names() -> dict[str, set[str]]: + """Every test function per module, read from the source rather than pytest. + + Parsing beats collecting here: collection would import every module and run + every fixture, and this check must be able to run when something else in the + suite is broken. + """ + found: dict[str, set[str]] = {} + for path in sorted(TESTS_DIR.glob("test_*.py")): + tree = ast.parse(path.read_text()) + names = { + node.name + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("test_") + } + found[path.name] = names + return found + + +def _core_test_names() -> set[str]: + if not CORE_TESTS.exists(): + return set() + return set(re.findall(r"async fn (\w+)\(", CORE_TESTS.read_text())) | set( + re.findall(r"\bfn (\w+)\(\)", CORE_TESTS.read_text()) + ) + + +def unresolved(node_ids: list[str]) -> list[str]: + """The mapped tests that do not exist. Empty means the mapping is honest.""" + python = _python_test_names() + core = _core_test_names() + missing = [] + for node_id in node_ids: + module, _, name = node_id.partition("::") + if module.endswith(".rs"): + if core and name not in core: + missing.append(node_id) + elif module not in python: + missing.append(f"{node_id} (no such module)") + elif name not in python[module]: + missing.append(node_id) + return missing diff --git a/tests/contrib/external_workflow_streams/memory_backend.py b/tests/contrib/external_workflow_streams/memory_backend.py new file mode 100644 index 0000000000..7be8c3e4aa --- /dev/null +++ b/tests/contrib/external_workflow_streams/memory_backend.py @@ -0,0 +1,228 @@ +"""A conforming in-memory backend, used to validate the conformance suite. + +Its offsets are shaped like Redis stream ids -- ``-`` compared as a +numeric pair -- so the checks that catch lexical comparison and width-crossing +boundaries are meaningful here rather than only against a real Redis. The +millisecond component comes from :attr:`now_ms`, which tests set directly, so +those boundaries are reachable without sleeping. +""" + +from __future__ import annotations + +import asyncio +import time +from datetime import timedelta +from typing import ClassVar + +from temporalio.contrib.external_workflow_streams._backend import ( + DEFAULT_WATCH_BLOCK, + AppendConflictError, + ParkIntent, + ParkIntentRemoval, + StreamBackend, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._record import ( + Cursor, + IdempotencyKey, + Offset, + StreamRecord, +) + + +def parse(offset: Offset) -> tuple[int, int]: + """``"12-3"`` as ``(12, 3)``. The provider's ordering rule, in one place.""" + ms, _, seq = offset.token.partition("-") + return int(ms), int(seq or 0) + + +class MemoryStreamBackend(StreamBackend): + """A correct reference implementation.""" + + guarantees_immutability: ClassVar[bool | None] = True + provider_id = "memory" + provider_format_version = 1 + + def __init__(self) -> None: + self.now_ms = 1 + self._records: dict[StreamKey, list[StreamRecord]] = {} + self._by_key: dict[tuple[StreamKey, IdempotencyKey], StreamRecord] = {} + self._appended = asyncio.Event() + #: Every read_range call, for tests that assert replay's call count. + self.range_reads: list[tuple[Offset, Offset]] = [] + #: Park intents, keyed `(stream key, wait_id)` -- never by stream alone. + self._intents: dict[tuple[StreamKey, int], ParkIntent] = {} + #: `(stream key, wait_id) -> (claimant, generation, expires_at)`. + self._claims: dict[tuple[StreamKey, int], tuple[str, int, float]] = {} + + # --- required operations ------------------------------------------------ + + async def append(self, key: StreamKey, record: StreamRecord) -> StreamRecord: + existing = self._by_key.get((key, record.idempotency_key)) + if existing is not None: + # Idempotent on *identity*: same bytes is a no-op returning the + # original offset, different bytes is an error. + if existing.to_fields() == record.to_fields(): + return existing + raise AppendConflictError(record.idempotency_key) + + stream = self._records.setdefault(key, []) + seq = sum( + 1 + for record in stream + if record.offset is not None and parse(record.offset)[0] == self.now_ms + ) + placed = record.placed_at(Offset(f"{self.now_ms}-{seq}")) + stream.append(placed) + self._by_key[(key, record.idempotency_key)] = placed + + self._appended.set() + self._appended.clear() + return placed + + async def read_range( + self, key: StreamKey, first: Offset, last: Offset + ) -> list[StreamRecord]: + self.range_reads.append((first, last)) + lo, hi = parse(first), parse(last) + return [ + r + for r in self._records.get(key, []) + if lo <= parse(r.offset) <= hi # type: ignore[arg-type] + ] + + async def read_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = DEFAULT_WATCH_BLOCK, + ) -> list[StreamRecord]: + deadline = None if block is None else block.total_seconds() + while True: + found = self._after(key, after)[:max_records] + if found or deadline is None or deadline <= 0: + return found + waiter = asyncio.ensure_future(self._appended.wait()) + try: + await asyncio.wait_for(waiter, deadline) + except asyncio.TimeoutError: + return [] + finally: + # `wait_for` cancels the outer future but the inner `Event.wait` + # task survives it, and a leaked one keeps the event's waiter + # list growing for the life of the process. + if not waiter.done(): + waiter.cancel() + + def _after(self, key: StreamKey, after: Cursor) -> list[StreamRecord]: + stream = self._records.get(key, []) + if after.is_beginning: + return list(stream) + bound = parse(after.offset) # type: ignore[arg-type] + return [r for r in stream if parse(r.offset) > bound] # type: ignore[arg-type] + + def compare_offsets(self, left: Offset, right: Offset) -> int: + a, b = parse(left), parse(right) + return (a > b) - (a < b) + + # --- parking ------------------------------------------------------------ + + supports_leased_claims = True + + async def install_park_intent(self, key: StreamKey, intent: ParkIntent) -> None: + self._intents[(key, intent.wait_id)] = intent + + async def remove_park_intent(self, key: StreamKey, wait_id: int) -> None: + self._intents.pop((key, wait_id), None) + self._claims.pop((key, wait_id), None) + + async def remove_park_intent_if_matches( + self, + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + intent = self._intents.get((key, wait_id)) + if intent is None: + return ParkIntentRemoval.ABSENT + if intent.run_id != run_id or intent.park_generation != park_generation: + return ParkIntentRemoval.MISMATCH + self._intents.pop((key, wait_id), None) + self._claims.pop((key, wait_id), None) + return ParkIntentRemoval.REMOVED + + async def parked_wait_ids(self, key: StreamKey) -> list[int]: + return sorted(wait_id for (stored, wait_id) in self._intents if stored == key) + + async def park_intent(self, key: StreamKey, wait_id: int) -> ParkIntent | None: + return self._intents.get((key, wait_id)) + + async def recheck(self, key: StreamKey, wait_id: int) -> bool: + intent = self._intents.get((key, wait_id)) + if intent is None: + return False + return bool(self._after(key, intent.cursor)) + + async def claim_park_generation( + self, + key: StreamKey, + wait_id: int, + park_generation: int, + *, + claimant: str, + lease: timedelta, + ) -> bool: + now = time.monotonic() + held = self._claims.get((key, wait_id)) + if held is not None: + holder, generation, expires_at = held + # A live claim held by someone else, for this generation, blocks. + # An expired one is taken over -- that is the whole point of a + # lease, and without it a producer that crashed here would strand + # the generation forever. + if ( + generation == park_generation + and holder != claimant + and expires_at > now + ): + return False + self._claims[(key, wait_id)] = ( + claimant, + park_generation, + now + lease.total_seconds(), + ) + return True + + async def current_park_generation(self, key: StreamKey, wait_id: int) -> int | None: + intent = self._intents.get((key, wait_id)) + return None if intent is None else intent.park_generation + + # --- test affordances --------------------------------------------------- + + async def expire_claims_for_test(self) -> None: + """Ages every claim past its lease. + + Stands in for a producer that crashed between claiming and signaling -- + the failure the lease exists for -- without making the test wait out a + real lease. + """ + self._claims = { + key: (holder, generation, 0.0) + for key, (holder, generation, _) in self._claims.items() + } + + async def delete_for_test(self, key: StreamKey, offset: Offset) -> None: + """Removes a record, standing in for XDEL, trimming, or retention loss. + + Deletion is permitted -- what immutability forbids is *rewriting* a + record in place, which is why replay only has to detect absence. + """ + stream = self._records.get(key, []) + self._records[key] = [r for r in stream if r.offset != offset] + + def all_records(self, key: StreamKey) -> list[StreamRecord]: + return list(self._records.get(key, [])) diff --git a/tests/contrib/external_workflow_streams/observations.py b/tests/contrib/external_workflow_streams/observations.py new file mode 100644 index 0000000000..f2755b3cb1 --- /dev/null +++ b/tests/contrib/external_workflow_streams/observations.py @@ -0,0 +1,34 @@ +"""A sink Workflow code can write its own observations to. + +A Workflow's return value is only visible for the *live* run: the ``Replayer`` +runs the same code and reports whether it failed, not what it produced. So a +test that wants to compare what a predicate saw live against what it saw on +replay needs somewhere outside the Workflow to put it. + +This module is imported through :py:func:`workflow.unsafe.imports_passed_through`, +so Workflow code inside the sandbox reaches the *same* module object the test +does. Recording is append-only per Run: the live execution appends first and the +replay appends second, which is what makes the two comparable at all. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +#: ``run_id -> [observations of each execution, in the order they ran]``. +#: +#: Deliberately untyped in its element: what is worth comparing between a live +#: run and its replay differs per test -- the states a predicate saw, the +#: records that were delivered -- and narrowing it here would only push a cast +#: into every caller. +OBSERVED: dict[str, list[list[Any]]] = {} + + +def record(run_id: str, states: Sequence[Any]) -> None: + """Appends one execution's observed sequence for a Run.""" + OBSERVED.setdefault(run_id, []).append(list(states)) + + +def executions(run_id: str) -> list[list[Any]]: + return OBSERVED.get(run_id, []) diff --git a/tests/contrib/external_workflow_streams/test_annotation.py b/tests/contrib/external_workflow_streams/test_annotation.py new file mode 100644 index 0000000000..08a7265db0 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_annotation.py @@ -0,0 +1,755 @@ +"""P5 — the replay annotation codec.""" + +from __future__ import annotations + +import pytest + +import temporalio.exceptions +from temporalio.contrib.external_workflow_streams._annotation import ( + MAX_ANNOTATION_BYTES, + ROLLOVER_HIGH_WATER, + SCHEMA_VERSION, + Annotation, + AnnotationAccumulator, + AnnotationBudgetExceeded, + AnnotationDecodeError, + AnnotationHeader, + Run, + Segment, + SegmentEndReason, + StreamBinding, + decode_annotation, + encode_annotation, + encode_bindings, + encode_segment, + encode_terminal, +) +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + Offset, +) + +KEY = StreamKey("ns", "wf", "run-1", "tokens") +OTHER_KEY = StreamKey("ns", "wf", "run-1", "tool-events") + + +def binding(key: StreamKey, start_cursor: Cursor = BEGINNING) -> StreamBinding: + """A binding carrying the configured backend's provider identity.""" + return StreamBinding( + stream_key=key, + start_cursor=start_cursor, + provider_id="redis-streams", + provider_format_version=1, + ) + + +def header(streams: dict[int, StreamBinding] | None = None) -> AnnotationHeader: + return AnnotationHeader( + streams=streams if streams is not None else {1: binding(KEY)}, + ) + + +def offset(ms: int, seq: int = 0) -> Offset: + return Offset(f"{ms}-{seq}") + + +# --- round trips ------------------------------------------------------------ + + +def test_a_full_annotation_round_trips() -> None: + annotation = Annotation( + header=header({1: binding(KEY, AFTER(offset(50)))}), + segments=( + Segment( + runs=(Run(1, offset(100), offset(199), 100, (7, 42)),), + end_reason=SegmentEndReason.NO_DATA_AVAILABLE, + ), + Segment( + runs=(Run(1, offset(200), offset(205), 6),), + end_reason=SegmentEndReason.FENCE_REACHED, + ), + ), + terminal={1: AFTER(offset(205))}, + ) + + assert decode_annotation(encode_annotation(annotation)) == annotation + + +def test_schema_version_leads_the_encoding() -> None: + """Readable before anything version-dependent is interpreted.""" + encoded = encode_annotation(Annotation(header(), terminal={1: BEGINNING})) + + assert encoded[0] == SCHEMA_VERSION + assert decode_annotation(encoded).header.schema_version == SCHEMA_VERSION + + +def test_an_unknown_schema_version_is_refused_by_name() -> None: + encoded = bytearray(encode_annotation(Annotation(header(), terminal={}))) + encoded[0] = 99 + + with pytest.raises(AnnotationDecodeError, match="schema version 99"): + decode_annotation(bytes(encoded)) + + +def test_an_empty_segment_round_trips() -> None: + """An activation that drained and found nothing still ran one drain. + + Dropping it would make ``wait_condition`` predicates fire a different + number of times on replay than they did live. + """ + annotation = Annotation( + header(), + segments=(Segment(runs=(), end_reason=SegmentEndReason.NO_DATA_AVAILABLE),), + terminal={1: BEGINNING}, + ) + + decoded = decode_annotation(encode_annotation(annotation)) + + assert decoded == annotation + assert decoded.segments[0].runs == () + + +def test_an_annotation_with_no_segments_round_trips() -> None: + """A subscription to an empty stream: header and terminal carry everything.""" + annotation = Annotation(header(), segments=(), terminal={1: BEGINNING}) + + decoded = decode_annotation(encode_annotation(annotation)) + + assert decoded == annotation + assert decoded.segments == () + assert decoded.header.streams[1].start_cursor == BEGINNING + + +def test_cursors_and_offsets_stay_distinct_types() -> None: + """A boundary and a record identity are not interchangeable. + + ``AFTER(100-0)`` in the terminal and the record offset ``100-0`` in a run + must decode as a Cursor and an Offset respectively, not as each other. + """ + annotation = Annotation( + header(), + segments=( + Segment( + (Run(1, offset(100), offset(100), 1),), SegmentEndReason.BATCH_LIMIT + ), + ), + terminal={1: AFTER(offset(100))}, + ) + + decoded = decode_annotation(encode_annotation(annotation)) + + assert decoded.segments[0].runs[0].first_offset == offset(100) + assert decoded.terminal is not None + assert decoded.terminal[1] == AFTER(offset(100)) + assert decoded.terminal[1] != offset(100) + + +def test_beginning_is_not_the_same_as_after_a_zero_offset() -> None: + annotation = Annotation(header(), terminal={1: BEGINNING, 2: AFTER(Offset("0-0"))}) + + decoded = decode_annotation(encode_annotation(annotation)) + + assert decoded.terminal is not None + assert decoded.terminal[1].is_beginning + assert not decoded.terminal[2].is_beginning + + +def test_sparse_control_positions_round_trip() -> None: + run = Run(1, offset(100), offset(999), 500, (0, 250, 499)) + + decoded = decode_annotation( + encode_annotation( + Annotation( + header(), + segments=(Segment((run,), SegmentEndReason.FENCE_REACHED),), + terminal={1: AFTER(offset(999))}, + ) + ) + ) + + assert decoded.segments[0].runs[0].control_positions == (0, 250, 499) + + +@pytest.mark.parametrize( + ("positions", "count", "reason"), + [ + ((5,), 3, "outside a run"), + ((2, 1), 5, "strictly increasing"), + ((1, 1), 5, "strictly increasing"), + ], +) +def test_malformed_control_positions_are_rejected_at_construction( + positions: tuple[int, ...], count: int, reason: str +) -> None: + with pytest.raises(ValueError, match=reason): + Run(1, offset(1), offset(9), count, positions) + + +def test_a_run_covers_at_least_one_record() -> None: + with pytest.raises(ValueError, match="at least one record"): + Run(1, offset(1), offset(1), 0) + + +@pytest.mark.parametrize( + "corrupt", + [b"", b"\x01", b"\x01\x02", b"\x01\x01\x05abc"], + ids=["empty", "version-only", "wrong-first-frame", "truncated-header"], +) +def test_corrupt_bytes_raise_a_decode_error(corrupt: bytes) -> None: + with pytest.raises(AnnotationDecodeError): + decode_annotation(corrupt) + + +def test_two_terminals_are_refused() -> None: + doubled = encode_annotation(Annotation(header(), terminal={1: BEGINNING})) + doubled += doubled[doubled.index(b"\x03") :] + + with pytest.raises(AnnotationDecodeError, match="one terminal, not two"): + decode_annotation(doubled) + + +# --- runs are what keep markers small --------------------------------------- + + +def test_a_large_single_stream_batch_encodes_as_one_run() -> None: + """100,000 records, one run. This is the claim the whole design rests on.""" + annotation = Annotation( + header(), + segments=( + Segment( + (Run(1, offset(1), offset(100_000), 100_000),), + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(offset(100_000))}, + ) + + decoded = decode_annotation(encode_annotation(annotation)) + + assert len(decoded.segments[0].runs) == 1 + assert decoded.segments[0].runs[0].count == 100_000 + + +def test_encoded_size_stays_flat_as_a_single_stream_batch_grows() -> None: + """Byte size, not run count. + + A run-count assertion says nothing about what a future per-record field + would cost; measuring bytes is what actually catches one being added. + """ + + def encoded_size(count: int) -> int: + return len( + encode_annotation( + Annotation( + header(), + segments=( + Segment( + (Run(1, offset(1), offset(count), count),), + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(offset(count))}, + ) + ) + ) + + small, large = encoded_size(100), encoded_size(100_000) + + # Three orders of magnitude more records. The only growth permitted is in + # the varints holding the count and the offset tokens themselves. + assert large - small <= 16, ( + f"{small} bytes for 100 records but {large} for 100,000 -- something in " + "the encoding is per-record" + ) + + +def test_alternating_streams_cost_one_run_per_delivery() -> None: + """The honest worst case, stated as a test rather than assumed away.""" + runs = tuple(Run(1 + (i % 2), offset(i), offset(i), 1) for i in range(1, 21)) + annotation = Annotation( + header({1: binding(KEY), 2: binding(OTHER_KEY)}), + segments=(Segment(runs, SegmentEndReason.NO_DATA_AVAILABLE),), + terminal={1: AFTER(offset(19)), 2: AFTER(offset(20))}, + ) + + decoded = decode_annotation(encode_annotation(annotation)) + + assert len(decoded.segments[0].runs) == 20 + assert [r.wait_id for r in decoded.segments[0].runs[:4]] == [2, 1, 2, 1] + + +# --- accumulation: deltas concatenate into the annotation ------------------- + + +def test_a_delta_sequence_concatenates_into_the_annotation() -> None: + """What Core accumulates by byte-append must equal what encoding produces. + + Core never parses a delta, so if these two ever diverged nothing would + notice until replay read a marker that could not be decoded. + """ + head = header() + segments = ( + Segment((Run(1, offset(100), offset(110), 11),), SegmentEndReason.BATCH_LIMIT), + Segment((), SegmentEndReason.NO_DATA_AVAILABLE), + Segment( + (Run(1, offset(111), offset(120), 10),), SegmentEndReason.FENCE_REACHED + ), + ) + terminal = {1: AFTER(offset(120))} + + accumulator = AnnotationAccumulator(head) + deltas = [accumulator.add_segment(s) for s in segments] + deltas.append(accumulator.add_terminal(terminal)) + + core_would_hold = accumulator.accumulated() + + assert core_would_hold == encode_annotation(Annotation(head, segments, terminal)) + assert decode_annotation(core_would_hold) == Annotation(head, segments, terminal) + # And the header rode the accumulator's first emission, not a delta. + assert core_would_hold.endswith(b"".join(deltas)) + + +def test_the_accumulator_reports_the_size_the_marker_will_carry() -> None: + accumulator = AnnotationAccumulator(header()) + accumulator.add_segment( + Segment((Run(1, offset(1), offset(9), 9),), SegmentEndReason.BATCH_LIMIT) + ) + accumulator.add_terminal({1: AFTER(offset(9))}) + + assert accumulator.size == len(accumulator.accumulated()) + + +# --- a wait bound after the header ------------------------------------------ + + +def test_a_wait_bound_after_the_header_decodes_into_it() -> None: + """A subscription can be created at any activation of a retained task. + + The header frame for that task has usually already gone to Core by then, + and Core appends deltas rather than rewriting them, so the binding rides a + frame of its own. What replay reads is still one flat table: a wait that + joined late is indistinguishable, once decoded, from one that was there + from the start -- which is what lets `prepare_replay` resolve its backend + and stream key at all. + """ + accumulator = AnnotationAccumulator(header()) + accumulator.add_segment( + Segment((Run(1, offset(100), offset(100), 1),), SegmentEndReason.BATCH_LIMIT) + ) + accumulator.add_bindings({2: binding(OTHER_KEY)}) + accumulator.add_segment( + Segment( + (Run(2, offset(200), offset(200), 1),), SegmentEndReason.NO_DATA_AVAILABLE + ) + ) + accumulator.add_terminal({1: AFTER(offset(100)), 2: AFTER(offset(200))}) + + decoded = decode_annotation(accumulator.accumulated()) + + assert decoded.header.streams == {1: binding(KEY), 2: binding(OTHER_KEY)} + assert decoded.terminal == {1: AFTER(offset(100)), 2: AFTER(offset(200))} + assert [run.wait_id for segment in decoded.segments for run in segment.runs] == [ + 1, + 2, + ] + + +def test_a_late_binding_keeps_its_own_start_cursor() -> None: + """Not wherever the waits already in the header have reached. + + A wait registered part-way through a Workflow Task begins at the boundary + its own `subscribe()` was given. Recording the annotation-wide position + instead would start replay of that wait past records it in fact received. + """ + accumulator = AnnotationAccumulator(header({1: binding(KEY, AFTER(offset(100)))})) + accumulator.add_bindings({2: binding(OTHER_KEY)}) + accumulator.add_terminal({1: AFTER(offset(100)), 2: BEGINNING}) + + decoded = decode_annotation(accumulator.accumulated()) + + assert decoded.header.streams[1].start_cursor == AFTER(offset(100)) + assert decoded.header.streams[2].start_cursor == BEGINNING + + +def test_a_bindings_frame_is_the_header_frame_body_under_its_own_tag() -> None: + """Locked to exact bytes, like the golden annotation and for the reason. + + A second *header* frame would have been the smaller change and is what the + encoding deliberately does not do: a header frame also carries the schema + version, so a decoder meeting two of them has to decide whether the later + replaces the earlier. The distinct tag says one thing only. + """ + assert encode_bindings({2: binding(OTHER_KEY)}).hex() == "".join( + [ + "04", # bindings frame + "01", # one stream + "02", # wait id 2 + "02" + "ns".encode().hex(), + "02" + "wf".encode().hex(), + "05" + "run-1".encode().hex(), + "0b" + "tool-events".encode().hex(), + "00", # start cursor BEGINNING + "0d" + "redis-streams".encode().hex(), # provider id + "01", # provider format version 1 + ] + ) + + +def test_a_wait_bound_twice_is_refused() -> None: + """Two bindings for one wait leave replay choosing between stream keys. + + Whichever it chose could be the one the recorded records were not written + to, and the range read would then fail as integrity loss against a backend + nothing is wrong with. + """ + accumulator = AnnotationAccumulator(header()) + accumulator.add_bindings({1: binding(OTHER_KEY)}) + accumulator.add_terminal({1: BEGINNING}) + + with pytest.raises(AnnotationDecodeError, match="bound twice"): + decode_annotation(accumulator.accumulated()) + + +def test_bindings_after_the_terminal_are_refused() -> None: + """Both halves: the encoder cannot emit one and the decoder rejects one. + + The annotation ends at its terminal. A binding appended past it belongs to + a Workflow Task whose marker has already been decided, and Core would write + it into that marker rather than the next one. + """ + accumulator = AnnotationAccumulator(header()) + accumulator.add_terminal({1: BEGINNING}) + with pytest.raises(ValueError, match="after the terminal"): + accumulator.add_bindings({2: binding(OTHER_KEY)}) + + with pytest.raises(AnnotationDecodeError, match="follows the terminal"): + decode_annotation( + encode_annotation(Annotation(header(), terminal={1: BEGINNING})) + + encode_bindings({2: binding(OTHER_KEY)}) + ) + + +def test_a_bindings_frame_binds_at_least_one_wait() -> None: + """An empty frame costs bytes and says nothing, so it is a caller error.""" + accumulator = AnnotationAccumulator(header()) + with pytest.raises(ValueError, match="at least one wait"): + accumulator.add_bindings({}) + + +def test_a_segment_after_the_terminal_is_refused() -> None: + accumulator = AnnotationAccumulator(header()) + accumulator.add_terminal({1: BEGINNING}) + + with pytest.raises(ValueError, match="after the terminal"): + accumulator.add_segment(Segment((), SegmentEndReason.NO_DATA_AVAILABLE)) + + +def test_a_second_terminal_is_refused() -> None: + accumulator = AnnotationAccumulator(header()) + accumulator.add_terminal({1: BEGINNING}) + + with pytest.raises(ValueError, match="one terminal, not two"): + accumulator.add_terminal({1: BEGINNING}) + + +# --- the byte budget -------------------------------------------------------- + + +def test_rollover_is_requested_before_the_budget_is_reached() -> None: + accumulator = AnnotationAccumulator(header()) + assert not accumulator.request_rollover + + run = Run(1, offset(1), offset(2), 2) + while not accumulator.request_rollover: + accumulator.add_segment(Segment((run,), SegmentEndReason.BATCH_LIMIT)) + + assert accumulator.size >= MAX_ANNOTATION_BYTES * ROLLOVER_HIGH_WATER + # The margin is the point: the request has to reach Core and come back, so + # asking at 100% would arrive too late to prevent the overflow. + assert accumulator.size < MAX_ANNOTATION_BYTES + + +def test_an_alternating_two_stream_batch_asks_for_rollover_rather_than_overflowing() -> ( + None +): + """Bounded marker size is bought with additional Workflow Tasks. + + An alternating workload has a schedule transition per record, so it cannot + be range-compressed. The encoder must hit the high-water mark and ask for a + rollover, not grow past the budget. + """ + accumulator = AnnotationAccumulator( + header({1: binding(KEY), 2: binding(OTHER_KEY)}) + ) + + delivered = 0 + while not accumulator.request_rollover: + runs = tuple( + Run(1 + (i % 2), offset(delivered + i), offset(delivered + i), 1) + for i in range(50) + ) + delivered += 50 + accumulator.add_segment(Segment(runs, SegmentEndReason.BATCH_LIMIT)) + + assert accumulator.size <= MAX_ANNOTATION_BYTES + assert delivered > 0 + + +def test_exceeding_the_hard_budget_raises_rather_than_growing() -> None: + """The last resort, reached only if every prevention above it failed. + + The alternative is the server rejecting an oversized event, which surfaces + as an unexplained Workflow Task failure a long way from the cause. + """ + tiny = AnnotationAccumulator(header(), max_bytes=120, high_water=0.9) + + with pytest.raises(AnnotationBudgetExceeded, match="past the"): + for i in range(100): + tiny.add_segment( + Segment( + (Run(1, offset(i), offset(i + 1), 2),), SegmentEndReason.BATCH_LIMIT + ) + ) + + +def test_the_budget_error_fails_the_workflow_rather_than_the_task() -> None: + """A budget overflow may not become a Workflow Task that retries forever. + + ADR-007 rejects check-and-fail precisely because the encoding that overflowed + overflows again on the retry, and the server retries Workflow Task failures + regardless of cause: no marker, no terminal, no rollover, forever. A + non-retryable application failure is bounded and reported once instead. + """ + tiny = AnnotationAccumulator(header(), max_bytes=120, high_water=0.9) + + with pytest.raises(AnnotationBudgetExceeded) as caught: + for i in range(100): + tiny.add_segment( + Segment( + (Run(1, offset(i), offset(i + 1), 2),), SegmentEndReason.BATCH_LIMIT + ) + ) + + assert isinstance(caught.value, temporalio.exceptions.ApplicationError) + assert caught.value.non_retryable, ( + "a retryable budget overflow is the permanent Workflow Task retry loop " + "ADR-007 exists to rule out" + ) + + +def test_a_reserved_terminal_always_fits() -> None: + """The frame that closes the annotation may never be the one refused. + + An annotation Core writes with no terminal is durable and cannot be decoded + past the frame after it, so "the terminal did not fit" has to be arithmetically + impossible rather than merely unlikely. Reserving what it costs is what makes + it so -- segments are then refused *before* the space the terminal needs is + gone. + """ + blocked = {1: AFTER(offset(9)), 2: AFTER(offset(9))} + terminal_bytes = len(encode_terminal(blocked)) + + accumulator = AnnotationAccumulator(header(), max_bytes=4096, high_water=0.9) + accumulator.reserve(terminal_bytes) + + refused = False + for i in range(4096): + segment = Segment( + (Run(1, offset(i), offset(i + 1), 2),), SegmentEndReason.BATCH_LIMIT + ) + if not accumulator.fits(len(encode_segment(segment))): + refused = True + break + accumulator.add_segment(segment) + + assert refused, "the loop must have run into the reserve, or it proves nothing" + assert accumulator.headroom == 0 or accumulator.headroom < terminal_bytes + 32 + # The whole point: with the segments stopped, the terminal still goes in. + accumulator.add_terminal(blocked) + assert accumulator.size <= 4096 + decoded = decode_annotation(accumulator.accumulated()) + assert decoded.terminal == blocked + + +def test_the_reserve_is_only_spendable_by_the_frames_it_was_set_aside_for() -> None: + """A segment may not eat the terminal's space, however much room *looks* left.""" + accumulator = AnnotationAccumulator(header(), max_bytes=4096, high_water=0.9) + accumulator.reserve(4096 - accumulator.size) + + assert accumulator.headroom == 0 + with pytest.raises(AnnotationBudgetExceeded): + accumulator.add_segment( + Segment((Run(1, offset(0), offset(1), 2),), SegmentEndReason.BATCH_LIMIT) + ) + + +# --- golden file ------------------------------------------------------------ + +#: A fixed annotation and its exact bytes. Regenerating this to make a test +#: pass is the mistake it exists to catch: any change here is a wire-format +#: change, and markers already in History were written by the old encoder. +#: +#: The feature is private and unreleased, so this covers only the current format +#: and deliberately provides no compatibility fixture for earlier prototypes. +GOLDEN_ANNOTATION = Annotation( + header=AnnotationHeader( + streams={ + 1: StreamBinding( + stream_key=StreamKey("ns", "wf", "run-1", "tokens"), + start_cursor=BEGINNING, + provider_id="redis-streams", + provider_format_version=1, + ) + }, + ), + segments=( + Segment( + runs=(Run(1, Offset("100-0"), Offset("104-0"), 5, (2,)),), + end_reason=SegmentEndReason.FENCE_REACHED, + ), + Segment(runs=(), end_reason=SegmentEndReason.NO_DATA_AVAILABLE), + ), + terminal={1: AFTER(Offset("104-0"))}, +) + +GOLDEN_BYTES = bytes.fromhex( + "".join( + [ + "01", # schema version 1 + "01", # header frame + "01", # one stream + "01", # wait id 1 + "02" + "ns".encode().hex(), + "02" + "wf".encode().hex(), + "05" + "run-1".encode().hex(), + "06" + "tokens".encode().hex(), + "00", # start cursor BEGINNING + "0d" + "redis-streams".encode().hex(), # provider id + "01", # provider format version 1 + "02", # segment frame + "01", # one run + "01", # wait id 1 + "05" + "100-0".encode().hex(), + "05" + "104-0".encode().hex(), + "05", # count 5 + "01", # one control position + "02", # at relative index 2 + "03", # FENCE_REACHED + "02", # segment frame + "00", # no runs + "01", # NO_DATA_AVAILABLE + "03", # terminal frame + "01", # one entry + "01", # wait id 1 + "01", # AFTER + "05" + "104-0".encode().hex(), + ] + ) +) + + +def test_golden_bytes_are_unchanged() -> None: + """Silent format drift is what this catches. + + If this fails, the encoding changed. Bump `SCHEMA_VERSION` and keep the old + decoder working -- do not regenerate the constant. + """ + assert encode_annotation(GOLDEN_ANNOTATION).hex() == GOLDEN_BYTES.hex() + + +def test_golden_bytes_decode_to_the_golden_annotation() -> None: + assert decode_annotation(GOLDEN_BYTES) == GOLDEN_ANNOTATION + + +def test_encoded_size_stays_flat_with_sparse_control_records() -> None: + """The realistic shape, and the one where a per-record cost could hide. + + ``control_positions`` is the only field that carries per-record information + at all, which makes it the natural place for the encoding to start scaling + with record count. It stays sparse -- a handful of fence positions in a batch + of any size -- so the marker's cost must track the number of *control* + records, not the number of records. + + Measured in bytes rather than by inspecting the field, because what matters + is what reaches History. + """ + + def encoded_size(count: int) -> int: + # A fence every ~10,000 records: what a few long-running producers + # finishing at intervals actually looks like. + controls = tuple(range(0, count, max(1, count // 10))) + return len( + encode_annotation( + Annotation( + header(), + segments=( + Segment( + ( + Run( + 1, + offset(1), + offset(count), + count, + control_positions=controls, + ), + ), + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(offset(count))}, + ) + ) + ) + + small, large = encoded_size(100), encoded_size(100_000) + + # Both carry ten control positions; only the positions themselves grow, and + # only as varints. + assert large - small <= 32, ( + f"{small} bytes for 100 records but {large} for 100,000, both with ten " + "control records -- the control encoding is scaling with record count " + "rather than with control count" + ) + + +def test_encoded_size_grows_only_with_the_number_of_control_records() -> None: + """The other half: a marker *may* grow with fences, and only with them. + + Stated as a bound rather than left implicit, because "sparse" is a claim + about the producer's behaviour and this is what makes the encoding's side of + it checkable. + """ + + def encoded_size(controls: int) -> int: + return len( + encode_annotation( + Annotation( + header(), + segments=( + Segment( + ( + Run( + 1, + offset(1), + offset(1000), + 1000, + control_positions=tuple(range(controls)), + ), + ), + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(offset(1000))}, + ) + ) + ) + + per_control = encoded_size(101) - encoded_size(1) + + assert 100 <= per_control <= 300, ( + f"100 extra control records cost {per_control} bytes; a sparse position " + "list should cost a small constant each" + ) diff --git a/tests/contrib/external_workflow_streams/test_api.py b/tests/contrib/external_workflow_streams/test_api.py new file mode 100644 index 0000000000..cb257cb0f9 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_api.py @@ -0,0 +1,846 @@ +"""P9 — the Workflow-facing API and its wait-id assignment.""" + +from __future__ import annotations + +import asyncio +import inspect +from datetime import timedelta +from typing import Any + +import pytest + +import temporalio.converter +from temporalio.contrib.external_workflow_streams import _api +from temporalio.contrib.external_workflow_streams._api import ( + DEFAULT_IDLE_TIMEOUT, + MAX_RECORDS_PER_ACTIVATION, + ExternalStreamOptions, + ExternalStreamSubscription, + ExternalStreamTopic, + _install_runtime, + external_stream, + merge, +) +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._errors import ( + ConcurrentStreamConsumerError, + StreamDecodeError, +) +from temporalio.contrib.external_workflow_streams._manager import PreparedRecord +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + Cursor, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._wake import channel_for + + +class FakeRuntime: + """The Worker's half, standing in for the real manager handle. + + Deliberately only offers what the protocol names -- no provider instance, + no way to reach one -- because that is the point of the boundary. + """ + + def __init__(self) -> None: + self.registrations: list[tuple[int, StreamKey]] = [] + #: `wait_id -> configured idle timeout`, so a test can see that the + #: value `with_options` was given actually reached the Worker. + self.idle_timeouts: dict[int, timedelta] = {} + self.start_cursors: dict[int, Cursor | None] = {} + self.buffers: dict[int, list[StreamRecord]] = {} + self.deliveries: list[tuple[int, StreamRecord]] = [] + self.consumed: list[tuple[int, StreamRecord]] = [] + self.blocked: list[tuple[int, bool]] = [] + #: The wait ids `close()` asked the Worker to stop serving. + self.unsubscribed: list[int] = [] + self.pending: dict[int, asyncio.Future[None]] = {} + self.budget = MAX_RECORDS_PER_ACTIVATION + #: Overrides what `codec_for` hands back, so a test can control when -- + #: and whether -- decoding a record succeeds. + self.codec: Any = None + #: Offsets handed out by `drain`, so buffered records come back in the + #: read-path shape whatever a test put in. + self._placed = 0 + + def stream_key(self, stream_name: str) -> StreamKey: + return StreamKey("ns", "wf", "first-run", stream_name) + + def register( + self, + *, + wait_id: int, + stream_key: StreamKey, + idle_timeout: timedelta, + start_cursor: Cursor | None = None, + start_at_tail: Any = None, + ) -> None: + self.registrations.append((wait_id, stream_key)) + self.idle_timeouts[wait_id] = idle_timeout + # A tail start stands in for the cursor the Worker would resolve. + self.start_cursors[wait_id] = ( + start_cursor if start_at_tail is None else start_at_tail + ) + + def drain(self, wait_id: int, max_records: int | None = None) -> list[StreamRecord]: + buffered = self.buffers.get(wait_id, []) + if max_records is not None: + buffered, self.buffers[wait_id] = ( + buffered[:max_records], + buffered[max_records:], + ) + else: + self.buffers[wait_id] = [] + # A provider places a record when it appends it, so nothing without an + # offset ever reaches a reader. Place what a test left bare rather than + # let the fake deliver a shape the real path cannot produce. + placed = [] + for record in buffered: + if record.offset is not None: + placed.append(record) + continue + self._placed += 1 + at = record.placed_at(Offset(f"{self._placed:020d}")) + if isinstance(record, PreparedRecord): + at = PreparedRecord.of( + at, record.prepared_payload, record.prepare_error + ) + placed.append(at) + return placed + + def delivery_budget_remaining(self) -> int: + return self.budget + + def record_consumption(self, wait_id: int, record: StreamRecord) -> None: + self.consumed.append((wait_id, record)) + + def codec_for( + self, + value_type: type | None, + wait_id: int, # pyright: ignore[reportUnusedParameter] + ) -> StreamPayloadCodec[Any]: + if self.codec is not None: + return self.codec + return StreamPayloadCodec( + temporalio.converter.DataConverter.default, value_type + ) + + def new_readiness_future(self) -> asyncio.Future[None]: + return asyncio.get_event_loop().create_future() + + def record_delivery(self, wait_id: int, record: StreamRecord) -> None: + self.deliveries.append((wait_id, record)) + # Charged where the real runtime charges it: the drain that moves a record + # into a ready list is the reservation, not the consumption that follows. + self.budget = max(0, self.budget - 1) + + def note_blocked(self, wait_id: int, blocked: bool) -> None: + self.blocked.append((wait_id, blocked)) + + def unsubscribe(self, wait_id: int) -> None: + self.unsubscribed.append(wait_id) + + def register_pending(self, wait_id: int, future: asyncio.Future[None]) -> None: + self.pending[wait_id] = future + + def discard_pending(self, wait_id: int) -> None: + self.pending.pop(wait_id, None) + + +class FakeInstance: + """Stands in for the user's Workflow object, which holds the per-Run state.""" + + +@pytest.fixture +def workflow_instance(monkeypatch: pytest.MonkeyPatch) -> FakeInstance: + instance = FakeInstance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: instance, + ) + return instance + + +@pytest.fixture +def runtime(workflow_instance: FakeInstance) -> FakeRuntime: + runtime = FakeRuntime() + _install_runtime(workflow_instance, runtime) + return runtime + + +# --- options ------------------------------------------------------------------ + + +def test_the_default_idle_timeout_is_one_second() -> None: + assert external_stream.idle_timeout == DEFAULT_IDLE_TIMEOUT == timedelta(seconds=1) + + +def test_with_options_returns_a_copy() -> None: + """The module-level entry point must not be mutated by configuring it.""" + configured = external_stream.with_options(idle_timeout=timedelta(seconds=5)) + + assert configured.idle_timeout == timedelta(seconds=5) + assert external_stream.idle_timeout == timedelta(seconds=1) + assert isinstance(configured, ExternalStreamOptions) + + +@pytest.mark.parametrize("bad", [timedelta(0), timedelta(seconds=-1)]) +def test_a_non_positive_idle_timeout_is_rejected(bad: timedelta) -> None: + """A configuration error, not a request to park immediately.""" + with pytest.raises(ValueError, match="must be positive"): + external_stream.with_options(idle_timeout=bad) + + +def test_topics_inherit_their_options( + runtime: FakeRuntime, # pyright: ignore[reportUnusedParameter] +) -> None: + configured = external_stream.with_options(idle_timeout=timedelta(seconds=3)) + + subscription = configured.topic("tokens").subscribe() + + assert subscription.idle_timeout == timedelta(seconds=3) + + +def test_a_subscription_may_name_the_boundary_it_starts_after( + runtime: FakeRuntime, +) -> None: + """A named boundary reaches the runtime; an unnamed one leaves the default to it.""" + boundary = AFTER(Offset("1700000000000-3")) + + seeded = external_stream.topic("tokens").subscribe(start_cursor=boundary) + default = external_stream.topic("tokens").subscribe() + + assert runtime.start_cursors[seeded.wait_id] == boundary + assert runtime.start_cursors[default.wait_id] is None + + +# --- topics ------------------------------------------------------------------- + + +def test_a_workflow_never_sees_the_configured_backend(runtime: FakeRuntime) -> None: + topic = external_stream.topic("tokens", type=str) + + subscription = topic.subscribe() + + assert isinstance(subscription, ExternalStreamSubscription) + assert runtime.registrations == [(1, StreamKey("ns", "wf", "first-run", "tokens"))] + + +def test_a_topic_needs_a_name() -> None: + with pytest.raises(ValueError, match="non-empty name"): + external_stream.topic("") + + +def test_subscribing_without_a_configured_worker_says_so( + workflow_instance: FakeInstance, # pyright: ignore[reportUnusedParameter] +) -> None: + with pytest.raises(RuntimeError, match="external_stream_backend"): + external_stream.topic("tokens").subscribe() + + +# --- wait id assignment ------------------------------------------------------- + + +def test_wait_ids_are_assigned_from_one_in_subscribe_call_order( + runtime: FakeRuntime, +) -> None: + first = external_stream.topic("a").subscribe() + second = external_stream.topic("b").subscribe() + third = external_stream.topic("c").subscribe() + + assert [first.wait_id, second.wait_id, third.wait_id] == [1, 2, 3] + assert [wait_id for wait_id, _ in runtime.registrations] == [1, 2, 3] + + +def test_wait_ids_reproduce_across_two_runs_of_the_same_code( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Replay depends on this exactly as it depends on timer sequence numbers. + + A renumbered wait produces an annotation mismatch rather than a silently + different stream result -- but only because the numbering is reproducible + in the first place. + """ + + def run_once() -> list[int]: + instance = FakeInstance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: instance, + ) + _install_runtime(instance, FakeRuntime()) + return [ + external_stream.topic(name).subscribe().wait_id + for name in ("tokens", "tool-events", "tokens") + ] + + assert run_once() == run_once() == [1, 2, 3] + + +def test_two_subscriptions_to_one_stream_get_distinct_wait_ids( + runtime: FakeRuntime, +) -> None: + """They are two independent waits, each with its own cursor and park intent. + + Delivery is broadcast, so each sees every record from its own position -- + which is only expressible if they are numbered apart. + """ + topic = external_stream.topic("tokens") + + first = topic.subscribe() + second = topic.subscribe() + + assert first.wait_id != second.wait_id + assert first.stream_key == second.stream_key + assert [wait_id for wait_id, _ in runtime.registrations] == [1, 2] + + +def test_a_second_run_restarts_the_counter(monkeypatch: pytest.MonkeyPatch) -> None: + """Per-Run state lives on the instance, so an evicted Run takes it with it. + + A module global would outlive the Run and hand its wait ids to the next one. + """ + first_instance = FakeInstance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: first_instance, + ) + _install_runtime(first_instance, FakeRuntime()) + external_stream.topic("tokens").subscribe() + + second_instance = FakeInstance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: second_instance, + ) + _install_runtime(second_instance, FakeRuntime()) + + assert external_stream.topic("tokens").subscribe().wait_id == 1 + + +# --- iteration ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_iteration_yields_decoded_values_from_the_buffer( + runtime: FakeRuntime, +) -> None: + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode(v), "s", i) + for i, v in enumerate(["a", "b"]) + ] + + seen = [] + async for value in subscription: + seen.append(value) + if len(seen) == 2: + break + + assert seen == ["a", "b"] + + +@pytest.mark.asyncio +async def test_control_records_are_never_yielded(runtime: FakeRuntime) -> None: + """A fence advances the cursor but is the runtime's, not the Workflow's.""" + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode("a"), "s", 0), + StreamRecord(RecordKind.WRITE_FENCE, b"", "s", 1), + StreamRecord(RecordKind.DATA, await codec.encode("b"), "s", 2), + ] + + seen = [] + async for value in subscription: + seen.append(value) + if len(seen) == 2: + break + + assert seen == ["a", "b"] + + +@pytest.mark.asyncio +async def test_an_empty_buffer_blocks_on_a_readiness_future( + runtime: FakeRuntime, # pyright: ignore[reportUnusedParameter] +) -> None: + """Iteration never polls the backend; only Core can say when to look again.""" + subscription = external_stream.topic("tokens", type=str).subscribe() + + iterator = subscription.__aiter__() + pending = asyncio.ensure_future(iterator.__anext__()) + await asyncio.sleep(0.05) + + assert not pending.done(), "iteration must block rather than spin on the backend" + pending.cancel() + try: + await pending + except asyncio.CancelledError: + pass + + +def test_closing_a_subscription_unsubscribes_it(runtime: FakeRuntime) -> None: + """The Workflow side going quiet is only half of closing. + + Everything the Workflow itself can observe is already right without this + call -- iteration ends, the wait leaves the blocked set, undelivered records + stay unconsumed -- so nothing on this side of the boundary would notice it + missing. What is left behind is on the Worker: a watcher still prefetching + into a buffer nobody will drain, and a park intent still in the backend for + every producer that asks. + """ + subscription = external_stream.topic("tokens", type=str).subscribe() + + subscription.close() + + assert runtime.unsubscribed == [subscription.wait_id], ( + "closing left the Worker serving a wait no Workflow code is reading" + ) + # Idempotent, like the rest of `close`: the ordinary shape is a `finally` + # that cannot know whether the iterator already ended, and by a second call + # the Worker has already dropped this wait. + subscription.close() + assert runtime.unsubscribed == [subscription.wait_id] + + +# --- one subscription, one consumer ------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_second_coroutine_waiting_on_one_subscription_is_refused( + runtime: FakeRuntime, +) -> None: + """Sharing a single-slot wait between two waiters strands one of them forever. + + ``__aiter__`` returns a new generator every time, but everything a blocked + wait is found through is the subscription's: one ``_pending_future`` and one + entry in the runtime's pending map, keyed by wait id. A second waiter + therefore *replaced* the first in both places -- readiness resolved only the + newer one, its ``finally`` removed the map entry, and the older future became + unreachable by the readiness activation and by ``close()`` alike. Observed as + a coroutine still pending after a record had been buffered, readiness + resolved, and the subscription closed: permanently stuck, with the shared + blocked flag saying the wait was not even blocked. + """ + subscription = external_stream.topic("tokens", type=str).subscribe() + + first = asyncio.ensure_future(subscription.__aiter__().__anext__()) + await asyncio.sleep(0.05) + assert subscription.wait_id in runtime.pending, ( + "the first consumer has to be registered and blocked before a second one " + "can overwrite it" + ) + + with pytest.raises(ConcurrentStreamConsumerError, match="single consumer"): + await subscription.__aiter__().__anext__() + + # The refusal changed nothing about the consumer that was already there. + assert runtime.pending[subscription.wait_id] is subscription._pending_future + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode("a"), "s", 0) + ] + runtime.pending[subscription.wait_id].set_result(None) + + assert await asyncio.wait_for(first, 1) == "a" + assert not runtime.pending, "the resolved wait was left registered" + assert subscription._pending_future is None + + +@pytest.mark.asyncio +async def test_iterating_again_after_the_first_consumer_stopped_is_allowed( + runtime: FakeRuntime, +) -> None: + """The shape the refusal must not catch, which is why it is not on the iterator. + + Taking a few records, doing something else, and coming back to the same + subscription is ordinary code, and a ``break`` leaves the generator *suspended* + rather than closed -- so a guard that claimed the subscription for an iterator + could not tell that shape from two live consumers, and would refuse it. + """ + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode(value), "s", i) + for i, value in enumerate(["a", "b"]) + ] + + async for value in subscription: + assert value == "a" + break + + # The abandoned generator is still suspended at its `yield`; nothing closed + # it. A second pass must still work, and must resume rather than restart. + async for value in subscription: + assert value == "b" + break + + +@pytest.mark.asyncio +async def test_a_merge_cannot_take_a_wait_another_consumer_is_blocked_on( + runtime: FakeRuntime, +) -> None: + """``merge()`` registers the same single slot, so it is the same defect. + + And it fails *before* registering anything, because a merge that refused + half-way would leave its earlier waits blocked with no coroutine behind them + -- which is the state that asks Core to retain a Workflow Task for nobody. + """ + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + + solo = asyncio.ensure_future(second.__aiter__().__anext__()) + await asyncio.sleep(0.05) + assert set(runtime.pending) == {second.wait_id} + + async def consume_merged() -> None: + async for _ in merge(first, second): + pass + + with pytest.raises(ConcurrentStreamConsumerError): + await consume_merged() + + assert set(runtime.pending) == {second.wait_id}, ( + "the refused merge left a wait registered, or took the one that was " + "already blocked" + ) + assert first._pending_future is None + solo.cancel() + try: + await solo + except asyncio.CancelledError: + pass + + +# --- names -------------------------------------------------------------------- + + +def test_no_exported_name_collides_with_the_shipped_contrib_feature() -> None: + """The two coexist and are mirror images; nothing here may shadow that one.""" + import temporalio.contrib.external_workflow_streams as package + + for module in (package, _api): + for name in dir(module): + assert not name.startswith("__temporal_workflow_stream"), ( + f"{module.__name__}.{name} collides with the reserved namespace of " + "temporalio.contrib.workflow_streams" + ) + + +def test_the_reserved_instance_attribute_is_in_this_features_namespace() -> None: + assert _api._RUN_STATE_ATTR.startswith("__temporal_external_stream") + assert not _api._RUN_STATE_ATTR.startswith("__temporal_workflow_stream") + + +def test_the_handle_types_are_named_as_the_design_says() -> None: + assert ExternalStreamTopic.__name__ == "ExternalStreamTopic" + assert not hasattr(ExternalStreamTopic, "publish") + assert hasattr(ExternalStreamTopic, "subscribe") + + +def test_the_package_exports_both_directions_without_merging_their_roles() -> None: + import temporalio.contrib.external_workflow_streams as package + + assert { + "ExternalOutputStreamClient", + "ExternalOutputStreamProducer", + "ExternalOutputStreamTopic", + "OutputStreamBackend", + "StreamDirection", + "external_output_stream", + } <= set(package.__all__) + assert not hasattr(package.ExternalOutputStreamTopic, "subscribe") + assert hasattr(package.ExternalOutputStreamTopic, "publish") + + +@pytest.mark.asyncio +async def test_a_record_buffered_while_not_iterating_is_still_delivered( + runtime: FakeRuntime, +) -> None: + """The readiness for it was reported to nobody, and none is coming. + + A record that arrives while Workflow code is doing something else -- a + timer, an activity, another stream -- is buffered and its readiness is + reported and consumed immediately. By the time the Workflow comes back and + asks for the next value, no further notification will ever be sent: the + watcher has moved its prefetch cursor past that record. Blocking without + looking at the buffer first strands the Workflow forever on a record that is + already sitting in front of it. + """ + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode("first"), "s", 0) + ] + + iterator = subscription.__aiter__() + assert await iterator.__anext__() == "first" + + # Arrives while the Workflow is off doing something else. Nothing resolves a + # readiness future for it, because nothing was waiting when it landed. + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode("second"), "s", 1) + ] + + assert await asyncio.wait_for(iterator.__anext__(), 1) == "second", ( + "the iterator blocked on a readiness notification that had already been " + "spent, with the record buffered in front of it" + ) + + +@pytest.mark.asyncio +async def test_a_record_buffered_after_blocking_begins_still_resolves( + runtime: FakeRuntime, +) -> None: + """The other side of the window, which must keep working. + + Looking at the buffer before blocking must not replace the readiness path: + a record that genuinely arrives later has no buffered copy to find, and the + future is what delivers it. + """ + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + subscription = external_stream.topic("tokens", type=str).subscribe() + + iterator = subscription.__aiter__() + pending = asyncio.ensure_future(iterator.__anext__()) + await asyncio.sleep(0.05) + assert not pending.done() + + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode("late"), "s", 0) + ] + runtime.pending[subscription.wait_id].set_result(None) + + assert await asyncio.wait_for(pending, 1) == "late" + + +# --- consumption is committed only once a value exists ------------------------ + + +class FailingCodec: + """A converter mismatch: the stream is fine, the configuration is not. + + Fails in :meth:`convert`, the half that runs on the Workflow thread, since + that is where a type hint meets a payload the producer did not write. + """ + + def __init__(self, value_type: type | None = str) -> None: + self._inner: StreamPayloadCodec[Any] = StreamPayloadCodec( + temporalio.converter.DataConverter.default, value_type + ) + self.calls = 0 + + def parse_unprepared(self, payload: bytes) -> Any: + return self._inner.parse_unprepared(payload) + + def convert( + self, + prepared: Any, # pyright: ignore[reportUnusedParameter] + ) -> Any: + self.calls += 1 + raise RuntimeError("this converter cannot read this payload") + + +@pytest.mark.asyncio +async def test_decoding_on_the_workflow_thread_cannot_suspend( + runtime: FakeRuntime, +) -> None: + """The Workflow thread converts; it does not await a converter. + + ``DataConverter.decode`` is retrieval, then a user ``PayloadCodec``, then + ``from_payloads``. The first two are arbitrary asynchronous work -- a + network fetch, a KMS round trip -- and the Worker awaits them on its own + loop before a record is buffered, exactly as it does for every other payload + an activation carries. What is left here is the third, which needs the + topic's type and performs no I/O. + + Asserted structurally as well as behaviourally, because the property is + structural: a coroutine on this path is a suspension point inside + ``activate()``, and a suspension point is where I/O, a synthesized Workflow + command, or a deadlock timeout can appear. + """ + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode("a"), "s", 0) + ] + + assert not inspect.iscoroutinefunction(ExternalStreamSubscription._decode), ( + "decoding a record awaits on the Workflow thread, so a payload codec " + "or an external payload fetch runs inside activate()" + ) + + record = runtime.buffers[subscription.wait_id][0] + subscription._fill() + assert subscription._decode(record) == "a" + + +@pytest.mark.asyncio +async def test_a_cancelled_delivery_leaves_the_record_unconsumed( + runtime: FakeRuntime, +) -> None: + """Consumption is a claim about what Workflow code received. + + Decoding no longer suspends, so the one place a cancellation can still land + between a record arriving and Workflow code receiving it is the readiness + wait. A cancellation there -- a stream raced against a timer, most often -- + must leave the record exactly where a later pass finds it: recording + consumption for a value nothing yielded makes the claim false in the case + that matters, because the consumption cursor is what a Continue-As-New + successor resumes from. + """ + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + subscription = external_stream.topic("tokens", type=str).subscribe() + + iterator = subscription.__aiter__() + pending = asyncio.ensure_future(iterator.__anext__()) + # Blocked: nothing is buffered yet, and only Core can say when to look. + await asyncio.sleep(0.05) + assert not pending.done() + + # Buffered while the wait is outstanding, then cancelled before the + # readiness that would have delivered it. + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode("a"), "s", 0) + ] + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + + assert runtime.consumed == [], ( + "a cancelled wait recorded a consumption, so the cursor claims the " + "Workflow received a record it never saw" + ) + + # And it is still there to be taken. + assert await asyncio.wait_for(subscription.__aiter__().__anext__(), 1) == "a" + assert [wait_id for wait_id, _ in runtime.consumed] == [subscription.wait_id] + + +@pytest.mark.asyncio +async def test_a_failed_decode_leaves_the_record_unconsumed( + runtime: FakeRuntime, +) -> None: + """The same ordering, reached by the failure a converter mismatch produces. + + The budget is spent on the record either way; the question is whether the + Run also records having handed it over. + """ + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + runtime.codec = FailingCodec() + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await codec.encode("a"), "s", 0) + ] + + with pytest.raises(StreamDecodeError, match="could not be decoded"): + await subscription.__aiter__().__anext__() + + assert runtime.consumed == [], ( + "a record whose decode raised was marked consumed, so the value is lost " + "and the cursor claims the Workflow received it" + ) + + +@pytest.mark.asyncio +async def test_a_preparation_failure_is_raised_where_the_record_would_arrive( + runtime: FakeRuntime, +) -> None: + """A codec that fails on the Worker's loop still fails *this* Workflow. + + Preparation happens in the watcher, which has no Workflow Task to fail and + no Workflow to tell -- and which may be preparing a record the Workflow + never asks for. So the failure travels with the record and is raised by the + delivery that would have yielded its value, as the taxonomy's third row: + the bytes are intact and the consumer's converter cannot read them. + + Raised, and *not* consumed: nothing was received, so the cursor must not say + otherwise. + """ + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.buffers[subscription.wait_id] = [ + PreparedRecord.of( + StreamRecord(RecordKind.DATA, await codec.encode("a"), "s", 0), + None, + RuntimeError("the codec could not decrypt this payload"), + ) + ] + + with pytest.raises(StreamDecodeError, match="could not be decoded") as caught: + await subscription.__aiter__().__anext__() + assert isinstance(caught.value.__cause__, RuntimeError) + + assert runtime.consumed == [], ( + "a record whose preparation failed was marked consumed, so the cursor " + "claims the Workflow received a value that never existed" + ) + + +@pytest.mark.asyncio +async def test_an_unprepared_record_is_refused_rather_than_decoded_late( + runtime: FakeRuntime, +) -> None: + """A codec-bearing converter has an asynchronous half that must have run. + + If a record reaches the Workflow thread without it, running it here is the + defect this split exists to remove, and converting the raw bytes anyway + would hand Workflow code whatever the codec's output happens to look like. + Refused instead -- and refused as a decode failure, because that is what the + Workflow can be told. + """ + + class NeverDecodes(temporalio.converter.PayloadCodec): + async def encode(self, payloads: Any) -> Any: + return list(payloads) + + async def decode(self, payloads: Any) -> Any: + raise AssertionError("the Workflow thread ran a payload codec") + + plain = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + runtime.codec = StreamPayloadCodec( + temporalio.converter.DataConverter(payload_codec=NeverDecodes()), str + ) + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.buffers[subscription.wait_id] = [ + StreamRecord(RecordKind.DATA, await plain.encode("a"), "s", 0) + ] + + with pytest.raises(StreamDecodeError, match="could not be decoded"): + await subscription.__aiter__().__anext__() + assert runtime.consumed == [] + + +# --- the stream's channel ----------------------------------------------------- + + +def test_a_subscription_asks_the_run_to_listen_on_its_streams_channel( + runtime: FakeRuntime, workflow_instance: FakeInstance +) -> None: + """Every ``subscribe()`` asks; the Run keeps it to one command per channel.""" + asked: list[object] = [] + + def subscribe_channel(channel: object) -> bool: + asked.append(channel) + return True + + workflow_instance.subscribe_stream_channel = subscribe_channel # type: ignore[attr-defined] + + external_stream.topic("tokens").subscribe() + external_stream.topic("tokens").subscribe() + external_stream.topic("events").subscribe() + + tokens = channel_for(runtime.stream_key("tokens")) + events = channel_for(runtime.stream_key("events")) + assert asked == [tokens, tokens, events] + + +def test_a_run_that_cannot_listen_still_subscribes(runtime: FakeRuntime) -> None: + # The bare FakeInstance has no subscribe_stream_channel, like a Run object + # that predates the channel; the subscription itself is unaffected. + assert external_stream.topic("tokens").subscribe().wait_id == 1 + assert runtime.registrations[0][0] == 1 diff --git a/tests/contrib/external_workflow_streams/test_backend_conformance.py b/tests/contrib/external_workflow_streams/test_backend_conformance.py new file mode 100644 index 0000000000..9adf60d3aa --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_backend_conformance.py @@ -0,0 +1,318 @@ +"""P2 — the conformance suite, and proof that it fails a broken backend. + +The first half runs every check against the conforming reference backend. The +second half runs a *single* check against a stub broken in exactly one way and +asserts it fails with the message naming that obligation. A suite that only +ever passes is not evidence of anything. +""" + +from __future__ import annotations + +import uuid +from collections.abc import AsyncGenerator +from datetime import timedelta +from typing import cast + +import pytest + +from temporalio.contrib.external_workflow_streams._backend import ( + StreamBackend, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + Cursor, + Offset, + RecordKind, + StreamRecord, +) +from tests.contrib.external_workflow_streams import conformance +from tests.contrib.external_workflow_streams.conformance import ( + CONFORMANCE_CHECKS, + Check, +) +from tests.contrib.external_workflow_streams.memory_backend import ( + MemoryStreamBackend, + parse, +) + + +@pytest.fixture +def stream_key() -> StreamKey: + return StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +# --- the reference backend conforms ----------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("check", CONFORMANCE_CHECKS, ids=lambda c: c.__name__) +async def test_reference_backend_conforms( + check: Check, backend: MemoryStreamBackend, stream_key: StreamKey +) -> None: + await check(backend, stream_key) + + +def test_the_suite_is_not_silently_empty() -> None: + assert len(CONFORMANCE_CHECKS) >= 16 + assert len({c.__name__ for c in CONFORMANCE_CHECKS}) == len(CONFORMANCE_CHECKS) + + +# --- deliberately broken backends ------------------------------------------- + + +class ExclusiveRangeBackend(MemoryStreamBackend): + """Implements the inclusive range read with exclusive-of-first semantics. + + The realistic mistake: reaching for the same primitive the watch uses. + """ + + async def read_range( + self, key: StreamKey, first: Offset, last: Offset + ) -> list[StreamRecord]: + got = await super().read_range(key, first, last) + return [r for r in got if r.offset != first] + + +class LexicalOffsetBackend(MemoryStreamBackend): + """Compares offsets as strings.""" + + def compare_offsets(self, left: Offset, right: Offset) -> int: + return (left.token > right.token) - (left.token < right.token) + + +class NameableCursorBackend(MemoryStreamBackend): + """Validates the resume token against a record that must follow it. + + Stands for a provider whose cursor is a record identity rather than a + boundary: a position with nothing after it is not a position it can + express, so a consumer that has drained to the tail has nothing legal to + hold. This is the state parking puts every subscription in, which is why + the contract forbids it. + """ + + async def read_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = None, + ) -> list[StreamRecord]: + if not after.is_beginning: + bound = parse(after.offset) # type: ignore[arg-type] + if not any(parse(r.offset) > bound for r in self.all_records(key)): # type: ignore[arg-type] + raise ValueError( + f"cursor {after.offset} names no position: nothing follows it" + ) + return await super().read_after( + key, after, max_records=max_records, block=block + ) + + +class KeyOnlyIdempotencyBackend(MemoryStreamBackend): + """Idempotent on the key alone, ignoring whether the bytes match.""" + + async def append(self, key: StreamKey, record: StreamRecord) -> StreamRecord: + existing = self._by_key.get((key, record.idempotency_key)) + if existing is not None: + return existing + return await super().append(key, record) + + +class InclusiveWatchBackend(MemoryStreamBackend): + """Re-delivers the record its boundary names.""" + + async def read_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = None, + ) -> list[StreamRecord]: + if after.is_beginning: + return await super().read_after( + key, after, max_records=max_records, block=block + ) + bound = parse(after.offset) # type: ignore[arg-type] + return [r for r in self.all_records(key) if parse(r.offset) >= bound][ # type: ignore[arg-type] + :max_records + ] + + +class CountingRangeBackend(MemoryStreamBackend): + """Reads "however many records fit between the endpoints, starting at first". + + The realistic mistake: treating ``last`` as a length hint rather than as a + closing endpoint, so a gap is silently filled from beyond the range and a + trimmed stream reads back with the right count. + """ + + async def read_range( + self, key: StreamKey, first: Offset, last: Offset + ) -> list[StreamRecord]: + want = len( + [ + r + for r in self.all_records(key) + if parse(first) <= parse(r.offset) <= parse(last) # type: ignore[arg-type] + ] + ) + # Whatever was asked for, deliver `want` records starting at `first` -- + # reaching past `last` when the interval has holes in it. + from_first = [ + r + for r in self.all_records(key) + if parse(r.offset) >= parse(first) # type: ignore[arg-type] + ] + return from_first[: max(want, 3)] + + +class UndeclaredImmutabilityBackend(MemoryStreamBackend): + """Never considered the question.""" + + guarantees_immutability = None + + +class DeniedImmutabilityBackend(MemoryStreamBackend): + """Considered it and cannot make the guarantee.""" + + guarantees_immutability = False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("broken", "check", "reason"), + [ + pytest.param( + ExclusiveRangeBackend, + conformance.check_range_read_includes_both_endpoints, + "inclusive of both endpoints", + id="exclusive-range-read", + ), + pytest.param( + LexicalOffsetBackend, + conformance.check_offsets_are_not_compared_lexically, + "lexically", + id="lexical-offset-comparison", + ), + pytest.param( + InclusiveWatchBackend, + conformance.check_watch_is_exclusive_of_its_boundary, + "strictly after", + id="inclusive-watch", + ), + pytest.param( + KeyOnlyIdempotencyBackend, + conformance.check_reappend_with_different_bytes_is_rejected, + "must raise AppendConflictError", + id="idempotent-on-key-alone", + ), + pytest.param( + CountingRangeBackend, + conformance.check_range_read_reports_a_missing_record_as_missing, + "rather than substituting another", + id="substitutes-for-a-deleted-record", + ), + pytest.param( + UndeclaredImmutabilityBackend, + conformance.check_immutability_is_declared, + "must declare guarantees_immutability", + id="immutability-undeclared", + ), + pytest.param( + DeniedImmutabilityBackend, + conformance.check_immutability_is_declared, + "must declare guarantees_immutability", + id="immutability-denied", + ), + ], +) +async def test_a_broken_backend_fails_for_the_right_reason( + broken: type[StreamBackend], check: Check, reason: str, stream_key: StreamKey +) -> None: + with pytest.raises(AssertionError, match=reason): + await check(broken(), stream_key) + + +@pytest.mark.asyncio +async def test_a_backend_needing_a_nameable_cursor_fails_the_tail_check( + stream_key: StreamKey, +) -> None: + """Fails, though not with an assertion -- it cannot express the state at all. + + Worth its own case: the failure mode is "the provider refuses the cursor", + not "the provider returned the wrong records", and a suite that only caught + AssertionError would let it through. + """ + with pytest.raises((AssertionError, ValueError)): + await conformance.check_a_consumer_parked_at_the_tail_resumes( + NameableCursorBackend(), stream_key + ) + + +# --- the derived helpers on the ABC ----------------------------------------- + + +@pytest.mark.asyncio +async def test_watch_iterates_across_batches( + backend: MemoryStreamBackend, stream_key: StreamKey +) -> None: + """The cursor advances past what was delivered, not to the current tail.""" + for i, payload in enumerate([b"a", b"b", b"c"]): + await backend.append(stream_key, StreamRecord(RecordKind.DATA, payload, "s", i)) + + seen = [] + watcher = cast( + AsyncGenerator[StreamRecord, None], + backend.watch( + stream_key, Cursor(), max_records=1, block=timedelta(milliseconds=10) + ), + ) + async for record in watcher: + seen.append(record.payload) + if len(seen) == 3: + break + await watcher.aclose() + + assert seen == [b"a", b"b", b"c"] + + +@pytest.mark.asyncio +async def test_strictly_increasing_uses_the_provider_comparator( + backend: MemoryStreamBackend, +) -> None: + assert backend.strictly_increasing([Offset("9-0"), Offset("10-0")]) + assert not backend.strictly_increasing([Offset("10-0"), Offset("9-0")]) + assert not backend.strictly_increasing([Offset("9-0"), Offset("9-0")]) + + +@pytest.mark.asyncio +async def test_read_after_blocks_until_a_record_arrives( + backend: MemoryStreamBackend, stream_key: StreamKey +) -> None: + """Blocking is the provider's, so latency never reaches the Workflow thread.""" + import asyncio + + placed = await backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"a", "s", 0) + ) + assert placed.offset is not None + + async def append_soon() -> None: + await asyncio.sleep(0.05) + await backend.append(stream_key, StreamRecord(RecordKind.DATA, b"b", "s", 1)) + + task = asyncio.ensure_future(append_soon()) + got = await backend.read_after( + stream_key, AFTER(placed.offset), max_records=10, block=timedelta(seconds=2) + ) + await task + + assert [r.payload for r in got] == [b"b"] diff --git a/tests/contrib/external_workflow_streams/test_bridge.py b/tests/contrib/external_workflow_streams/test_bridge.py new file mode 100644 index 0000000000..a383741d5e --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_bridge.py @@ -0,0 +1,221 @@ +"""P7 — the readiness call and the run-status probe, reachable from Python. + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +from __future__ import annotations + +import asyncio +import threading +import uuid + +import pytest + +from temporalio import workflow +from temporalio.bridge.worker import ( + ExternalStreamReadyResult, + ExternalStreamRunStatus, +) +from temporalio.client import Client +from temporalio.worker import Worker + + +@workflow.defn +class IdleWorkflow: + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: False) + + +def bridge_worker(worker: Worker): # type: ignore[no-untyped-def] + """The bridge worker underneath a Python Worker.""" + return worker._workflow_worker._bridge_worker() # type: ignore[union-attr] + + +async def _wait_for_first_task_completed(handle, timeout: float = 30) -> None: # type: ignore[no-untyped-def] + """Waits until this Worker has run the Workflow's first task and cached the Run. + + Polled rather than slept: how long a Worker takes to poll, execute, and + complete one Workflow Task is a property of the machine, and a fixed wait is a + guess about it that a loaded machine invalidates. + """ + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + events = [event async for event in handle.fetch_history_events()] + if any(e.HasField("workflow_task_completed_event_attributes") for e in events): + return + await asyncio.sleep(0.05) + raise AssertionError( + "the Workflow's first task never completed, so the Run was never cached " + "and the probe has no settled state to be repeatable about" + ) + + +@pytest.fixture +def unknown_run_id() -> str: + return f"no-such-run-{uuid.uuid4()}" + + +async def test_readiness_for_an_unknown_run_returns_rather_than_raising( + client: Client, unknown_run_id: str +) -> None: + """A watcher must be able to *handle* a missing Run, not catch an exception. + + ``RunNotFound`` is a normal, expected answer -- it is what a watcher sees + every time its Run is evicted -- so surfacing it as an exception would put + ordinary lifecycle in the error path. + """ + async with Worker( + client, task_queue=f"tq-{uuid.uuid4()}", workflows=[IdleWorkflow] + ) as worker: + bridge = bridge_worker(worker) + + result = await bridge.notify_external_stream_ready(unknown_run_id, 1, 0) + + assert result is ExternalStreamReadyResult.RUN_NOT_FOUND + assert result.needs_wake_signal + + +async def test_status_for_an_unknown_run_returns_rather_than_raising( + client: Client, unknown_run_id: str +) -> None: + async with Worker( + client, task_queue=f"tq-{uuid.uuid4()}", workflows=[IdleWorkflow] + ) as worker: + status = await bridge_worker(worker).external_stream_run_status(unknown_run_id) + + assert status is ExternalStreamRunStatus.RUN_NOT_FOUND + + +async def test_both_calls_surface_their_full_result_enums( + client: Client, # pyright: ignore[reportUnusedParameter] +) -> None: + """The five and four values reach Python as values, not strings. + + A watcher branches on all five, and a stringly-typed result would let a + typo silently take the "do nothing" branch for a Run that needed a Signal. + """ + assert {r.value for r in ExternalStreamReadyResult} == { + "Accepted", + "Stale", + "Parked", + "NoOpenWorkflowTask", + "RunNotFound", + } + assert {s.value for s in ExternalStreamRunStatus} == { + "WftOpen", + "Parked", + "NoOpenWorkflowTask", + "RunNotFound", + } + + +async def test_the_readiness_call_is_safe_from_several_threads( + client: Client, unknown_run_id: str +) -> None: + """One watcher per subscription means concurrent callers by construction. + + Each thread drives its own event loop, so this exercises the call from + genuinely separate threads rather than from concurrent tasks on one. The + joins run off the main loop for the reason + :py:meth:`notify_external_stream_ready_sync` documents: blocking the loop + that owns the Worker would stop it polling, and the answer comes from a + lane that Worker drives. + """ + async with Worker( + client, task_queue=f"tq-{uuid.uuid4()}", workflows=[IdleWorkflow] + ) as worker: + bridge = bridge_worker(worker) + # One call first, so the Worker is provably polling before anything + # blocks -- otherwise this test could pass or hang on timing alone. + assert ( + await bridge.notify_external_stream_ready(unknown_run_id, 1, 0) + ) is ExternalStreamReadyResult.RUN_NOT_FOUND + + results: list[object] = [] + errors: list[BaseException] = [] + lock = threading.Lock() + + def call_from_own_loop(wait_id: int) -> None: + async def go() -> None: + for _ in range(5): + got = await bridge.notify_external_stream_ready( + unknown_run_id, wait_id, 0 + ) + with lock: + results.append(got) + + try: + asyncio.run(go()) + except BaseException as err: # noqa: BLE001 -- recorded and asserted on + with lock: + errors.append(err) + + def run_all_threads() -> None: + threads = [ + threading.Thread(target=call_from_own_loop, args=(wait_id,)) + for wait_id in range(1, 9) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=20) + with lock: + errors.extend( + RuntimeError(f"thread {i} did not finish") + for i, thread in enumerate(threads) + if thread.is_alive() + ) + + await asyncio.to_thread(run_all_threads) + + assert not errors, f"concurrent readiness calls failed: {errors}" + assert len(results) == 8 * 5 + assert all(r is ExternalStreamReadyResult.RUN_NOT_FOUND for r in results) + + +async def test_the_status_probe_is_repeatable(client: Client) -> None: + """Asking must change nothing, however many times it is asked. + + The Run has to be **settled** before the first question, or the test is about + something else. `start_workflow` returns when the server accepts the Workflow, + which is before this Worker has polled it, run its first Workflow Task, and + cached the Run -- and until that has happened the probe honestly answers + `RunNotFound`. Answers of `RunNotFound` then `NoOpenWorkflowTask` are Core + telling the truth twice about two different states, not a probe that changed + something. + + So the first Workflow Task's completion is awaited first. `IdleWorkflow` blocks + forever, so after it the Run is cached with no task open, and that state does + not move again on its own -- which is what makes "the same answer every time" a + property of the probe rather than of the timing. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, task_queue=task_queue, workflows=[IdleWorkflow] + ) as worker: + handle = await client.start_workflow( + IdleWorkflow.run, id=f"wf-{uuid.uuid4()}", task_queue=task_queue + ) + try: + description = await handle.describe() + bridge = bridge_worker(worker) + + await _wait_for_first_task_completed(handle) + settled = await bridge.external_stream_run_status(description.run_id) + assert settled is ExternalStreamRunStatus.NO_OPEN_WORKFLOW_TASK, ( + "the Run is cached and blocked with no Workflow Task open, so this " + f"is the only honest answer; got {settled}" + ) + + answers = [ + await bridge.external_stream_run_status(description.run_id) + for _ in range(5) + ] + + # No subscriptions exist, so the honest answer is the same one every + # time -- and in particular the probe never manufactured a task. + assert answers == [settled] * 5 + finally: + await handle.terminate() diff --git a/tests/contrib/external_workflow_streams/test_channel_report.py b/tests/contrib/external_workflow_streams/test_channel_report.py new file mode 100644 index 0000000000..42130e9e1b --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_channel_report.py @@ -0,0 +1,426 @@ +"""The channel report: what a run tells Core its readers listen on. + +Driven with activations directly, the way Core drives the instance, over a +real stream runtime on the memory backend, so the report is built from the +readers the runtime holds open exactly as the Worker's would be. Core keeps +the server's subscriptions in step with the report, so what is pinned down +here is the report alone: when it goes out, what it carries and in which +order, and that the external runtime no longer issues the subscribe command +itself. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator, Iterable +from datetime import datetime, timedelta, timezone +from typing import Any + +import pytest + +import temporalio.api.common.v1 +import temporalio.api.notification.v1 +import temporalio.bridge.proto.workflow_activation +import temporalio.bridge.proto.workflow_completion +import temporalio.common +import temporalio.converter +from temporalio import workflow +from temporalio.contrib.external_workflow_streams import external_stream +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._runtime import WorkflowStreamRuntime +from temporalio.contrib.external_workflow_streams._wake import channel_for +from temporalio.worker._workflow import _WorkflowWorker +from temporalio.worker._workflow_instance import ( + UnsandboxedWorkflowRunner, + WorkflowInstance, + WorkflowInstanceDetails, + _WorkflowLogicFlag, +) +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +WorkflowActivation = temporalio.bridge.proto.workflow_activation.WorkflowActivation +WorkflowActivationJob = ( + temporalio.bridge.proto.workflow_activation.WorkflowActivationJob +) +WorkflowActivationCompletion = ( + temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion +) +Notification = temporalio.api.notification.v1.Notification +INDEPENDENT = _WorkflowLogicFlag.SUBSCRIBE_NOTIFICATION_CHANNELS +LINKED = _WorkflowLogicFlag.LINKED_NOTIFICATION_CHANNELS +RUN_ID = "run" +WORKFLOW_ID = "wf" + + +@workflow.defn +class OpenThenWait: + """Opens a reader without blocking on it, waits, closes it, waits again. + + The reader is open across the first task and gone across the second, + while neither task blocks on a stream, so both completions leave with a + timer and no quiescent snapshot. + """ + + @workflow.run + async def run(self) -> None: + reader = external_stream.topic("tokens", type=str).subscribe() + await workflow.sleep(60) + reader.close() + await workflow.sleep(60) + + +@workflow.defn +class ReadOne: + """Blocks on a reader until its first value arrives.""" + + @workflow.run + async def run(self) -> str: + async for value in external_stream.topic("tokens", type=str).subscribe(): + return value + return "" + + +@workflow.defn +class SleepThenRead: + """Waits on a timer before opening any reader.""" + + @workflow.run + async def run(self) -> str: + await workflow.sleep(60) + async for value in external_stream.topic("tokens", type=str).subscribe(): + return value + return "" + + +@workflow.defn +class ThreeReaders: + """Opens readers on two streams, the first of them twice, then waits.""" + + @workflow.run + async def run(self) -> None: + external_stream.topic("b", type=str).subscribe() + external_stream.topic("a", type=str).subscribe() + external_stream.topic("b", type=str).subscribe() + await workflow.sleep(60) + + +async def _accept(run_id: str, wait_id: int, generation: int) -> str: + del run_id, wait_id, generation + return ReadinessResult.ACCEPTED + + +@pytest.fixture +async def manager() -> AsyncIterator[StreamSubscriptionManager]: + manager = StreamSubscriptionManager( + backend=MemoryStreamBackend(), + notify_ready=_accept, + watch_block=timedelta(milliseconds=10), + ) + try: + yield manager + finally: + await manager.shutdown() + + +def _runtime(manager: StreamSubscriptionManager) -> WorkflowStreamRuntime: + return WorkflowStreamRuntime( + manager=manager, + backend=manager._backend, + run_id=RUN_ID, + namespace="default", + workflow_id=WORKFLOW_ID, + first_execution_run_id=RUN_ID, + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + ) + + +def _instance( + workflow_class: type, + runtime: WorkflowStreamRuntime, + flags: Iterable[_WorkflowLogicFlag] = (), +) -> WorkflowInstance: + """Build an instance the way the worker does, with its stream runtime. + + Needs a running event loop, since the constructor puts the runtime on it. + ``flags`` are the lang flags the Worker would default on from its probe. + """ + defn = workflow._Definition.must_from_class(workflow_class) + now = datetime.now(timezone.utc) + info = workflow.Info( + attempt=1, + continued_run_id=None, + cron_schedule=None, + execution_timeout=None, + first_execution_run_id=RUN_ID, + headers={}, + namespace="default", + original_execution_run_id=RUN_ID, + parent=None, + root=None, + priority=temporalio.common.Priority.default, + raw_memo={}, + retry_policy=None, + run_id=RUN_ID, + run_timeout=None, + search_attributes={}, + start_time=now, + task_queue="tq", + task_timeout=timedelta(seconds=10), + typed_search_attributes=temporalio.common.TypedSearchAttributes.empty, + workflow_id=WORKFLOW_ID, + workflow_start_time=now, + workflow_type=defn.name or "", + ) + converter = temporalio.converter.DataConverter.default + return UnsandboxedWorkflowRunner().create_instance( + WorkflowInstanceDetails( + payload_converter_factory=converter._new_internal_payload_converter, + failure_converter_class=converter.failure_converter_class, + interceptor_classes=[], + defn=defn, + info=info, + randomness_seed=0, + extern_functions={}, + disable_eager_activity_execution=False, + worker_level_failure_exception_types=[], + patch_activation_callback=None, + last_completion_result=temporalio.api.common.v1.Payloads(), + last_failure=None, + default_workflow_logic_flags=frozenset(flags), + external_stream_runtime=runtime, + external_streams_configured=True, + ) + ) + + +def _start( + workflow_class: type, recorded_flags: Iterable[_WorkflowLogicFlag] = () +) -> WorkflowActivation: + """The run's first activation, a replay when ``recorded_flags`` is given.""" + job = WorkflowActivationJob() + init = job.initialize_workflow + init.workflow_type = workflow._Definition.must_from_class(workflow_class).name or "" + init.workflow_id = WORKFLOW_ID + init.first_execution_run_id = RUN_ID + recorded = [int(flag) for flag in recorded_flags] + return WorkflowActivation( + run_id=RUN_ID, + jobs=[job], + is_replaying=bool(recorded), + available_internal_flags=recorded, + ) + + +def _timer_fired(seq: int) -> WorkflowActivation: + job = WorkflowActivationJob() + job.fire_timer.seq = seq + return WorkflowActivation(run_id=RUN_ID, jobs=[job]) + + +def _notified(*notifications: Notification) -> WorkflowActivation: + job = WorkflowActivationJob() + job.notifications_received.notifications.extend(notifications) + return WorkflowActivation(run_id=RUN_ID, jobs=[job]) + + +def _variants(completion: WorkflowActivationCompletion) -> list[str]: + assert completion.HasField("successful"), completion.failed.failure.message + return [ + command.WhichOneof("variant") or "" + for command in completion.successful.commands + ] + + +def _reported(completion: WorkflowActivationCompletion) -> list[str] | None: + """The channel report, or ``None`` when the completion carries none.""" + assert completion.HasField("successful"), completion.failed.failure.message + reports = [ + list(command.workflow_stream_channels.channels) + for command in completion.successful.commands + if command.HasField("workflow_stream_channels") + ] + assert len(reports) <= 1, "one report per completion" + return reports[0] if reports else None + + +def _subscribed(completion: WorkflowActivationCompletion) -> list[str]: + return [ + command.subscribe_notification_channel.channel + for command in completion.successful.commands + if command.HasField("subscribe_notification_channel") + ] + + +def _channel(runtime: WorkflowStreamRuntime, stream: str) -> str: + return channel_for(runtime.stream_key(stream)).channel + + +async def test_the_report_goes_out_on_a_completion_without_a_snapshot( + manager: StreamSubscriptionManager, +) -> None: + """A reader that is open but not blocked on is still listened for.""" + runtime = _runtime(manager) + instance = _instance(OpenThenWait, runtime, flags=[INDEPENDENT]) + completion = instance.activate(_start(OpenThenWait)) + assert "start_timer" in _variants(completion) + assert "workflow_stream_quiescent" not in _variants(completion) + assert _reported(completion) == [_channel(runtime, "tokens")] + assert _subscribed(completion) == [] + + +async def test_the_report_goes_out_with_a_snapshot( + manager: StreamSubscriptionManager, +) -> None: + runtime = _runtime(manager) + instance = _instance(ReadOne, runtime, flags=[INDEPENDENT]) + completion = instance.activate(_start(ReadOne)) + assert "workflow_stream_quiescent" in _variants(completion) + assert _reported(completion) == [_channel(runtime, "tokens")] + assert _subscribed(completion) == [] + + +async def test_the_report_is_absent_before_any_reader_opened( + manager: StreamSubscriptionManager, +) -> None: + """Absent means unchanged to Core, and nothing has changed yet.""" + runtime = _runtime(manager) + instance = _instance(SleepThenRead, runtime, flags=[INDEPENDENT]) + completion = instance.activate(_start(SleepThenRead)) + assert _variants(completion) == ["start_timer"] + assert _reported(completion) is None + completion = instance.activate(_timer_fired(1)) + assert _reported(completion) == [_channel(runtime, "tokens")] + + +async def test_the_report_carries_the_empty_set_after_the_last_reader_closes( + manager: StreamSubscriptionManager, +) -> None: + """Empty is a report, not an absence: it is what Core unsubscribes from.""" + runtime = _runtime(manager) + instance = _instance(OpenThenWait, runtime, flags=[INDEPENDENT]) + instance.activate(_start(OpenThenWait)) + completion = instance.activate(_timer_fired(1)) + assert "start_timer" in _variants(completion) + assert _reported(completion) == [] + + +async def test_the_report_orders_channels_by_reader_and_names_each_once( + manager: StreamSubscriptionManager, +) -> None: + runtime = _runtime(manager) + instance = _instance(ThreeReaders, runtime, flags=[INDEPENDENT]) + completion = instance.activate(_start(ThreeReaders)) + assert _reported(completion) == [_channel(runtime, "b"), _channel(runtime, "a")] + + +async def test_a_linked_server_reports_nothing_for_the_runs_own_streams( + manager: StreamSubscriptionManager, +) -> None: + """The owner is the listener by construction, so there is nothing to subscribe.""" + runtime = _runtime(manager) + instance = _instance(ReadOne, runtime, flags=[INDEPENDENT, LINKED]) + completion = instance.activate(_start(ReadOne)) + assert "workflow_stream_quiescent" in _variants(completion) + assert _reported(completion) == [] + assert _subscribed(completion) == [] + + +async def test_a_server_without_channels_gets_no_report( + manager: StreamSubscriptionManager, +) -> None: + runtime = _runtime(manager) + instance = _instance(ReadOne, runtime) + completion = instance.activate(_start(ReadOne)) + assert "workflow_stream_quiescent" in _variants(completion) + assert _reported(completion) is None + assert _subscribed(completion) == [] + + +async def test_a_replay_reports_what_the_live_run_did( + manager: StreamSubscriptionManager, +) -> None: + """Core matches the recorded subscribed event from the replayed report.""" + runtime = _runtime(manager) + instance = _instance(ReadOne, runtime) + completion = instance.activate(_start(ReadOne, recorded_flags=[INDEPENDENT])) + assert _reported(completion) == [_channel(runtime, "tokens")] + assert _subscribed(completion) == [] + + +class _BareWorker: + """Only the two things ``_handle_external_stream_jobs`` reaches for.""" + + def __init__(self, runtime: WorkflowStreamRuntime, manager: object) -> None: + self._external_stream_runtimes = {RUN_ID: runtime} + self._manager = manager + + def _stream_manager(self) -> object: + return self._manager + + +def _park_job(quiescence_generation: int, *wait_ids: int) -> WorkflowActivation: + activation = WorkflowActivation(run_id=RUN_ID) + job = activation.jobs.add().prepare_external_stream_park + job.quiescence_generation = quiescence_generation + for wait_id in wait_ids: + job.waits.add().wait_id = wait_id + return activation + + +def _finalize_job(quiescence_generation: int) -> WorkflowActivation: + activation = WorkflowActivation(run_id=RUN_ID) + job = activation.jobs.add().finalize_external_streams + job.quiescence_generation = quiescence_generation + return activation + + +async def test_the_park_and_finalize_answers_carry_no_report( + manager: StreamSubscriptionManager, +) -> None: + """Those answers run no Workflow code, so Core keeps the last report.""" + runtime = _runtime(manager) + instance = _instance(ReadOne, runtime, flags=[INDEPENDENT]) + assert _reported(instance.activate(_start(ReadOne))) is not None + worker: Any = _BareWorker(runtime, manager) + + parked = await _WorkflowWorker._handle_external_stream_jobs( + worker, + _park_job(1, 1), + None, # type: ignore[arg-type] + ) + assert parked is not None + assert _variants(parked) == ["external_stream_park_result"] + + finalized = await _WorkflowWorker._handle_external_stream_jobs( + worker, + _finalize_job(1), + None, # type: ignore[arg-type] + ) + assert finalized is not None + assert _variants(finalized) == ["external_stream_finalized"] + + +async def test_a_notification_on_a_reported_channel_is_the_runs_concern( + manager: StreamSubscriptionManager, caplog: pytest.LogCaptureFixture +) -> None: + """The report, not a command, says which channels the run listens on.""" + runtime = _runtime(manager) + instance = _instance(ReadOne, runtime, flags=[INDEPENDENT]) + instance.activate(_start(ReadOne)) + with caplog.at_level(logging.DEBUG, logger="temporalio.worker._workflow_instance"): + instance.activate( + _notified( + Notification(channel=_channel(runtime, "tokens"), counter=1), + Notification(channel="other", counter=1), + ) + ) + dropped = [ + record.getMessage() + for record in caplog.records + if "Dropping a notification" in record.getMessage() + ] + assert len(dropped) == 1 and "'other'" in dropped[0], dropped diff --git a/tests/contrib/external_workflow_streams/test_channels.py b/tests/contrib/external_workflow_streams/test_channels.py new file mode 100644 index 0000000000..22c6dae7e9 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_channels.py @@ -0,0 +1,780 @@ +"""The notification channel surface: the command, the delivery and the client calls. + +The workflow instance is driven with activations directly, the way Core +drives it, because the dev server this chain tests against does not accept +the subscribe command. The live cases at the end need a server that does and +skip otherwise; the ones on the linked kind need a server with it. The +external-stream runtime's own subscriptions go through the same instance +behind a gate, which is driven here too, and are covered beside the stream +tests as well. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import uuid +from collections.abc import Iterable +from datetime import datetime, timedelta, timezone +from typing import Any + +import pytest + +import temporalio.api.common.v1 +import temporalio.api.enums.v1 +import temporalio.api.notification.v1 +import temporalio.api.workflowservice.v1 +import temporalio.bridge.proto.workflow_activation +import temporalio.bridge.proto.workflow_completion +import temporalio.common +import temporalio.converter +from temporalio import workflow +from temporalio.api.enums.v1 import EventType +from temporalio.client import Callback, ChannelKind, Client +from temporalio.client._client import _channel_execution +from temporalio.client._impl import _channel_owner +from temporalio.common import Execution, ExecutionType +from temporalio.contrib.external_workflow_streams._wake import ChannelAddress +from temporalio.service import RPCError, RPCStatusCode +from temporalio.worker._workflow_instance import ( + UnsandboxedWorkflowRunner, + WorkflowInstance, + WorkflowInstanceDetails, + _WorkflowLogicFlag, +) +from temporalio.workflow._context import _Runtime +from tests.contrib.external_workflow_streams.conftest import server_channel_support +from tests.helpers import assert_eventually, new_worker + +WorkflowActivation = temporalio.bridge.proto.workflow_activation.WorkflowActivation +WorkflowActivationJob = ( + temporalio.bridge.proto.workflow_activation.WorkflowActivationJob +) +WorkflowActivationCompletion = ( + temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion +) +Notification = temporalio.api.notification.v1.Notification +ExecutionProto = temporalio.api.common.v1.Execution +WORKFLOW_TYPE = temporalio.api.enums.v1.ExecutionType.EXECUTION_TYPE_WORKFLOW +INDEPENDENT = _WorkflowLogicFlag.SUBSCRIBE_NOTIFICATION_CHANNELS +LINKED = _WorkflowLogicFlag.LINKED_NOTIFICATION_CHANNELS + + +def _linked_to(notification: workflow.Notification) -> dict[str, Any] | None: + if notification.linked_to is None: + return None + return { + "type": notification.linked_to.type.name, + "business_id": notification.linked_to.business_id, + "run_id": notification.linked_to.run_id, + } + + +@workflow.defn +class ReceiveOne: + """Subscribes to one channel and reports the first notification.""" + + @workflow.run + async def run(self, channel: str) -> dict[str, Any]: + notification = await workflow.subscribe_channel(channel).receive() + return { + "channel": notification.channel, + "counter": notification.counter, + "position": notification.position.decode(), + "topic": ( + workflow.payload_converter().from_payload( + notification.metadata["topic"], str + ) + if "topic" in notification.metadata + else None + ), + } + + +@workflow.defn +class CountToTwo: + """Subscribes twice to one channel and counts notifications up to counter two.""" + + @workflow.run + async def run(self, channel: str) -> int: + first = workflow.subscribe_channel(channel) + second = workflow.subscribe_channel(channel) + assert first is second + seen = 0 + async for notification in second: + seen += 1 + if notification.counter >= 2: + break + return seen + + +@workflow.defn +class EmptyChannel: + """Asks for a channel with no name.""" + + @workflow.run + async def run(self) -> None: + workflow.subscribe_channel("") + + +@workflow.defn +class ReceiveLinked: + """Reads its own linked channel and reports the first notification.""" + + @workflow.run + async def run(self, channel: str) -> dict[str, Any]: + subscription = workflow.linked_channel(channel) + assert subscription is workflow.linked_channel(channel) + assert subscription.linked + notification = await subscription.receive() + return { + "channel": notification.channel, + "counter": notification.counter, + "linked_to": _linked_to(notification), + } + + +@workflow.defn +class BothKinds: + """Listens on the independent and the linked channel of one name.""" + + @workflow.run + async def run(self, channel: str) -> list[str]: + independent = workflow.subscribe_channel(channel) + linked = workflow.linked_channel(channel) + seen: list[str] = [] + + async def take(kind: str, subscription: workflow.ChannelSubscription) -> None: + notification = await subscription.receive() + assert (notification.linked_to is not None) == (kind == "linked") + seen.append(kind) + + await asyncio.gather(take("independent", independent), take("linked", linked)) + return seen + + +@workflow.defn +class ListenOnStream: + """Asks the run to listen for a stream the way the stream runtime does.""" + + @workflow.run + async def run(self, channel: str, owner: str) -> bool: + instance: Any = _Runtime.current() + return instance.subscribe_stream_channel( + ChannelAddress( + channel=channel, + execution=Execution.workflow(owner) if owner else None, + ) + ) + + +def _instance( + workflow_class: type, flags: Iterable[_WorkflowLogicFlag] = () +) -> WorkflowInstance: + """Build an instance the way the worker does, without a worker. + + Needs a running event loop, since the constructor puts the runtime on it. + ``flags`` are the lang flags the Worker would default on from its probe. + """ + defn = workflow._Definition.must_from_class(workflow_class) + now = datetime.now(timezone.utc) + info = workflow.Info( + attempt=1, + continued_run_id=None, + cron_schedule=None, + execution_timeout=None, + first_execution_run_id="run", + headers={}, + namespace="default", + original_execution_run_id="run", + parent=None, + root=None, + priority=temporalio.common.Priority.default, + raw_memo={}, + retry_policy=None, + run_id="run", + run_timeout=None, + search_attributes={}, + start_time=now, + task_queue="tq", + task_timeout=timedelta(seconds=10), + typed_search_attributes=temporalio.common.TypedSearchAttributes.empty, + workflow_id="wf", + workflow_start_time=now, + workflow_type=defn.name or "", + ) + converter = temporalio.converter.DataConverter.default + return UnsandboxedWorkflowRunner().create_instance( + WorkflowInstanceDetails( + payload_converter_factory=converter._new_internal_payload_converter, + failure_converter_class=converter.failure_converter_class, + interceptor_classes=[], + defn=defn, + info=info, + randomness_seed=0, + extern_functions={}, + disable_eager_activity_execution=False, + worker_level_failure_exception_types=[], + patch_activation_callback=None, + last_completion_result=temporalio.api.common.v1.Payloads(), + last_failure=None, + default_workflow_logic_flags=frozenset(flags), + ) + ) + + +def _start( + workflow_class: type, + *args: Any, + recorded_flags: Iterable[_WorkflowLogicFlag] = (), +) -> WorkflowActivation: + """The run's first activation. + + With ``recorded_flags`` it is a replay, and those are the flags History + says the live run used. + """ + job = WorkflowActivationJob() + init = job.initialize_workflow + init.workflow_type = workflow._Definition.must_from_class(workflow_class).name or "" + init.workflow_id = "wf" + init.arguments.extend( + temporalio.converter.PayloadConverter.default.to_payloads(args) + ) + recorded = [int(flag) for flag in recorded_flags] + return WorkflowActivation( + run_id="run", + jobs=[job], + is_replaying=bool(recorded), + available_internal_flags=recorded, + ) + + +def _notified(*notifications: Notification) -> WorkflowActivation: + job = WorkflowActivationJob() + job.notifications_received.notifications.extend(notifications) + return WorkflowActivation(run_id="run", jobs=[job]) + + +def _subscribed(completion: WorkflowActivationCompletion) -> list[str]: + assert completion.HasField("successful"), completion.failed.failure.message + return [ + command.subscribe_notification_channel.channel + for command in completion.successful.commands + if command.HasField("subscribe_notification_channel") + ] + + +def _completed(completion: WorkflowActivationCompletion) -> bool: + assert completion.HasField("successful"), completion.failed.failure.message + return any( + command.HasField("complete_workflow_execution") + for command in completion.successful.commands + ) + + +def _result(completion: WorkflowActivationCompletion) -> Any: + assert completion.HasField("successful"), completion.failed.failure.message + [done] = [ + command + for command in completion.successful.commands + if command.HasField("complete_workflow_execution") + ] + return temporalio.converter.PayloadConverter.default.from_payload( + done.complete_workflow_execution.result + ) + + +async def test_the_first_subscription_is_a_command_and_the_second_shares_it(): + instance = _instance(CountToTwo) + completion = instance.activate(_start(CountToTwo, "orders")) + assert _subscribed(completion) == ["orders"] + assert not _completed(completion) + + +async def test_a_notifications_received_job_wakes_the_receiver(): + instance = _instance(ReceiveOne) + assert _subscribed(instance.activate(_start(ReceiveOne, "orders"))) == ["orders"] + [topic] = temporalio.converter.PayloadConverter.default.to_payloads(["inputs"]) + completion = instance.activate( + _notified( + Notification( + channel="orders", position=b"7-0", counter=7, metadata={"topic": topic} + ) + ) + ) + assert _result(completion) == { + "channel": "orders", + "counter": 7, + "position": "7-0", + "topic": "inputs", + } + + +async def test_notifications_arrive_in_order_and_other_channels_are_dropped(): + instance = _instance(CountToTwo) + instance.activate(_start(CountToTwo, "orders")) + # A channel this run never subscribed to is not the workflow's concern. + completion = instance.activate(_notified(Notification(channel="other", counter=9))) + assert not _completed(completion) + completion = instance.activate( + _notified( + Notification(channel="orders", counter=1), + Notification(channel="orders", counter=2), + ) + ) + assert _result(completion) == 2 + + +async def test_an_empty_channel_name_is_refused(): + completion = _instance(EmptyChannel).activate(_start(EmptyChannel)) + assert completion.HasField("failed") + assert "channel must not be empty" in completion.failed.failure.message + + +# --- the linked kind ---------------------------------------------------------- + + +def _linked(channel: str, counter: int, run_id: str = "run") -> Notification: + return Notification( + channel=channel, + counter=counter, + linked_to=ExecutionProto(type=WORKFLOW_TYPE, business_id="wf", run_id=run_id), + ) + + +async def test_a_linked_channel_issues_no_command_and_gets_its_notification(): + instance = _instance(ReceiveLinked) + completion = instance.activate(_start(ReceiveLinked, "orders")) + assert _subscribed(completion) == [] + assert not _completed(completion) + completion = instance.activate(_notified(_linked("orders", 7))) + assert _result(completion) == { + "channel": "orders", + "counter": 7, + "linked_to": {"type": "WORKFLOW", "business_id": "wf", "run_id": "run"}, + } + + +async def test_a_notification_without_an_owner_does_not_reach_the_linked_channel(): + instance = _instance(ReceiveLinked) + instance.activate(_start(ReceiveLinked, "orders")) + # The independent channel of the same name is somebody else's. + completion = instance.activate(_notified(Notification(channel="orders", counter=1))) + assert not _completed(completion) + assert _result(instance.activate(_notified(_linked("orders", 2))))["counter"] == 2 + + +async def test_the_two_kinds_under_one_name_are_told_apart_by_the_owner(): + instance = _instance(BothKinds) + completion = instance.activate(_start(BothKinds, "orders")) + assert _subscribed(completion) == ["orders"] + completion = instance.activate( + _notified(Notification(channel="orders", counter=1), _linked("orders", 1)) + ) + assert sorted(_result(completion)) == ["independent", "linked"] + + +async def test_a_linked_notification_nobody_asked_for_is_dropped(): + instance = _instance(ReceiveLinked) + instance.activate(_start(ReceiveLinked, "orders")) + completion = instance.activate(_notified(_linked("other", 9))) + assert completion.HasField("successful") + assert not _completed(completion) + + +def _used_flags(completion: WorkflowActivationCompletion) -> set[int]: + assert completion.HasField("successful"), completion.failed.failure.message + return set(completion.successful.used_internal_flags) + + +async def test_a_stream_the_run_owns_listens_on_its_linked_channel_without_a_command(): + """The three-way probe's answer, recorded on the first task as two flags.""" + instance = _instance(ListenOnStream, flags=[INDEPENDENT, LINKED]) + completion = instance.activate(_start(ListenOnStream, "external-stream/x", "wf")) + assert _result(completion) is True + assert _subscribed(completion) == [] + assert _used_flags(completion) == {int(INDEPENDENT), int(LINKED)} + + +async def test_a_stream_with_another_owner_is_listened_on_by_report_on_a_linked_server(): + """No command of its own either way: the run's channel report, built from + the stream runtime's open readers on every completion, is what Core + subscribes from. See ``test_channel_report`` for the report itself.""" + # Both built before either runs: a completed run leaves no loop to build on. + another_owner = _instance(ListenOnStream, flags=[INDEPENDENT, LINKED]) + no_owner = _instance(ListenOnStream, flags=[INDEPENDENT, LINKED]) + completion = another_owner.activate( + _start(ListenOnStream, "external-stream/x", "other") + ) + assert _result(completion) is True + assert _subscribed(completion) == [] + # A stream nobody owns does the same. + completion = no_owner.activate(_start(ListenOnStream, "standalone/x", "")) + assert _result(completion) is True + assert _subscribed(completion) == [] + + +async def test_a_server_with_only_independent_channels_listens_by_report(): + instance = _instance(ListenOnStream, flags=[INDEPENDENT]) + completion = instance.activate(_start(ListenOnStream, "external-stream/x", "wf")) + assert _result(completion) is True + assert _subscribed(completion) == [] + assert _used_flags(completion) == {int(INDEPENDENT)} + + +async def test_a_server_without_channels_leaves_the_run_to_the_signal(): + instance = _instance(ListenOnStream) + completion = instance.activate(_start(ListenOnStream, "external-stream/x", "wf")) + assert _result(completion) is False + assert _subscribed(completion) == [] + assert _used_flags(completion) == set() + + +async def test_a_replay_takes_the_path_the_live_run_recorded(): + without_channels = _instance(ListenOnStream) + with_linked = _instance(ListenOnStream, flags=[INDEPENDENT, LINKED]) + # Replayed on a Worker that found no channels: the flags come from History. + completion = without_channels.activate( + _start( + ListenOnStream, + "external-stream/x", + "wf", + recorded_flags=[INDEPENDENT, LINKED], + ) + ) + assert _result(completion) is True + assert _subscribed(completion) == [] + # And a live run on a linked server replays the independent path it took. + completion = with_linked.activate( + _start(ListenOnStream, "external-stream/x", "wf", recorded_flags=[INDEPENDENT]) + ) + assert _result(completion) is True + assert _subscribed(completion) == [] + + +def test_a_channel_call_names_the_execution_it_is_linked_to(): + # The short form names a workflow. The long form names any execution. + assert _channel_execution(None, None, None) is None + assert _channel_execution(None, "wf", None) == Execution.workflow("wf") + assert _channel_execution(None, "wf", "run") == Execution.workflow("wf", "run") + activity = Execution.activity("act") + assert _channel_execution(activity, None, None) is activity + with pytest.raises(ValueError, match="workflow_id"): + _channel_execution(None, None, "run") + with pytest.raises(ValueError, match="not both"): + _channel_execution(activity, "wf", None) + with pytest.raises(ValueError, match="not both"): + _channel_execution(activity, None, "run") + # Unset, the request addresses the independent channel of that name. + request = temporalio.api.workflowservice.v1.DescribeChannelRequest( + channel="c", execution=_channel_owner(None) + ) + assert not request.HasField("execution") + owner = _channel_owner(Execution.workflow("wf")) + assert owner is not None + assert (owner.type, owner.business_id, owner.run_id) == (WORKFLOW_TYPE, "wf", "") + owner = _channel_owner(Execution.workflow("wf", "run")) + assert owner is not None + assert (owner.type, owner.business_id, owner.run_id) == ( + WORKFLOW_TYPE, + "wf", + "run", + ) + assert Execution.from_proto(owner) == Execution.workflow("wf", "run") + assert Execution.from_proto(_channel_owner(activity)) == activity # type: ignore[arg-type] + assert Execution.from_proto(ExecutionProto(business_id="x")) == Execution( + ExecutionType.UNSPECIFIED, "x" + ) + + +async def test_the_client_describes_a_linked_channel_by_its_owner( + client: Client, monkeypatch: pytest.MonkeyPatch +): + described: list[temporalio.api.workflowservice.v1.DescribeChannelRequest] = [] + + async def describe(request, **kwargs): # type: ignore[no-untyped-def] + described.append(request) + return temporalio.api.workflowservice.v1.DescribeChannelResponse( + kind=temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_LINKED, + linked_to=ExecutionProto( + type=WORKFLOW_TYPE, business_id="wf", run_id="run" + ), + latest=_linked("orders", 3), + ) + + monkeypatch.setattr(client.workflow_service, "describe_channel", describe) + description = await client.describe_channel( + "orders", workflow_id="wf", run_id="run" + ) + [request] = described + assert request.channel == "orders" + assert request.execution.type == WORKFLOW_TYPE + assert request.execution.business_id == "wf" + assert request.execution.run_id == "run" + assert description.kind is ChannelKind.LINKED + assert description.linked_to == Execution.workflow("wf", "run") + assert description.latest is not None + assert _linked_to(description.latest) == { + "type": "WORKFLOW", + "business_id": "wf", + "run_id": "run", + } + # The long form carries any execution as given. + activity = Execution.activity("act", "run-2") + await client.describe_channel("orders", execution=activity) + assert Execution.from_proto(described[-1].execution) == activity + with pytest.raises(ValueError, match="not both"): + await client.describe_channel("orders", execution=activity, workflow_id="wf") + # A server that predates the kinds reports none. + monkeypatch.setattr( + client.workflow_service, + "describe_channel", + lambda request, **kwargs: _answer( # type: ignore[no-untyped-def] + temporalio.api.workflowservice.v1.DescribeChannelResponse() + ), + ) + description = await client.describe_channel("orders") + assert description.kind is ChannelKind.UNSPECIFIED + assert description.linked_to is None + + +async def _answer(response: Any) -> Any: + return response + + +async def _require_linked(client: Client) -> None: + from temporalio.contrib.external_workflow_streams._wake import ChannelSupport + + if await server_channel_support(client) is not ChannelSupport.LINKED: + pytest.skip("the server does not serve channels linked to a workflow") + + +@pytest.mark.needs_linked_server +async def test_a_workflow_receives_a_notification_on_its_linked_channel(client: Client): + await _require_linked(client) + channel = f"orders-{uuid.uuid4()}" + worker = new_worker(client, ReceiveLinked) + running = asyncio.create_task(worker.run()) + handle = await client.start_workflow( + ReceiveLinked.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + try: + # The channel exists with the run: no listener to register, nothing + # retained yet, and the owner named. + description = await client.describe_channel(channel, workflow_id=handle.id) + assert description.kind is ChannelKind.LINKED + assert description.linked_to is not None + assert description.linked_to.type is ExecutionType.WORKFLOW + assert description.linked_to.business_id == handle.id + assert description.listeners == [] + assert description.latest is None + await client.notify_channel( + channel, position=b"1-0", counter=1, workflow_id=handle.id + ) + result = await asyncio.wait_for(handle.result(), 30) + assert (result["channel"], result["counter"]) == (channel, 1) + # The owner and the run that received it, so a listener holding both + # kinds under one name can route it. + assert result["linked_to"]["type"] == "WORKFLOW" + assert result["linked_to"]["business_id"] == handle.id + assert result["linked_to"]["run_id"] + events = [event.event_type async for event in handle.fetch_history_events()] + assert ( + EventType.EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED not in events + ) + # The channel's state dies with the run. + with pytest.raises(RPCError) as closed: + await client.notify_channel(channel, counter=2, workflow_id=handle.id) + assert closed.value.status == RPCStatusCode.NOT_FOUND + finally: + with contextlib.suppress(RPCError): + await handle.terminate() + await asyncio.wait_for(worker.shutdown(), 15) + await running + + +@pytest.mark.needs_linked_server +async def test_a_linked_channel_retains_for_pollers_and_takes_callbacks(client: Client): + await _require_linked(client) + channel = f"orders-{uuid.uuid4()}" + # Nobody polls this queue, so the run stays open for the calls below. + handle = await client.start_workflow( + ReceiveLinked.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=f"nobody-polls-{uuid.uuid4()}", + ) + try: + callback = Callback(url="http://localhost:1/never-called", headers={}) + listener_id = await client.register_channel_listener( + channel, callback, workflow_id=handle.id + ) + description = await client.describe_channel(channel, workflow_id=handle.id) + # The owner listens by construction and the server may list it beside + # the callback once the channel holds state; the callback is the one + # listener that was registered. + assert {listener.workflow_id for listener in description.listeners} <= { + None, + handle.id, + } + [registered] = [ + listener for listener in description.listeners if listener.callback + ] + assert (registered.listener_id, registered.callback) == (listener_id, callback) + await client.unregister_channel_listener( + channel, listener_id, workflow_id=handle.id + ) + description = await client.describe_channel(channel, workflow_id=handle.id) + assert [ + listener for listener in description.listeners if listener.callback + ] == [] + # The owner is woken, so a notify counts no registered callback but + # still retains for pollers. + await client.notify_channel( + channel, position=b"2-0", counter=2, workflow_id=handle.id + ) + polled = await client.poll_channel(channel, workflow_id=handle.id, wait=False) + assert [(n.counter, _linked_to(n) is not None) for n in polled] == [(2, True)] + description = await client.describe_channel(channel, workflow_id=handle.id) + assert description.latest is not None and description.latest.counter == 2 + # The independent channel of the same name is untouched. + with pytest.raises(RPCError) as untouched: + await client.describe_channel(channel) + assert untouched.value.status == RPCStatusCode.NOT_FOUND + finally: + with contextlib.suppress(RPCError): + await handle.terminate() + + +@pytest.mark.needs_channel_server +async def test_a_workflow_receives_a_client_notification(client: Client): + channel = f"orders-{uuid.uuid4()}" + worker = new_worker(client, ReceiveOne) + running = asyncio.create_task(worker.run()) + handle = await client.start_workflow( + ReceiveOne.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + try: + + async def subscribed() -> None: + # The worker's Core has to carry the subscribe command to the + # server. One that refuses it fails every completion, and the task + # times out instead; say so rather than wait on the result forever. + events = [event.event_type async for event in handle.fetch_history_events()] + if EventType.EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT in events: + pytest.fail( + "the worker could not complete the task that subscribes; the Core " + "the bridge pins must carry the subscribe command" + ) + assert ( + EventType.EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED in events + ) + + await assert_eventually(subscribed, timeout=timedelta(seconds=30)) + + async def listening() -> None: + # The channel exists once the subscribe lands, so a describe that + # races it is answered with not found. + try: + description = await client.describe_channel(channel) + except RPCError as err: + assert err.status != RPCStatusCode.NOT_FOUND, "channel not created yet" + raise + assert [listener.workflow_id for listener in description.listeners] == [ + handle.id + ] + + await assert_eventually(listening) + listeners = await client.notify_channel( + channel, position=b"1-0", counter=1, metadata={"topic": "inputs"} + ) + assert listeners == 1 + assert await asyncio.wait_for(handle.result(), 30) == { + "channel": channel, + "counter": 1, + "position": "1-0", + "topic": "inputs", + } + polled = await client.poll_channel(channel, wait=False) + assert [n.counter for n in polled] == [1] + description = await client.describe_channel(channel) + assert description.latest is not None and description.latest.counter == 1 + finally: + # A Core that refuses the subscribe command leaves the task in a + # timeout loop and the worker's shutdown waiting on it, so end the run + # first and give the shutdown a bound. + with contextlib.suppress(RPCError): + await handle.terminate() + try: + await asyncio.wait_for(worker.shutdown(), 15) + except asyncio.TimeoutError: + running.cancel() + with contextlib.suppress(asyncio.CancelledError): + await running + + +@pytest.mark.needs_channel_server +async def test_a_callback_listener_registers_and_unregisters(client: Client): + channel = f"orders-{uuid.uuid4()}" + callback = Callback(url="http://localhost:1/never-called", headers={}) + listener_id = await client.register_channel_listener(channel, callback) + description = await client.describe_channel(channel) + assert [listener.listener_id for listener in description.listeners] == [listener_id] + assert description.listeners[0].callback == callback + await client.unregister_channel_listener(channel, listener_id) + description = await client.describe_channel(channel) + assert description.listeners == [] + + +@pytest.mark.needs_channel_server +async def test_a_channel_retains_notifications_for_pollers(client: Client): + channel = f"orders-{uuid.uuid4()}" + # A channel nobody has touched does not exist. + with pytest.raises(RPCError) as untouched: + await client.describe_channel(f"untouched-{uuid.uuid4()}") + assert untouched.value.status == RPCStatusCode.NOT_FOUND + # Nobody listens yet: the notification is kept for pollers and the count + # says zero. + assert await client.notify_channel(channel, position=b"2-0", counter=2) == 0 + description = await client.describe_channel(channel) + assert description.listeners == [] + assert description.latest is not None + assert (description.latest.position, description.latest.counter) == (b"2-0", 2) + assert description.retained_count == 1 + polled = await client.poll_channel(channel, wait=False) + assert [(n.position, n.counter) for n in polled] == [(b"2-0", 2)] + # At or below the latest counter a notify changes nothing and is not kept. + assert await client.notify_channel(channel, position=b"1-0", counter=1) == 0 + assert await client.notify_channel(channel, position=b"2-0", counter=2) == 0 + description = await client.describe_channel(channel) + assert description.latest is not None and description.latest.counter == 2 + assert description.retained_count == 1 + # Above it the notification is kept, metadata and all, and a poll after + # the earlier counter sees only the new one. + assert ( + await client.notify_channel( + channel, position=b"3-0", counter=3, metadata={"topic": "inputs"} + ) + == 0 + ) + [newest] = await client.poll_channel(channel, after_counter=2, wait=False) + assert newest.counter == 3 + assert ( + client.data_converter.payload_converter.from_payload( + newest.metadata["topic"], str + ) + == "inputs" + ) + polled = await client.poll_channel(channel, wait=False) + assert [n.counter for n in polled] == [2, 3] + # A poll above the latest waits its bound out and comes back empty. + polled = await client.poll_channel( + channel, after_counter=3, wait=timedelta(seconds=1) + ) + assert polled == [] diff --git a/tests/contrib/external_workflow_streams/test_codec.py b/tests/contrib/external_workflow_streams/test_codec.py new file mode 100644 index 0000000000..4b25393673 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_codec.py @@ -0,0 +1,119 @@ +"""P4 — payload encoding through the Workflow's DataConverter.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import pytest + +import temporalio.api.common.v1 +import temporalio.converter +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._record import ( + Offset, + RecordKind, + StreamRecord, +) + + +@dataclass +class Token: + """A custom type, to prove the topic's declared type reaches the converter.""" + + text: str + index: int + + +DEFAULT = temporalio.converter.DataConverter.default + + +@pytest.mark.asyncio +async def test_str_round_trips() -> None: + codec = StreamPayloadCodec(DEFAULT, str) + + assert await codec.decode(await codec.encode("hello")) == "hello" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("value", ["", "unicode: é中", "a" * 10_000]) +async def test_awkward_strings_round_trip(value: str) -> None: + codec = StreamPayloadCodec(DEFAULT, str) + + assert await codec.decode(await codec.encode(value)) == value + + +@pytest.mark.asyncio +async def test_a_custom_type_round_trips_as_that_type() -> None: + codec = StreamPayloadCodec(DEFAULT, Token) + + decoded = await codec.decode(await codec.encode(Token("hi", 3))) + + # Not merely equal-looking: the declared type is what makes this a `Token` + # rather than the `dict` the default converter would otherwise infer. + assert isinstance(decoded, Token) + assert decoded == Token("hi", 3) + + +@pytest.mark.asyncio +async def test_without_a_declared_type_the_converter_infers() -> None: + untyped: StreamPayloadCodec[Any] = StreamPayloadCodec(DEFAULT, None) + + decoded = await untyped.decode(await untyped.encode(Token("hi", 3))) + + assert decoded == {"text": "hi", "index": 3} + + +@pytest.mark.asyncio +async def test_encoded_bytes_are_a_payload_envelope() -> None: + """The envelope is what lets a consumer decode without out-of-band metadata.""" + codec = StreamPayloadCodec(DEFAULT, str) + + payload = temporalio.api.common.v1.Payload() + payload.ParseFromString(await codec.encode("hello")) + + assert payload.metadata["encoding"] == b"json/plain" + + +@pytest.mark.asyncio +async def test_with_type_rebinds_without_touching_the_converter() -> None: + codec = StreamPayloadCodec(DEFAULT, str) + rebound = codec.with_type(Token) + + assert rebound.data_converter is codec.data_converter + assert rebound.value_type is Token + assert codec.value_type is str + + +@pytest.mark.asyncio +async def test_a_payload_encoded_for_one_type_decodes_under_another() -> None: + """A converter mismatch surfaces here, on the consumer, not as damage. + + The bytes are intact; only the reader's expectation is wrong. Classifying + that as a decode failure rather than integrity loss is what keeps an + operator from being sent to repair an undamaged backend. + """ + produced = await StreamPayloadCodec(DEFAULT, str).encode("not a token") + + with pytest.raises(Exception): + await StreamPayloadCodec(DEFAULT, Token).decode(produced) + + +@pytest.mark.asyncio +async def test_undecodable_bytes_raise_rather_than_returning_a_value() -> None: + with pytest.raises(Exception): + await StreamPayloadCodec(DEFAULT, str).decode(b"\xff\xfe not a payload") + + +@pytest.mark.asyncio +async def test_a_record_carries_the_encoded_payload_unchanged() -> None: + """The codec's output survives the record's own field round trip.""" + codec = StreamPayloadCodec(DEFAULT, Token) + encoded = await codec.encode(Token("round", 1)) + + record = StreamRecord(RecordKind.DATA, encoded, "session-a", 0).placed_at( + Offset("1-0") + ) + rebuilt = StreamRecord.from_fields(Offset("1-0"), record.to_fields()) + + assert await codec.decode(rebuilt.payload) == Token("round", 1) diff --git a/tests/contrib/external_workflow_streams/test_continuation.py b/tests/contrib/external_workflow_streams/test_continuation.py new file mode 100644 index 0000000000..f3dba0b607 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_continuation.py @@ -0,0 +1,969 @@ +"""P15 — the Continue-As-New cursor. + +A stream spans a whole chain, so a new Run must resume where its predecessor +stopped. The position travels in a reserved internal header and is restored from +History, because a cursor derived from mutable backend state would give replay +whatever the stream holds *now* rather than what the Run started from -- and two +replays of one history could then diverge (ADR-022). + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +# pyright: reportMissingParameterType=false, reportUnusedFunction=false +from __future__ import annotations + +import asyncio +import dataclasses +import uuid +from datetime import timedelta +from typing import Any + +import pytest + +import temporalio.converter +from temporalio import workflow +from temporalio.bridge.proto.workflow_activation import ( + InitializeWorkflow, + WorkflowActivation, +) +from temporalio.client import Client, WorkflowHandle +from temporalio.contrib.external_workflow_streams._annotation import ( + AnnotationDecodeError, +) +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._continuation import ( + CONTINUATION_HEADER, + Continuation, + decode_continuation, + encode_continuation, + read_continuation_header, + write_continuation_header, +) +from temporalio.contrib.external_workflow_streams._errors import StreamStorageError +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._runtime import WorkflowStreamRuntime +from temporalio.exceptions import ApplicationError +from temporalio.worker import Replayer, Worker +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams._api import external_stream + + +async def _notify( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] +) -> str: + return ReadinessResult.ACCEPTED + + +def continuation( + cursors, + stream_names, + provider_ids=None, + provider_format_versions=None, +): # type: ignore[no-untyped-def] + provider_ids = provider_ids or { + wait_id: MemoryStreamBackend.provider_id for wait_id in cursors + } + provider_format_versions = provider_format_versions or { + wait_id: MemoryStreamBackend.provider_format_version for wait_id in cursors + } + return Continuation( + cursors=cursors, + stream_names=stream_names, + provider_ids=provider_ids, + provider_format_versions=provider_format_versions, + ) + + +def make_runtime(manager, backend, continuation=None): # type: ignore[no-untyped-def] + return WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id="run-2", + namespace="ns", + workflow_id="wf", + first_execution_run_id="first-run", + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + continuation=continuation, + ) + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +class OtherProviderBackend(MemoryStreamBackend): + """The same name, mapped to a different implementation. + + A deployment can do this without touching Workflow code, which is exactly + why the continuation has to carry what the cursor was produced by rather + than only what it was produced for. + """ + + provider_id = "other-memory" + + +class NewerFormatBackend(MemoryStreamBackend): + """The recorded provider, at a format version that reads offsets differently.""" + + provider_format_version = 2 + + +class StubManager: + def cancel_from_workflow_thread(self, run_id, wait_id): # type: ignore[no-untyped-def] + pass + + """Records registrations and starts no watcher. + + What these tests are about is the cursor a subscription is registered *with*, + which is decided before any watching happens. A real manager here would spawn + prefetch loops against a backend nothing is writing to, and its failures + would be noise rather than signal. + """ + + def __init__(self) -> None: + self.registered: list[tuple[int, object]] = [] + + def register(self, *, run_id, wait_id, stream_key, start_cursor): # type: ignore[no-untyped-def] + self.registered.append((wait_id, start_cursor)) + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + +@pytest.fixture +def manager() -> StubManager: + return StubManager() + + +# --- the header's encoding ---------------------------------------------------- + + +def test_a_continuation_round_trips() -> None: + """Binding included: a cursor without one cannot be checked on restoration.""" + original = continuation( + cursors={1: AFTER(Offset("100-3")), 2: BEGINNING}, + stream_names={1: "tokens", 2: "tool-events"}, + provider_ids={1: "memory", 2: "redis"}, + provider_format_versions={1: 1, 2: 3}, + ) + + assert decode_continuation(encode_continuation(original)) == original + + +def test_the_encoding_is_stable_for_one_state() -> None: + """The header is re-derived from live state every time the command is built. + + Encoding the same state two different ways -- by iterating a dict in + insertion order, say -- would hand the successor a different cursor on a + Workflow Task the server retried. + """ + forwards = continuation({1: BEGINNING, 2: BEGINNING}, {1: "a", 2: "b"}) + backwards = continuation({2: BEGINNING, 1: BEGINNING}, {2: "b", 1: "a"}) + + assert encode_continuation(forwards) == encode_continuation(backwards) + + +def test_a_newer_schema_version_is_reported_not_guessed_at() -> None: + """Silently starting at BEGINNING would redeliver the whole stream.""" + future = bytearray(encode_continuation(continuation({1: BEGINNING}, {1: "a"}))) + future[0] = 99 + + with pytest.raises(AnnotationDecodeError, match="schema version 99"): + decode_continuation(bytes(future)) + + +def test_the_header_bypasses_the_user_data_converter() -> None: + """One Run writes it and the next reads it, possibly on different config. + + A chain whose Runs were deployed with different converter configuration + would otherwise restart at an unreadable cursor -- silently, since an + unreadable cursor looks exactly like no cursor at all. + """ + payload = write_continuation_header(continuation({1: BEGINNING}, {1: "a"})) + + assert payload.metadata["encoding"] == b"binary/plain" + assert decode_continuation(payload.data).cursors == {1: BEGINNING} + + +def test_the_header_name_is_in_this_features_namespace() -> None: + assert CONTINUATION_HEADER.startswith("__temporal_external_stream") + assert not CONTINUATION_HEADER.startswith("__temporal_workflow_stream") + + +def test_no_header_is_a_first_execution_not_a_failure() -> None: + assert read_continuation_header(None) is None + assert read_continuation_header({}) is None + assert ( + read_continuation_header( + {"other": write_continuation_header(continuation({1: BEGINNING}, {1: "a"}))} + ) + is None + ) + + +# --- restoration -------------------------------------------------------------- + + +def test_a_first_execution_starts_at_the_beginning( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """The same field, filled the same way -- replay reads a boundary either way. + + Including when the stream was empty for the subscription's entire life, + which is the case an implicit "start wherever" would leave unrecorded. + """ + runtime = make_runtime(manager, backend) + + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + + assert runtime._subscriptions[1].start_cursor == BEGINNING + + +def test_a_successor_run_restores_its_predecessors_cursor( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + runtime = make_runtime( + manager, + backend, + continuation({1: AFTER(Offset("100-3"))}, {1: "tokens"}), + ) + + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + + assert runtime._subscriptions[1].start_cursor == AFTER(Offset("100-3")) + + +def test_restoration_reads_nothing_from_the_backend( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """The criterion, and the reason the header exists at all. + + A cursor read from the backend is whatever the stream holds now, so two + replays of one history could restore different positions and diverge. + """ + runtime = make_runtime( + manager, backend, continuation({1: AFTER(Offset("100-3"))}, {1: "tokens"}) + ) + + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + + assert backend.range_reads == [], "restoration must not read the stream" + + +def test_two_same_stream_subscriptions_restore_independently( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """They are separate waits with their own cursors. + + A stream-keyed continuation would restart one of them at the other's + position, replaying records it had already consumed or skipping records it + had not. + """ + runtime = make_runtime( + manager, + backend, + continuation( + {1: AFTER(Offset("100-0")), 2: AFTER(Offset("500-0"))}, + {1: "tokens", 2: "tokens"}, + ), + ) + key = runtime.stream_key("tokens") + + runtime.register(wait_id=1, stream_key=key) + runtime.register(wait_id=2, stream_key=key) + + assert runtime._subscriptions[1].start_cursor == AFTER(Offset("100-0")) + assert runtime._subscriptions[2].start_cursor == AFTER(Offset("500-0")) + + +def test_a_subscription_the_predecessor_lacked_starts_at_the_beginning( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """Adding one on a path the chain has not reached yet is a supported change.""" + runtime = make_runtime( + manager, backend, continuation({1: AFTER(Offset("100-0"))}, {1: "tokens"}) + ) + + runtime.register(wait_id=2, stream_key=runtime.stream_key("later")) + + assert runtime._subscriptions[2].start_cursor == BEGINNING + + +def test_a_renumbered_subscription_is_caught_rather_than_resumed_wrong( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """Restoring a cursor onto a differently-numbered wait resumes the wrong stream. + + The backend would accept the offset and no later check would catch it, so + this is reported here -- as nondeterminism, with the same remedy an inserted + timer needs. Not as integrity loss: the cursor is exactly what the + predecessor committed, and it is the Workflow code that moved. + """ + runtime = make_runtime( + manager, backend, continuation({1: AFTER(Offset("100-0"))}, {1: "tokens"}) + ) + + with pytest.raises(workflow.NondeterminismError, match="workflow.patched"): + runtime.register( + wait_id=1, + stream_key=runtime.stream_key("tool-events"), + ) + + +def test_a_different_configured_provider_is_a_storage_failure( + manager: StubManager, +) -> None: + """The Workflow is unchanged and neither store is damaged. + + Marker replay reports this against a recorded range; here it is reported + against a restored cursor, which is the same question one Workflow Task + earlier -- and before any read, so the wrong implementation never gets to + interpret the boundary at all. + """ + backend = OtherProviderBackend() + runtime = make_runtime( + manager, + backend, + continuation( + {1: AFTER(Offset("200-0"))}, + {1: "tokens"}, + {1: "memory"}, + {1: 1}, + ), + ) + + with pytest.raises(StreamStorageError, match="'other-memory'"): + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + + assert manager.registered == [] + assert backend.range_reads == [], "raised before any backend read" + + +def test_a_newer_provider_format_version_is_a_storage_failure( + manager: StubManager, +) -> None: + """The implementation is the recorded one but reads offsets differently. + + Resuming would interpret a boundary under a format it was not written in, + which is not something the offset itself can reveal. + """ + backend = NewerFormatBackend() + runtime = make_runtime( + manager, + backend, + continuation( + {1: AFTER(Offset("200-0"))}, + {1: "tokens"}, + {1: "memory"}, + {1: 1}, + ), + ) + + with pytest.raises(StreamStorageError, match="format version"): + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + + assert manager.registered == [] + assert backend.range_reads == [], "raised before any backend read" + + +def test_a_recorded_format_version_of_zero_is_still_compared( + manager: StubManager, +) -> None: + """Zero is a format version, not a "nothing was recorded" sentinel. + + The backend contract types ``provider_format_version`` as a plain integer + and reserves no value, and the header represents zero exactly. Deciding + whether a version was recorded by truthiness therefore skipped the + comparison for the one value that looks falsey, which made Continue-As-New + less safe than marker replay for the same binding: replay compares exactly. + """ + backend = MemoryStreamBackend() + assert type(backend).provider_format_version == 1, ( + "the fixture must declare a version the recorded zero disagrees with" + ) + runtime = make_runtime( + manager, + backend, + continuation( + {1: AFTER(Offset("200-0"))}, + {1: "tokens"}, + {1: "memory"}, + {1: 0}, + ), + ) + + with pytest.raises(StreamStorageError, match="format version"): + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + + assert manager.registered == [], "the cursor must not reach the manager" + assert backend.range_reads == [], "raised before any backend read" + + +async def test_a_worker_without_a_backend_still_decodes_the_continuation( + client: Client, +) -> None: + async with Worker( + client, + task_queue=f"tq-{uuid.uuid4()}", + workflows=[ChainedConsumerWorkflow], + ) as worker: + assert worker._workflow_worker is not None + init = InitializeWorkflow( + workflow_id="workflow", first_execution_run_id="first" + ) + init.headers[CONTINUATION_HEADER].data = b"\x03" + + with pytest.raises(AnnotationDecodeError, match="schema version 3"): + worker._workflow_worker._create_external_stream_runtime( + WorkflowActivation(run_id="run"), init + ) + + init.headers[CONTINUATION_HEADER].CopyFrom( + write_continuation_header(continuation({1: BEGINNING}, {1: "tokens"})) + ) + with pytest.raises(RuntimeError, match="external_stream_backend"): + worker._workflow_worker._create_external_stream_runtime( + WorkflowActivation(run_id="run"), init + ) + + +# --- what gets committed ------------------------------------------------------ + + +def test_the_continuation_reports_what_was_consumed_not_what_was_delivered( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """The successor must not start past records the Workflow never took. + + Delivery advances by whole drained batches, because that is what the + annotation records and what replay must reproduce. A Workflow that stops + iterating part-way through a batch has consumed only its prefix, and the + rest dies with the Run's buffer -- so a continuation taken from the delivery + cursor would step over records nothing had ever shown to Workflow code. + """ + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + batch = [ + StreamRecord(RecordKind.DATA, b"a", "s", i).placed_at(Offset(f"10-{i}")) + for i in range(3) + ] + for record in batch: + runtime.record_delivery(1, record) + # The Workflow took only the first of the three, then stopped iterating. + runtime.record_consumption(1, batch[0]) + + assert runtime.continuation().cursors[1] == AFTER(Offset("10-0")), ( + "the successor must resume at the first record this Run did not consume" + ) + + +def test_a_control_record_still_advances_consumption( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """It is never yielded, but it is finished with. + + Leaving it behind would restart the successor at a fence it cannot act on, + and it would be redelivered on every Run of the chain. + """ + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + fence = StreamRecord(RecordKind.WRITE_FENCE, b"", "s", 0).placed_at(Offset("10-0")) + + runtime.record_consumption(1, fence) + + assert runtime.continuation().cursors[1] == AFTER(Offset("10-0")) + + +def test_a_run_that_delivered_nothing_still_reports_its_start( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """Otherwise the successor would restart at BEGINNING and redeliver.""" + runtime = make_runtime( + manager, backend, continuation({1: AFTER(Offset("100-0"))}, {1: "tokens"}) + ) + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + + assert runtime.continuation().cursors[1] == AFTER(Offset("100-0")) + + +# --- through a real chain ----------------------------------------------------- + + +@workflow.defn +class ChainedConsumerWorkflow: + """Consumes two records, continues as new, consumes two more. + + The whole point is the second Run must not see the first Run's records. A + Workflow that returned everything it saw would hide that: only counting what + the *successor* observed makes a redelivery visible. + """ + + @workflow.run + async def run(self, remaining: int) -> list[str]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + seen: list[str] = [] + async for token in tokens.subscribe(): + seen.append(token) + if len(seen) >= 2: + break + if remaining > 1: + workflow.continue_as_new(remaining - 1) + return seen + + +async def publish(backend: MemoryStreamBackend, key: StreamKey, values: list[str]): + from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec + + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + for i, value in enumerate(values): + await backend.append( + key, + StreamRecord(RecordKind.DATA, await codec.encode(value), "producer", i), + ) + + +async def test_a_chain_resumes_where_its_predecessor_stopped( + client: Client, backend: MemoryStreamBackend +) -> None: + """The successor consumes the records the first Run did not, and no others.""" + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[ChainedConsumerWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + ChainedConsumerWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await asyncio.sleep(1) + await publish(backend, key, ["a", "b", "c", "d"]) + + assert await asyncio.wait_for(handle.result(), 60) == ["c", "d"], ( + "the successor Run redelivered records its predecessor had already " + "consumed, so the continuation cursor did not survive Continue-As-New" + ) + + +async def test_a_chain_carries_the_provider_binding_through_history( + client: Client, backend: MemoryStreamBackend +) -> None: + """The binding reaches the successor as History, and is checked on the way in. + + This shows that a live Worker puts the binding on the command, the server + persists it into the successor's ``WorkflowExecutionStarted``, and the + successor validates it rather than merely tolerating it. + + The last of those is asserted against the *recorded* bytes with only the + provider identity moved, so the run that must fail differs from the run that + must succeed in exactly the thing the binding exists to detect. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[ChainedConsumerWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + ChainedConsumerWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + first_run_id = description.raw_description.workflow_execution_info.first_run_id + key = StreamKey(client.namespace, handle.id, first_run_id, "tokens") + await asyncio.sleep(1) + # One call, carrying its own sequence: a second call would restart the + # producer sequence at zero and the backend would reject the appends as + # non-identical repeats, which presents as a hung Workflow. + await publish(backend, key, ["a", "b", "c", "d"]) + + assert await asyncio.wait_for(handle.result(), 60) == ["c", "d"] + predecessor = await client.get_workflow_handle( + handle.id, run_id=first_run_id + ).fetch_history() + + continued = [ + e + for e in predecessor.events + if e.HasField("workflow_execution_continued_as_new_event_attributes") + ] + assert continued, "the predecessor did not continue as new" + attributes = continued[0].workflow_execution_continued_as_new_event_attributes + recorded = decode_continuation(attributes.header.fields[CONTINUATION_HEADER].data) + + assert recorded.provider_ids == {1: MemoryStreamBackend.provider_id} + assert recorded.provider_format_versions == { + 1: MemoryStreamBackend.provider_format_version + } + + # The bytes the successor actually reads. `read_continuation_header` is + # handed this event's headers, not the command's, so a header the server + # dropped or rewrote between the two would leave the successor starting at + # BEGINNING with nothing to say so. + successor = await client.get_workflow_handle( + handle.id, run_id=attributes.new_execution_run_id + ).fetch_history() + started = successor.events[0].workflow_execution_started_event_attributes + assert ( + decode_continuation(started.header.fields[CONTINUATION_HEADER].data) == recorded + ) + + # Replayed against the provider that produced the cursor, the recorded + # binding has to agree; this is also what makes the mismatch below evidence + # about the binding rather than about the history. + matched = await Replayer( + workflows=[ChainedConsumerWorkflow], + external_stream_backend=backend, + ).replay_workflow(successor, raise_on_replay_failure=False) + + assert matched.replay_failure is None, ( + f"the successor's own history no longer replays: {matched.replay_failure}" + ) + + # Same history, same records, and same stream -- only the configured + # implementation has moved. Accepting the cursor would skip records in + # silence because the boundary means nothing in the replacement store. + # + # Reported by *marker* replay, not by the continuation: the successor's + # `ReplayExternalStreams` job is handled before its Workflow code runs, so on + # a history that already carries a marker the marker binding is checked + # first. This is therefore the wrong layer to ask the continuation's own + # check about -- disabling `_verify_restored_provider` does not change this + # outcome, which is why that check is asked for separately below instead of + # being inferred from this failure. + foreign = OtherProviderBackend() + foreign._records = {k: list(v) for k, v in backend._records.items()} + moved = await Replayer( + workflows=[ChainedConsumerWorkflow], + external_stream_backend=foreign, + ).replay_workflow(successor, raise_on_replay_failure=False) + + assert moved.replay_failure is not None, ( + "the successor read records out of a store that did not produce them" + ) + assert "other-memory" in str(moved.replay_failure), ( + f"the failure does not name the provider that was found: {moved.replay_failure}" + ) + + # The continuation's own check, on the one Workflow Task that has to make it: + # the successor's *first*, which restores the cursor before any marker exists + # for marker replay to check in its place. Asked with the bytes History + # actually holds rather than with a hand-built `Continuation`, so what is + # validated is the recorded binding and not a shape a test chose. + first_task = make_runtime( + StubManager(), + foreign, + recorded, + ) + + with pytest.raises(StreamStorageError, match="'other-memory'"): + first_task.register( + wait_id=1, + stream_key=first_task.stream_key("tokens"), + ) + + assert foreign.range_reads == [], ( + "the mismatch was reported after a read, so the wrong implementation " + "already interpreted the recorded boundary" + ) + + +# --- the boundary the header is taken at -------------------------------------- +# +# Creating the Continue-As-New command does not end the activation. The event +# loop keeps draining what is already ready, and a stream consumer that runs in +# that tail consumes records the predecessor's final marker *does* record -- so a +# header taken when the command was created describes an earlier boundary than +# History does, and the successor is handed those records a second time. + + +@workflow.defn +class LateConsumerContinueAsNewWorkflow: + """Schedules a consumer, then continues as new without yielding to it. + + The scheduled task takes a record that is already buffered, so it finishes + without blocking -- it just does so after the terminal command exists. + + Only the successor's records are returned. A Run that reported everything it + saw would hide the redelivery inside the predecessor's own list. + """ + + @workflow.run + async def run(self, remaining: int) -> list[str]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + subscription = tokens.subscribe() + iterator = subscription.__aiter__() + first = await iterator.__anext__() + if remaining <= 1: + return [first, await iterator.__anext__()] + _require_a_buffered_record(subscription) + + async def take_one() -> None: + await iterator.__anext__() + + # On the ready queue, and able to finish without blocking. `_run_once` + # gives it its turn after the command below is created. + asyncio.create_task(take_one()) + workflow.continue_as_new(remaining - 1) + + +@workflow.defn +class SignalContinueAsNewWorkflow: + """A signal handler continues as new while a consumer is being unblocked. + + The consumer is parked on a condition the handler sets, and conditions are + re-checked *after* the ready queue drains -- which is after the terminal + command was created. So this reaches the same tail by the route ordinary + application structure reaches it by, rather than by scheduling a task next + to the Continue-As-New call. + """ + + def __init__(self) -> None: + self._release = False + self._staged = False + + @workflow.run + async def run(self, remaining: int) -> list[str]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + subscription = tokens.subscribe() + iterator = subscription.__aiter__() + first = await iterator.__anext__() + if remaining <= 1: + return [first, await iterator.__anext__()] + _require_a_buffered_record(subscription) + + async def take_one() -> None: + await workflow.wait_condition(lambda: self._release) + await iterator.__anext__() + + asyncio.create_task(take_one()) + self._staged = True + # The handler ends this Run; nothing else does. + await workflow.wait_condition(lambda: False) + raise AssertionError("unreachable") + + @workflow.query + def staged(self) -> bool: + """Whether the consumer is parked yet. + + Signalling before it is would continue as new with nothing else ready, + and the test would pass without exercising anything. + """ + return self._staged + + @workflow.signal + async def wrap_up(self, remaining: int) -> None: + self._release = True + workflow.continue_as_new(remaining) + + +def _require_a_buffered_record(subscription: object) -> None: + """Fails the Run rather than letting a thin test pass. + + The whole shape depends on a record still sitting in the subscription's + ready list when the terminal command is created. If the batch arrived one + record at a time there is nothing for the tail to consume, and the + assertions below would hold for the wrong reason. + """ + if not subscription._ready: # type: ignore[attr-defined] + raise ApplicationError( + "no record was buffered when the terminal command was created, so " + "nothing was consumed after it and this Run proves nothing", + non_retryable=True, + ) + + +async def stage_stream( + client: Client, + backend: MemoryStreamBackend, + handle: WorkflowHandle[Any, Any], + values: list[str], +) -> StreamKey: + """Puts the whole batch in the stream before any Worker can read it. + + A record that arrives an activation later is delivered to the successor by + the ordinary path and says nothing about this boundary. Publishing before + the Worker exists is what makes the Run's first drain take the batch whole. + """ + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await publish(backend, key, values) + return key + + +async def _until_staged(handle: WorkflowHandle[Any, Any]) -> None: + for _ in range(300): + if await handle.query(SignalContinueAsNewWorkflow.staged): + return + await asyncio.sleep(0.1) + raise AssertionError("the consumer never parked on its condition") + + +async def test_a_consumer_that_runs_after_the_terminal_command_is_still_consumed( + client: Client, backend: MemoryStreamBackend +) -> None: + """The predecessor's marker and the successor's header are one boundary. + + Both records the predecessor took are recorded in its final marker. A + continuation snapshot taken when the command was created holds only the + first, and the successor then starts one record too early. + """ + task_queue = f"tq-{uuid.uuid4()}" + handle = await client.start_workflow( + LateConsumerContinueAsNewWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + await stage_stream(client, backend, handle, ["a", "b", "c", "d"]) + + async with Worker( + client, + task_queue=task_queue, + workflows=[LateConsumerContinueAsNewWorkflow], + external_stream_backend=backend, + ): + assert await asyncio.wait_for(handle.result(), 30) == ["c", "d"], ( + "the successor was handed a record its predecessor consumed after " + "the Continue-As-New command was created, so the continuation " + "header was taken before the activation had finished" + ) + + +async def test_a_signal_handler_continuing_as_new_waits_for_the_consumer( + client: Client, backend: MemoryStreamBackend +) -> None: + """The same boundary, reached through a condition rather than a new task.""" + task_queue = f"tq-{uuid.uuid4()}" + handle = await client.start_workflow( + SignalContinueAsNewWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + await stage_stream(client, backend, handle, ["a", "b", "c", "d"]) + + async with Worker( + client, + task_queue=task_queue, + workflows=[SignalContinueAsNewWorkflow], + external_stream_backend=backend, + ): + await _until_staged(handle) + await handle.signal(SignalContinueAsNewWorkflow.wrap_up, 1) + + assert await asyncio.wait_for(handle.result(), 30) == ["c", "d"], ( + "the consumer the signal handler unblocked consumed a record after " + "the Continue-As-New command was created, and the successor " + "received it again" + ) + + +async def test_a_history_written_at_the_earlier_boundary_still_replays( + client: Client, backend: MemoryStreamBackend +) -> None: + """A chain that continued as new before this Worker was deployed. + + Replay regenerates the header at the *later* boundary, so it no longer + matches the recorded one. That is allowed and needs no compatibility flag: + Core matches a Continue-As-New command to its + ``WorkflowExecutionContinuedAsNew`` event by command type alone and never + compares headers. The successor of such a chain is unaffected either way -- + it reads its cursor from its own ``WorkflowExecutionStarted``, which + replaying the predecessor does not rewrite. + """ + task_queue = f"tq-{uuid.uuid4()}" + handle = await client.start_workflow( + LateConsumerContinueAsNewWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + key = await stage_stream(client, backend, handle, ["a", "b", "c", "d"]) + first_run_id = handle.first_execution_run_id + assert first_run_id is not None + + async with Worker( + client, + task_queue=task_queue, + workflows=[LateConsumerContinueAsNewWorkflow], + external_stream_backend=backend, + ): + await asyncio.wait_for(handle.result(), 30) + history = await client.get_workflow_handle( + handle.id, run_id=first_run_id + ).fetch_history() + + continued = [ + e + for e in history.events + if e.HasField("workflow_execution_continued_as_new_event_attributes") + ] + assert continued, "the predecessor did not continue as new" + attributes = continued[0].workflow_execution_continued_as_new_event_attributes + recorded = decode_continuation(attributes.header.fields[CONTINUATION_HEADER].data) + + # What the earlier snapshot held: the first record, taken before the tail of + # the activation consumed the second. + first_offset = backend._records[key][0].offset + assert first_offset is not None + early = dataclasses.replace( + recorded, cursors={wait_id: AFTER(first_offset) for wait_id in recorded.cursors} + ) + assert early != recorded, ( + "the recorded header already holds the earlier boundary, so this " + "history is not the pre-fix one it is meant to stand in for" + ) + attributes.header.fields[CONTINUATION_HEADER].CopyFrom( + write_continuation_header(early) + ) + + result = await Replayer( + workflows=[LateConsumerContinueAsNewWorkflow], + external_stream_backend=backend, + ).replay_workflow(history, raise_on_replay_failure=False) + + assert result.replay_failure is None, ( + "a history whose Continue-As-New header holds the earlier boundary no " + f"longer replays: {result.replay_failure}" + ) diff --git a/tests/contrib/external_workflow_streams/test_delivery_budget.py b/tests/contrib/external_workflow_streams/test_delivery_budget.py new file mode 100644 index 0000000000..06681da962 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_delivery_budget.py @@ -0,0 +1,1402 @@ +"""The per-activation delivery budget. + +A producer that never lets a subscription's buffer run dry makes one +``activate()`` call never return: the iterator re-fills before every record and +always finds one. Activations run on a thread-pool executor under a **2-second +deadlock timeout**, so the Workflow Task fails -- and every retry meets the same +producer, so the Workflow is stuck permanently rather than merely slow. Driving +the real iterator against a runtime whose ``drain`` always returned a full batch +consumed 316,086 records in two seconds and blocked zero times. + +Everything here asserts the *mechanism*: that the iterator gives control back +after a fixed number of records, that the count spans a merged set rather than +each member of it, that the records the budget left behind get their readiness +re-reported, and that replay -- whose boundaries are already recorded -- is not +subject to any of it. +""" + +from __future__ import annotations + +import asyncio +import time +import uuid +from collections import Counter +from datetime import timedelta +from typing import Any, cast + +import pytest + +import temporalio.converter +from temporalio.contrib.external_workflow_streams import _runtime as _runtime_module +from temporalio.contrib.external_workflow_streams._annotation import ( + MAX_ANNOTATION_BYTES, + SegmentEndReason, + decode_annotation, +) +from temporalio.contrib.external_workflow_streams._api import ( + MAX_RECORDS_PER_ACTIVATION, + _install_runtime, + external_stream, + merge, +) +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._record import ( + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._runtime import WorkflowStreamRuntime +from temporalio.worker._workflow_instance import _WorkflowInstanceImpl +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +RUN_ID = "run-1" + +#: How far past the budget an unbounded fake is allowed to run before the test +#: gives up. Large enough that a missing cap is unmistakable, small enough that +#: the failure arrives quickly. +_UNBOUNDED_BATCH = 1000 + + +# --- fixtures ----------------------------------------------------------------- + + +class FakeInstance: + """Stands in for the user's Workflow object, which holds the per-Run state.""" + + +@pytest.fixture +def workflow_instance(monkeypatch: pytest.MonkeyPatch) -> FakeInstance: + instance = FakeInstance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: instance, + ) + return instance + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +def make_runtime(manager: Any, backend: MemoryStreamBackend) -> WorkflowStreamRuntime: + return WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id="first-run", + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + ) + + +async def encoded(value: str) -> bytes: + codec: StreamPayloadCodec[str] = StreamPayloadCodec( + temporalio.converter.DataConverter.default, str + ) + return await codec.encode(value) + + +class NeverEmptyManager: + """A manager whose buffers never run dry. + + The exact shape that makes an activation never return. It honours + ``max_records`` because a real manager does; when asked for an unbounded + drain it hands back a large batch, so a budget that failed to reach the drain + shows up as an iterator that runs past the cap rather than as a fake that + quietly stops. + """ + + def __init__(self, payload: bytes) -> None: + self.payload = payload + self.max_records_seen: list[int | None] = [] + #: Every wait the runtime actually asked about. A wait that never + #: appears was not merely served nothing -- the Worker was never + #: enquired of on its behalf at all. + self.drained_waits: list[int] = [] + self.rearmed: list[str] = [] + self._next: dict[int, int] = {} + + def register(self, *, run_id, wait_id, stream_key, start_cursor): # type: ignore[no-untyped-def] + self._next.setdefault(wait_id, 0) + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + def drain( + self, + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, + max_records: int | None = None, + ) -> list[StreamRecord]: + self.max_records_seen.append(max_records) + self.drained_waits.append(wait_id) + count = _UNBOUNDED_BATCH if max_records is None else max_records + start = self._next.get(wait_id, 0) + self._next[wait_id] = start + count + return [ + StreamRecord(RecordKind.DATA, self.payload, f"p{wait_id}", i).placed_at( + Offset(f"{wait_id:02d}-{i:08d}") + ) + for i in range(start, start + count) + ] + + def rearm_ready(self, run_id: str) -> None: + self.rearmed.append(run_id) + + +async def consume(iterator: Any, seen: list[Any]) -> None: + async for value in iterator: + seen.append(value) + + +async def settle(task: asyncio.Task[None], seen: list[Any], expected: int) -> None: + """Runs the consumer until it stops making progress, then a little longer. + + The extra window is what turns "it has not finished yet" into "it blocked": + an iterator that ignored the budget would keep going right through it. + """ + deadline = time.monotonic() + 5 + while len(seen) < expected and not task.done() and time.monotonic() < deadline: + await asyncio.sleep(0.005) + await asyncio.sleep(0.1) + + +async def cancel(task: asyncio.Task[None]) -> None: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +# --- one subscription --------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_never_empty_subscription_yields_control_after_the_budget( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """The measured failure, and the cap that ends it. + + Without a bound this loop never returns and the Workflow Task dies on the + deadlock timeout -- on every retry, because the producer is still there. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + subscription = external_stream.topic("tokens", type=str).subscribe() + + seen: list[str] = [] + task = asyncio.ensure_future(consume(subscription.__aiter__(), seen)) + await settle(task, seen, MAX_RECORDS_PER_ACTIVATION) + + assert not task.done(), ( + "the iterator ran past the budget against a buffer that never empties; " + "this activation would never return and the Workflow Task would fail on " + "the 2-second deadlock timeout" + ) + assert len(seen) == MAX_RECORDS_PER_ACTIVATION + assert runtime.delivery_budget_exhausted() + assert manager.max_records_seen and all( + limit is not None for limit in manager.max_records_seen + ), "every drain must carry the remaining budget" + assert 0 not in manager.max_records_seen, ( + "a spent budget must stop the drain from being attempted at all, rather " + "than reaching the manager and asking it for nothing" + ) + await cancel(task) + + +@pytest.mark.asyncio +async def test_the_budget_blocks_even_though_records_are_buffered( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """Blocking is the whole point: it is what ends the activation. + + The double-check refill inside ``_await_readiness`` is the hazard here. It + exists to catch a record buffered while the wait was being registered, and if + it ignored the budget it would hand this iterator a record immediately and + the activation would go right back to never returning. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + subscription = external_stream.topic("tokens", type=str).subscribe() + + seen: list[str] = [] + task = asyncio.ensure_future(consume(subscription.__aiter__(), seen)) + await settle(task, seen, MAX_RECORDS_PER_ACTIVATION) + + # Blocked on a readiness future, with the manager still holding records. + assert runtime.resolve_all_pending() == 1, ( + "the iterator must be parked on a readiness future, which is what lets " + "the activation return" + ) + await cancel(task) + + +@pytest.mark.asyncio +async def test_the_next_activation_gets_a_fresh_budget( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """The cap bounds one activation, not the Run. + + A budget that were never reset would deliver 256 records and then never + another one, which is a different way of hanging the same Workflow. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + subscription = external_stream.topic("tokens", type=str).subscribe() + + seen: list[str] = [] + task = asyncio.ensure_future(consume(subscription.__aiter__(), seen)) + await settle(task, seen, MAX_RECORDS_PER_ACTIVATION) + assert len(seen) == MAX_RECORDS_PER_ACTIVATION + + # The next activation begins: budget reset, readiness resolved. + runtime.begin_activation() + assert runtime.delivery_budget_remaining() == MAX_RECORDS_PER_ACTIVATION + runtime.resolve_all_pending() + await settle(task, seen, 2 * MAX_RECORDS_PER_ACTIVATION) + + assert len(seen) == 2 * MAX_RECORDS_PER_ACTIVATION + assert not task.done() + await cancel(task) + + +# --- the budget spans subscriptions ------------------------------------------- + + +@pytest.mark.asyncio +async def test_merge_spends_one_budget_across_every_subscription( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """A per-subscription budget would let n streams run n times as long. + + That is the same deadlock with a larger constant in front of it, which is + why the counter lives on the runtime rather than on a subscription. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + + seen: list[tuple[int, str]] = [] + + async def consume_merged() -> None: + async for subscription, value in merge(first, second): + seen.append((subscription.wait_id, value)) + + task = asyncio.ensure_future(consume_merged()) + await settle(task, seen, MAX_RECORDS_PER_ACTIVATION) # type: ignore[arg-type] + + assert not task.done() + assert len(seen) == MAX_RECORDS_PER_ACTIVATION, ( + "the budget covers the whole merged set; two never-empty streams must " + "not buy twice the records" + ) + await cancel(task) + + +@pytest.mark.asyncio +async def test_merge_blocks_on_every_wait_once_the_budget_is_spent( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """``_await_any_readiness`` takes the same last look, under the same cap. + + A refill there that ignored the budget would resume the merge immediately and + the activation would never end. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + + seen: list[tuple[int, str]] = [] + + async def consume_merged() -> None: + async for subscription, value in merge(first, second): + seen.append((subscription.wait_id, value)) + + task = asyncio.ensure_future(consume_merged()) + await settle(task, seen, MAX_RECORDS_PER_ACTIVATION) # type: ignore[arg-type] + + assert runtime.resolve_all_pending() == 2, ( + "the complete set must be blocked, not just the stream that happened to " + "spend the last of the budget" + ) + await cancel(task) + + +async def settle_all( + tasks: list[asyncio.Task[None]], counted: list[list[Any]], expected: int +) -> None: + """`settle`, over several consumers that share one budget between them.""" + deadline = time.monotonic() + 5 + while ( + sum(len(seen) for seen in counted) < expected + and not any(task.done() for task in tasks) + and time.monotonic() < deadline + ): + await asyncio.sleep(0.005) + await asyncio.sleep(0.1) + + +@pytest.mark.asyncio +async def test_two_independent_consumers_share_one_budget( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """The same property as the merge case, without `merge()` to enforce it. + + `merge()` fills one record at a time, so it spends the budget in the same + places it checks it. Two ordinary subscriptions consumed by two ordinary + coroutines do not: each one drains a whole batch into its own ready list, and + a budget that is *checked* at the drain but *charged* at consumption is no + budget at all. The first subscription takes all 256 slots into its private + list and yields one, so the second sees 255 still available and takes those, + and both then drain lists nothing will check again. Measured at 511 records + before either coroutine blocked, and it is `n` times the cap for `n` + consumers -- the same deadlock the merge case is about, reached without + merging anything. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + + seen_first: list[str] = [] + seen_second: list[str] = [] + + async def consume_yielding(subscription: Any, seen: list[str]) -> None: + # Awaiting after every value is what lets the other coroutine reach its + # own drain, which is the whole shape: two private ready lists, filled + # from one budget, consumed independently. + async for value in subscription: + seen.append(value) + await asyncio.sleep(0) + + tasks = [ + asyncio.ensure_future(consume_yielding(first, seen_first)), + asyncio.ensure_future(consume_yielding(second, seen_second)), + ] + await settle_all(tasks, [seen_first, seen_second], MAX_RECORDS_PER_ACTIVATION) + + try: + assert len(seen_first) + len(seen_second) == MAX_RECORDS_PER_ACTIVATION, ( + "two independently consumed subscriptions spent more than one " + "activation's budget between them; with enough streams this is the " + "activation that never returns" + ) + assert all(not task.done() for task in tasks) + assert runtime.delivery_budget_remaining() == 0 + assert runtime.delivery_budget_exhausted() + # Both are blocked, which is what ends the activation. A consumer still + # holding a ready list would not be, and the assertion above would have + # been reached with records left to hand over. + assert runtime.resolve_all_pending() == 2, ( + "both consumers must be parked on a readiness future once the budget " + "is spent, not sitting on a private ready list the cap cannot see" + ) + # And what the activation *recorded* matches what it delivered: the + # segment is the schedule replay reproduces, so a cap that bounded + # consumption while the annotation recorded delivery bounded the wrong + # quantity. + delta = runtime.take_observation_delta() + assert delta is not None + encoded_annotation = delta + runtime.add_terminal() + annotation = decode_annotation(encoded_annotation) + recorded = sum( + run.count for segment in annotation.segments for run in segment.runs + ) + assert recorded == MAX_RECORDS_PER_ACTIVATION + # And it still fits. Delivering two speculative batches put both in the + # current annotation before the counter caught up, so the same schedule + # could also overrun the affordability calculation and turn the intended + # rollover into a Workflow Task failure that repeats on every attempt. + assert len(encoded_annotation) <= MAX_ANNOTATION_BYTES + finally: + for task in tasks: + await cancel(task) + + +@pytest.mark.asyncio +async def test_a_carried_over_ready_list_is_charged_to_the_next_activation( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """The same hole from the other side, and the reason the reset is not to zero. + + A batch is delivered whole and consumed one record at a time, so a consumer + that stops part-way through leaves the rest in its ready list -- where the + next activation takes it with no drain, and so with no budget check. Charging + only at the drain would make that remainder free, and it accumulates: one + subscription per activation can leave a nearly full list behind, so `n` of + them arrive holding `n` times the cap and hand all of it over in one + `activate()` call. + + Nothing is redelivered here and no drain is needed, so what the budget must + bound is what Workflow code can *take*: the carry-over plus whatever is newly + delivered, together, is one cap. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + subscription = external_stream.topic("tokens", type=str).subscribe() + + # One activation that delivers a whole batch and consumes a single record, + # which is the ordinary shape of a consumer that does real work per record: + # the loop body blocks on something else and the activation ends. + seen: list[str] = [] + iterator = subscription.__aiter__() + seen.append(await iterator.__anext__()) + assert runtime.delivery_budget_remaining() == 0 + carried = MAX_RECORDS_PER_ACTIVATION - 1 + + runtime.begin_activation() + + assert ( + runtime.delivery_budget_remaining() == MAX_RECORDS_PER_ACTIVATION - carried + ), ( + f"the next activation was handed a whole fresh budget on top of {carried} " + "records already in Workflow code's hands; that remainder is consumed " + "with no drain and so with no check, and it accumulates across streams" + ) + # Everything carried over is still handed over -- the records are already here + # and refusing them would strand them, since nothing re-announces a record + # already popped out of the manager's buffer -- and the whole activation still + # totals one cap rather than the carry-over plus another one. + task = asyncio.ensure_future(consume(iterator, seen)) + await settle(task, seen, MAX_RECORDS_PER_ACTIVATION + 1) + assert len(seen) - 1 == MAX_RECORDS_PER_ACTIVATION, ( + "the second activation handed over the carry-over and a fresh cap on top " + "of it; what it may hand over is one cap in total" + ) + assert not task.done() + await cancel(task) + + +class OneBusyOneQuietManager: + """Wait 1 never runs dry; wait 2 holds a single record and then nothing. + + The shape a merge has to survive: one stream a producer keeps saturated, and + one that has something to say exactly once. A merge that drains the busy + stream to the bottom of the budget before it looks at the quiet one never + reaches the quiet one at all, and never will -- every fresh budget starts at + the same lowest wait id. + """ + + def __init__(self, payload: bytes) -> None: + self.payload = payload + self.rearmed: list[str] = [] + self._next = 0 + self._quiet_pending = True + + def register(self, *, run_id, wait_id, stream_key, start_cursor): # type: ignore[no-untyped-def] + pass + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + def drain( + self, + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, + max_records: int | None = None, + ) -> list[StreamRecord]: + if max_records is not None and max_records <= 0: + return [] + if wait_id == 2: + if not self._quiet_pending: + return [] + self._quiet_pending = False + return [ + StreamRecord(RecordKind.DATA, self.payload, "p2", 0).placed_at( + Offset("02-00000000") + ) + ] + count = _UNBOUNDED_BATCH if max_records is None else max_records + start = self._next + self._next = start + count + return [ + StreamRecord(RecordKind.DATA, self.payload, "p1", i).placed_at( + Offset(f"01-{i:08d}") + ) + for i in range(start, start + count) + ] + + def rearm_ready(self, run_id: str) -> None: + self.rearmed.append(run_id) + + +@pytest.mark.asyncio +async def test_merge_does_not_starve_a_stream_behind_a_saturated_one( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """A merge that is not fair is not a merge. + + Draining one subscription's whole ready list before looking at the next lets + the lowest wait id spend the entire activation budget by itself, and the next + activation begins the same pass in the same order. A record sitting ready on + a later wait is then never yielded -- deterministically, but that only makes + it reproducible rather than acceptable. + + Three budget cycles is well past any plausible "it was just slow": one pass + over the set is enough for a fair merge. + """ + manager = OneBusyOneQuietManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + busy = external_stream.topic("busy", type=str).subscribe() + quiet = external_stream.topic("quiet", type=str).subscribe() + assert (busy.wait_id, quiet.wait_id) == (1, 2) + + seen: list[tuple[int, str]] = [] + + async def consume_merged() -> None: + async for subscription, value in merge(busy, quiet): + seen.append((subscription.wait_id, value)) + + task = asyncio.ensure_future(consume_merged()) + for cycle in range(1, 4): + await settle(task, seen, cycle * MAX_RECORDS_PER_ACTIVATION) # type: ignore[arg-type] + # What the Worker's completion path does: a fresh budget, and readiness + # re-armed for the records the budget left buffered. + runtime.begin_activation() + runtime.resolve_all_pending() + + await cancel(task) + assert quiet.wait_id in {wait_id for wait_id, _ in seen}, ( + "the quiet stream's one ready record was never yielded across three " + "whole activation budgets; the busy stream spent every one of them" + ) + assert len(seen) <= 3 * MAX_RECORDS_PER_ACTIVATION + 1, ( + "the budget must still bound the merged set as a whole" + ) + + +# --- fairness across activations ---------------------------------------------- + +#: The budget the two fairness tests below run against, in place of the real +#: 256. What they prove is a relationship between the *subscription count* and +#: the budget -- more ready waits than the budget can pay for, or a count that +#: does not divide it -- and the real constant only sets the scale at which that +#: relationship bites. Standing up 257 subscriptions and handing out 768 records +#: to watch it costs more than the property is worth, and destabilised the +#: end-to-end tests that share the run. +_SMALL_BUDGET = 8 + + +@pytest.fixture +def small_budget(monkeypatch: pytest.MonkeyPatch) -> int: + """Shrinks the delivery budget for one test, and returns what it is now. + + Patched on ``_runtime`` rather than on ``_api``, which is where the constant + is defined: ``_runtime`` binds the name at import, so it reads its own + module global, and a patch on the defining module would leave every check + that actually enforces the budget looking at 256. + """ + monkeypatch.setattr(_runtime_module, "MAX_RECORDS_PER_ACTIVATION", _SMALL_BUDGET) + return _SMALL_BUDGET + + +#: A subscription count that does not divide the budget, at the shrunk scale: 8 +#: records over 3 always-ready waits is two whole passes and two waits of a +#: third, so the pass is always cut in the middle -- which is the arrangement a +#: start position fixed at the lowest wait id turns into permanent, growing +#: unfairness. The real-scale case is 100 waits against 256 records: two whole +#: passes and 56 waits of a third, the same cut in the same place. +_SKEWED_STREAMS = 3 + +#: How many activations the skew is watched over. Whatever the scale, a pass +#: cut in the middle hands the favoured waits exactly one record more per +#: activation than the rest, so the gap is the activation count and nothing +#: bounds it: five is five times the tolerance rather than a coin toss. +_SKEW_CYCLES = 5 + + +async def run_budget_cycles( + runtime: WorkflowStreamRuntime, + task: asyncio.Task[None], + seen: list[Any], + cycles: int, + budget: int, +) -> None: + """Runs `cycles` whole activations, doing what the Worker's completion does. + + A fresh budget and readiness re-armed for the records the budget left + buffered -- which is what makes the next pass a *new activation* rather + than a continuation of this one, and so is the only place a start position + that never moves can show itself. + + `budget` is passed rather than read from the constant because these callers + run against a patched one; a never-empty set delivers exactly that many + records per activation, which is how each cycle knows it has finished. + """ + for cycle in range(1, cycles + 1): + await settle(task, seen, cycle * budget) + runtime.begin_activation() + runtime.resolve_all_pending() + + +@pytest.mark.asyncio +async def test_merge_reaches_a_wait_the_budget_cannot_pay_for_in_one_pass( + workflow_instance: FakeInstance, + backend: MemoryStreamBackend, + small_budget: int, +) -> None: + """More ready subscriptions than one activation has records to hand out. + + The budget is charged per record handed to Workflow code, once per + subscription per pass, so a pass over *budget + 1* ready waits runs out + exactly one short. A pass that began at the lowest wait id on every + activation ran out in the same place on every one of them: the last wait was + never yielded, and -- because ``_fill`` returns on the spent budget before it + reaches the manager -- never even drained. The Worker was not asked about + that stream at all, for the life of the Run. + + Run against a budget of ``_SMALL_BUDGET`` rather than the real 256, because + "one more ready wait than the budget can pay for" is the whole property and + the constant only says how many that is: at full scale the starved case is + 257 subscriptions, and it starves for exactly the reason nine do here. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + # The shrunk budget has to have reached the binding the runtime reads, or + # the counts below stand in a relationship to 256 that they were not chosen + # for and the test passes without exercising anything. + assert runtime.delivery_budget_remaining() == small_budget + subscriptions = [ + external_stream.topic(f"s{i}", type=str).subscribe() + for i in range(small_budget + 1) + ] + last = subscriptions[-1] + + seen: list[int] = [] + + async def consume_merged() -> None: + async for subscription, _ in merge(*subscriptions): + seen.append(subscription.wait_id) + + task = asyncio.ensure_future(consume_merged()) + await run_budget_cycles(runtime, task, seen, 3, small_budget) + await cancel(task) + + assert last.wait_id in set(seen), ( + "the wait one past the budget was never yielded across three whole " + "activations; every one of them spent itself on the same prefix" + ) + assert last.wait_id in manager.drained_waits, ( + "the starved wait was never even drained -- the spent budget stopped " + "`_fill` before it reached the manager, so the Worker was never asked " + "whether that stream had anything at all" + ) + assert len(seen) <= 3 * small_budget, ( + "the budget must still bound the merged set as a whole" + ) + + +@pytest.mark.asyncio +async def test_merge_keeps_the_skew_bounded_when_the_count_splits_the_budget( + workflow_instance: FakeInstance, + backend: MemoryStreamBackend, + small_budget: int, +) -> None: + """The general defect, of which total starvation is only the extreme. + + Nothing has to be starved outright for the merge to be unfair: a count that + does not divide the budget is enough. At full scale, 256 records over 100 + always-ready waits leaves the pass cut at the 56th, and a pass that restarted + at the lowest wait id was cut there on every activation -- the first 56 + taking three records per activation and the other 44 taking two, forever. The + gap between the best- and worst-served stream then grows by one per + activation and nothing bounds it: measured at 8 after eight activations, and + it would be 800 after eight hundred. + + Run here at ``_SMALL_BUDGET`` over ``_SKEWED_STREAMS`` waits, which is the + same arrangement in miniature -- two whole passes and a cut in the middle of + a third -- because what makes the skew unbounded is the remainder, not its + size. The gap still grows by exactly one per activation, so + ``_SKEW_CYCLES`` activations separate a merge that resumes where the last + pass stopped from one that does not. + + Taking at most one record per turn bounds the skew *within* a pass. Only + resuming after the wait that last took one bounds it *across* passes, which + is what the single-record bound was always claimed to be. + """ + manager = NeverEmptyManager(await encoded("x")) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + # The shrunk budget has to have reached the binding the runtime reads, or + # the counts below stand in a relationship to 256 that they were not chosen + # for and the test passes without exercising anything. + assert runtime.delivery_budget_remaining() == small_budget + subscriptions = [ + external_stream.topic(f"s{i}", type=str).subscribe() + for i in range(_SKEWED_STREAMS) + ] + + seen: list[int] = [] + + async def consume_merged() -> None: + async for subscription, _ in merge(*subscriptions): + seen.append(subscription.wait_id) + + task = asyncio.ensure_future(consume_merged()) + await run_budget_cycles(runtime, task, seen, _SKEW_CYCLES, small_budget) + await cancel(task) + + per_wait = Counter(seen) + counts = [per_wait[s.wait_id] for s in subscriptions] + assert max(counts) - min(counts) <= 1, ( + f"after {_SKEW_CYCLES} activations the best-served stream had " + f"{max(counts)} records and the worst-served {min(counts)}; the skew " + "grows by one per activation and is bounded by nothing, so a long-" + "running merge reads its later streams arbitrarily far behind its " + "earlier ones" + ) + assert min(counts) > 0 + assert len(seen) <= _SKEW_CYCLES * small_budget, ( + "the budget must still bound the merged set as a whole" + ) + + +# --- re-arming readiness ------------------------------------------------------ + + +class RecordingNotifier: + """Stands in for Core's readiness call, recording every notification.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, int, int]] = [] + self.notified = asyncio.Event() + + async def __call__(self, run_id: str, wait_id: int, wait_generation: int) -> str: + self.calls.append((run_id, wait_id, wait_generation)) + self.notified.set() + return ReadinessResult.ACCEPTED + + +@pytest.mark.asyncio +async def test_rearming_re_reports_readiness_for_a_non_empty_buffer() -> None: + """Without this the Workflow waits forever on records already in front of it. + + Readiness is reported once, when the watcher buffers a record, and that + watcher has long since moved its prefetch cursor past whatever the budget + left behind. Nothing else will ever announce those records. + """ + stream_key = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=notifier, + watch_block=timedelta(milliseconds=20), + ) + try: + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"a", "session", 0) + ) + await asyncio.wait_for(notifier.notified.wait(), 2) + # The activation drained nothing: the budget was already spent elsewhere. + assert subscription.buffered == 1 + first_round = len(notifier.calls) + + manager.rearm_ready(RUN_ID) + await asyncio.sleep(0.1) + + assert len(notifier.calls) > first_round, ( + "the records the budget left buffered got no second notification, so " + "the Workflow would block on data that is already at the Worker" + ) + assert notifier.calls[-1] == (RUN_ID, 1, 0) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_rearming_says_nothing_about_an_empty_buffer() -> None: + """Readiness means *buffered*, and re-arming may not weaken that. + + A notification for an empty buffer produces an activation whose drain finds + nothing, which is the spurious Workflow Task the "only after buffered" rule + exists to prevent. + """ + stream_key = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=notifier, + watch_block=timedelta(milliseconds=20), + ) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await asyncio.sleep(0.05) + assert notifier.calls == [] + + manager.rearm_ready(RUN_ID) + await asyncio.sleep(0.1) + + assert notifier.calls == [] + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_record_arriving_after_the_last_drain_is_re_announced() -> None: + """The stall this closes, driven through the real manager and completion path. + + A record that arrives after an activation's final drain is announced to Core + at the wait generation that was current when the watcher buffered it. The + Workflow then re-blocks on the way out of that activation, which bumps the + generation -- and Core keeps readiness across a quiescent snapshot only where + the generations match, because a bumped one ordinarily means the Workflow + drained and re-blocked and the readiness refers to a block already resolved. + Here it means the opposite, and Core cannot tell the difference: the record was + never seen. + + Nothing else would announce it. The watcher reports only after a *new* + non-empty read and its prefetch cursor is already past this record; the idle + timer would park a Workflow Task whose data is sitting at the Worker. So the + completion has to re-announce, and it has to do so whether or not a delivery + budget was involved -- which is what this asserts. + """ + stream_key = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=notifier, + watch_block=timedelta(milliseconds=20), + ) + try: + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=stream_key) + # A wait is blocked from creation, so reaching generation 1 means taking a + # record and blocking again -- which is the ordinary shape and the one the + # generation counter exists for. + runtime.note_blocked(1, False) + runtime.note_blocked(1, True) + + await backend.append( + stream_key, StreamRecord(RecordKind.DATA, await encoded("late"), "s", 0) + ) + await asyncio.wait_for(notifier.notified.wait(), 5) + announced_at = list(notifier.calls) + assert announced_at and announced_at[-1][2] == 1, ( + f"the watcher did not announce under generation 1: {announced_at}" + ) + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + assert subscription.buffered == 1, "the record is not buffered at the Worker" + + # This activation delivered nothing -- the record landed after its last + # drain -- and the Workflow re-blocks on the way out. That is the + # generation bump Core reads as "already resolved". + runtime.note_blocked(1, False) + runtime.note_blocked(1, True) + assert subscription.current_wait_generation() == 2 + + before = len(notifier.calls) + _WorkflowInstanceImpl._emit_external_stream_commands( + cast(Any, _CompletionStub(runtime)) + ) + await asyncio.sleep(0.2) + + assert len(notifier.calls) > before, ( + "the completion announced nothing, so the record Core dropped the " + "readiness for is announced by nobody and the Workflow waits on data " + "it is already holding" + ) + assert notifier.calls[-1] == (RUN_ID, 1, 2), ( + "the re-announcement has to name the *current* generation, or Core " + f"discards it for the same reason: {notifier.calls[-1]}" + ) + finally: + await manager.shutdown() + + +class _CompletionStub: + """Just enough of the Workflow instance to drive the completion path. + + The real method is reached unbound, so what is under test is the code that + ships rather than a copy of it. + """ + + def __init__(self, runtime: Any, *, replaying: bool = False) -> None: + self._external_stream_runtime = runtime + self._deleting = False + self._is_replaying = replaying + # No reader has been opened on a server with channels, so the + # completion carries no channel report; the report has its own tests. + self._stream_channels_reporting = False + import temporalio.bridge.proto.workflow_completion + + self._current_completion = ( + temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() + ) + self._current_completion.successful.SetInParent() + + def _add_command(self) -> Any: + return self._current_completion.successful.commands.add() + + # Reached unbound for the same reason `_emit_external_stream_commands` is: + # the completion path calls these, and a stub that reimplemented them would + # let the shipped versions drift. Neither reads the runtime unless the + # completion carries a Continue-As-New command, which none of these do. + def _refresh_external_stream_continuations(self) -> None: + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + _WorkflowInstanceImpl._refresh_external_stream_continuations(self) # type: ignore[arg-type] + + def _attach_external_stream_continuation(self, command: Any) -> None: + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + _WorkflowInstanceImpl._attach_external_stream_continuation(self, command) # type: ignore[arg-type] + + +class _RecordingRuntime: + """Answers the two questions the completion path asks, and records the call.""" + + def __init__(self, *, exhausted: bool) -> None: + self._exhausted = exhausted + self.rearms = 0 + + def delivery_budget_exhausted(self) -> bool: + return self._exhausted + + def rearm_readiness(self) -> None: + self.rearms += 1 + + def take_observation_delta(self) -> bytes | None: + return None + + def quiescent_snapshot(self) -> None: + return None + + def start_new_annotation(self) -> None: + pass + + @property + def annotation_started(self) -> bool: + # Nothing was ever accumulated, so there is no annotation to terminate. + return False + + @property + def request_rollover(self) -> bool: + return False + + +@pytest.mark.parametrize("exhausted", [True, False]) +def test_every_completion_rearms_readiness_not_only_a_budget_stop( + exhausted: bool, +) -> None: + """The window must be closed on **every** completion, budget or not. + + It has to happen on the completion path: the waits involved are marked + blocked, so they enter the quiescent snapshot, and a snapshot is what lets + Core start the idle timer and eventually park. A Workflow Task parked with + records in the local buffer would wait out an idle timeout for data that had + already arrived. + + Gating it on the budget was a real stall, not a tidiness point, because the + budget is not the only way an activation ends with a full buffer. Core keeps + readiness accepted during an open activation **only if the quiescent snapshot + that follows reports the same wait generation** -- same generation means the + Workflow never saw the record, a bumped one means it drained and re-blocked. + So a record arriving after this activation's last drain is accepted at + generation G, the wait re-blocks to G+1 on the way out, and the snapshot + legitimately drops a readiness that now refers to a resolved block. The + watcher will not report it again either: it only reports after a *new* + non-empty read, and its prefetch cursor is already past that record. Nothing + else announces it, so the Workflow waits on data it is already holding. + + The re-arm is cheap where it is not needed -- the manager skips every + subscription whose buffer is empty, which is the ordinary case. + """ + runtime = _RecordingRuntime(exhausted=exhausted) + stub = _CompletionStub(runtime) + + _WorkflowInstanceImpl._emit_external_stream_commands(stub) # type: ignore[arg-type] + + assert runtime.rearms == 1, ( + "a completion that did not hit the delivery budget skipped the re-arm, " + "which is the only thing that re-announces a record Core dropped the " + "readiness for" + ) + + +def test_the_completion_rearms_even_when_it_emits_nothing_else() -> None: + """The early returns below it must not swallow the re-arm. + + A completion carrying the Workflow's own commands returns before asking for + retention. If the re-arm sat behind that return, a Workflow with records still + buffered while also starting a timer would never hear about them again. + """ + runtime = _RecordingRuntime(exhausted=True) + stub = _CompletionStub(runtime) + stub._add_command() # something server-bound rides along + + _WorkflowInstanceImpl._emit_external_stream_commands(stub) # type: ignore[arg-type] + + assert runtime.rearms == 1 + + +def test_a_terminal_completion_does_not_rearm_buffered_readiness() -> None: + """A Workflow that is ending needs no wake for records it leaves behind. + + Re-arming after the terminal command has already been produced reports the + remaining buffer after Core closes the Workflow Task. ``NoOpenWorkflowTask`` + then sends a fresh Signal into the execution's closing window, where it can + contend with the terminal report and force another replay of the same + completion. Complete, fail, cancel, and Continue-As-New all end this Run; + none can consume a later activation, so none has a liveness reason to wake. + """ + runtime = _RecordingRuntime(exhausted=False) + stub = _CompletionStub(runtime) + stub._add_command().complete_workflow_execution.SetInParent() + + _WorkflowInstanceImpl._emit_external_stream_commands(stub) # type: ignore[arg-type] + + assert runtime.rearms == 0 + + +def test_a_wait_the_budget_stopped_is_not_immediately_parkable( + backend: MemoryStreamBackend, +) -> None: + """Blocked is not quiescent when the budget is what stopped it. + + ``immediately_parkable`` invites Core to park without waiting out the idle + timer. A subscription the budget stopped still has records at the Worker, so + parking it would strand them. + """ + + class StubManager: + def cancel_from_workflow_thread(self, run_id, wait_id): # type: ignore[no-untyped-def] + pass + + def register( + self, + **kwargs: Any, # pyright: ignore[reportUnusedParameter] + ) -> None: + pass + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + runtime = make_runtime(StubManager(), backend) + runtime.begin_activation() + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + record = StreamRecord(RecordKind.WRITE_FENCE, b"", "s", 0).placed_at(Offset("1")) + runtime.record_delivery(1, record) + runtime.note_blocked(1, True) + + fenced = runtime.quiescent_snapshot() + assert fenced is not None and fenced[0].immediately_parkable + + # Spent through `record_delivery`, which is where the budget is charged: a + # record moving into a subscription's ready list is the reservation, and the + # consumption that follows it costs nothing. + for _ in range(MAX_RECORDS_PER_ACTIVATION): + runtime.record_delivery(1, record) + + stopped = runtime.quiescent_snapshot() + assert stopped is not None and not stopped[0].immediately_parkable + + +# --- the double-check refill still works -------------------------------------- + + +class LateArrivalManager: + """Empty on the first look, holding a record on the second. + + Exactly the window the double-check inside ``_await_readiness`` exists for: + the record landed after ``_iterate`` looked and before the wait was + registered, so its readiness was reported to nobody and none is coming. + """ + + def __init__(self, record: StreamRecord) -> None: + self.record: StreamRecord | None = record + self.calls = 0 + + def register( + self, + **kwargs: Any, # pyright: ignore[reportUnusedParameter] + ) -> None: + pass + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + def drain( + self, + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + max_records: int | None = None, # pyright: ignore[reportUnusedParameter] + ) -> list[StreamRecord]: + self.calls += 1 + if self.calls == 1 or self.record is None: + return [] + record, self.record = self.record, None + return [record] + + +@pytest.mark.asyncio +async def test_the_double_check_refill_still_delivers_a_late_record( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """Budgeting the double-check must not disable it. + + It fixes a separate stall: a record buffered while Workflow code was + elsewhere has already had its readiness reported and consumed, so blocking + without looking again strands the Workflow on a record sitting in front of + it. + """ + record = StreamRecord(RecordKind.DATA, await encoded("late"), "s", 0).placed_at( + Offset("1") + ) + manager = LateArrivalManager(record) + runtime = make_runtime(manager, backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + subscription = external_stream.topic("tokens", type=str).subscribe() + + iterator = subscription.__aiter__() + + assert await asyncio.wait_for(iterator.__anext__(), 1) == "late" + assert manager.calls >= 2, ( + "the record must have been found by the look taken *after* the wait was " + "registered, which is the window that double-check closes" + ) + + +# --- replay is unaffected ----------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_replay_segment_larger_than_the_budget_is_delivered_in_full( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """The recorded boundaries already say how many records each activation got. + + Re-cutting them at the budget would deliver a different schedule than the + live run recorded, which is divergence rather than protection: replay reads + from a finite recorded segment, not from a producer that can outrun it. + """ + payload = await encoded("r") + count = MAX_RECORDS_PER_ACTIVATION + 44 + + class StubManager: + def cancel_from_workflow_thread(self, run_id, wait_id): # type: ignore[no-untyped-def] + pass + + def register( + self, + **kwargs: Any, # pyright: ignore[reportUnusedParameter] + ) -> None: + pass + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + def drain(self, run_id: str, wait_id: int, max_records: int | None = None): # type: ignore[no-untyped-def] + return [] + + runtime = make_runtime(StubManager(), backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + subscription = external_stream.topic("tokens", type=str).subscribe() + runtime.begin_replay_segment( + [ + ( + subscription.wait_id, + StreamRecord(RecordKind.DATA, payload, "s", i).placed_at( + Offset(f"{i:08d}") + ), + ) + for i in range(count) + ] + ) + + seen: list[str] = [] + task = asyncio.ensure_future(consume(subscription.__aiter__(), seen)) + await settle(task, seen, count) + + assert len(seen) == count, ( + "the recorded segment was cut short by the live delivery budget; replay " + "would then diverge from the run it is replaying" + ) + + # Back on the live buffer, which is where the budget applies again. + runtime.end_replay() + assert runtime.delivery_budget_remaining() == MAX_RECORDS_PER_ACTIVATION, ( + "replayed records must not spend the live budget either -- a later live " + "delivery in the same activation would start with none left, and the " + "completion would re-arm readiness for a purely replayed Workflow Task" + ) + assert not runtime.delivery_budget_exhausted() + await cancel(task) + + +@pytest.mark.asyncio +async def test_replay_delivers_in_full_even_when_the_live_budget_is_spent( + workflow_instance: FakeInstance, backend: MemoryStreamBackend +) -> None: + """The bypass has to be the replay state, not an empty counter. + + A budget that merely happened to be untouched would work by accident. What + must hold is that a recorded segment is delivered whole no matter what the + live counter says, because its boundaries are already in History and cutting + them somewhere else is divergence. + """ + payload = await encoded("r") + count = MAX_RECORDS_PER_ACTIVATION + 44 + + class StubManager: + def cancel_from_workflow_thread(self, run_id, wait_id): # type: ignore[no-untyped-def] + pass + + def register( + self, + **kwargs: Any, # pyright: ignore[reportUnusedParameter] + ) -> None: + pass + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + def drain(self, run_id: str, wait_id: int, max_records: int | None = None): # type: ignore[no-untyped-def] + return [] + + runtime = make_runtime(StubManager(), backend) + _install_runtime(workflow_instance, runtime) + runtime.begin_activation() + subscription = external_stream.topic("tokens", type=str).subscribe() + + # Spend the whole live budget first, through the delivery that charges it. + spent = StreamRecord(RecordKind.DATA, payload, "s", 0).placed_at(Offset("live")) + for _ in range(MAX_RECORDS_PER_ACTIVATION): + runtime.record_delivery(subscription.wait_id, spent) + runtime.record_consumption(subscription.wait_id, spent) + assert runtime.delivery_budget_remaining() == 0 + + runtime.begin_replay_segment( + [ + ( + subscription.wait_id, + StreamRecord(RecordKind.DATA, payload, "s", i).placed_at( + Offset(f"{i:08d}") + ), + ) + for i in range(count) + ] + ) + + seen: list[str] = [] + task = asyncio.ensure_future(consume(subscription.__aiter__(), seen)) + await settle(task, seen, count) + + assert len(seen) == count, ( + "a spent live budget stopped a recorded segment, so replay delivered " + "fewer records than the marker says the live run received" + ) + await cancel(task) + + +# --- the recorded reason ------------------------------------------------------ + + +def _annotation_reasons(runtime: WorkflowStreamRuntime) -> list[SegmentEndReason]: + delta = runtime.take_observation_delta() + assert delta is not None + annotation = decode_annotation(delta + runtime.add_terminal()) + return [segment.end_reason for segment in annotation.segments] + + +def test_a_budget_stop_records_batch_limit(backend: MemoryStreamBackend) -> None: + """``NO_DATA_AVAILABLE`` would be a false statement, and a durable one. + + The end reason is the only thing in the annotation that says why an + activation stopped. Recording "the stream ran dry" for an activation that + stopped with records still buffered puts a claim in History that was never + true. + """ + + class StubManager: + def cancel_from_workflow_thread(self, run_id, wait_id): # type: ignore[no-untyped-def] + pass + + def register( + self, + **kwargs: Any, # pyright: ignore[reportUnusedParameter] + ) -> None: + pass + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + runtime = make_runtime(StubManager(), backend) + runtime.begin_activation() + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + for i in range(MAX_RECORDS_PER_ACTIVATION): + record = StreamRecord(RecordKind.DATA, b"x", "s", i).placed_at( + Offset(f"{i:08d}") + ) + runtime.record_delivery(1, record) + runtime.record_consumption(1, record) + + assert _annotation_reasons(runtime)[0] is SegmentEndReason.BATCH_LIMIT + + +def test_an_activation_that_ran_out_of_records_still_records_no_data( + backend: MemoryStreamBackend, +) -> None: + """The other direction, without which "always BATCH_LIMIT" would pass.""" + + class StubManager: + def cancel_from_workflow_thread(self, run_id, wait_id): # type: ignore[no-untyped-def] + pass + + def register( + self, + **kwargs: Any, # pyright: ignore[reportUnusedParameter] + ) -> None: + pass + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + runtime = make_runtime(StubManager(), backend) + runtime.begin_activation() + runtime.register(wait_id=1, stream_key=runtime.stream_key("tokens")) + for i in range(5): + record = StreamRecord(RecordKind.DATA, b"x", "s", i).placed_at( + Offset(f"{i:08d}") + ) + runtime.record_delivery(1, record) + runtime.record_consumption(1, record) + + assert _annotation_reasons(runtime)[0] is SegmentEndReason.NO_DATA_AVAILABLE diff --git a/tests/contrib/external_workflow_streams/test_errors.py b/tests/contrib/external_workflow_streams/test_errors.py new file mode 100644 index 0000000000..6368b0fd22 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_errors.py @@ -0,0 +1,197 @@ +"""P18 — the failure taxonomy, its metrics, and the classification rule.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import pytest + +import temporalio.common +from temporalio.contrib.external_workflow_streams._errors import ( + METRIC_DECODE, + METRIC_INTEGRITY, + METRIC_SHUTDOWN_WAKE_FAILED, + METRIC_STORAGE, + StreamDecodeError, + StreamError, + StreamIntegrityError, + StreamMetrics, + StreamStorageError, + classify_read_failure, +) + + +@dataclass +class RecordingCounter(temporalio.common.MetricCounter): + counter_name: str + added: list[int] = field(default_factory=list) + + @property + def name(self) -> str: + return self.counter_name + + @property + def description(self) -> str | None: + return None + + @property + def unit(self) -> str | None: + return None + + def add( + self, + value: int, + additional_attributes: temporalio.common.MetricAttributes | None = None, + ) -> None: + self.added.append(value) + + def with_additional_attributes( + self, additional_attributes: temporalio.common.MetricAttributes + ) -> RecordingCounter: + return self + + +class RecordingMeter(temporalio.common.MetricMeter): + def __init__(self) -> None: + self.counters: dict[str, RecordingCounter] = {} + + def create_counter( + self, name: str, description: str | None = None, unit: str | None = None + ) -> temporalio.common.MetricCounter: + counter = RecordingCounter(name) + self.counters[name] = counter + return counter + + def create_histogram(self, name, description=None, unit=None): # type: ignore[no-untyped-def] + raise NotImplementedError + + def create_histogram_float(self, name, description=None, unit=None): # type: ignore[no-untyped-def] + raise NotImplementedError + + def create_histogram_timedelta(self, name, description=None, unit=None): # type: ignore[no-untyped-def] + raise NotImplementedError + + def create_gauge(self, name, description=None, unit=None): # type: ignore[no-untyped-def] + raise NotImplementedError + + def create_gauge_float(self, name, description=None, unit=None): # type: ignore[no-untyped-def] + raise NotImplementedError + + def with_additional_attributes(self, additional_attributes): # type: ignore[no-untyped-def] + return self + + +# --- the types -------------------------------------------------------------- + + +def test_the_two_types_are_distinct_and_neither_subclasses_the_other() -> None: + """Collapsing them would send an operator to repair an undamaged backend.""" + assert issubclass(StreamIntegrityError, StreamError) + assert issubclass(StreamDecodeError, StreamError) + assert not issubclass(StreamIntegrityError, StreamDecodeError) + assert not issubclass(StreamDecodeError, StreamIntegrityError) + + +# --- the classification rule, in both directions ---------------------------- + + +def test_a_failure_after_the_range_validated_is_a_decode_failure() -> None: + cause = ValueError("bad JSON") + + err = classify_read_failure(range_validated=True, cause=cause) + + assert isinstance(err, StreamDecodeError) + assert err.__cause__ is cause + + +def test_a_failure_before_the_range_validated_is_integrity_loss() -> None: + cause = LookupError("offset 100-0 is missing") + + err = classify_read_failure(range_validated=False, cause=cause) + + assert isinstance(err, StreamIntegrityError) + assert err.__cause__ is cause + + +def test_the_same_cause_classifies_differently_by_validation_alone() -> None: + """The rule is mechanical: only `range_validated` decides which row it is.""" + cause = ValueError("undecodable") + + assert isinstance( + classify_read_failure(range_validated=True, cause=cause), StreamDecodeError + ) + assert isinstance( + classify_read_failure(range_validated=False, cause=cause), StreamIntegrityError + ) + + +def test_a_storage_failure_is_never_reclassified() -> None: + """There was no range to read, so validation has nothing to say about it.""" + cause = StreamStorageError("connection refused") + + for validated in (True, False): + assert classify_read_failure(range_validated=validated, cause=cause) is cause + + +# --- metrics ---------------------------------------------------------------- + + +def test_each_class_gets_its_own_counter() -> None: + meter = RecordingMeter() + metrics = StreamMetrics.create(meter) + + assert set(meter.counters) == { + METRIC_INTEGRITY, + METRIC_DECODE, + METRIC_STORAGE, + METRIC_SHUTDOWN_WAKE_FAILED, + } + # Separate series, not one counter with a `reason` attribute -- otherwise an + # integrity alert is diluted by a backend outage sharing the series. + assert len({id(c) for c in meter.counters.values()}) == 4 + assert metrics.integrity is not metrics.decode + + +def test_recording_increments_only_the_matching_counter() -> None: + meter = RecordingMeter() + metrics = StreamMetrics.create(meter) + + metrics.record(StreamIntegrityError("gone")) + metrics.record(StreamDecodeError("garbled")) + metrics.record(StreamDecodeError("garbled again")) + metrics.record(StreamStorageError("timeout")) + + assert meter.counters[METRIC_INTEGRITY].added == [1] + assert meter.counters[METRIC_DECODE].added == [1, 1] + assert meter.counters[METRIC_STORAGE].added == [1] + assert meter.counters[METRIC_SHUTDOWN_WAKE_FAILED].added == [] + + +def test_an_unrelated_error_increments_nothing() -> None: + meter = RecordingMeter() + metrics = StreamMetrics.create(meter) + + metrics.record(RuntimeError("something else entirely")) + + assert all(not c.added for c in meter.counters.values()) + + +# --- no terminal-failure opt-in --------------------------------------------- + + +@pytest.mark.parametrize( + "err_type", [StreamIntegrityError, StreamDecodeError, StreamStorageError] +) +def test_no_stream_error_is_a_temporal_failure(err_type: type[StreamError]) -> None: + """Integrity loss blocks a Workflow; it does not terminate one (ADR-014). + + A blocked Workflow can be resumed once the backend is repaired, and a + failed one cannot. These stay plain exceptions rather than + ``FailureError``s so that the SDK's failure conversion cannot turn one into + a Workflow *execution* failure -- which is what a + ``workflow_failure_exception_types`` registration would do. + """ + import temporalio.exceptions + + assert not issubclass(err_type, temporalio.exceptions.FailureError) + assert not issubclass(err_type, temporalio.exceptions.ApplicationError) diff --git a/tests/contrib/external_workflow_streams/test_m1_gate.py b/tests/contrib/external_workflow_streams/test_m1_gate.py new file mode 100644 index 0000000000..78ee8854a1 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_m1_gate.py @@ -0,0 +1,409 @@ +"""P16a/P16b — the milestone gates. + +The required-test lists are what "the milestone is met" means. This module makes +them enforceable rather than aspirational: each case in the plan is mapped to the +test that covers it, and the mapping is checked against both the plan and the +test suite. + +A case that no test fully covers yet maps to its concrete missing boundary in +the corresponding gaps table. That is deliberately louder than leaving it out: +an unmapped case would pass by omission, while an open one is counted and +reported every time the gate runs. +""" + +from __future__ import annotations + +import pytest + +from tests.contrib.external_workflow_streams.m1_gate import ( + declared_count, + required_cases, + unresolved, +) + +#: Case number -> the test(s) that fully cover it. A case may need +#: more than one test: several of them name two independent properties, and +#: mapping such a case to whichever half happened to be written first would +#: claim coverage the suite does not have. +#: Numbers follow `tests-m1.md` in document order. +M1_COVERAGE: dict[int, str | tuple[str, ...]] = { + 1: "external_streams.rs::a_normal_completion_commits_its_marker", + 2: ( + "external_streams.rs::an_activity_command_writes_its_marker_ordered_before_it", + "external_streams.rs::replaying_a_marker_before_an_activity_delivers_its_record_exactly_once", + ), + 3: ( + "external_streams.rs::a_failed_workflow_writes_its_marker_ordered_before_the_failure", + "external_streams.rs::a_continue_as_new_writes_its_marker_ordered_before_it", + ), + 4: "external_streams.rs::several_progress_reports_collapse_into_one_marker", + 5: "external_streams.rs::every_completion_path_writes_exactly_one_marker_ending_in_a_terminal", + 6: "test_runtime.py::test_a_first_subscription_to_an_empty_stream_still_emits", + 7: "test_runtime.py::test_an_activation_that_drained_nothing_emits_an_empty_segment", + 8: ( + "external_streams.rs::a_budget_driven_split_writes_two_markers_rather_than_one_oversized_one", + "test_replay.py::test_two_markers_reassemble_in_workflow_task_order", + ), + 9: "test_annotation.py::test_a_large_single_stream_batch_encodes_as_one_run", + 10: ( + "test_annotation.py::test_encoded_size_stays_flat_with_sparse_control_records", + "test_annotation.py::test_encoded_size_grows_only_with_the_number_of_control_records", + ), + 11: "external_streams.rs::readiness_before_the_idle_timer_expires_cancels_it", + 12: "external_streams.rs::a_confirmed_idle_park_writes_one_marker_and_completes_the_task", + 13: "test_runtime_only_jobs.py::test_a_recheck_that_finds_records_abandons_the_whole_park", + 14: ( + "test_wake.py::test_a_producer_that_finds_a_wakeable_generation_sends_exactly_one_signal", + "test_runtime.py::test_readiness_after_a_re_block_names_the_current_generation", + ), + 15: "external_streams.rs::a_stale_generation_produces_no_activation", + 16: "external_streams.rs::an_all_fenced_snapshot_parks_without_waiting_out_the_idle_timeout", + 17: "external_streams.rs::a_core_decided_boundary_asks_for_a_terminal_before_writing_anything", + 18: "external_streams.rs::a_core_decided_boundary_asks_for_a_terminal_before_writing_anything", + 19: "test_rollover_integration.py::test_a_continuously_fed_stream_survives_a_rollover", + 20: "external_streams.rs::the_rollover_deadline_fires_on_a_workflow_only_worker", + 21: "test_rollover_integration.py::test_a_signal_into_a_retained_task_lands_by_the_rollover_deadline", + 22: ( + "test_worker_integration.py::test_an_append_with_no_open_task_wakes_the_workflow", + "test_api.py::test_a_record_buffered_while_not_iterating_is_still_delivered", + ), + 23: "test_rollover_integration.py::test_an_append_after_a_rollover_completion_wakes_the_subscription", + 24: "test_manager.py::test_undeliverable_readiness_owes_a_wake_and_keeps_the_right_watchers", + 25: "external_streams.rs::an_unknown_envelope_version_is_ignored_harmlessly", + 26: ( + "test_wake.py::test_two_producers_retrying_one_wake_derive_the_identical_request_id", + "test_wake.py::test_two_producers_retrying_one_wake_are_deduplicated_by_the_server", + ), + 27: ( + "test_wake.py::test_two_unparked_wakes_from_different_senders_differ", + "test_wake.py::test_two_workers_unparked_wakes_are_both_delivered", + "test_wake.py::test_one_senders_retry_stays_a_single_wake", + ), + 28: "external_streams.rs::a_second_worker_reconstructs_the_subscription_from_the_shutdown_marker", + 29: "test_worker_handoff.py::test_shutdown_with_no_open_task_hands_the_run_to_another_worker", + 30: "test_worker_handoff.py::test_a_finalization_that_cannot_be_answered_writes_no_marker", + 31: "external_streams.rs::an_unwritten_annotation_exists_only_while_a_workflow_task_is_open", + 32: ( + "test_shutdown_sweep.py::test_a_momentarily_failing_wake_is_retried_and_succeeds", + "test_shutdown_sweep.py::test_the_retry_is_bounded_and_then_reported", + "test_shutdown_sweep.py::test_the_retry_is_the_same_wake_not_a_second_one", + "test_shutdown_sweep.py::test_an_unacknowledged_wake_surfaces_on_the_metric", + ), + 33: "test_runtime_only_jobs.py::test_finalization_touches_no_provider_at_all", + 34: "test_replay.py::test_replay_performs_no_live_waiting", + 35: "test_replay_end_to_end.py::test_replaying_a_stream_history_reproduces_the_same_observations", + 36: ( + "test_replay_end_to_end.py::test_an_empty_stream_parked_and_evicted_replays_from_the_recorded_cursor", + "test_replay_end_to_end.py::test_an_empty_stream_replays_from_its_recorded_boundary", + ), + 37: "test_replay.py::test_an_unreachable_backend_fails_as_transient_storage", + 38: "test_replay.py::test_each_deletion_position_fails_a_different_check", + 39: "test_replay.py::test_an_intact_but_undecodable_record_is_a_decode_error", + 40: "test_replay.py::test_a_marker_naming_an_unknown_wait_is_nondeterminism_not_integrity_loss", + 41: "test_worker_crash.py::test_a_crash_before_the_marker_makes_the_next_worker_re_read", + 42: "test_continuation.py::test_a_first_execution_starts_at_the_beginning", + 43: "external_streams.rs::a_pending_timer_suppresses_retention_but_its_subscriptions_survive_it", + 44: "test_backend_conformance.py::test_a_broken_backend_fails_for_the_right_reason", + 45: "test_backend_conformance.py::test_a_backend_needing_a_nameable_cursor_fails_the_tail_check", + 46: "test_redis_backend.py::test_offsets_compare_numerically_not_lexically", + 47: "test_producer.py::test_republishing_different_content_under_one_key_is_an_error", + 48: "test_manager.py::test_a_backend_slower_than_the_deadlock_timeout_delays_readiness", + 49: "test_runtime_only_jobs.py::test_a_park_slower_than_the_deadlock_timeout_is_still_answered", + 50: "test_manager.py::test_the_provider_is_never_called_from_the_workflow_thread", + 51: "test_manager.py::test_a_full_buffer_stops_prefetch_without_dropping_or_blocking", + 52: "test_manager.py::test_an_evicted_run_re_delivers_the_records_it_had_already_seen", + 53: "test_shutdown_sweep.py::test_teardown_removes_every_run", + 54: "test_registry.py::test_a_backend_without_the_guarantee_fails_worker_construction", + 55: ( + "test_redis_backend.py::test_a_trimmed_range_is_integrity_loss_not_a_storage_failure", + "test_redis_backend.py::test_a_deleted_write_fence_is_integrity_loss", + ), + # Cases 56-62 -- "Boundaries an implementation review found unprotected". + # These are mapped here before the vendored copy of the list carries them: + # the gate parses the plan out of `temporalio/bridge/sdk-core`, so it counts + # 55 cases until the submodule pointer moves to the commit that adds them. + # Mapping ahead of that is deliberate -- an unmapped case is what the gate is + # designed to fail on, so the map has to arrive no later than the list does. + 56: "test_replay.py::test_the_first_live_drain_after_a_replay_needs_no_loop_turn", + 57: "test_replay.py::test_a_marker_drains_once_per_segment_including_the_activations_own", + 58: "test_runtime.py::test_the_segment_that_crosses_the_mark_asks_for_rollover_itself", + 59: ( + "test_runtime.py::test_a_frame_larger_than_the_slack_rolls_over_instead_of_raising", + "test_runtime.py::test_a_subscription_set_too_large_to_record_is_refused_at_subscribe", + ), + 60: ( + "test_wake.py::test_a_coordination_failure_after_the_append_is_still_unacknowledged", + "test_wake.py::test_a_coordination_failure_after_a_fence_is_still_unacknowledged", + ), + 61: ( + "test_shutdown_sweep.py::test_a_grace_period_expiry_counts_every_wake_it_abandons", + "test_shutdown_sweep.py::test_a_probe_that_cannot_answer_is_not_reported_as_nothing_owed", + "test_shutdown_sweep.py::test_a_hanging_probe_counts_the_runs_it_never_answered_for", + "test_shutdown_sweep.py::test_a_run_with_nothing_owed_is_not_counted_as_a_failure", + ), + 62: "test_replay.py::test_an_unreachable_payload_store_is_a_storage_failure_not_a_decode_one", + # 63 and 64 came out of reviewing the fixes for 56-62 rather than out of the + # review itself: the first fix for the byte budget stopped delivery without + # obliging the same completion to ask for the rollover, and the first fix for + # the sweep counted subscriptions on a manager that had no sweep to perform. + 63: "test_runtime.py::test_an_activation_the_annotation_budget_stopped_is_not_wedged_by_it", + 64: "test_shutdown_sweep.py::test_a_manager_with_no_probe_owes_nothing_and_reports_nothing", + 65: "test_replay.py::test_a_marker_with_no_segments_closes_before_the_activations_own_drain", + # 66-69 came out of an independent review of the fixes for 56-62, which found + # four more defects in this round's own code. The shape is the one worth + # remembering: each was correct for the case it was aimed at. + 66: "test_runtime.py::test_the_first_record_of_an_activation_is_priced_from_a_measurement", + 67: "test_runtime.py::test_a_run_too_large_for_any_annotation_says_so_and_fails_the_workflow", + 68: "test_runtime.py::test_the_capacity_floor_covers_everything_an_empty_annotation_carries", + 69: ( + "test_replay.py::test_a_failing_activation_reports_its_own_error_not_the_replays", + "test_shutdown_sweep.py::test_a_wake_the_live_path_delivered_is_not_counted_as_abandoned", + ), + # 70-71 pin both halves of replay-to-live repositioning: a read already in + # flight cannot refill a retracted buffer, and replay delivers no record the + # marker omitted. + 70: "test_replay.py::test_a_read_in_flight_across_a_reposition_cannot_be_appended", + 71: "test_replay.py::test_a_replay_activation_never_delivers_a_record_the_marker_omits", + # 72-76 are the fourth review's five findings. Four of them are the same shape + # as the third round's: a guarantee written for the path it was aimed at and + # not for the adjacent one -- a budget spent where it was not reserved, a key + # drawn after an await that reorders it, a handler for `Exception` on a path + # cancellation also takes, and a retry whose answer was discarded. + 72: ( + "test_delivery_budget.py::test_two_independent_consumers_share_one_budget", + "test_delivery_budget.py::test_a_carried_over_ready_list_is_charged_to_the_next_activation", + ), + 73: ( + "test_producer.py::test_concurrent_publishes_take_their_sequence_in_invocation_order", + "test_producer.py::test_reordered_encodes_cannot_duplicate_across_two_topics", + ), + 74: ( + "test_wake.py::test_cancellation_after_the_append_is_an_unacknowledged_wake", + "test_wake.py::test_cancellation_after_a_fence_is_an_unacknowledged_wake", + "test_wake.py::test_cancellation_before_the_append_stays_a_cancellation", + ), + 75: ( + "test_api.py::test_a_second_coroutine_waiting_on_one_subscription_is_refused", + "test_api.py::test_iterating_again_after_the_first_consumer_stopped_is_allowed", + "test_api.py::test_a_merge_cannot_take_a_wait_another_consumer_is_blocked_on", + ), + 76: "test_manager.py::test_a_stale_retry_that_finds_the_run_gone_tears_the_watcher_down", + # 77 came out of reviewing case 74's fix, and is the same shape once more: a + # recovery written for the boundary it was aimed at -- cancellation after + # `append()` returned -- and not for the one just before it, where a remote + # backend has already committed and the answer is what was lost. + 77: ( + "test_wake.py::test_cancellation_after_backend_commit_before_append_ack_is_recoverable", + "test_wake.py::test_a_lost_append_response_is_recoverable_the_same_way", + "test_wake.py::test_an_unresolved_append_refuses_the_publish_that_would_duplicate_it", + "test_wake.py::test_settling_an_unacknowledged_append_wakes_exactly_once", + "test_wake.py::test_an_unacknowledged_fence_settles_to_exactly_one_fence", + "test_wake.py::test_settling_an_append_that_never_landed_appends_it_once", + "test_wake.py::test_a_conflicting_append_is_a_refusal_not_an_unknown_outcome", + ), + # 78 is case 77's own review. The outcome was right and its recovery was + # under-bound: it carried the record but not the operation, and validated + # neither the stream the record belongs to nor the producer instance that + # still holds the session's counters. + 78: ( + "test_wake.py::test_a_refused_append_preserves_the_unresolved_operations_recovery", + "test_wake.py::test_an_unknown_append_can_only_be_resolved_on_its_originating_topic", + "test_wake.py::test_recovery_is_bound_to_the_producer_instance_that_made_the_append", + ), + # 79 follows the same operation through a second unknown outcome. The + # recovery attempt's effective wake and lease supersede the original + # publish's, while cancellation delivered on either attempt remains owed. + 79: "test_wake.py::test_a_reinterrupted_append_recovery_preserves_its_latest_wake", + # 80-101 promote Workflow-originated output. Only cases with every named + # boundary asserted belong here; partial unit coverage stays in M1_GAPS. + 80: "test_output_worker_integration.py::test_reader_waits_at_pending_barrier_until_marker_commit", + 81: "test_output_worker_integration.py::test_rejected_post_stage_completion_never_exposes_phantom_output", + 82: "external_streams.rs::speculative_output_redelivery_accepts_a_fresh_token_once", + 83: "test_output_client.py::test_first_task_boundary_without_token_aborts_pending_stage", + 84: "external_streams.rs::exact_output_floor_excludes_the_previous_workflow_task_close", + 85: ( + "test_worker_crash.py::test_disconnected_update_leaves_live_staged_output_undecided_until_timeout", + "test_output_client.py::test_disconnected_speculative_update_stays_pending_until_history_loss", + ), + 86: "test_worker_crash.py::test_real_crash_after_output_stage_aborts_old_token_and_exposes_one_retry", + 87: "test_output_worker_integration.py::test_cold_client_repairs_post_report_commit_failure", + 88: ( + "test_redis_output.py::test_exact_stage_retry_returns_actual_status_after_terminal", + "test_output_client.py::test_repeating_the_same_history_reconciliation_is_idempotent", + ), + 89: "test_redis_output.py::test_reusing_stage_identity_with_another_manifest_conflicts_atomically", + 90: "test_output_runtime.py::test_output_replay_performs_no_io_token_mint_or_live_policy_split", + 91: "test_output_runtime.py::test_replay_keeps_recorded_segments_when_codec_wire_size_changes", + 92: "test_output_worker_integration.py::test_output_latency_flushes_a_retained_workflow_task", + 93: "test_output_worker_integration.py::test_three_retained_output_windows_write_three_markers_and_wfts", + 94: "test_redis_output.py::test_pending_stage_blocks_later_direct_output_until_commit", + 95: "test_output_producer.py::test_ambiguous_append_retries_exact_record_without_duplicate", + 96: "test_output_runtime.py::test_changed_codec_bytes_keep_logical_retry_identity_and_first_bytes", + 97: "test_output_codec.py::test_metadata_insertion_order_does_not_change_frame_or_fingerprint", + 98: "test_output_worker_integration.py::test_concurrent_updates_preserve_their_own_turn_ids", + 99: "test_output_worker_integration.py::test_stage_outage_blocks_wft_and_reports_external_storage_cause", + 100: ( + "test_redis_output.py::test_missing_staged_record_is_an_integrity_failure", + "test_output_client.py::test_history_loss_is_integrity_but_an_outage_is_storage", + ), + 101: ( + "test_output_runtime.py::test_oversized_marker_manifest_is_rejected_before_external_io", + "test_output_runtime.py::test_one_oversized_logical_record_is_refused_without_poisoning_batch", + ), + # 102 restores the public default to the required contract after the design + # documents were split into guide, specification, and rationale sets. + 102: "test_api.py::test_the_default_idle_timeout_is_one_second", +} + +#: Case number -> what is still missing, for cases the suite does not fully +#: cover. A case is in exactly one of the two maps: claiming a partially covered +#: case as covered is how a gate stops meaning anything. +M1_GAPS: dict[int, str] = {} + +#: The same, for `tests-m2.md`. +M2_COVERAGE: dict[int, str | tuple[str, ...]] = { + 1: "test_m2_required.py::test_readiness_on_one_of_several_streams_resets_global_quiescence", + 2: "test_multi_stream.py::test_one_idle_stream_cannot_park_while_another_is_active", + 3: "test_multi_stream.py::test_a_fence_on_one_stream_alone_does_not_make_the_set_parkable", + 4: "test_multi_stream.py::test_all_fenced_streams_make_the_whole_set_parkable", + 5: ( + "test_multi_stream.py::test_an_alternating_two_stream_batch_encodes_one_run_per_delivery", + "test_m2_required.py::test_an_alternating_two_stream_batch_rolls_over_within_budget", + ), + 6: "test_m2_required.py::test_simultaneously_ready_streams_are_drained_in_one_pass", + 7: "test_runtime.py::test_alternating_streams_produce_one_run_per_delivery", + 8: ( + "test_multi_stream.py::test_each_same_stream_subscription_receives_every_record", + "test_continuation.py::test_two_same_stream_subscriptions_restore_independently", + ), + 9: ( + "test_multi_stream.py::test_two_same_stream_subscriptions_install_distinct_park_intents", + "test_m2_required.py::test_two_same_stream_subscriptions_never_overwrite_each_other", + ), + 10: ( + "test_multi_stream.py::test_differing_idle_timeouts_reduce_to_the_minimum", + "test_m2_required.py::test_the_idle_timeout_reduction_reproduces_exactly", + ), + 11: ( + "test_m2_required.py::test_a_wake_for_one_stream_resolves_every_blocked_wait", + "test_m2_required.py::test_resolving_twice_is_harmless", + ), + 12: "test_continuation.py::test_a_chain_resumes_where_its_predecessor_stopped", + 13: "test_replay_end_to_end.py::test_live_and_replay_share_one_input_output_drain_schedule", + 14: "test_output_worker_integration.py::test_cursor_resume_crosses_rollover_and_continue_as_new", + 15: "test_redis_output.py::test_input_and_output_with_same_user_identity_are_physically_isolated", + 16: ( + "test_output_worker_integration.py::test_output_deadline_wins_park_race_and_removes_every_intent", + "external_streams.rs::confirmed_park_flushes_output_in_its_marker_without_forcing_a_task", + "external_streams.rs::output_deadline_invalidates_an_issued_park_and_flushes_once", + ), + 17: "test_output_worker_integration.py::test_finished_output_survives_continue_as_new_without_backend_reads", +} + +#: The same, for `tests-m2.md`. +M2_GAPS: dict[int, str] = {} + + +def _coverage(list_name: str) -> dict[int, str | tuple[str, ...]]: + return M1_COVERAGE if list_name == "tests-m1.md" else M2_COVERAGE + + +def _gaps(list_name: str) -> dict[int, str]: + return M1_GAPS if list_name == "tests-m1.md" else M2_GAPS + + +def _node_ids(coverage: dict[int, str | tuple[str, ...]]) -> list[str]: + """Flattens the map, since a case may name more than one test.""" + flat: list[str] = [] + for value in coverage.values(): + flat.extend([value] if isinstance(value, str) else list(value)) + return flat + + +@pytest.mark.parametrize("list_name", ["tests-m1.md", "tests-m2.md"]) +def test_the_parsed_case_count_matches_the_plans_own_heading(list_name: str) -> None: + """Catches a case added to the plan without the heading being updated. + + The two numbers are maintained by hand in the same document, so they are + exactly the kind of pair that drifts. If they disagree, every count below is + measuring the wrong thing. + """ + cases = required_cases(list_name) + + assert len(cases) == declared_count(list_name), ( + f"{list_name} declares {declared_count(list_name)} cases but contains " + f"{len(cases)}" + ) + + +@pytest.mark.parametrize("list_name", ["tests-m1.md", "tests-m2.md"]) +def test_every_required_case_is_accounted_for(list_name: str) -> None: + """A new case must fail the gate rather than pass by not being mentioned.""" + cases = required_cases(list_name) + coverage = _coverage(list_name) + + gaps = _gaps(list_name) + unmapped = [c for c in cases if c.number not in coverage and c.number not in gaps] + + assert not unmapped, ( + "these required cases appear in neither map -- add the test that covers " + "each, or record what is still missing in the gaps map:\n" + + "\n".join(f" {c.number}. {c.text}" for c in unmapped) + ) + + +@pytest.mark.parametrize("list_name", ["tests-m1.md", "tests-m2.md"]) +def test_every_mapped_test_exists(list_name: str) -> None: + """Catches a rename or deletion that would leave the gate pointing at nothing. + + This is the failure the whole mechanism exists for: without it the map keeps + claiming coverage that was deleted, and the milestone stays "met" on paper. + """ + coverage = _coverage(list_name) + + missing = unresolved(_node_ids(coverage)) + + assert not missing, "the coverage map names tests that do not exist:\n" + "\n".join( + f" {node_id}" for node_id in missing + ) + + +@pytest.mark.parametrize("list_name", ["tests-m1.md", "tests-m2.md"]) +def test_no_case_is_both_covered_and_open(list_name: str) -> None: + """A case in both maps would be counted twice and read as covered. + + Claiming a partially covered case as covered is how a gate stops meaning + anything, so the two maps must partition the list rather than overlap. + """ + overlap = sorted(set(_coverage(list_name)) & set(_gaps(list_name))) + + assert not overlap, f"cases claimed as both covered and open: {overlap}" + + +@pytest.mark.parametrize("list_name", ["tests-m1.md", "tests-m2.md"]) +def test_every_open_case_says_what_is_missing(list_name: str) -> None: + """An empty reason is indistinguishable from "nobody looked".""" + empty = sorted(n for n, reason in _gaps(list_name).items() if not reason.strip()) + + assert not empty, f"open cases with no stated gap: {empty}" + + +def test_the_gate_reports_how_far_each_milestone_is_from_met() -> None: + """The gate's actual output, and the thing a human reads. + + Coverage and open cases are reported independently of the consistency + assertions above. If a future requirement is added before its test, its gap + remains visible without turning this reporting test into the failure itself. + """ + for list_name in ("tests-m1.md", "tests-m2.md"): + cases = {c.number: c.text for c in required_cases(list_name)} + gaps = _gaps(list_name) + covered = len(cases) - len(gaps) + blocked = {n for n, why in gaps.items() if why.startswith("BLOCKED")} + print( + f"\n{list_name}: {covered}/{len(cases)} covered, {len(gaps)} open " + f"({len(blocked)} of them blocked on a deliverable)" + ) + for number in sorted(gaps): + print(f" {number}. {cases[number][:70]}") + print(f" {gaps[number]}") diff --git a/tests/contrib/external_workflow_streams/test_m2_required.py b/tests/contrib/external_workflow_streams/test_m2_required.py new file mode 100644 index 0000000000..f5cf37c4a8 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_m2_required.py @@ -0,0 +1,341 @@ +"""P16b — the Milestone 2 cases not already covered elsewhere. + +The rest of the twelve are covered by the deliverables that produced them (P21, +P15, P10b); these are the ones that fall between deliverables, which is exactly +why a milestone gate is a separate list rather than a sum of "done when"s. +""" + +# pyright: reportMissingParameterType=false +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest +import pytest_asyncio + +import temporalio.converter +from temporalio.contrib.external_workflow_streams._annotation import ( + MAX_ANNOTATION_BYTES, +) +from temporalio.contrib.external_workflow_streams._backend import ParkIntent, StreamKey +from temporalio.contrib.external_workflow_streams._continuation import Continuation +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._runtime import WorkflowStreamRuntime +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +RUN_ID = "run-1" + + +async def _notify( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] +) -> str: + return ReadinessResult.ACCEPTED + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +@pytest_asyncio.fixture # type: ignore[reportUntypedFunctionDecorator] +async def manager(backend: MemoryStreamBackend): + mgr = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + yield mgr + await mgr.shutdown() + + +def make_runtime(manager, backend, continuation=None): # type: ignore[no-untyped-def] + return WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id=uuid.uuid4().hex, + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + continuation=continuation, + ) + + +def data(offset: str) -> StreamRecord: + return StreamRecord(RecordKind.DATA, b"x", "s", 0).placed_at(Offset(offset)) + + +# --- case 1: readiness on one stream resets global quiescence ----------------- + + +@pytest.mark.asyncio +async def test_readiness_on_one_of_several_streams_resets_global_quiescence( + manager, backend: MemoryStreamBackend +) -> None: + """The idle timer covers the set, so any member's record restarts it. + + Core starts a fresh idle timer when the generations it is retaining for + change. If a record on one stream left every generation untouched, the timer + that was already running would keep counting down toward parking a Workflow + that had just been given work. + """ + runtime = make_runtime(manager, backend) + for wait_id, name in ((1, "a"), (2, "b")): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(name), + ) + runtime.note_blocked(wait_id, True) + before = {w.wait_id: w.generation for w in runtime.quiescent_snapshot()} # type: ignore[union-attr] + + # A record arrives on the first stream: it stops being blocked, Workflow code + # consumes it, and then blocks again. + runtime.note_blocked(1, False) + runtime.record_delivery(1, data("10-0")) + runtime.note_blocked(1, True) + + after = {w.wait_id: w.generation for w in runtime.quiescent_snapshot()} # type: ignore[union-attr] + assert after[1] > before[1], ( + "the wait that received a record must re-enter the blocked state with a " + "new generation, which is what makes Core restart the idle timer" + ) + assert after[2] == before[2], ( + "the other stream did not move and must keep its generation, or Core " + "could not tell a stale readiness for it from a live one" + ) + + +# --- case 5: an alternating batch rolls over rather than exceeding the budget -- + + +@pytest.mark.asyncio +async def test_an_alternating_two_stream_batch_rolls_over_within_budget( + manager, backend: MemoryStreamBackend +) -> None: + """Alternation is the worst case for annotation size, and must still be bounded. + + Every delivery starts a new run, so an alternating batch grows the annotation + per *record* rather than per batch -- the one shape where the byte budget is + reachable in normal use. It must ask for rollover rather than grow past the + limit, because an oversized marker cannot be written at all. + """ + runtime = make_runtime(manager, backend) + for wait_id, name in ((1, "a"), (2, "b")): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(name), + ) + + delivered = 0 + while not runtime.request_rollover and delivered < 20_000: + wait_id = 1 + (delivered % 2) + runtime.record_delivery(wait_id, data(f"{delivered}-0")) + runtime.take_observation_delta() + delivered += 1 + + assert runtime.request_rollover, ( + f"an alternating batch of {delivered} records never asked for rollover" + ) + annotation = b"".join( + [runtime.take_observation_delta() or b"", runtime.add_terminal()] + ) + assert len(annotation) <= MAX_ANNOTATION_BYTES, ( + f"the annotation reached {len(annotation)} bytes, past the " + f"{MAX_ANNOTATION_BYTES} budget: rollover was requested too late to be " + "of any use" + ) + + +# --- case 6: simultaneously ready streams are observable together ------------- + + +@pytest.mark.asyncio +async def test_simultaneously_ready_streams_are_drained_in_one_pass( + manager, backend: MemoryStreamBackend +) -> None: + """Two records that arrived together cost one activation, not two. + + Waiting for readiness between them would turn every simultaneous arrival + into a second Workflow Task -- and the whole point of retention is that a + batch costs one. + """ + from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec + + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + first = StreamKey("ns", "wf", "chain", "a") + second = StreamKey("ns", "wf", "chain", "b") + for wait_id, key in ((1, first), (2, second)): + manager.register( + run_id=RUN_ID, + wait_id=wait_id, + stream_key=key, + start_cursor=BEGINNING, + ) + await backend.append( + first, StreamRecord(RecordKind.DATA, await codec.encode("a1"), "p", 0) + ) + await backend.append( + second, StreamRecord(RecordKind.DATA, await codec.encode("b1"), "p", 0) + ) + await asyncio.sleep(0.2) + + # Both are buffered at the same moment, so one drain pass sees both. + assert manager.drain(RUN_ID, 1) and manager.drain(RUN_ID, 2), ( + "both streams must be buffered together; if one had to wait for a " + "separate readiness round trip the batch would cost two activations" + ) + + +# --- case 9: two same-stream subscriptions stay independent throughout -------- + + +@pytest.mark.asyncio +async def test_two_same_stream_subscriptions_never_overwrite_each_other( + manager, backend: MemoryStreamBackend +) -> None: + """Park, wake, cancel, and Continue-As-New, verified against the backend. + + Every step is a chance for one subscription's state to land under the + other's key. The park intents are read back rather than assumed, because a + collapsed intent looks like success from the caller's side -- the second + install simply returns. + """ + key = StreamKey("ns", "wf", "chain", "tokens") + for wait_id, cursor in ((1, BEGINNING), (2, AFTER(Offset("50-0")))): + await backend.install_park_intent( + key, + ParkIntent( + wait_id=wait_id, cursor=cursor, park_generation=3, run_id="run-1" + ), + ) + + # Parked: two intents, two cursors. + first = await backend.park_intent(key, 1) + second = await backend.park_intent(key, 2) + assert first is not None and second is not None + assert first.cursor != second.cursor + assert sorted(await backend.parked_wait_ids(key)) == [1, 2] + + # Cancelled: removing one leaves the other exactly as it was. + await backend.remove_park_intent(key, 1) + assert await backend.park_intent(key, 1) is None + assert await backend.park_intent(key, 2) == second + assert await backend.parked_wait_ids(key) == [2] + + # Continued as new: each restores its own cursor, not the other's. + runtime = make_runtime( + manager, + backend, + Continuation( + {1: AFTER(Offset("10-0")), 2: AFTER(Offset("90-0"))}, + {1: "tokens", 2: "tokens"}, + {1: "memory", 2: "memory"}, + {1: 1, 2: 1}, + ), + ) + runtime.register(wait_id=1, stream_key=key) + runtime.register(wait_id=2, stream_key=key) + assert runtime._subscriptions[1].start_cursor == AFTER(Offset("10-0")) + assert runtime._subscriptions[2].start_cursor == AFTER(Offset("90-0")) + + +# --- case 10: the reduction reproduces on replay ------------------------------ + + +@pytest.mark.asyncio +async def test_the_idle_timeout_reduction_reproduces_exactly( + manager, backend: MemoryStreamBackend +) -> None: + """It rides on a command replay compares against History. + + The inputs are the quiescent set's configured values and nothing else, which + is what makes the result a pure function of state replay reconstructs. A + reduction that consulted wall-clock time, arrival order, or anything else + live would produce a different `WorkflowStreamQuiescent` on replay and fail + as nondeterminism. + """ + + def reduce_over(order: list[int]) -> timedelta: + runtime = make_runtime(manager, backend) + for wait_id in order: + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(f"s{wait_id}"), + idle_timeout=timedelta(seconds=wait_id), + ) + runtime.note_blocked(wait_id, True) + return runtime.effective_idle_timeout() + + # Registration order differs; the set does not. Replay reconstructs the set, + # so only the set may matter. + assert reduce_over([1, 2, 3]) == reduce_over([3, 2, 1]) == timedelta(seconds=1) + + +# --- case 11: a wake for one stream rechecks all ------------------------------ + + +@pytest.mark.asyncio +async def test_a_wake_for_one_stream_resolves_every_blocked_wait( + manager, backend: MemoryStreamBackend +) -> None: + """The Signal names one stream; the runtime rechecks all of them. + + A wake can only name the stream whose producer sent it, but by the time the + Workflow Task exists other streams may have records too. Resolving only the + named wait would leave those buffered until something else happened to wake + the Workflow -- and the wake it needed had already been spent. + """ + runtime = make_runtime(manager, backend) + loop = asyncio.get_running_loop() + futures = {} + for wait_id, name in ((1, "named"), (2, "other")): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(name), + ) + runtime.note_blocked(wait_id, True) + future: asyncio.Future[None] = loop.create_future() + runtime.register_pending(wait_id, future) + futures[wait_id] = future + + # What the resolve activation does, regardless of which wait it names. + runtime.resolve_all_pending() + + assert all(f.done() for f in futures.values()), ( + "every blocked wait must be resolved, not only the one the Signal named" + ) + + +@pytest.mark.asyncio +async def test_resolving_twice_is_harmless( + manager, backend: MemoryStreamBackend +) -> None: + """Duplicate wake Signals are expected, and must not raise. + + Two producers racing one generation both signal; the server deduplicates the + request but a stale Signal can still arrive alongside a live one. + """ + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=runtime.stream_key("a")) + future: asyncio.Future[None] = asyncio.get_running_loop().create_future() + runtime.register_pending(1, future) + + runtime.resolve_all_pending() + runtime.resolve_all_pending() + + assert future.done() diff --git a/tests/contrib/external_workflow_streams/test_manager.py b/tests/contrib/external_workflow_streams/test_manager.py new file mode 100644 index 0000000000..e3f4b35948 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_manager.py @@ -0,0 +1,2241 @@ +"""P8 — the subscription manager, driven directly. + +No Workflow API and no Core activations: neither is in this deliverable's +closure, and both would obscure what is actually under test -- that backend I/O +never reaches the thread ``_apply`` runs on, and that readiness means *buffered*. +""" + +from __future__ import annotations + +import asyncio +import inspect +import threading +import time +import uuid +from collections.abc import Awaitable, Callable +from datetime import timedelta + +import pytest + +from temporalio.contrib.external_workflow_streams._backend import ( + ParkIntent, + ParkIntentRemoval, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._errors import ( + StreamStorageError, +) +from temporalio.contrib.external_workflow_streams._manager import ( + PARK_REMOVAL_ATTEMPTS, + ReadinessResult, + StreamSubscriptionManager, + Subscription, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.worker.workflow_sandbox._restrictions import SandboxRestrictions +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +RUN_ID = "run-1" + + +@pytest.fixture +def stream_key() -> StreamKey: + return StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + + +class RecordingNotifier: + """Stands in for Core's readiness call, recording every notification.""" + + def __init__(self, answer: str = ReadinessResult.ACCEPTED) -> None: + self.answer = answer + self.calls: list[tuple[str, int, int]] = [] + self.notified = asyncio.Event() + + async def __call__(self, run_id: str, wait_id: int, wait_generation: int) -> str: + self.calls.append((run_id, wait_id, wait_generation)) + self.notified.set() + return self.answer + + +def make_manager( + backend: MemoryStreamBackend, + notifier: RecordingNotifier, + *, + buffer_size: int = 256, + watch_block: timedelta = timedelta(milliseconds=20), + send_wake: object | None = None, +) -> StreamSubscriptionManager: + return StreamSubscriptionManager( + backend=backend, + notify_ready=notifier, + send_wake=send_wake, # type: ignore[arg-type] + buffer_size=buffer_size, + watch_block=watch_block, + ) + + +async def append( + backend: MemoryStreamBackend, key: StreamKey, *payloads: bytes +) -> None: + for i, payload in enumerate(payloads): + await backend.append( + key, StreamRecord(RecordKind.DATA, payload, f"s{payload.decode()}", i) + ) + + +async def until( + condition: Callable[[], object | Awaitable[object]], + message: str, + timeout: float = 2.0, +) -> None: + """Waits for something a background task is on its way to doing. + + Retries, reconciliations and watchers all run on the manager's own loop, so + a single ``sleep`` long enough to be reliable is also long enough to make + every one of these tests slow. Accepts an awaitable condition because most + of what is being waited for is a question for the backend. + """ + deadline = time.monotonic() + timeout + while True: + outcome = condition() + if inspect.isawaitable(outcome): + outcome = await outcome + if outcome: + return + if time.monotonic() >= deadline: + pytest.fail(message) + await asyncio.sleep(0.01) + + +# --- readiness means buffered ------------------------------------------------ + + +@pytest.mark.asyncio +async def test_readiness_is_reported_only_after_a_record_is_buffered( + stream_key: StreamKey, +) -> None: + """The rule the whole structure rests on. + + Readiness for an unbuffered record would produce an activation whose drain + must block -- exactly the deadlock hazard the out-of-thread buffer removes. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await asyncio.sleep(0.05) + assert notifier.calls == [], "nothing is buffered, so nothing is ready" + + await append(backend, stream_key, b"a") + await asyncio.wait_for(notifier.notified.wait(), 2) + + assert notifier.calls == [(RUN_ID, 1, 0)] + assert subscription.buffered == 1, ( + "the record must already be in the buffer when readiness is reported" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_drain_returns_buffered_records_without_touching_the_backend( + stream_key: StreamKey, +) -> None: + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await append(backend, stream_key, b"a", b"b") + await asyncio.wait_for(notifier.notified.wait(), 2) + reads_before = len(backend.range_reads) + + drained = manager.drain(RUN_ID, 1) + + assert [r.payload for r in drained] == [b"a", b"b"] + assert len(backend.range_reads) == reads_before + assert manager.drain(RUN_ID, 1) == [] + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_draining_advances_only_the_delivery_cursor( + stream_key: StreamKey, +) -> None: + """Consuming is not committing. Only a marker moves ``committed``.""" + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await append(backend, stream_key, b"a") + await asyncio.wait_for(notifier.notified.wait(), 2) + manager.drain(RUN_ID, 1) + + assert subscription.committed_cursor == BEGINNING + assert subscription.delivery_cursor != BEGINNING + assert subscription.prefetch_cursor != BEGINNING + finally: + await manager.shutdown() + + +# --- backend latency never reaches the Workflow thread ----------------------- + + +class SlowBackend(MemoryStreamBackend): + """Every read takes longer than the Workflow deadlock timeout.""" + + delay_seconds = 2.5 + + async def read_after(self, key, after, *, max_records, block=None): # type: ignore[no-untyped-def] + await asyncio.sleep(self.delay_seconds) + return await super().read_after( + key, after, max_records=max_records, block=block + ) + + +@pytest.mark.asyncio +async def test_a_backend_slower_than_the_deadlock_timeout_delays_readiness( + stream_key: StreamKey, +) -> None: + """It delays the *report*, not the caller. + + The Workflow thread runs under a 2-second deadlock timeout. A provider + slower than that must therefore be invisible to it -- and it is, because + the drain the Workflow thread performs only ever touches the buffer. + """ + backend = SlowBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier, watch_block=timedelta(milliseconds=20)) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await append(backend, stream_key, b"a") + + # Well past the deadlock timeout: no readiness yet, because nothing is + # buffered yet. + await asyncio.sleep(2.2) + assert notifier.calls == [] + + # And the Workflow thread's own call returns immediately regardless. + start = asyncio.get_running_loop().time() + assert manager.drain(RUN_ID, 1) == [] + assert asyncio.get_running_loop().time() - start < 0.5, ( + "the drain must not wait on the provider" + ) + + await asyncio.wait_for(notifier.notified.wait(), 5) + assert manager.drain(RUN_ID, 1) != [] + finally: + await manager.shutdown() + + +# --- backpressure ------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_full_buffer_stops_prefetch_without_dropping_or_blocking( + stream_key: StreamKey, +) -> None: + """Backpressure *is* the buffer bound. Nothing is dropped, nobody blocks.""" + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier, buffer_size=3) + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await append(backend, stream_key, *[bytes([i]) for i in range(10)]) + await asyncio.sleep(0.2) + + assert subscription.buffered == 3, ( + f"prefetch must stop at the bound, buffered {subscription.buffered}" + ) + + # Nothing was dropped: draining frees room and the rest arrives, in order. + seen: list[bytes] = [] + for _ in range(10): + seen.extend(r.payload for r in manager.drain(RUN_ID, 1)) + if len(seen) == 10: + break + await asyncio.sleep(0.1) + + assert seen == [bytes([i]) for i in range(10)] + finally: + await manager.shutdown() + + +# --- the manager's loop owns every provider call ----------------------------- + + +class ThreadCheckingBackend(MemoryStreamBackend): + """Raises if called from any thread other than the one it was made on.""" + + def __init__(self) -> None: + super().__init__() + self.owning_thread = threading.get_ident() + self.foreign_calls: list[str] = [] + + def _check(self, name: str) -> None: + if threading.get_ident() != self.owning_thread: + self.foreign_calls.append(name) + raise AssertionError( + f"{name} was called from thread {threading.get_ident()}, not the " + f"manager's loop thread {self.owning_thread}" + ) + + async def read_after(self, key, after, *, max_records, block=None): # type: ignore[no-untyped-def] + self._check("read_after") + return await super().read_after( + key, after, max_records=max_records, block=block + ) + + async def read_range(self, key, first, last): # type: ignore[no-untyped-def] + self._check("read_range") + return await super().read_range(key, first, last) + + +@pytest.mark.asyncio +async def test_the_provider_is_never_called_from_the_workflow_thread( + stream_key: StreamKey, +) -> None: + """Driven from a real second thread, standing in for the executor. + + ``activate()`` runs on a thread-pool executor, so "the Workflow thread" is + genuinely a different OS thread -- not merely a different task. + """ + backend = ThreadCheckingBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await append(backend, stream_key, b"a", b"b") + await asyncio.wait_for(notifier.notified.wait(), 2) + + drained: list[bytes] = [] + + def workflow_thread() -> None: + for record in manager.drain(RUN_ID, 1): + drained.append(record.payload) + + thread = threading.Thread(target=workflow_thread) + thread.start() + thread.join(timeout=5) + + assert drained == [b"a", b"b"] + assert backend.foreign_calls == [], ( + f"the provider was called off the manager's loop: {backend.foreign_calls}" + ) + finally: + await manager.shutdown() + + +# --- eviction ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_eviction_discards_prefetch_state_and_restarts_from_committed( + stream_key: StreamKey, +) -> None: + """Reading is not consuming, and consuming is not committing. + + Nothing prefetched or delivered was ever a claim, which is exactly why "no + cursor advances unless the marker commits" is safe to state. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a", b"b", b"c") + await asyncio.wait_for(notifier.notified.wait(), 2) + manager.drain(RUN_ID, 1, max_records=1) + + assert subscription.delivery_cursor != BEGINNING + assert subscription.prefetch_cursor != BEGINNING + + await manager.evict_run(RUN_ID) + + assert subscription.buffered == 0 + assert subscription.delivery_cursor == subscription.committed_cursor == BEGINNING + assert subscription.prefetch_cursor == BEGINNING + assert manager.subscriptions(RUN_ID) == [] + + +@pytest.mark.asyncio +async def test_a_committed_cursor_is_where_a_restart_resumes( + stream_key: StreamKey, +) -> None: + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await append(backend, stream_key, b"a", b"b") + await asyncio.wait_for(notifier.notified.wait(), 2) + drained = manager.drain(RUN_ID, 1) + assert drained[0].offset is not None + + # A marker commits the first record only. + subscription.commit(AFTER(drained[0].offset)) + subscription.reset_to_committed() + + assert subscription.delivery_cursor == AFTER(drained[0].offset) + assert subscription.prefetch_cursor == AFTER(drained[0].offset) + finally: + await manager.shutdown() + + +# --- what the watcher does with each readiness answer ------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("answer", "keeps_watcher"), + [ + (ReadinessResult.PARKED, True), + (ReadinessResult.NO_OPEN_WORKFLOW_TASK, True), + (ReadinessResult.RUN_NOT_FOUND, False), + ], +) +async def test_undeliverable_readiness_owes_a_wake_and_keeps_the_right_watchers( + stream_key: StreamKey, answer: str, keeps_watcher: bool +) -> None: + """All three send a Signal; they differ in what happens afterwards. + + Tearing a watcher down for ``NoOpenWorkflowTask`` would remove the only + thing watching during the normal window between Workflow Tasks. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier(answer) + woken: list[int] = [] + + async def send_wake(subscription) -> None: # type: ignore[no-untyped-def] + woken.append(subscription.wait_id) + + manager = make_manager(backend, notifier, send_wake=send_wake) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await append(backend, stream_key, b"a") + await asyncio.wait_for(notifier.notified.wait(), 2) + await asyncio.sleep(0.1) + + assert woken == [1] + assert bool(manager.subscription(RUN_ID, 1)) is keeps_watcher + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("answer", [ReadinessResult.ACCEPTED, ReadinessResult.STALE]) +async def test_deliverable_readiness_owes_no_wake( + stream_key: StreamKey, answer: str +) -> None: + """A Signal sent here would be a wake nobody needed. + + ``Stale`` in particular must not signal: the wait moved on, so the right + response is to re-probe, not to manufacture a Workflow Task. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier(answer) + woken: list[int] = [] + + async def send_wake(subscription) -> None: # type: ignore[no-untyped-def] + woken.append(subscription.wait_id) + + manager = make_manager(backend, notifier, send_wake=send_wake) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await append(backend, stream_key, b"a") + await asyncio.wait_for(notifier.notified.wait(), 2) + await asyncio.sleep(0.1) + + assert woken == [] + assert manager.subscription(RUN_ID, 1) is not None + finally: + await manager.shutdown() + + +# --- the blocked snapshot ---------------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_blocked_snapshot_is_read_from_manager_state_alone( + stream_key: StreamKey, +) -> None: + """Finalization performs no backend I/O (ADR-010). + + The boundary is not "wherever the stream is now"; it is where this Workflow + Task's deliveries stopped, which is already fixed. Refreshing it against the + backend could name a position replay must not reproduce. + """ + backend = ThreadCheckingBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + manager.register(run_id=RUN_ID, wait_id=2, stream_key=stream_key) + try: + await append(backend, stream_key, b"a") + await asyncio.wait_for(notifier.notified.wait(), 2) + drained = manager.drain(RUN_ID, 1) + reads_before = len(backend.range_reads) + + snapshot = manager.blocked_snapshot(RUN_ID) + + assert set(snapshot) == {1, 2} + assert drained[0].offset is not None + assert snapshot[1] == AFTER(drained[0].offset) + assert snapshot[2] == BEGINNING, ( + "a subscription that delivered nothing is blocked at its start cursor" + ) + assert len(backend.range_reads) == reads_before + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_start_cursor_is_where_a_continued_chain_resumes( + stream_key: StreamKey, +) -> None: + """All three cursors begin at the restored boundary, not at BEGINNING.""" + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + restored: Cursor = AFTER(Offset("500-0")) + subscription = manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=stream_key, + start_cursor=restored, + ) + try: + assert subscription.committed_cursor == restored + assert subscription.delivery_cursor == restored + assert subscription.prefetch_cursor == restored + finally: + await manager.shutdown() + + +# --- teardown ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_cancelling_one_subscription_leaves_the_others( + stream_key: StreamKey, +) -> None: + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + manager.register(run_id=RUN_ID, wait_id=2, stream_key=stream_key) + try: + await manager.cancel(RUN_ID, 1) + + assert manager.subscription(RUN_ID, 1) is None + assert manager.subscription(RUN_ID, 2) is not None + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_cancelling_a_wait_takes_back_its_park_intent( + stream_key: StreamKey, +) -> None: + """The intent has to go here, because nothing later can be asked to retry. + + The resolve path can afford to log a failure and try again, since the + subscription it is working on stays registered. This one drops it, so an + intent left behind is left behind for good -- and a stale intent is what + `current_park_generation` answers to every producer that asks. A producer + naming a generation Core has discarded sends a wake Core ignores as stale: + the record is appended, the Signal is sent, and the Workflow is never woken. + """ + backend = MemoryStreamBackend() + manager = make_manager(backend, RecordingNotifier()) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + assert await backend.parked_wait_ids(stream_key) == [1] + + await manager.cancel(RUN_ID, 1) + + assert await backend.parked_wait_ids(stream_key) == [], ( + "the closed wait's intent is still in the backend, advertising a " + "park generation Core has discarded" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_cancelling_a_wait_stops_its_watcher(stream_key: StreamKey) -> None: + """Dropping the subscription is what makes an orphaned watcher invisible. + + Once it is out of the Run's map nothing can reach it to notice, and it goes + on prefetching into a buffer nobody can drain -- holding a backend + connection for the rest of the Worker's life. + """ + backend = MemoryStreamBackend() + manager = make_manager(backend, RecordingNotifier()) + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + await asyncio.sleep(0.05) + assert subscription._watcher is not None and not subscription._watcher.done() + + await manager.cancel(RUN_ID, 1) + + assert subscription._cancelled, "the cancelled subscription must be marked dead" + assert subscription._watcher.done(), ( + "its watcher outlived it, so it is still reading a stream nothing " + "is reading back" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_cancel_from_the_workflow_thread_reaches_a_loop_with_nothing_to_do( + stream_key: StreamKey, +) -> None: + """Driven from a real second thread, standing in for the executor. + + ``close()`` runs inside the synchronous ``activate()``, and ``create_task`` + from there does not raise: the task is appended to the loop's ready queue + and the loop is never woken, so it runs only if something else happens to + wake the loop. Between activations nothing does -- the watchers are sitting + in blocking reads -- which is why the watch block below is long and why the + other thread, not this coroutine, is what waits. Awaiting anything with a + timeout here would arm a timer, and the wake that timer produces would run + the cancel whether or not the loop was ever told about it. + """ + backend = MemoryStreamBackend() + manager = make_manager( + backend, RecordingNotifier(), watch_block=timedelta(seconds=30) + ) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + try: + # Long enough for the watcher to settle into its blocking read. + await asyncio.sleep(0.1) + + def workflow_thread() -> bool: + # Waited out here rather than on the loop, and waited out at all: + # a task appended while the loop is still on its way into the + # select is picked up on the way in, which hides the missing wake. + time.sleep(0.2) + manager.cancel_from_workflow_thread(RUN_ID, 1) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + if manager.subscription(RUN_ID, 1) is None: + return True + time.sleep(0.01) + return False + + cancelled = await asyncio.get_running_loop().run_in_executor( + None, workflow_thread + ) + + assert cancelled, ( + "the cancel was queued onto a loop nothing woke, so a watcher that " + "was supposed to stop keeps running with nothing to say so" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_shutdown_tears_down_every_run(stream_key: StreamKey) -> None: + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + for run_id in ("run-a", "run-b"): + manager.register(run_id=run_id, wait_id=1, stream_key=stream_key) + + await manager.shutdown() + + assert manager.runs_with_subscriptions() == [] + + +@pytest.mark.asyncio +async def test_manager_state_is_keyed_by_run(stream_key: StreamKey) -> None: + """So a stale Run cannot leak connections into a live one.""" + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + manager.register(run_id="run-a", wait_id=1, stream_key=stream_key) + manager.register(run_id="run-b", wait_id=1, stream_key=stream_key) + try: + await manager.evict_run("run-a") + + assert manager.subscriptions("run-a") == [] + assert len(manager.subscriptions("run-b")) == 1 + finally: + await manager.shutdown() + + +# --- the sandbox ------------------------------------------------------------- + + +def test_the_manager_module_passes_through_the_sandbox() -> None: + """Re-importing it inside would give the Workflow a manager watching nothing. + + The real one owns the Worker's connections, watcher tasks, and buffers; only + an opaque handle to it may cross the boundary. + """ + assert ( + "temporalio.contrib.external_workflow_streams._manager" + in SandboxRestrictions.passthrough_modules_minimum + ) + assert ( + "temporalio.contrib.external_workflow_streams._manager" + in SandboxRestrictions.default.passthrough_modules + ) + + +@pytest.mark.asyncio +async def test_re_registering_a_wait_stops_the_watcher_it_replaced( + stream_key: StreamKey, +) -> None: + """The replaced subscription is unreachable and must not keep polling. + + Nothing else would notice: the new subscription works, records are + delivered, and the only symptom is a backend connection that never closes + and a task that outlives its Run -- visible, if at all, as a Worker whose + memory grows. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + try: + first = manager.register( + run_id="run-1", + wait_id=1, + stream_key=stream_key, + ) + await asyncio.sleep(0.05) + assert first._watcher is not None and not first._watcher.done() + + second = manager.register( + run_id="run-1", + wait_id=1, + stream_key=stream_key, + ) + await asyncio.sleep(0.05) + + assert first._cancelled, "the replaced subscription must be marked dead" + assert first._watcher.done(), "its watcher must be stopped, not orphaned" + assert second._watcher is not None and not second._watcher.done(), ( + "the replacement's own watcher must still be running" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_an_evicted_run_re_delivers_the_records_it_had_already_seen( + stream_key: StreamKey, +) -> None: + """Discarding prefetch state is only half of it; the records must come back. + + Cursors reset to the committed boundary is the mechanism, but what makes it + correct is the outcome: a Run that consumed records without committing a + marker must see those same records again, or eviction would silently drop + everything between the last marker and the eviction. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier() + manager = make_manager(backend, notifier) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a", b"b", b"c") + await asyncio.wait_for(notifier.notified.wait(), 2) + first_pass = [r.offset for r in manager.drain(RUN_ID, 1)] + assert len(first_pass) == 3 + + # Evicted with no marker committed: nothing it saw was ever a claim. + await manager.evict_run(RUN_ID) + + notifier.notified.clear() + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await asyncio.wait_for(notifier.notified.wait(), 2) + second_pass = [r.offset for r in manager.drain(RUN_ID, 1)] + + try: + assert second_pass == first_pass, ( + "the restarted subscription must re-read the same offsets; resuming " + "past them would drop every record between the last marker and the " + "eviction" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_stale_answer_is_re_reported_rather_than_dropped( + stream_key: StreamKey, +) -> None: + """`Stale` means Core held a newer generation, not that the record landed. + + The watcher calls back here only after a *new* non-empty read, and its + prefetch cursor is already past the buffered record, so a `Stale` treated as + delivered announces that record to nobody. The Workflow then blocks forever + on data it is already holding. + """ + backend = MemoryStreamBackend() + + class StaleThenAccepted(RecordingNotifier): + async def __call__(self, run_id: str, wait_id: int, generation: int) -> str: + self.calls.append((run_id, wait_id, generation)) + self.notified.set() + return ( + ReadinessResult.STALE + if len(self.calls) == 1 + else ReadinessResult.ACCEPTED + ) + + notifier = StaleThenAccepted() + manager = make_manager(backend, notifier) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a") + await asyncio.wait_for(notifier.notified.wait(), 2) + await asyncio.sleep(0.3) + + assert len(notifier.calls) >= 2, ( + "a stale answer must be re-reported; the watcher will not announce " + f"this record again, got {len(notifier.calls)} report(s)" + ) + assert manager.subscriptions(RUN_ID)[0].buffered == 1, ( + "the record must still be there to deliver" + ) + finally: + await manager.shutdown() + + +@pytest.mark.parametrize("position", [1, 2, 3]) +@pytest.mark.asyncio +async def test_a_stale_retry_that_finds_the_run_gone_tears_the_watcher_down( + stream_key: StreamKey, position: int +) -> None: + """The retries' answer is the answer, and one of them requires a teardown. + + A first report can race a wait-generation change and be answered `Stale`; the + Run can then be evicted before the delayed retry that follows. The retry loop + kept only whether *some* attempt was accepted and threw the rest away, so + control returned with the original `Stale` still in hand: the owed wake went + out (right) and the `RunNotFound` teardown never ran (wrong). What is left + behind is a watcher, a buffer, a backend read loop and a `_runs` entry for a + Run this Worker no longer owns -- and every later readiness report and wake + attempt is made on its behalf. + + Parameterized over which retry discovers the eviction, because nothing makes + the first one special. + """ + backend = MemoryStreamBackend() + + class StaleThenGone(RecordingNotifier): + async def __call__(self, run_id: str, wait_id: int, generation: int) -> str: + self.calls.append((run_id, wait_id, generation)) + self.notified.set() + if len(self.calls) < position + 1: + return ReadinessResult.STALE + return ReadinessResult.RUN_NOT_FOUND + + notifier = StaleThenGone() + wake = CountingWake(failures=0) + manager = make_manager(backend, notifier, send_wake=wake) + try: + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a") + + await until( + lambda: manager.subscription(RUN_ID, 1) is None, + "a `RunNotFound` reached on a stale retry left the subscription in " + "`_runs`, so this Worker keeps a watcher, a buffer and a read loop " + "for a Run it no longer holds", + ) + assert subscription._cancelled, "the dropped subscription is still live" + await until( + lambda: subscription._watcher is not None and subscription._watcher.done(), + "the watcher for a Run that is gone is still reading from the backend", + ) + assert len(wake.counters) == 1, ( + "the wake `RunNotFound` requires must still be sent exactly once, " + f"got {len(wake.counters)}" + ) + assert len(notifier.calls) == position + 1, ( + "the retries did not stop at `RunNotFound`; a Run that is gone cannot " + "come back, and each further attempt delays the wake the record needs" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_readiness_transport_failure_does_not_kill_the_watcher( + stream_key: StreamKey, +) -> None: + """An exception is a transport problem, not an answer. + + Letting it escape ends the watcher for good: the subscription stays + registered, its buffer keeps its records, and nothing announces them again. + Swallowing it would instead claim the record was announced. + """ + backend = MemoryStreamBackend() + + class FailsOnce(RecordingNotifier): + async def __call__(self, run_id: str, wait_id: int, generation: int) -> str: + self.calls.append((run_id, wait_id, generation)) + self.notified.set() + if len(self.calls) == 1: + raise ConnectionError("core unreachable") + return ReadinessResult.ACCEPTED + + notifier = FailsOnce() + manager = make_manager(backend, notifier) + try: + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a") + await asyncio.wait_for(notifier.notified.wait(), 2) + await asyncio.sleep(0.5) + + assert len(notifier.calls) >= 2, "the failing report must be retried" + watcher = subscription._watcher + assert watcher is not None and not watcher.done(), ( + "the watcher must survive a readiness failure; if it ends, this " + "subscription is never served again" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_wake_failure_does_not_kill_the_watcher( + stream_key: StreamKey, +) -> None: + """The wake sender reports failure by raising, and that must stop here. + + An owed wake that could not be sent stays owed for the shutdown sweep. An + exception escaping instead takes the watcher with it, which loses every + later record too. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier(answer=ReadinessResult.NO_OPEN_WORKFLOW_TASK) + + async def failing_wake(subscription) -> None: # type: ignore[no-untyped-def] + raise ConnectionError("service unavailable") + + manager = make_manager(backend, notifier, send_wake=failing_wake) + try: + subscription = manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a") + await asyncio.wait_for(notifier.notified.wait(), 2) + await asyncio.sleep(0.3) + + watcher = subscription._watcher + assert watcher is not None and not watcher.done(), ( + "a failed wake must not end the watcher" + ) + assert subscription.wakes_owed >= 1, "the wake stays owed for the sweep" + finally: + await manager.shutdown() + + +# --- the park handshake's wait set (P19) -------------------------------------- +# +# One Core activation is constructed here, and only here. The set that gets +# parked is decided in two places -- the Worker's job handler turns Core's +# `PrepareExternalStreamPark.waits` into the map it hands over, and the manager +# installs and rechecks against it -- so a test on either half alone cannot show +# that the two agree on what Core asked for. + + +def _park_job(run_id: str, quiescence_generation: int, *wait_ids: int): # type: ignore[no-untyped-def] + """The activation Core sends to open a park handshake.""" + from temporalio.bridge.proto.workflow_activation import WorkflowActivation + + activation = WorkflowActivation() + activation.run_id = run_id + job = activation.jobs.add().prepare_external_stream_park + job.quiescence_generation = quiescence_generation + for wait_id in wait_ids: + job.waits.add().wait_id = wait_id + return activation + + +class _StubRuntime: + """The runtime's half of the park path: cursors in, terminal out.""" + + def __init__(self, cursors: dict[int, Cursor]) -> None: + self._cursors = cursors + self.terminals = 0 + + def blocked_snapshot(self) -> dict[int, Cursor]: + return dict(self._cursors) + + def add_terminal(self) -> bytes: + self.terminals += 1 + return b"terminal" + + +class _StubWorker: + """Only the two things ``_handle_external_stream_jobs`` reaches for.""" + + def __init__(self, run_id: str, runtime: _StubRuntime, manager: object) -> None: + self._external_stream_runtimes = {run_id: runtime} + self._manager = manager + + def _stream_manager(self) -> object: + return self._manager + + +@pytest.mark.asyncio +async def test_the_park_set_is_cores_wait_set_and_not_every_registration() -> None: + """A registered subscription is not necessarily one Core is parking. + + ``PrepareExternalStreamPark.waits`` is Core's complete *blocked* snapshot. + The runtime's own registration list is a superset of it: a subscription that + delivered a record and was not awaited again is registered and not blocked, + so it is absent from the quiescent snapshot Core parks. + + Parking the superset is wrong in both directions. The recheck for a wait + Core is not parking finds that wait's records -- which is *not* news, since + nothing is waiting on them -- and aborts a park that was entirely + legitimate, so the Workflow Task never parks and the handshake runs again on + the next idle timeout, and again. And an intent installed for a wait outside + the park set is an intent with no park behind it, which is precisely the + thing `backend-contract.md` forbids leaving in a backend. + """ + from temporalio.worker._workflow import _WorkflowWorker + + backend = MemoryStreamBackend() + manager = make_manager(backend, RecordingNotifier()) + parked = StreamKey("ns", "wf", "first-run", "tokens") + driving = StreamKey("ns", "wf", "first-run", "tool-events") + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=parked) + manager.register(run_id=RUN_ID, wait_id=2, stream_key=driving) + # Wait 2 has a record sitting in it. Workflow code is not blocked on it, + # so it is not in the set Core asks to park. + await append(backend, driving, b"b") + + runtime = _StubRuntime({1: BEGINNING, 2: BEGINNING}) + worker = _StubWorker(RUN_ID, runtime, manager) + completion = await _WorkflowWorker._handle_external_stream_jobs( + worker, # type: ignore[arg-type] + _park_job(RUN_ID, 4, 1), + None, # type: ignore[arg-type] + ) + + assert completion is not None + result = completion.successful.commands[0].external_stream_park_result + assert result.WhichOneof("outcome") == "confirmed", ( + "a wait Core is not parking aborted the park of the wait it is" + ) + assert await backend.park_intent(parked, 1) is not None + assert await backend.park_intent(driving, 2) is None, ( + "an intent was installed for a wait Core never asked to park, so a " + "producer on that stream reads a park generation nothing is sitting in" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failing", ["install", "recheck"]) +async def test_a_failed_park_leaves_no_externally_visible_half(failing: str) -> None: + """Parking is all-or-nothing, and a raised activation is not an exception to it. + + Intents are installed one at a time, so any failure part-way through -- the + second install, or a recheck once every install has landed -- leaves the + completed ones visible to producers while the activation itself fails. Core + parks nothing, so those intents describe a park that does not exist; an + eviction then takes the local bookkeeping away and nothing can remove them + at all. + """ + + class Failing(MemoryStreamBackend): + async def install_park_intent(self, key, intent): # type: ignore[no-untyped-def] + if failing == "install" and intent.wait_id == 2: + raise ConnectionError("backend unavailable") + return await super().install_park_intent(key, intent) + + async def recheck(self, key, wait_id): # type: ignore[no-untyped-def] + if failing == "recheck": + raise ConnectionError("backend unavailable") + return await super().recheck(key, wait_id) + + backend = Failing() + manager = make_manager(backend, RecordingNotifier()) + first = StreamKey("ns", "wf", "first-run", "tokens") + second = StreamKey("ns", "wf", "first-run", "tool-events") + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=first) + manager.register(run_id=RUN_ID, wait_id=2, stream_key=second) + + # Reported as the taxonomy's transient row -- a backend that is + # unreachable is nothing for an operator to do anything about -- with + # the provider's own error kept as the cause, so the log still says + # which call failed and why. + with pytest.raises(StreamStorageError) as failure: + await manager.prepare_park(RUN_ID, 4, {1: BEGINNING, 2: BEGINNING}) + assert isinstance(failure.value.__cause__, ConnectionError) + + assert await backend.parked_wait_ids(first) == [], ( + "the park failed, and wait 1's intent is still in the backend " + "advertising a park Core never confirmed" + ) + assert await backend.parked_wait_ids(second) == [] + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_cancelled_park_takes_back_the_intents_it_had_installed() -> None: + """Cancellation is the *most* likely way a half-installed park is abandoned. + + Core withdrawing the activation and the Worker shutting down both arrive as + ``CancelledError``, which derives from ``BaseException`` -- so a rollback + reached only by ``except Exception`` is skipped for exactly the failures + most likely to leave a park half-installed. The intents that stay describe a + park Core never confirmed, and the eviction that follows takes the local + mirror away while they remain. + """ + + class Blocking(MemoryStreamBackend): + """Holds the second install open, so the first one is already visible.""" + + def __init__(self) -> None: + super().__init__() + self.reached = asyncio.Event() + self.release = asyncio.Event() + + async def install_park_intent(self, key, intent): # type: ignore[no-untyped-def] + if intent.wait_id == 2: + self.reached.set() + await self.release.wait() + return await super().install_park_intent(key, intent) + + backend = Blocking() + manager = make_manager(backend, RecordingNotifier()) + key = StreamKey("ns", "wf", "first-run", "tokens") + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=key) + manager.register(run_id=RUN_ID, wait_id=2, stream_key=key) + parking = asyncio.get_running_loop().create_task( + manager.prepare_park(RUN_ID, 4, {1: BEGINNING, 2: BEGINNING}) + ) + await asyncio.wait_for(backend.reached.wait(), 2) + assert await backend.parked_wait_ids(key) == [1], ( + "this case is only meaningful once one intent is installed" + ) + + parking.cancel() + with pytest.raises(asyncio.CancelledError): + await parking + + assert await backend.parked_wait_ids(key) == [], ( + "the park was abandoned mid-install and wait 1's intent is still in " + "the backend, advertising a park Core never confirmed" + ) + finally: + backend.release.set() + await manager.shutdown() + + +# --- the removal a failure leaves owed ---------------------------------------- +# +# A park intent is durable backend state; the manager's knowledge of it is not. +# `installed_park_generation` is a mirror that only the Worker which installed +# the park holds, and every removal path reaches its intent *through* the +# Subscription that carries it -- which the close, the eviction and the hand-off +# all throw away. So a failed removal recorded on the Subscription is a failed +# removal recorded on the object the next step discards, and the manager keeps a +# per-Run ledger of owed removals instead. + + +class FailingRemovals(MemoryStreamBackend): + """A backend whose park-intent removals fail on demand. + + ``failures`` counts down, so a test can ask for a single blip or -- with a + number no retry can reach -- for a window it closes itself by setting it + back to zero. + """ + + def __init__(self, failures: int = 0) -> None: + super().__init__() + self.failures = failures + self.removal_attempts = 0 + + async def remove_park_intent_if_matches( + self, + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + self.removal_attempts += 1 + if self.failures > 0: + self.failures -= 1 + raise ConnectionError("backend unavailable") + return await super().remove_park_intent_if_matches( + key, + wait_id, + run_id=run_id, + park_generation=park_generation, + ) + + +async def _nothing_parked(backend: MemoryStreamBackend, key: StreamKey) -> bool: + return await backend.parked_wait_ids(key) == [] + + +async def _only_wait_two_is_parked( + backend: MemoryStreamBackend, key: StreamKey +) -> bool: + return await backend.parked_wait_ids(key) == [2] + + +async def _inherit(backend: MemoryStreamBackend, key: StreamKey, run_id: str) -> None: + """Leaves behind the intent of a park whose Worker is gone. + + Installed straight into the backend rather than through a manager, because + that is exactly what makes it inherited: no mirror of it exists anywhere on + the Worker that finds it. + """ + await backend.install_park_intent( + key, + ParkIntent(wait_id=1, cursor=BEGINNING, park_generation=7, run_id=run_id), + ) + + +@pytest.mark.asyncio +async def test_one_blip_does_not_end_the_inherited_park_reconciliation( + stream_key: StreamKey, +) -> None: + """Registration is the only moment an inherited intent is looked for. + + It runs at most once per Subscription, so a single transient backend error + used to leave the intent installed with nothing left to try it again. What + that costs is the invariant's whole point: ``current_park_generation`` goes + on answering a generation Core has discarded, and because a parked wake's + request ID ignores sender identity the wake naming it is byte-identical to + the one that already ended that generation -- so the server deduplicates it + and the Workflow is never told about a record that is durably present. + """ + backend = FailingRemovals(failures=1) + await _inherit(backend, stream_key, RUN_ID) + manager = make_manager(backend, RecordingNotifier()) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + + await until( + lambda: _nothing_parked(backend, stream_key), + "the inherited intent is still installed after one failed removal, " + "and only another registration of this same wait would try again", + ) + assert backend.removal_attempts >= 2, "the failed removal must be retried" + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_an_inherited_intent_is_retried_autonomously_after_recovery( + stream_key: StreamKey, +) -> None: + """Retries are the cheap first line; the ledger is what makes them optional. + + Bounding the retries is only survivable because giving up records the + removal rather than forgetting it. Waiting instead for "the next time this + wait is registered" is a coincidence of eviction, not a mechanism -- and the + Run that most needs the removal is the one cached and blocked on something + other than this stream, which never registers this wait again at all. + """ + backend = FailingRemovals(failures=99) + await _inherit(backend, stream_key, RUN_ID) + manager = make_manager(backend, RecordingNotifier()) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await until( + lambda: ( + not manager._reconciliations + and backend.removal_attempts >= PARK_REMOVAL_ATTEMPTS + ), + "the reconciliation must exhaust its attempts before the ledger is " + "the only thing left holding the removal", + ) + backend.failures = 0 + + await until( + lambda: _nothing_parked(backend, stream_key), + "the inherited intent stayed installed after backend recovery even " + "though no park, resolve, registration, or eviction occurred", + ) + + assert await backend.parked_wait_ids(stream_key) == [], ( + "the reconciliation exhausted its inline attempts and its ledger " + "had no autonomous retry" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_close_whose_removal_fails_leaves_the_removal_owed( + stream_key: StreamKey, +) -> None: + """A close drops the subscription, and used to drop the removal with it. + + Nothing could retry afterwards: the resolve path iterates *registered* + subscriptions, eviction and the shutdown sweep remove no intents, and + another wait's park cannot touch a per-wait key. + + And the damage is not confined to the closed wait. A stale intent keeps + ``parked_wait_ids`` non-empty, which suppresses the unparked-wake fallback + for the whole stream: with no live wait parked, the producer sends only the + dead generation, Core discards it as stale, and dedup silences every later + publish -- so wait 2 here loses its wakes to wait 1's leftover. + """ + backend = FailingRemovals(failures=99) + manager = make_manager(backend, RecordingNotifier()) + try: + for wait_id in (1, 2): + manager.register( + run_id=RUN_ID, + wait_id=wait_id, + stream_key=stream_key, + ) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING, 2: BEGINNING}) + + await manager.cancel(RUN_ID, 1) + + assert manager.subscription(RUN_ID, 1) is None, ( + "the close must still drop the subscription -- one kept registered " + "while a removal is retried is an orphaned watcher" + ) + assert await backend.parked_wait_ids(stream_key) == [1, 2], ( + "this case is only meaningful while the removal is genuinely failing" + ) + backend.failures = 0 + + await manager.resolve_park(RUN_ID) + + assert await backend.parked_wait_ids(stream_key) == [], ( + "the closed wait's intent survived the only path that could still " + "have removed it, and it suppresses the unparked wake for wait 2 too" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_failed_resolve_removal_retries_without_another_event( + stream_key: StreamKey, +) -> None: + """Backend recovery alone retires a resolved park's stale intent.""" + backend = FailingRemovals(failures=99) + manager = make_manager(backend, RecordingNotifier()) + try: + manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=stream_key, + ) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + + await manager.resolve_park(RUN_ID) + assert await backend.current_park_generation(stream_key, 1) == 4 + backend.failures = 0 + + await until( + lambda: _nothing_parked(backend, stream_key), + "the resolved intent stayed installed after backend recovery even " + "though no park, resolve, registration, or eviction occurred", + ) + assert manager._owed_removals == {} + finally: + await manager.shutdown() + + +class BlockingAutonomousRemoval(MemoryStreamBackend): + """Fails the eager attempt, then blocks the manager-owned retry.""" + + def __init__(self) -> None: + super().__init__() + self.attempts = 0 + self.retry_started = asyncio.Event() + self.retry_cancelled = asyncio.Event() + + async def remove_park_intent_if_matches( + self, + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + self.attempts += 1 + if self.attempts == 1: + raise ConnectionError("backend unavailable") + self.retry_started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + self.retry_cancelled.set() + raise + return ParkIntentRemoval.ABSENT + + +@pytest.mark.asyncio +async def test_shutdown_cancels_and_awaits_an_owed_removal_retry( + stream_key: StreamKey, +) -> None: + backend = BlockingAutonomousRemoval() + manager = make_manager(backend, RecordingNotifier()) + manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=stream_key, + ) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + await manager.resolve_park(RUN_ID) + await asyncio.wait_for(backend.retry_started.wait(), 2) + + await asyncio.wait_for(manager.shutdown(grace=timedelta(milliseconds=200)), 1) + + assert backend.retry_cancelled.is_set() + assert manager._owed_removal_retries == {} + + +class HangsOnceThenRecovers(MemoryStreamBackend): + """One removal call that never returns, and healthy ones after it.""" + + def __init__(self) -> None: + super().__init__() + self.attempts = 0 + self.hanging = asyncio.Event() + + async def remove_park_intent_if_matches( # type: ignore[override] + self, + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + self.attempts += 1 + if self.attempts == 1: + raise ConnectionError("backend unavailable") + if self.attempts == 2: + # A call that hangs rather than raises: no exception to catch, no + # return to act on, and nothing counting attempts can bound it. + self.hanging.set() + await asyncio.Event().wait() + return await super().remove_park_intent_if_matches( + key, wait_id, run_id=run_id, park_generation=park_generation + ) + + +@pytest.mark.asyncio +async def test_a_removal_call_that_hangs_does_not_end_the_autonomous_retry( + stream_key: StreamKey, +) -> None: + """The loop's own liveness, which counting attempts cannot give it. + + The retry loop is the only thing left holding a removal after the inline + attempts are spent, so a single call that never returns is not one lost + attempt -- it is the mechanism stopping. Nothing restarts the loop either: + `_owe_removal` starts one only when the task it finds is absent or done, and + a task awaiting a hung call is neither. The intent would stay installed on a + backend that had already recovered. + + It also holds the Run's park lock while it waits, which every park, resolve + and eviction of this Run needs. + """ + backend = HangsOnceThenRecovers() + manager = make_manager(backend, RecordingNotifier()) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + await manager.resolve_park(RUN_ID) + await asyncio.wait_for(backend.hanging.wait(), 2) + assert await backend.parked_wait_ids(stream_key) == [1] + + await until( + lambda: _nothing_parked(backend, stream_key), + "the retry gave up its park lock and came back around after the " + "call that hung, or it did not", + timeout=4.0, + ) + # And the lock an activation needs is reachable again: nothing is owed + # by now, so this resolve wants the lock and nothing else. + await asyncio.wait_for(manager.resolve_park(RUN_ID), 2) + finally: + await manager.shutdown(grace=timedelta(milliseconds=200)) + + +@pytest.mark.asyncio +async def test_shutdown_makes_the_last_attempt_the_retry_loop_cannot( + stream_key: StreamKey, +) -> None: + """A backend that recovered during a backoff, on a Worker that is exiting. + + The retry loop is cancelled by shutdown, and cancelling it during its + backoff throws away an attempt that would have succeeded -- the failure that + started the backoff is not the state of the backend now. Eviction used to be + that last attempt and deliberately stands aside while shutting down, so the + pass after the loops are cancelled is what has to make it. + """ + backend = FailingRemovals(failures=99) + manager = make_manager(backend, RecordingNotifier()) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + await manager.resolve_park(RUN_ID) + # Long enough for the backoff to have grown past the grace period below, so + # a sleeping loop cannot be what removes it. + await until( + lambda: backend.removal_attempts >= 4, + "the retry loop must have failed enough times to be backing off", + timeout=4.0, + ) + attempts = backend.removal_attempts + backend.failures = 0 + + await asyncio.wait_for(manager.shutdown(grace=timedelta(seconds=1)), 3) + + assert backend.removal_attempts > attempts, ( + "shutdown cancelled the retry and exited without trying once itself" + ) + assert await backend.parked_wait_ids(stream_key) == [], ( + "the intent outlived a Worker whose backend was healthy again" + ) + + +@pytest.mark.asyncio +async def test_cleanup_re_announces_the_record_the_stale_intent_silenced( + stream_key: StreamKey, +) -> None: + """Removing the intent is not the same as delivering what it suppressed. + + Every wake sent while the stale intent was installed named the generation + behind it, and Core discards a non-zero generation that is not the park it + holds. The record that arrived in that window is buffered here, its wake is + counted as sent, and the watcher will not report it again without a *new* + append -- so retiring the intent stops the suppression and announces + nothing. The Run stays blocked on a record this Worker is already holding. + """ + backend = FailingRemovals(failures=99) + sent: list[int] = [] + + async def send_wake(subscription: Subscription) -> None: + # What the Worker's sender does: read the generation at send time. + sent.append( + await subscription.backend.current_park_generation( + subscription.stream_key, subscription.wait_id + ) + or 0 + ) + + # No open Workflow Task, so local readiness cannot be delivered and each + # announcement becomes a wake Signal. + manager = make_manager( + backend, + RecordingNotifier(ReadinessResult.NO_OPEN_WORKFLOW_TASK), + send_wake=send_wake, + ) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + await manager.resolve_park(RUN_ID) + assert await backend.current_park_generation(stream_key, 1) == 4 + + await append(backend, stream_key, b"1") + await until(lambda: sent, "the record's wake must be sent") + assert sent == [4], "the wake named the generation Core has discarded" + + backend.failures = 0 + await until( + lambda: _nothing_parked(backend, stream_key), + "the autonomous retry must retire the stale intent", + ) + await until( + lambda: len(sent) > 1, + "nothing announced the record again once the suppression ended", + ) + assert sent[1] == 0, ( + "the wake that follows the cleanup must be the unparked one Core " + "accepts unconditionally" + ) + finally: + await manager.shutdown(grace=timedelta(milliseconds=200)) + + +class DelaysTheIntentRead(MemoryStreamBackend): + """Holds the reconciliation's read until the watcher has sent its wake.""" + + def __init__(self) -> None: + super().__init__() + self.may_read = asyncio.Event() + + async def park_intent(self, key: StreamKey, wait_id: int) -> ParkIntent | None: + await self.may_read.wait() + return await super().park_intent(key, wait_id) + + +@pytest.mark.asyncio +async def test_reconciling_an_inherited_intent_re_announces_what_it_silenced( + stream_key: StreamKey, +) -> None: + """The ordinary reconciliation races the watcher, and can lose it a record. + + `_start_watcher` starts the reconciliation and the watcher as two tasks, so + the watcher can read a record, find no open Workflow Task and send its wake + -- naming the inherited generation, which Core discards -- while the + reconciliation is still in flight. The removal then succeeds *first time*, + which is the path that never goes near the owed-removal ledger, so the + reannouncement the retry path does would never happen here. The record sits + buffered with its prefetch cursor already past it. + """ + backend = DelaysTheIntentRead() + sent: list[int] = [] + + async def send_wake(subscription: Subscription) -> None: + sent.append( + await subscription.backend.current_park_generation( + subscription.stream_key, subscription.wait_id + ) + or 0 + ) + # Only now may the reconciliation read: the wake naming the inherited + # generation is already gone. + backend.may_read.set() + + manager = make_manager( + backend, + RecordingNotifier(ReadinessResult.NO_OPEN_WORKFLOW_TASK), + send_wake=send_wake, + ) + try: + await _inherit(backend, stream_key, RUN_ID) + await append(backend, stream_key, b"1") + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + + await until(lambda: sent, "the watcher must announce the buffered record") + assert sent[0] == 7, "the wake named the inherited generation" + + await until( + lambda: _nothing_parked(backend, stream_key), + "the reconciliation must remove the intent it inherited", + ) + await until( + lambda: len(sent) > 1, + "the ordinary reconciliation removed the intent and left the record " + "it had silenced unannounced", + ) + assert sent[1] == 0, "the wake after cleanup must be the unparked one" + finally: + await manager.shutdown(grace=timedelta(milliseconds=200)) + + +class LosesTheReplyAfterCommitting(MemoryStreamBackend): + """Deletes the intent, then fails before the reply reaches the Worker.""" + + def __init__(self) -> None: + super().__init__() + self.calls = 0 + self.removing = asyncio.Event() + self.wake_sent = asyncio.Event() + + async def remove_park_intent_if_matches( # type: ignore[override] + self, + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + self.calls += 1 + if self.calls == 1: + self.removing.set() + await self.wake_sent.wait() + await super().remove_park_intent_if_matches( + key, wait_id, run_id=run_id, park_generation=park_generation + ) + raise ConnectionError("the reply never arrived") + return await super().remove_park_intent_if_matches( + key, wait_id, run_id=run_id, park_generation=park_generation + ) + + +@pytest.mark.asyncio +async def test_a_removal_whose_reply_was_lost_still_re_announces( + stream_key: StreamKey, +) -> None: + """An absent key is cleanup that happened, not a park to leave alone. + + A provider can commit the compare-and-delete and lose the connection before + its reply arrives. The removal raises, so it stays owed; the retry finds the + key clear. Reading that as "someone else's intent is there" would leave the + record whose wake named the dead generation buffered and unannounced -- + which is why the outcome is three-valued rather than a Boolean. + """ + backend = LosesTheReplyAfterCommitting() + sent: list[int] = [] + + async def send_wake(subscription: Subscription) -> None: + sent.append( + await subscription.backend.current_park_generation( + subscription.stream_key, subscription.wait_id + ) + or 0 + ) + + manager = make_manager( + backend, + RecordingNotifier(ReadinessResult.NO_OPEN_WORKFLOW_TASK), + send_wake=send_wake, + ) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + + # Core has ended the park, so generation 4 is dead the moment this + # resolve starts -- and its removal is in flight when the record lands. + resolve = asyncio.ensure_future(manager.resolve_park(RUN_ID)) + await asyncio.wait_for(backend.removing.wait(), 2) + await append(backend, stream_key, b"1") + await until(lambda: sent, "the record's wake must be sent") + assert sent == [4], "the wake named the generation Core has discarded" + + backend.wake_sent.set() + await asyncio.wait_for(resolve, 2) + + await until( + lambda: len(sent) > 1, + "the retry read the committed delete as a mismatch and left the " + "record it had silenced unannounced", + ) + assert sent[1] == 0, "the wake after cleanup must be the unparked one" + assert manager._owed_removals == {} + finally: + await manager.shutdown(grace=timedelta(milliseconds=200)) + + +class FailingIntentReads(MemoryStreamBackend): + """Cannot say whether an intent exists until it is told to recover.""" + + def __init__(self) -> None: + super().__init__() + self.healthy = False + self.reads = 0 + + async def park_intent(self, key: StreamKey, wait_id: int) -> ParkIntent | None: + self.reads += 1 + if not self.healthy: + raise ConnectionError("backend unavailable") + return await super().park_intent(key, wait_id) + + +@pytest.mark.asyncio +async def test_discovering_an_inherited_intent_is_retried_after_recovery( + stream_key: StreamKey, +) -> None: + """The ledger cannot hold what was never read, so the read needs the retry. + + An owed removal is recorded from an intent's *identity*, and a read that + never succeeds produces no identity -- so a reconciliation that gave up + after its cheap attempts left no ledger entry, no retry task and no owner, + while logging that an autonomous retry was continuing. Recovery then needed + exactly the unrelated Workflow or Core event the ledger exists to stop + needing. + """ + backend = FailingIntentReads() + manager = make_manager(backend, RecordingNotifier()) + try: + await _inherit(backend, stream_key, RUN_ID) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await until( + lambda: backend.reads >= PARK_REMOVAL_ATTEMPTS, + "the inline attempts must be spent before recovery", + ) + assert manager._owed_removals == {}, "nothing was read, so nothing can be owed" + backend.healthy = True + + await until( + lambda: _nothing_parked(backend, stream_key), + "the inherited intent outlived a backend that recovered, with no " + "park, resolve, registration or eviction to prompt another read", + timeout=4.0, + ) + finally: + await manager.shutdown(grace=timedelta(milliseconds=200)) + + +@pytest.mark.asyncio +async def test_closing_a_wait_removes_an_intent_it_only_inherited( + stream_key: StreamKey, +) -> None: + """The second door, which needs no failure of its own to open. + + ``installed_park_generation`` is set only by a park *this* Worker installed, + so for an inherited intent it is ``None`` and the removal short-circuits + silently -- the close attempts nothing at all. That is why the close is not + a backstop for a reconciliation that has already failed: it is disabled by + exactly the condition that made the reconciliation necessary. + """ + backend = FailingRemovals(failures=99) + await _inherit(backend, stream_key, RUN_ID) + manager = make_manager(backend, RecordingNotifier()) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await until( + lambda: backend.removal_attempts >= PARK_REMOVAL_ATTEMPTS, + "the reconciliation must have given up before the close is asked to " + "cover for it", + ) + backend.failures = 0 + + await manager.cancel(RUN_ID, 1) + + assert await backend.parked_wait_ids(stream_key) == [], ( + "the close attempted no removal, because the intent it inherited is " + "mirrored in no generation of its own" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_an_owed_removal_outlives_eviction_and_keeps_retrying( + stream_key: StreamKey, +) -> None: + """Cache ownership ending does not end backend-cleanup ownership. + + Installed intents are deliberately untouched by the same path: an eviction + is not the end of a park, and the intents of one that really is outstanding + have to survive the Worker losing the Run. Only a removal already decided on + is retried after it. + """ + backend = FailingRemovals(failures=99) + manager = make_manager(backend, RecordingNotifier()) + try: + for wait_id in (1, 2): + manager.register( + run_id=RUN_ID, + wait_id=wait_id, + stream_key=stream_key, + ) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING, 2: BEGINNING}) + await manager.cancel(RUN_ID, 1) + + await manager.evict_run(RUN_ID) + assert await backend.parked_wait_ids(stream_key) == [1, 2] + backend.failures = 0 + + await until( + lambda: _only_wait_two_is_parked(backend, stream_key), + "wait 1's owed removal stopped retrying when eviction dropped the " + "Run, while wait 2's outstanding park still had to survive", + ) + + assert await backend.parked_wait_ids(stream_key) == [2], ( + "the owed removal did not outlive the cached Run" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_cleanup_after_eviction_still_announces_what_it_silenced( + stream_key: StreamKey, +) -> None: + """The removal outlives the Subscription; the wake it owes has to as well. + + Eviction is what makes the ledger worth having -- it drops the Subscription + and deliberately leaves the entry and its retry alive -- so by the time a + recovered backend lets the removal through, the object every announcement + path goes through is gone. Announcing only when the wait happens to still be + registered therefore kept the leak fixed and put the *record loss* straight + back: while the stale intent was installed a producer saw a non-empty parked + set and sent only generation 4, Core discarded it, and with no cached Run + there was no local watcher to send an unparked wake of its own. + + The wake is unconditional on purpose. From out here the buffer is gone and a + remote producer's append was never visible at all, so "a record arrived" and + "none did" are the same observation -- and only one of them is silent if the + guess goes the wrong way. The cost is one empty Workflow Task. + """ + backend = FailingRemovals(failures=99) + sent: list[tuple[int, int]] = [] + + async def send_wake(target: object) -> None: + # What the Worker's real sender composes: the manager decides which park + # a wake names, because only it can tell an intent Core is still parked + # on from one already decided against. + sent.append( + ( + target.wait_id, # type: ignore[attr-defined] + await manager.wake_park_generation(target) or 0, # type: ignore[arg-type] + ) + ) + + manager = make_manager(backend, RecordingNotifier(), send_wake=send_wake) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + assert not await manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + await manager.resolve_park(RUN_ID) + assert await backend.current_park_generation(stream_key, 1) == 4, ( + "the failed removal must leave the stale intent installed" + ) + + await manager.evict_run(RUN_ID) + assert manager.subscription(RUN_ID, 1) is None, ( + "eviction must drop the Subscription -- that is the premise" + ) + assert await backend.parked_wait_ids(stream_key) == [1], ( + "and must leave the intent and its ledger entry behind" + ) + + # The record that gets lost. Appended by a producer this Worker cannot + # see, into a stream whose parked set is non-empty because of the stale + # intent -- so the only wake it sends names generation 4. + await append(backend, stream_key, b"1") + + backend.failures = 0 + await until( + lambda: _nothing_parked(backend, stream_key), + "the autonomous retry must retire the stale intent once the backend " + "recovers", + ) + await until( + lambda: sent, + "the removal ended the suppression and announced nothing, so the " + "record stays undelivered until some unrelated event happens to " + "create a Workflow Task", + ) + assert sent == [(1, 0)], ( + "the cleanup handoff must be the unparked wake Core accepts " + "unconditionally, since the generation it would otherwise name is " + "the one it just removed" + ) + finally: + await manager.shutdown(grace=timedelta(milliseconds=200)) + + +@pytest.mark.asyncio +async def test_a_drain_never_removes_the_intent_of_a_park_that_replaced_it( + stream_key: StreamKey, +) -> None: + """An entry records a removal, not the claim that some intent exists. + + A Continue-As-New successor re-uses this stream key with wait ids that start + again at 1, so an entry a predecessor Run left behind can name a live park's + key. Retiring it on the key alone would take the intent out from under a + park that really is outstanding -- manufacturing exactly the unwakeable Run + the ledger exists to prevent. + """ + backend = FailingRemovals(failures=99) + manager = make_manager(backend, RecordingNotifier()) + try: + manager.register(run_id="run-a", wait_id=1, stream_key=stream_key) + assert not await manager.prepare_park("run-a", 4, {1: BEGINNING}) + await manager.cancel("run-a", 1) + backend.failures = 0 + + # The successor's park, installed directly: what matters is that it is a + # different Run's intent sitting at the key run-a still owes a removal + # for, not how it got there. + await backend.install_park_intent( + stream_key, + ParkIntent(wait_id=1, cursor=BEGINNING, park_generation=1, run_id="run-b"), + ) + + await manager.resolve_park("run-a") + + installed = await backend.park_intent(stream_key, 1) + assert installed is not None and installed.run_id == "run-b", ( + "run-a's owed removal was retired against whatever it found, taking " + "out the intent of a park run-b is still sitting in" + ) + finally: + await manager.shutdown() + + +# --- a wake owed on the live path --------------------------------------------- + + +class CountingWake: + """Records the counter each attempt would derive its request ID from. + + A retry that re-counts is not a retry: the request ID moves with the + counter, so the server deduplicates nothing and answers with a second, empty + Workflow Task instead of resolving the wake that may in fact have arrived. + """ + + def __init__(self, failures: int) -> None: + self.failures = failures + self.counters: list[int] = [] + #: Asked at each attempt, for tests about what has happened to the + #: subscription by the time an attempt is made. Assigned after the + #: manager exists, since that is what it usually asks about. + self.observe: Callable[[], object] | None = None + self.observed: list[object] = [] + + async def __call__(self, subscription) -> None: # type: ignore[no-untyped-def] + self.counters.append(subscription.wake_counter) + if self.observe is not None: + self.observed.append(self.observe()) + if self.failures > 0: + self.failures -= 1 + raise ConnectionError("service unavailable") + + +@pytest.mark.asyncio +async def test_a_failed_live_wake_is_retried_as_the_same_wake( + stream_key: StreamKey, +) -> None: + """One attempt on the live path is one attempt with nothing behind it. + + The watcher has already moved ``prefetch_cursor`` past the buffered record + and returns on an empty read, so it never comes back here without a *new* + append. ``rearm_ready`` needs the activation the lost wake was supposed to + cause. And the idle timer only runs while a Workflow Task is retained, which + ``NoOpenWorkflowTask`` says there is not -- so no park handshake happens + either. The shutdown sweep already does this correctly, which is why the + retry belongs to both and not just to it. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier(answer=ReadinessResult.NO_OPEN_WORKFLOW_TASK) + wake = CountingWake(failures=1) + manager = make_manager(backend, notifier, send_wake=wake) + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a") + + await until( + lambda: len(wake.counters) >= 2, + "the failed wake was logged and forgotten; nothing on this path " + "ever attempts it again, so the buffered record is never announced", + ) + assert wake.counters == [1, 1], ( + "the retry derived a different request ID, so it asks for a second " + "empty Workflow Task rather than re-sending the wake that was owed" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_replayed_readiness_is_coalesced_until_the_task_completes( + stream_key: StreamKey, +) -> None: + """A failed Workflow Task must not recursively ask for another task. + + Eviction discards the subscription and its speculative buffer. Replaying + the failed task rebuilds both, but the wake that caused it remains + outstanding because Core never accepted a successful completion. Once the + replayed task does complete, any buffer it leaves behind needs a fresh wake. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier(answer=ReadinessResult.NO_OPEN_WORKFLOW_TASK) + wake = CountingWake(failures=0) + manager = make_manager(backend, notifier, send_wake=wake) + try: + # The readiness races the completion of a task that was already open. + # That completion cannot have been caused by the wake and must not + # release its gate. + manager.note_workflow_task_started(RUN_ID) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a") + await until( + lambda: len(wake.counters) >= 1, + "the original buffered readiness never owed a wake", + ) + manager.note_workflow_task_completed(RUN_ID, terminal=False) + + await manager.evict_run(RUN_ID) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await until( + lambda: len(notifier.calls) >= 2, + "replay never re-reported the reconstructed buffered readiness", + ) + + assert wake.counters == [1], ( + "replay turned one outstanding wake into another, so each failed " + f"task can recursively create the next: {wake.counters}" + ) + + manager.note_workflow_task_started(RUN_ID) + manager.note_workflow_task_completed(RUN_ID, terminal=False) + await until( + lambda: len(wake.counters) >= 2, + "a buffer left after successful replay was never announced again", + ) + assert wake.counters == [1, 2] + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_failed_wake_attempt_cannot_claim_a_completion_for_its_retry( + stream_key: StreamKey, +) -> None: + """Each retry correlates only with activations that begin during that attempt. + + The first Signal attempt can lose a closing-window race while a different + Workflow Task completes. If the later successful retry inherited that + completion, the gate would open before the retry's Signal had produced its + own task and another buffered generation could send a competing wake. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier(answer=ReadinessResult.NO_OPEN_WORKFLOW_TASK) + started = [asyncio.Event() for _ in range(3)] + release = [asyncio.Event() for _ in range(3)] + finished = [asyncio.Event() for _ in range(3)] + counters: list[int] = [] + + async def raced_wake(subscription: Subscription) -> None: + attempt = len(counters) + counters.append(subscription.wake_counter) + started[attempt].set() + try: + await release[attempt].wait() + if attempt == 0: + raise ConnectionError("closing window") + finally: + finished[attempt].set() + + manager = make_manager(backend, notifier, send_wake=raced_wake) + try: + manager.note_workflow_task_started(RUN_ID) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a") + await asyncio.wait_for(started[0].wait(), 2) + + # A different task starts and completes while attempt 1 is in flight. + manager.note_workflow_task_started(RUN_ID) + manager.note_workflow_task_completed(RUN_ID, terminal=False) + release[0].set() + + await asyncio.wait_for(started[1].wait(), 2) + release[1].set() + await asyncio.wait_for(finished[1].wait(), 2) + + # The successful retry has not caused an activation yet. Re-reporting + # the buffer must remain coalesced behind it. + manager.rearm_ready(RUN_ID) + await asyncio.sleep(0.05) + assert counters == [1, 1] + + manager.note_workflow_task_started(RUN_ID) + manager.note_workflow_task_completed(RUN_ID, terminal=False) + await asyncio.wait_for(started[2].wait(), 2) + assert counters == [1, 1, 2] + release[2].set() + finally: + for event in release: + event.set() + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_new_buffered_range_after_eviction_gets_a_new_wake_counter( + stream_key: StreamKey, +) -> None: + """A later range is a new ask even when the Run was evicted between them. + + The unparked request ID is derived from the sender's identity and its + counter, and the identity is fixed for the Worker's lifetime -- so the + counter is the only thing keeping two of this Worker's wakes apart. A + counter held on the `Subscription` restarts at one every time an evicted Run + is rebuilt, which re-derives the request ID the previous incarnation already + used: the server deduplicates it, no Workflow Task is created, and a Run + holding buffered records waits for a wake that was thrown away as a + duplicate. The sequence therefore belongs to the manager, which outlives the + Run. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier(answer=ReadinessResult.NO_OPEN_WORKFLOW_TASK) + wake = CountingWake(failures=0) + manager = make_manager(backend, notifier, send_wake=wake) + try: + first = await backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"a", "sa", 0) + ) + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await until( + lambda: len(wake.counters) >= 1, + "the first incarnation never owed its wake, so there is nothing for " + "the second one to collide with", + ) + manager.drain(RUN_ID, 1) + manager.note_workflow_task_started(RUN_ID) + manager.note_workflow_task_completed(RUN_ID, terminal=False) + + # The Run is evicted and comes back after the first range's boundary. + # This is not replay of the first readiness: the committed start cursor + # and the range waiting behind it are both new. + await manager.evict_run(RUN_ID) + assert first.offset is not None + manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=stream_key, + start_cursor=AFTER(first.offset), + ) + await append(backend, stream_key, b"b") + await until( + lambda: len(wake.counters) >= 2, + "the rebuilt subscription never owed a wake for the record waiting for it", + ) + + assert wake.counters[0] != wake.counters[1], ( + "the rebuilt Run drew the counter its predecessor had already used, " + "so both wakes derive one request ID and the server deduplicates " + f"the second away: {wake.counters}" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_wake_owed_by_a_vanished_run_is_retried_before_it_is_dropped( + stream_key: StreamKey, +) -> None: + """The sharpest sub-case: after the pop there is no backstop at all. + + ``RunNotFound`` takes the subscription out of ``_runs``, and the shutdown + sweep is driven by ``_runs`` -- so a wake given up on before that pop is not + merely delayed, it is unreachable by every remaining mechanism. + """ + backend = MemoryStreamBackend() + notifier = RecordingNotifier(answer=ReadinessResult.RUN_NOT_FOUND) + wake = CountingWake(failures=1) + manager = make_manager(backend, notifier, send_wake=wake) + wake.observe = lambda: manager.subscription(RUN_ID, 1) is not None + try: + manager.register(run_id=RUN_ID, wait_id=1, stream_key=stream_key) + await append(backend, stream_key, b"a") + + await until( + lambda: len(wake.counters) >= 2, + "the wake failed once and the subscription was dropped on top of " + "it, putting it out of reach of the sweep that would have retried it", + ) + assert wake.counters == [1, 1] + assert wake.observed == [True, True], ( + "an attempt was made after the subscription had already left " + "`_runs`, where the sweep that is supposed to back it up cannot " + "see it either" + ) + await until( + lambda: manager.subscription(RUN_ID, 1) is None, + "a Run this Worker no longer holds must still be dropped once the " + "wake it owed has been attempted", + ) + finally: + await manager.shutdown() diff --git a/tests/contrib/external_workflow_streams/test_multi_stream.py b/tests/contrib/external_workflow_streams/test_multi_stream.py new file mode 100644 index 0000000000..2bdb5c90bf --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_multi_stream.py @@ -0,0 +1,1051 @@ +"""P21 — several streams, ``merge``, and two subscriptions to one stream. + +The idle timeout is a **Workflow-Task policy**, not a per-subscription one: it +applies to the complete set the Workflow is blocked on, so one idle stream must +not park the task while another is still delivering. Everything below is about +that set behaving as a set. + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +# pyright: reportMissingParameterType=false, reportUnusedVariable=false +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest +import pytest_asyncio + +import temporalio.converter +from temporalio import workflow +from temporalio.client import Client +from temporalio.contrib.external_workflow_streams._annotation import decode_annotation +from temporalio.contrib.external_workflow_streams._api import ( + MAX_RECORDS_PER_ACTIVATION, + _install_runtime, + external_stream, + merge, +) +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._record import ( + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._runtime import WorkflowStreamRuntime +from temporalio.worker import Worker +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +with workflow.unsafe.imports_passed_through(): + pass + +RUN_ID = "run-1" + + +async def _notify( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] +) -> str: + return ReadinessResult.ACCEPTED + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +class StubManager: + """Records registrations and cancellations, and starts no watcher. + + The snapshot and annotation tests are about what the runtime decides, which + happens before any watching. A real manager would spawn prefetch loops + against a backend nothing is writing to, and their failures would be noise. + """ + + def __init__(self) -> None: + #: `(run_id, wait_id)` for every wait the runtime asked to stop serving. + self.cancelled: list[tuple[str, int]] = [] + + def register(self, *, run_id, wait_id, stream_key, start_cursor): # type: ignore[no-untyped-def] + pass + + def cancel_from_workflow_thread(self, run_id, wait_id): # type: ignore[no-untyped-def] + self.cancelled.append((run_id, wait_id)) + + def note_wait_generation(self, run_id, wait_id, generation) -> None: # type: ignore[no-untyped-def] + pass + + def drain(self, run_id, wait_id, max_records=None): # type: ignore[no-untyped-def] + return [] + + +@pytest.fixture +def manager() -> StubManager: + return StubManager() + + +@pytest_asyncio.fixture # type: ignore[reportUntypedFunctionDecorator] +async def live_manager(backend: MemoryStreamBackend): + """A real manager, for the tests that need records to actually move.""" + mgr = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + yield mgr + await mgr.shutdown() + + +def make_runtime(manager, backend, idle=timedelta(seconds=1)): # type: ignore[no-untyped-def] + return WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id="first-run", + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=idle, + ) + + +def data(seq: int, offset: str) -> StreamRecord: + return StreamRecord(RecordKind.DATA, b"x", "s", seq).placed_at(Offset(offset)) + + +def fence(seq: int, offset: str) -> StreamRecord: + return StreamRecord(RecordKind.WRITE_FENCE, b"", "s", seq).placed_at(Offset(offset)) + + +# --- the idle timeout is a property of the set -------------------------------- + + +def test_one_idle_stream_cannot_park_while_another_is_active( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """The criterion the whole "complete set" rule exists for. + + Parking on the idle one alone would strand records already sitting in the + active one, and the Workflow would wait out a timeout for a stream that was + never quiet. + """ + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=runtime.stream_key("idle")) + runtime.register(wait_id=2, stream_key=runtime.stream_key("busy")) + runtime.note_blocked(1, True) + # The second is *not* blocked -- Workflow code is still working through it. + runtime.note_blocked(2, False) + + snapshot = runtime.quiescent_snapshot() + + assert snapshot is not None + assert [w.wait_id for w in snapshot] == [1], ( + "the quiescent set must name only the waits Workflow code is actually " + "blocked on; naming an active one would ask Core to park it" + ) + + +def test_the_quiescent_set_is_complete_when_everything_is_blocked( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """A set missing a member would let Core park a wait that was still live.""" + runtime = make_runtime(manager, backend) + for wait_id, name in ((1, "a"), (2, "b"), (3, "c")): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(name), + ) + runtime.note_blocked(wait_id, True) + + snapshot = runtime.quiescent_snapshot() + + assert snapshot is not None + assert [w.wait_id for w in snapshot] == [1, 2, 3] + + +def test_differing_idle_timeouts_reduce_to_the_minimum( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """``min``, in ``wait_id`` order, over the quiescent set and nothing else. + + Taking the maximum would let one lax subscription hold the Workflow Task + open past what another was configured to tolerate; taking the *configured + values of the whole set* rather than of the blocked subset would make the + result depend on state replay does not reproduce. + """ + runtime = make_runtime(manager, backend) + for wait_id, seconds in ((1, 5), (2, 2), (3, 9)): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(f"s{wait_id}"), + idle_timeout=timedelta(seconds=seconds), + ) + runtime.note_blocked(wait_id, True) + + assert runtime.effective_idle_timeout() == timedelta(seconds=2) + + +def test_with_options_timeouts_reach_the_reduction_through_the_public_api( + manager: StubManager, + backend: MemoryStreamBackend, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The reduction is unreachable unless ``subscribe()`` passes the value on. + + The test above registers with an explicit ``idle_timeout``, which proves + ``min`` reduces but says nothing about whether a configured value can ever + get there. ``with_options`` is the only route a user has, and a + ``subscribe()`` that dropped the option would leave every quiescent set on + the one-second default while that test, and + ``ExternalStreamSubscription.idle_timeout``, both kept passing -- the option + would be silently decorative. + """ + + class Instance: + """Stands in for the Workflow object the per-Run state hangs off.""" + + instance = Instance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: instance, + ) + runtime = make_runtime(manager, backend, idle=timedelta(seconds=1)) + _install_runtime(instance, runtime) # type: ignore[arg-type] + + for name, seconds in (("slow", 30), ("quick", 4), ("slower", 60)): + subscription = ( + external_stream.with_options(idle_timeout=timedelta(seconds=seconds)) + .topic(name, type=str) + .subscribe() + ) + runtime.note_blocked(subscription.wait_id, True) + + assert runtime.effective_idle_timeout() == timedelta(seconds=4), ( + "no configured timeout reached the runtime, so the whole quiescent set " + "fell back to the default and every with_options() call was ignored" + ) + + +def test_the_reduction_ignores_waits_that_are_not_blocked( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """The inputs are the quiescent set's configured values and nothing else.""" + runtime = make_runtime(manager, backend) + runtime.register( + wait_id=1, + stream_key=runtime.stream_key("a"), + idle_timeout=timedelta(seconds=7), + ) + runtime.register( + wait_id=2, + stream_key=runtime.stream_key("b"), + idle_timeout=timedelta(milliseconds=50), + ) + runtime.note_blocked(1, True) + runtime.note_blocked(2, False) + + assert runtime.effective_idle_timeout() == timedelta(seconds=7) + + +# --- fences are per stream, parking is per set -------------------------------- + + +def test_a_fence_on_one_stream_alone_does_not_make_the_set_parkable( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """It marks *that* stream parkable and says nothing about the others. + + A fence means only that one producer session has finished writing. Treating + it as a signal for the whole set would park a Workflow Task while another + stream was still being written to. + """ + runtime = make_runtime(manager, backend) + for wait_id, name in ((1, "fenced"), (2, "open")): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(name), + ) + runtime.note_blocked(wait_id, True) + runtime.record_delivery(1, fence(0, "10-0")) + + snapshot = runtime.quiescent_snapshot() + + assert snapshot is not None + parkable = {w.wait_id: w.immediately_parkable for w in snapshot} + assert parkable == {1: True, 2: False}, ( + "the fenced stream is immediately parkable and the open one is not; " + "Core parks the task early only when every wait says True" + ) + + +def test_all_fenced_streams_make_the_whole_set_parkable( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + runtime = make_runtime(manager, backend) + for wait_id, name in ((1, "a"), (2, "b")): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(name), + ) + runtime.note_blocked(wait_id, True) + runtime.record_delivery(wait_id, fence(0, f"1{wait_id}-0")) + + snapshot = runtime.quiescent_snapshot() + + assert snapshot is not None + assert all(w.immediately_parkable for w in snapshot) + + +def test_a_record_after_a_fence_reopens_the_stream( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """A later record does not violate a fence; it simply resumes consumption. + + The fence spoke only for one producer session. Treating the stream as closed + would strand every record another producer went on to write. + """ + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=runtime.stream_key("a")) + runtime.note_blocked(1, True) + runtime.record_delivery(1, fence(0, "10-0")) + assert runtime.quiescent_snapshot()[0].immediately_parkable # type: ignore[index] + + runtime.record_delivery(1, data(1, "10-1")) + + assert not runtime.quiescent_snapshot()[0].immediately_parkable # type: ignore[index] + + +# --- two subscriptions to one stream ------------------------------------------ + + +@pytest.mark.asyncio +async def test_two_same_stream_subscriptions_install_distinct_park_intents( + live_manager, backend: MemoryStreamBackend +) -> None: + """Verified by reading both back, not by trusting the call. + + Intents keyed by stream alone would have the second install overwrite the + first, after which only one of the two could ever be woken -- and the + surviving intent would name the wrong cursor for the lost one. + """ + runtime = make_runtime(live_manager, backend) + key = runtime.stream_key("tokens") + runtime.register(wait_id=1, stream_key=key) + runtime.register(wait_id=2, stream_key=key) + # Two different positions, so a collapsed intent is visible rather than + # merely suspected. + runtime.record_delivery(1, data(0, "10-0")) + runtime.note_blocked(1, True) + runtime.note_blocked(2, True) + + await live_manager.prepare_park(RUN_ID, 1, runtime.blocked_snapshot()) + + first = await backend.park_intent(key, 1) + second = await backend.park_intent(key, 2) + assert first is not None and second is not None + assert first.cursor != second.cursor, ( + "each subscription parks at its own position; a shared intent would " + "resume one of them at the other's cursor" + ) + + +@pytest.mark.asyncio +async def test_each_same_stream_subscription_receives_every_record( + live_manager, backend: MemoryStreamBackend +) -> None: + """Delivery is broadcast, not competing-consumer. + + Two subscriptions to one stream are two independent readers. Splitting the + records between them would make either one's view of the stream depend on + the other's existence. + """ + runtime = make_runtime(live_manager, backend) + key = runtime.stream_key("tokens") + runtime.register(wait_id=1, stream_key=key) + runtime.register(wait_id=2, stream_key=key) + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + for i, value in enumerate(["a", "b"]): + await backend.append( + key, StreamRecord(RecordKind.DATA, await codec.encode(value), "p", i) + ) + + await asyncio.sleep(0.2) + + first = live_manager.drain(RUN_ID, 1) + second = live_manager.drain(RUN_ID, 2) + assert len(first) == 2 and len(second) == 2, ( + f"each subscription must see both records, got {len(first)} and {len(second)}" + ) + + +# --- the recorded delivery schedule ------------------------------------------- + + +def test_an_alternating_two_stream_batch_encodes_one_run_per_delivery( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """A run is a maximal consecutive stretch from a *single* wait. + + Collapsing the alternation into two runs -- one per stream -- would record a + delivery order that never happened, and replay would hand Workflow code all + of one stream before any of the other. + """ + runtime = make_runtime(manager, backend) + for wait_id, name in ((1, "a"), (2, "b")): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(name), + ) + for i in range(2): + runtime.record_delivery(1, data(i, f"1{i}-0")) + runtime.record_delivery(2, data(i, f"2{i}-0")) + + delta = runtime.take_observation_delta() + assert delta is not None + annotation = decode_annotation(delta + runtime.add_terminal()) + + runs = [run for segment in annotation.segments for run in segment.runs] + assert [run.wait_id for run in runs] == [1, 2, 1, 2], ( + f"expected one run per delivery in the observed order, got " + f"{[(r.wait_id, r.count) for r in runs]}" + ) + assert all(run.count == 1 for run in runs) + + +def test_consecutive_records_from_one_stream_are_one_run( + manager: StubManager, backend: MemoryStreamBackend +) -> None: + """The other half of the same rule, and what keeps the annotation compact. + + If every record became its own run the annotation would grow per item, which + is the cost model this feature exists to avoid. + """ + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=runtime.stream_key("a")) + for i in range(5): + runtime.record_delivery(1, data(i, f"10-{i}")) + + delta = runtime.take_observation_delta() + assert delta is not None + annotation = decode_annotation(delta + runtime.add_terminal()) + + runs = [run for segment in annotation.segments for run in segment.runs] + assert len(runs) == 1 + assert runs[0].count == 5 + + +# --- merge -------------------------------------------------------------------- + + +class FakeRuntime: + """Enough of the runtime protocol for the merge iterator, and no more.""" + + def __init__(self) -> None: + self.buffers: dict[int, list[StreamRecord]] = {} + self.blocked: list[tuple[int, bool]] = [] + self.pending: dict[int, asyncio.Future[None]] = {} + self.deliveries: list[tuple[int, StreamRecord]] = [] + self.consumed: list[tuple[int, StreamRecord]] = [] + self.registrations: list[tuple[int, StreamKey]] = [] + self.budget = MAX_RECORDS_PER_ACTIVATION + + def stream_key(self, stream_name: str) -> StreamKey: + return StreamKey("ns", "wf", "first-run", stream_name) + + def register( + self, + *, + wait_id: int, + stream_key: StreamKey, + idle_timeout: timedelta, # pyright: ignore[reportUnusedParameter] + ): + self.registrations.append((wait_id, stream_key)) + + def drain(self, wait_id: int, max_records: int | None = None): + buffered = self.buffers.get(wait_id, []) + if max_records is not None: + buffered, self.buffers[wait_id] = ( + buffered[:max_records], + buffered[max_records:], + ) + else: + self.buffers[wait_id] = [] + return buffered + + def delivery_budget_remaining(self) -> int: + return self.budget + + def codec_for( + self, + value_type: type | None, + wait_id: int, # pyright: ignore[reportUnusedParameter] + ): + return StreamPayloadCodec( + temporalio.converter.DataConverter.default, value_type + ) + + def new_readiness_future(self) -> asyncio.Future[None]: + return asyncio.get_event_loop().create_future() + + def record_delivery(self, wait_id: int, record: StreamRecord) -> None: + self.deliveries.append((wait_id, record)) + + def record_consumption(self, wait_id: int, record: StreamRecord) -> None: + self.consumed.append((wait_id, record)) + self.budget = max(0, self.budget - 1) + + def note_blocked(self, wait_id: int, blocked: bool) -> None: + self.blocked.append((wait_id, blocked)) + + def register_pending(self, wait_id: int, future: asyncio.Future[None]) -> None: + self.pending[wait_id] = future + + def discard_pending(self, wait_id: int) -> None: + self.pending.pop(wait_id, None) + + +class FakeInstance: + pass + + +@pytest.fixture +def fake_runtime(monkeypatch: pytest.MonkeyPatch) -> FakeRuntime: + instance = FakeInstance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: instance, + ) + runtime = FakeRuntime() + _install_runtime(instance, runtime) # type: ignore[arg-type] + return runtime + + +async def encoded(*values: str) -> list[StreamRecord]: + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + return [ + StreamRecord(RecordKind.DATA, await codec.encode(v), "p", i) + for i, v in enumerate(values) + ] + + +@pytest.mark.asyncio +async def test_merge_yields_from_every_subscription(fake_runtime: FakeRuntime) -> None: + """One record per subscription per pass, in ``wait_id`` order. + + The second stream's record comes out between the first stream's two, not + behind them: a pass that emptied one subscription before looking at the next + would let a backlogged first stream spend a whole activation budget on + itself and reach the second one never. + """ + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + fake_runtime.buffers[first.wait_id] = await encoded("a1", "a2") + fake_runtime.buffers[second.wait_id] = await encoded("b1") + + seen = [] + async for subscription, value in merge(first, second): + seen.append((subscription.wait_id, value)) + if len(seen) == 3: + break + + assert seen == [(1, "a1"), (2, "b1"), (1, "a2")] + + +@pytest.mark.asyncio +async def test_merge_drains_in_wait_id_order_regardless_of_argument_order( + fake_runtime: FakeRuntime, +) -> None: + """Records that arrived in one batch have no inherent order between them. + + An order that depended on argument order -- or on which watcher happened to + run first -- would replay differently than it ran. + """ + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + fake_runtime.buffers[first.wait_id] = await encoded("a1") + fake_runtime.buffers[second.wait_id] = await encoded("b1") + + seen = [] + async for subscription, value in merge(second, first): + seen.append(value) + if len(seen) == 2: + break + + assert seen == ["a1", "b1"] + + +@pytest.mark.asyncio +async def test_merge_marks_every_wait_blocked_when_nothing_is_ready( + fake_runtime: FakeRuntime, +) -> None: + """The quiescent set must name the complete set. + + Blocking on only the first would leave the second out of the snapshot, and + Core would park a wait the Workflow was still waiting on. + """ + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + + iterator = merge(first, second) + pending = asyncio.ensure_future(iterator.__anext__()) + await asyncio.sleep(0.05) + + assert set(fake_runtime.pending) == {1, 2} + assert (1, True) in fake_runtime.blocked and (2, True) in fake_runtime.blocked + pending.cancel() + try: + await pending + except asyncio.CancelledError: + pass + assert dict(fake_runtime.blocked) == {1: False, 2: False}, ( + "cancelling the merged wait left members in the blocked set" + ) + assert not fake_runtime.pending, "cancelling left readiness futures registered" + + +@pytest.mark.asyncio +async def test_merge_resumes_when_any_one_wait_is_resolved( + fake_runtime: FakeRuntime, +) -> None: + """One record on one stream is enough; it need not wait for the others.""" + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + + iterator = merge(first, second) + pending = asyncio.ensure_future(iterator.__anext__()) + await asyncio.sleep(0.05) + + fake_runtime.buffers[second.wait_id] = await encoded("b1") + fake_runtime.pending[second.wait_id].set_result(None) + + assert await asyncio.wait_for(pending, 1) == (second, "b1") + + +@pytest.mark.asyncio +async def test_merging_a_subscription_with_itself_is_refused( + fake_runtime: FakeRuntime, # pyright: ignore[reportUnusedParameter] +) -> None: + """It would deliver every record to that wait twice.""" + only = external_stream.topic("a", type=str).subscribe() + + with pytest.raises(ValueError, match="same subscription twice"): + await merge(only, only).__anext__() + + +@pytest.mark.asyncio +async def test_merging_nothing_is_refused( + fake_runtime: FakeRuntime, # pyright: ignore[reportUnusedParameter] +) -> None: + with pytest.raises(ValueError, match="at least one"): + await merge().__anext__() + + +@pytest.mark.asyncio +async def test_merge_never_yields_control_records(fake_runtime: FakeRuntime) -> None: + """A fence advances the cursor but belongs to the runtime, not the Workflow.""" + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + records = await encoded("a1", "a2") + fake_runtime.buffers[first.wait_id] = [ + records[0], + StreamRecord(RecordKind.WRITE_FENCE, b"", "p", 2), + records[1], + ] + fake_runtime.buffers[second.wait_id] = await encoded("b1") + + seen = [] + async for subscription, value in merge(first, second): + seen.append(value) + if len(seen) == 3: + break + + # The fence spends the first stream's turn on the second pass, so its "a2" + # arrives on the third -- consumed, and never yielded. + assert seen == ["a1", "b1", "a2"] + # The fence was still consumed -- it occupies an offset inside a run. + assert any(r.is_control for _, r in fake_runtime.consumed) + + +@pytest.mark.asyncio +async def test_a_replayed_segment_is_yielded_in_its_recorded_order( + public_api_runtime: WorkflowStreamRuntime, +) -> None: + """Rotation changes which wait is *asked* first, never what is *served*. + + ``merge`` resumes each pass after the wait that last took a record, and + under replay the budget is unbounded, so the passes are not cut where they + were cut live and the rotation reaches positions the live run never started + from. That is safe only because a replay drain serves from the **front** of + the recorded segment and only while the front belongs to the asking wait: a + wait asked out of turn is told nothing and the record stays for whoever + asks next. Every active wait is still asked exactly once per pass, so the + front's owner is always reached and the recorded global order comes out + whole. + + A recorded order that interleaves three waits and repeats one of them is + what would expose an ask order leaking into a yield order -- and a yield + order that differed from History is nondeterminism, which is the one thing + a fairness fix may not buy its fairness with. + """ + recorded = [2, 1, 3, 1, 2, 3, 3] + subscriptions = [ + external_stream.topic(name, type=str).subscribe() for name in ("a", "b", "c") + ] + assert [s.wait_id for s in subscriptions] == [1, 2, 3] + records = await encoded(*(f"r{i}" for i in range(len(recorded)))) + public_api_runtime.begin_activation() + public_api_runtime.begin_replay_segment( + [ + (wait_id, record.placed_at(Offset(f"{index:08d}"))) + for index, (wait_id, record) in enumerate(zip(recorded, records)) + ] + ) + + iterator = merge(*subscriptions) + seen: list[tuple[int, str]] = [] + for _ in recorded: + subscription, value = await asyncio.wait_for(iterator.__anext__(), 1) + seen.append((subscription.wait_id, value)) + await iterator.aclose() + + assert [wait_id for wait_id, _ in seen] == recorded, ( + "the replayed segment came out in an order the live run never " + "delivered; the merge is asking in a different order than it recorded " + "in and the drain is obliging" + ) + assert [value for _, value in seen] == [f"r{i}" for i in range(len(recorded))] + + +# --- through a real Worker ---------------------------------------------------- + + +@workflow.defn +class TwoStreamWorkflow: + """Consumes from two streams as one wait set, reporting the order it saw.""" + + @workflow.run + async def run(self, expected: int) -> list[str]: + from temporalio.contrib.external_workflow_streams._api import ( + external_stream as es, + ) + from temporalio.contrib.external_workflow_streams._api import merge as m + + options = es.with_options(idle_timeout=timedelta(seconds=30)) + left = options.topic("left", type=str).subscribe() + right = options.topic("right", type=str).subscribe() + + seen: list[str] = [] + async for _, value in m(left, right): + seen.append(value) + if len(seen) >= expected: + break + return seen + + +async def test_a_workflow_consumes_two_streams_as_one_wait_set( + client: Client, backend: MemoryStreamBackend +) -> None: + """One idle stream must not stall the other. + + Only the ``right`` stream is ever written to. If the two were waited on + independently the Workflow would block on ``left`` forever and never see a + single record. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[TwoStreamWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + TwoStreamWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + first_run = description.raw_description.workflow_execution_info.first_run_id + right = StreamKey(client.namespace, handle.id, first_run, "right") + await asyncio.sleep(1) + + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + for i, value in enumerate(["r1", "r2"]): + await backend.append( + right, + StreamRecord(RecordKind.DATA, await codec.encode(value), "p", i), + ) + + assert await asyncio.wait_for(handle.result(), 60) == ["r1", "r2"] + + +# --- a wait nothing is awaiting ------------------------------------------------ + + +@pytest.fixture +def public_api_runtime( + manager: StubManager, + backend: MemoryStreamBackend, + monkeypatch: pytest.MonkeyPatch, +) -> WorkflowStreamRuntime: + """The real runtime, reachable through ``subscribe()``. + + The quiescent snapshot is the thing under test here, and only the real + runtime has one. + """ + + class Instance: + pass + + instance = Instance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: instance, + ) + runtime = make_runtime(manager, backend) + _install_runtime(instance, runtime) # type: ignore[arg-type] + return runtime + + +def test_a_subscription_nobody_has_iterated_is_not_blocked( + public_api_runtime: WorkflowStreamRuntime, +) -> None: + """Blocked means *Workflow code is waiting*, not *a subscription exists*. + + A snapshot is a request to Core to retain the Workflow Task and, once the + idle timer expires, to park it. Constructing a subscription and then doing + something else entirely -- a timer, an activity, a signal handler -- must not + on its own produce a wait for Core to hold the task open for. + """ + external_stream.topic("tokens", type=str).subscribe() + + assert public_api_runtime.quiescent_snapshot() is None, ( + "a subscription that has never been iterated reported itself as a " + "quiescent wait, so Core would retain and eventually park the Workflow " + "Task for a wait no coroutine is awaiting" + ) + + +@pytest.mark.asyncio +async def test_a_cancelled_wait_leaves_the_quiescent_set( + public_api_runtime: WorkflowStreamRuntime, +) -> None: + """Racing a stream against a timer is ordinary code, and the loser is cancelled. + + ``asyncio.wait(..., FIRST_COMPLETED)`` followed by cancelling the pending + half is the standard shape. If the cancelled half stays in the blocked set, + every such race leaves a ghost wait behind, and the ghosts accumulate. + """ + subscription = external_stream.topic("tokens", type=str).subscribe() + + pending = asyncio.ensure_future(subscription.__aiter__().__anext__()) + await asyncio.sleep(0.05) + assert public_api_runtime.quiescent_snapshot() is not None, ( + "a wait that is genuinely being awaited must be in the snapshot" + ) + + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + + assert public_api_runtime.quiescent_snapshot() is None, ( + "the cancelled wait stayed in the quiescent set, so Core is asked to " + "hold the Workflow Task open for a coroutine that no longer exists" + ) + + +@pytest.mark.asyncio +async def test_closing_a_subscription_ends_its_wait_and_its_iteration( + public_api_runtime: WorkflowStreamRuntime, +) -> None: + """``_finished`` has to be reachable, or iteration has no end at all.""" + subscription = external_stream.topic("tokens", type=str).subscribe() + + iterator = subscription.__aiter__() + pending = asyncio.ensure_future(iterator.__anext__()) + await asyncio.sleep(0.05) + assert not pending.done() + + subscription.close() + + with pytest.raises(StopAsyncIteration): + await asyncio.wait_for(pending, 1) + assert public_api_runtime.quiescent_snapshot() is None + # Idempotent: closing twice is not an error, because the ordinary shape is a + # `finally` that cannot know whether the iterator already ended. + subscription.close() + + +@pytest.mark.asyncio +async def test_closing_a_subscription_tells_the_worker_to_stop_serving_it( + public_api_runtime: WorkflowStreamRuntime, # pyright: ignore[reportUnusedParameter] + manager: StubManager, +) -> None: + """Ending the wait is what the Workflow sees; the Worker has to be told too. + + Nothing above this line would notice the message never being sent: iteration + ends, the wait leaves the quiescent set, and the Workflow runs on. What is + left behind is a watcher still prefetching into a buffer nobody can drain + and a park intent still in the backend, and only the manager can take either + of them back. + """ + subscription = external_stream.topic("tokens", type=str).subscribe() + + subscription.close() + + assert manager.cancelled == [(RUN_ID, subscription.wait_id)], ( + "closing left the Worker serving a wait no Workflow code is reading" + ) + # Cancelled once, however often it is closed: by the second call the Worker + # has already dropped this wait, and its state here is kept only so that + # replay and a Continue-As-New successor can still read it. + subscription.close() + assert manager.cancelled == [(RUN_ID, subscription.wait_id)] + + +@pytest.mark.asyncio +async def test_a_resolved_merge_leaves_no_nonwinning_wait_blocked( + public_api_runtime: WorkflowStreamRuntime, + manager: StubManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A successful group wait ends for every member, not only its winner. + + The readiness activation resolves the group's futures together, while a + record may exist for only one wait. The others have no coroutine behind + them once the group returns, so leaving one in the quiescent snapshot asks + Core to retain and eventually park a Workflow Task for nobody. + """ + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + iterator = merge(first, second) + pending = asyncio.ensure_future(iterator.__anext__()) + await asyncio.sleep(0.05) + + snapshot = public_api_runtime.quiescent_snapshot() + assert snapshot is not None and [wait.wait_id for wait in snapshot] == [1, 2] + + record = (await encoded("b1"))[0].placed_at(Offset("00000001")) + delivered = False + + def drain(run_id, wait_id, max_records=None): # type: ignore[no-untyped-def] + nonlocal delivered + if wait_id == second.wait_id and not delivered: + delivered = True + return [record] + return [] + + monkeypatch.setattr(manager, "drain", drain) + public_api_runtime.resolve_all_pending() + + assert await asyncio.wait_for(pending, 1) == (second, "b1") + snapshot_after_yield = public_api_runtime.quiescent_snapshot() + await iterator.aclose() + + assert snapshot_after_yield is None, ( + "the merge yielded one member but left its non-winning member blocked, " + "so Core would retain the Workflow Task for a discarded future" + ) + + +@pytest.mark.asyncio +async def test_a_merge_double_check_leaves_no_nonwinning_wait_blocked( + public_api_runtime: WorkflowStreamRuntime, + manager: StubManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The post-registration early return has the same group cleanup contract. + + A record can arrive after the initial pass found every stream empty but + before the pending futures are awaited. `_await_any_readiness()` finds it in + its double-check and returns early; that successful exit must still unblock + every member. + """ + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + record = (await encoded("b1"))[0].placed_at(Offset("00000001")) + drain_calls: dict[int, int] = {} + + def drain(run_id, wait_id, max_records=None): # type: ignore[no-untyped-def] + drain_calls[wait_id] = drain_calls.get(wait_id, 0) + 1 + # First call: the merge's initial pass. Second call: the registered + # wait's double-check immediately before `asyncio.wait()`. + if wait_id == second.wait_id and drain_calls[wait_id] == 2: + return [record] + return [] + + monkeypatch.setattr(manager, "drain", drain) + iterator = merge(first, second) + + assert await asyncio.wait_for(iterator.__anext__(), 1) == (second, "b1") + snapshot_after_yield = public_api_runtime.quiescent_snapshot() + await iterator.aclose() + + assert snapshot_after_yield is None, ( + "the double-check returned a record but left another group member in " + "the quiescent snapshot" + ) + + +@pytest.mark.asyncio +async def test_a_half_registered_merge_leaves_no_wait_blocked( + public_api_runtime: WorkflowStreamRuntime, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Registering the group is covered by the cleanup that ends it. + + The refusal pass is careful to check every member before claiming any, so + that a refused merge claims nothing. A raise part-way through the claim + itself reaches the same state by the other route: the members already + marked blocked have futures nothing can resolve, and the snapshot asks Core + to retain a Workflow Task on their behalf. + """ + first = external_stream.topic("a", type=str).subscribe() + second = external_stream.topic("b", type=str).subscribe() + + real = public_api_runtime.new_readiness_future + made = 0 + + def fail_on_the_second() -> asyncio.Future[None]: + nonlocal made + made += 1 + if made == 2: + raise RuntimeError("no future for you") + return real() + + monkeypatch.setattr(public_api_runtime, "new_readiness_future", fail_on_the_second) + + with pytest.raises(RuntimeError, match="no future for you"): + await merge(first, second).__anext__() + + assert public_api_runtime.quiescent_snapshot() is None, ( + "a merge that failed while claiming its group left a wait blocked with " + "no future behind it, so Core would retain the Workflow Task for it" + ) + assert public_api_runtime.resolve_all_pending() == 0, ( + "a merge that failed while claiming its group left a readiness future " + "registered, which the next activation would resolve for nobody" + ) + assert first._pending_future is None and second._pending_future is None + + # And the group is claimable again rather than permanently refused: the + # single-slot claim each member holds is what the refusal reads. + monkeypatch.setattr(public_api_runtime, "new_readiness_future", real) + pending = asyncio.ensure_future(merge(first, second).__anext__()) + await asyncio.sleep(0.05) + snapshot = public_api_runtime.quiescent_snapshot() + assert snapshot is not None and [wait.wait_id for wait in snapshot] == [1, 2] + pending.cancel() + try: + await pending + except asyncio.CancelledError: + pass diff --git a/tests/contrib/external_workflow_streams/test_output_backend.py b/tests/contrib/external_workflow_streams/test_output_backend.py new file mode 100644 index 0000000000..7243bc0758 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_output_backend.py @@ -0,0 +1,185 @@ +"""Provider-neutral output identity, records, and staging contract.""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from temporalio.contrib.external_workflow_streams._backend import ( + StreamBackend, + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputReadResult, + OutputStage, + OutputStageManifest, + OutputStageStatus, + OutputStreamBackend, + OutputStreamRecord, + PendingOutputBarrier, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._producer import WorkflowChainKey +from temporalio.contrib.external_workflow_streams._record import ( + Offset, + RecordKind, + StreamRecord, +) + + +def output_key(name: str = "events") -> StreamKey: + return StreamKey( + "namespace", + "workflow", + "first-run", + name, + direction=StreamDirection.OUTPUT, + ) + + +def manifest(**changes: object) -> OutputStageManifest: + values: dict[str, object] = { + "stream_key": output_key(), + "provider_id": "memory", + "provider_format_version": 1, + "stage_token": "stage-token", + "run_id": "current-run", + "history_floor_event_id": 7, + "sub_batch_id": 0, + "fingerprint_version": 1, + "fingerprint": b"f" * 32, + "record_count": 2, + "logical_byte_count": 41, + } + values.update(changes) + return OutputStageManifest(**values) # type: ignore[arg-type] + + +def test_stream_direction_defaults_to_the_existing_input_identity() -> None: + existing = StreamKey("namespace", "workflow", "first-run", "events") + + assert existing.direction is StreamDirection.INPUT + assert existing == StreamKey( + "namespace", + "workflow", + "first-run", + "events", + direction=StreamDirection.INPUT, + ) + assert str(existing) == "namespace/workflow/first-run/input/events" + + +def test_input_and_output_topics_with_the_same_name_are_distinct() -> None: + input_key = StreamKey("namespace", "workflow", "first-run", "events") + output = output_key() + + assert input_key != output + assert len({input_key, output}) == 2 + assert str(output) == "namespace/workflow/first-run/output/events" + + +def test_workflow_chain_key_builds_either_direction() -> None: + chain = WorkflowChainKey("namespace", "workflow", "first-run") + + assert chain.stream_key("events") == StreamKey( + "namespace", "workflow", "first-run", "events" + ) + assert chain.stream_key("events", direction=StreamDirection.OUTPUT) == output_key() + + +def test_stream_key_rejects_a_raw_direction_string() -> None: + with pytest.raises(TypeError, match="must be a StreamDirection"): + StreamKey( + "namespace", + "workflow", + "first-run", + "events", + direction="output", # type: ignore[arg-type] + ) + + +def test_output_manifest_is_bound_to_an_output_key() -> None: + with pytest.raises(ValueError, match="requires an OUTPUT stream key"): + manifest(stream_key=StreamKey("namespace", "workflow", "first-run", "events")) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("provider_id", "", "non-empty provider_id"), + ("stage_token", "", "non-empty stage_token"), + ("run_id", "", "non-empty run_id"), + ("provider_format_version", 0, "positive provider_format_version"), + ("fingerprint_version", 0, "positive fingerprint_version"), + ("record_count", 0, "positive record_count"), + ("history_floor_event_id", 0, "positive history_floor_event_id"), + ("sub_batch_id", -1, "non-negative sub_batch_id"), + ("logical_byte_count", -1, "non-negative logical_byte_count"), + ("fingerprint", b"short", "must be 32 bytes"), + ], +) +def test_output_manifest_rejects_incomplete_identity( + field: str, value: object, message: str +) -> None: + with pytest.raises(ValueError, match=message): + manifest(**{field: value}) + + +def test_staged_and_placed_control_records_carry_no_payload() -> None: + staged = StagedOutputRecord(0, RecordKind.FINISH, b"") + placed = OutputStreamRecord(RecordKind.FINISH, b"", Offset("1-0")) + + assert staged.kind is RecordKind.FINISH + assert placed.kind.is_control + with pytest.raises(ValueError, match="carries no payload"): + StagedOutputRecord(0, RecordKind.FINISH, b"not-empty") + with pytest.raises(ValueError, match="carries no payload"): + OutputStreamRecord(RecordKind.WRITE_FENCE, b"not-empty", Offset("1-1")) + + +def test_finish_round_trips_through_provider_neutral_record_fields() -> None: + finish = StreamRecord(RecordKind.FINISH, b"", "session", 3).placed_at(Offset("2-0")) + + assert finish.is_control + assert StreamRecord.from_fields(Offset("2-0"), finish.to_fields()) == finish + + +def test_output_stage_record_count_must_match_its_manifest() -> None: + records = ( + OutputStreamRecord(RecordKind.DATA, b"one", Offset("1-0")), + OutputStreamRecord(RecordKind.DATA, b"two", Offset("1-1")), + ) + + stage = OutputStage(manifest(), records, OutputStageStatus.PENDING) + assert stage.records == records + with pytest.raises(ValueError, match="record count does not match"): + OutputStage(manifest(record_count=1), records, OutputStageStatus.PENDING) + + +def test_output_models_are_frozen_and_use_immutable_record_collections() -> None: + stage_manifest = manifest(record_count=1) + record = OutputStreamRecord(RecordKind.DATA, b"data", Offset("1-0")) + stage = OutputStage(stage_manifest, (record,), OutputStageStatus.PENDING) + barrier = PendingOutputBarrier(stage_manifest, Offset("1-0")) + result = OutputReadResult((record,), barrier) + + assert isinstance(stage.records, tuple) + assert isinstance(result.records, tuple) + with pytest.raises(dataclasses.FrozenInstanceError): + result.pending = None # type: ignore[misc] + + +def test_output_backend_is_an_additive_capability() -> None: + assert not issubclass(OutputStreamBackend, StreamBackend) + assert OutputStreamBackend.__abstractmethods__ == { + "abort_output", + "append_output", + "commit_output", + "compare_offsets", + "output_stage", + "output_tail", + "read_output_after", + "stage_output", + } diff --git a/tests/contrib/external_workflow_streams/test_output_client.py b/tests/contrib/external_workflow_streams/test_output_client.py new file mode 100644 index 0000000000..ab8a7ea51c --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_output_client.py @@ -0,0 +1,543 @@ +"""Focused client and pending-stage reconciliation tests for output streams.""" + +from __future__ import annotations + +from collections import deque +from collections.abc import AsyncIterator, Sequence +from types import SimpleNamespace +from typing import Any + +import pytest + +import temporalio.api.history.v1 +import temporalio.converter +import temporalio.service +from temporalio.contrib.external_workflow_streams import _output_client +from temporalio.contrib.external_workflow_streams._backend import ( + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._errors import ( + StreamDecodeError, + StreamIntegrityError, + StreamStorageError, +) +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputReadResult, + OutputStage, + OutputStageManifest, + OutputStageStatus, + OutputStreamBackend, + OutputStreamRecord, + PendingOutputBarrier, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._output_client import ( + ExternalOutputStreamClient, + ExternalOutputStreamClientTopic, +) +from temporalio.contrib.external_workflow_streams._producer import WorkflowChainKey +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + Offset, + RecordKind, + StreamRecord, +) + + +class FakeOutputBackend(OutputStreamBackend): + guarantees_immutability = True + provider_id = "fake-output" + provider_format_version = 1 + + def __init__(self) -> None: + self.read_results: deque[OutputReadResult | BaseException] = deque() + self.read_cursors: list[Cursor] = [] + self.tail = BEGINNING + self.tail_calls = 0 + self.stages: dict[OutputStageManifest, OutputStage] = {} + self.committed: list[OutputStageManifest] = [] + self.aborted: list[OutputStageManifest] = [] + + async def stage_output( + self, + manifest: OutputStageManifest, + records: Sequence[StagedOutputRecord], + ) -> OutputStage: + raise NotImplementedError + + async def commit_output(self, manifest: OutputStageManifest) -> OutputStage: + self.committed.append(manifest) + stage = self.stages[manifest] + resolved = OutputStage(manifest, stage.records, OutputStageStatus.COMMITTED) + self.stages[manifest] = resolved + return resolved + + async def abort_output(self, manifest: OutputStageManifest) -> OutputStage: + self.aborted.append(manifest) + stage = self.stages[manifest] + resolved = OutputStage(manifest, stage.records, OutputStageStatus.ABORTED) + self.stages[manifest] = resolved + return resolved + + async def output_stage(self, manifest: OutputStageManifest) -> OutputStage | None: + return self.stages.get(manifest) + + async def append_output( + self, key: StreamKey, record: StreamRecord + ) -> OutputStreamRecord: + raise NotImplementedError + + async def read_output_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: Any = None, + ) -> OutputReadResult: + assert key.direction is StreamDirection.OUTPUT + assert max_records > 0 + self.read_cursors.append(after) + result = self.read_results.popleft() + if isinstance(result, BaseException): + raise result + return result + + async def output_tail(self, key: StreamKey) -> Cursor: + assert key.direction is StreamDirection.OUTPUT + self.tail_calls += 1 + return self.tail + + def compare_offsets(self, left: Offset, right: Offset) -> int: + return int(left.token) - int(right.token) + + +class FakeHandle: + def __init__( + self, + *, + first_run_id: str, + events: Sequence[temporalio.api.history.v1.HistoryEvent] = (), + history_error: BaseException | None = None, + ) -> None: + self._first_run_id = first_run_id + self._events = events + self._history_error = history_error + + async def describe(self) -> Any: + return SimpleNamespace( + raw_description=SimpleNamespace( + workflow_execution_info=SimpleNamespace(first_run_id=self._first_run_id) + ) + ) + + def fetch_history_events(self) -> AsyncIterator[Any]: + async def iterate() -> AsyncIterator[Any]: + if self._history_error is not None: + raise self._history_error + for event in self._events: + yield event + + return iterate() + + +class FakeClient: + namespace = "ns" + data_converter = temporalio.converter.DataConverter.default + + def __init__(self) -> None: + self.events: Sequence[temporalio.api.history.v1.HistoryEvent] = () + self.history_error: BaseException | None = None + self.history_run_ids: list[str] = [] + + def get_workflow_handle( + self, workflow_id: str, *, run_id: str | None = None + ) -> FakeHandle: + assert workflow_id == "wf" + if run_id is not None: + self.history_run_ids.append(run_id) + return FakeHandle( + first_run_id="first-run", + events=self.events if run_id is not None else (), + history_error=self.history_error if run_id is not None else None, + ) + + +CHAIN = WorkflowChainKey("ns", "wf", "first-run") +OUTPUT_KEY = CHAIN.stream_key("events", direction=StreamDirection.OUTPUT) + + +def history_event(event_id: int, attributes: str) -> Any: + event = temporalio.api.history.v1.HistoryEvent(event_id=event_id) + getattr(event, attributes).SetInParent() + return event + + +def manifest(*, floor: int = 1, token: str = "stage-1") -> OutputStageManifest: + return OutputStageManifest( + stream_key=OUTPUT_KEY, + provider_id="fake-output", + provider_format_version=1, + stage_token=token, + run_id="current-run", + history_floor_event_id=floor, + sub_batch_id=0, + fingerprint_version=1, + fingerprint=b"f" * 32, + record_count=1, + logical_byte_count=10, + ) + + +def output_record( + offset: int, payload: bytes = b"", *, kind: RecordKind = RecordKind.DATA +) -> OutputStreamRecord: + return OutputStreamRecord(kind=kind, payload=payload, offset=Offset(str(offset))) + + +async def connected( + backend: FakeOutputBackend, client: FakeClient +) -> ExternalOutputStreamClient: + return await ExternalOutputStreamClient.connect( + backend=backend, + workflow=CHAIN, + client=client, # type: ignore[arg-type] + ) + + +async def test_committed_read_is_typed_resumable_and_backend_only() -> None: + backend = FakeOutputBackend() + client = FakeClient() + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + first = output_record(4, await codec.encode("first")) + second = output_record(5, await codec.encode("second")) + finish = output_record(6, kind=RecordKind.FINISH) + backend.read_results.extend( + [ + OutputReadResult((first,)), + OutputReadResult((second, finish)), + ] + ) + + topic = (await connected(backend, client)).topic("events", type=str) + items = [item async for item in topic.subscribe(after=AFTER(Offset("3")))] + + assert [(item.data, item.offset) for item in items] == [ + ("first", Offset("4")), + ("second", Offset("5")), + ] + assert backend.read_cursors == [AFTER(Offset("3")), AFTER(Offset("4"))] + assert client.history_run_ids == [] + + +async def test_an_item_offset_can_be_passed_directly_as_the_resume_cursor() -> None: + backend = FakeOutputBackend() + client = FakeClient() + backend.read_results.append( + OutputReadResult((output_record(4, kind=RecordKind.FINISH),)) + ) + + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + assert [item async for item in topic.subscribe(after=Offset("3"))] == [] + + assert backend.read_cursors == [AFTER(Offset("3"))] + + +async def test_tail_is_the_backend_committed_boundary_without_history() -> None: + backend = FakeOutputBackend() + backend.tail = AFTER(Offset("12")) + client = FakeClient() + + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + + assert await topic.tail() == AFTER(Offset("12")) + assert backend.tail_calls == 1 + assert client.history_run_ids == [] + + +async def test_pending_token_in_exact_run_history_commits_before_read_continues( + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend = FakeOutputBackend() + client = FakeClient() + staged = manifest() + placed = output_record(10, b"payload") + backend.stages[staged] = OutputStage(staged, (placed,), OutputStageStatus.PENDING) + barrier = PendingOutputBarrier(staged, Offset("10")) + client.events = ( + history_event(1, "workflow_execution_started_event_attributes"), + history_event(2, "workflow_task_completed_event_attributes"), + history_event(3, "marker_recorded_event_attributes"), + ) + monkeypatch.setattr( + _output_client, + "_event_has_output_stage_token", + lambda event, token: event.event_id == 3 and token == "stage-1", + ) + + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + + assert await topic._resolve_pending(barrier) + assert backend.committed == [staged] + assert backend.aborted == [] + assert client.history_run_ids == ["current-run"] + + +async def test_repeating_the_same_history_reconciliation_is_idempotent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend = FakeOutputBackend() + client = FakeClient() + staged = manifest() + placed = output_record(10, b"payload") + backend.stages[staged] = OutputStage(staged, (placed,), OutputStageStatus.PENDING) + barrier = PendingOutputBarrier(staged, placed.offset) + client.events = ( + history_event(1, "workflow_execution_started_event_attributes"), + history_event(2, "workflow_task_completed_event_attributes"), + history_event(3, "marker_recorded_event_attributes"), + ) + monkeypatch.setattr( + _output_client, + "_event_has_output_stage_token", + lambda event, token: event.event_id == 3 and token == "stage-1", + ) + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + + assert await topic._resolve_pending(barrier) + assert await topic._resolve_pending(barrier) + assert backend.committed == [staged, staged] + assert backend.stages[staged].status is OutputStageStatus.COMMITTED + + +def test_real_shared_marker_envelope_exposes_the_output_stage_token() -> None: + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + + marker = ExternalStreamMarkerData() + marker.output.stage_token = "stage-1" + event = history_event(3, "marker_recorded_event_attributes") + attributes = event.marker_recorded_event_attributes + attributes.marker_name = "core_external_stream" + attributes.details[ + "external_stream" + ].payloads.add().data = marker.SerializeToString() + + assert _output_client._event_has_output_stage_token(event, "stage-1") + assert not _output_client._event_has_output_stage_token(event, "another-stage") + + +async def test_subscribe_stops_at_barrier_and_resumes_from_committed_prefix( + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend = FakeOutputBackend() + client = FakeClient() + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + prefix = output_record(5, await codec.encode("before")) + staged_record = output_record(10, await codec.encode("staged")) + finish = output_record(11, kind=RecordKind.FINISH) + staged = manifest() + backend.stages[staged] = OutputStage( + staged, (staged_record,), OutputStageStatus.PENDING + ) + backend.read_results.extend( + [ + OutputReadResult( + (prefix,), PendingOutputBarrier(staged, staged_record.offset) + ), + OutputReadResult((staged_record, finish)), + ] + ) + client.events = ( + history_event(1, "workflow_execution_started_event_attributes"), + history_event(2, "workflow_task_completed_event_attributes"), + history_event(3, "marker_recorded_event_attributes"), + ) + monkeypatch.setattr( + _output_client, + "_event_has_output_stage_token", + lambda event, token: event.event_id == 3 and token == "stage-1", + ) + topic = (await connected(backend, client)).topic("events", type=str) + + items = [item async for item in topic.subscribe()] + + assert [item.data for item in items] == ["before", "staged"] + assert backend.read_cursors == [BEGINNING, AFTER(prefix.offset)] + assert backend.committed == [staged] + + +async def test_first_task_boundary_without_token_aborts_pending_stage( + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend = FakeOutputBackend() + client = FakeClient() + staged = manifest() + backend.stages[staged] = OutputStage( + staged, (output_record(10),), OutputStageStatus.PENDING + ) + client.events = ( + history_event(1, "workflow_execution_started_event_attributes"), + history_event(2, "workflow_task_failed_event_attributes"), + ) + monkeypatch.setattr( + _output_client, + "_event_has_output_stage_token", + lambda event, token: False, + ) + + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + + assert await topic._resolve_pending(PendingOutputBarrier(staged, Offset("10"))) + assert backend.aborted == [staged] + assert backend.committed == [] + + +async def test_no_durable_boundary_leaves_the_stage_pending( + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend = FakeOutputBackend() + client = FakeClient() + staged = manifest() + backend.stages[staged] = OutputStage( + staged, (output_record(10),), OutputStageStatus.PENDING + ) + client.events = ( + history_event(1, "workflow_execution_started_event_attributes"), + history_event(2, "workflow_task_scheduled_event_attributes"), + history_event(3, "workflow_task_started_event_attributes"), + ) + monkeypatch.setattr( + _output_client, + "_event_has_output_stage_token", + lambda event, token: False, + ) + + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + + assert not await topic._resolve_pending(PendingOutputBarrier(staged, Offset("10"))) + assert backend.committed == [] and backend.aborted == [] + + +async def test_disconnected_speculative_update_stays_pending_until_history_loss( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Elapsed retries cannot guess the outcome of an unreported Update task.""" + backend = FakeOutputBackend() + client = FakeClient() + staged = manifest() + placed = output_record(10, b"speculative-update-output") + backend.stages[staged] = OutputStage(staged, (placed,), OutputStageStatus.PENDING) + barrier = PendingOutputBarrier(staged, placed.offset) + + # The admitted Update exists durably, but there is no deciding Workflow + # Task terminal event above the exact floor. This is the shape left by a + # client disconnect while the Update's speculative task is still in flight. + client.events = ( + history_event(1, "workflow_execution_started_event_attributes"), + history_event(2, "workflow_execution_update_admitted_event_attributes"), + history_event(3, "workflow_task_scheduled_event_attributes"), + history_event(4, "workflow_task_started_event_attributes"), + ) + monkeypatch.setattr( + _output_client, + "_event_has_output_stage_token", + lambda event, token: False, + ) + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + + assert not await topic._resolve_pending(barrier) + assert not await topic._resolve_pending(barrier) + assert backend.stages[staged].status is OutputStageStatus.PENDING + assert backend.committed == [] and backend.aborted == [] + assert client.history_run_ids == ["current-run", "current-run"] + + # Once retention removes that exact Run, lack of evidence becomes an + # integrity failure. It still must not be guessed into an abort. + client.history_error = temporalio.service.RPCError( + "history expired", + temporalio.service.RPCStatusCode.NOT_FOUND, + b"", + ) + with pytest.raises(StreamIntegrityError, match="no longer available"): + await topic._resolve_pending(barrier) + assert backend.stages[staged].status is OutputStageStatus.PENDING + assert backend.committed == [] and backend.aborted == [] + + +async def test_history_loss_is_integrity_but_an_outage_is_storage() -> None: + backend = FakeOutputBackend() + client = FakeClient() + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + + client.events = (history_event(2, "workflow_task_completed_event_attributes"),) + with pytest.raises(StreamIntegrityError, match="exact floor event"): + await topic._history_decision(manifest()) + + client.history_error = temporalio.service.RPCError( + "server unavailable", + temporalio.service.RPCStatusCode.UNAVAILABLE, + b"", + ) + with pytest.raises(StreamStorageError, match="Temporal History"): + await topic._history_decision(manifest()) + + client.history_error = temporalio.service.RPCError( + "history expired", + temporalio.service.RPCStatusCode.NOT_FOUND, + b"", + ) + with pytest.raises(StreamIntegrityError, match="no longer available"): + await topic._history_decision(manifest()) + + +async def test_provider_cannot_return_a_record_beyond_pending_barrier() -> None: + backend = FakeOutputBackend() + client = FakeClient() + staged = manifest() + backend.read_results.append( + OutputReadResult( + (output_record(11),), + PendingOutputBarrier(staged, Offset("10")), + ) + ) + topic: ExternalOutputStreamClientTopic[Any] = ( + await connected(backend, client) + ).topic("events") + + with pytest.raises(StreamIntegrityError, match="pending barrier"): + await anext(topic.subscribe()) + assert client.history_run_ids == [] + + +async def test_backend_and_decode_failures_keep_the_existing_taxonomy() -> None: + backend = FakeOutputBackend() + client = FakeClient() + topic = (await connected(backend, client)).topic("events", type=str) + backend.read_results.append(ConnectionError("backend down")) + + with pytest.raises(StreamStorageError, match="backend down"): + await anext(topic.subscribe()) + + backend.read_results.append(OutputReadResult((output_record(1, b"not-proto"),))) + with pytest.raises(StreamDecodeError): + await anext(topic.subscribe()) diff --git a/tests/contrib/external_workflow_streams/test_output_codec.py b/tests/contrib/external_workflow_streams/test_output_codec.py new file mode 100644 index 0000000000..f6385930b0 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_output_codec.py @@ -0,0 +1,150 @@ +"""Canonical pre-codec output frames and fingerprints.""" + +from __future__ import annotations + +import hashlib +import struct + +import pytest + +import temporalio.api.common.v1 +from temporalio.contrib.external_workflow_streams._output_codec import ( + LOGICAL_FINGERPRINT_VERSION, + canonical_logical_record_frame, + fingerprint_logical_frames, + fingerprint_logical_records, +) +from temporalio.contrib.external_workflow_streams._record import RecordKind + + +def lp(value: bytes) -> bytes: + return struct.pack(">Q", len(value)) + value + + +def test_canonical_frame_has_an_exact_version_one_encoding() -> None: + payload = temporalio.api.common.v1.Payload( + metadata={"z": b"last", "a": b"first"}, + data=b"payload", + ) + + frame = canonical_logical_record_frame("events", RecordKind.DATA, payload) + + assert frame == b"".join( + ( + lp(b"events"), + lp(b"\x01"), + struct.pack(">Q", 2), + lp(b"a"), + lp(b"first"), + lp(b"z"), + lp(b"last"), + lp(b"payload"), + ) + ) + + +def test_metadata_insertion_order_does_not_change_frame_or_fingerprint() -> None: + first = temporalio.api.common.v1.Payload(data=b"data") + first.metadata["z"] = b"last" + first.metadata["a"] = b"first" + second = temporalio.api.common.v1.Payload(data=b"data") + second.metadata["a"] = b"first" + second.metadata["z"] = b"last" + + first_frame = canonical_logical_record_frame("events", RecordKind.DATA, first) + second_frame = canonical_logical_record_frame("events", RecordKind.DATA, second) + + assert first_frame == second_frame + assert fingerprint_logical_frames([first_frame]) == fingerprint_logical_frames( + [second_frame] + ) + + +def test_batch_hashes_ordered_length_prefixed_frames() -> None: + frames = [b"a", b"bc"] + + fingerprint = fingerprint_logical_frames(frames) + + assert fingerprint.version == LOGICAL_FINGERPRINT_VERSION == 1 + assert fingerprint.digest == hashlib.sha256(lp(b"a") + lp(b"bc")).digest() + assert fingerprint.logical_byte_count == 3 + assert fingerprint.record_count == 2 + + +def test_fingerprint_changes_with_order_topic_kind_metadata_or_data() -> None: + base = temporalio.api.common.v1.Payload( + metadata={"encoding": b"json/plain"}, data=b"one" + ) + changed_metadata = temporalio.api.common.v1.Payload( + metadata={"encoding": b"binary/plain"}, data=b"one" + ) + changed_data = temporalio.api.common.v1.Payload( + metadata={"encoding": b"json/plain"}, data=b"two" + ) + variants: list[ + list[ + tuple[ + str, + RecordKind, + temporalio.api.common.v1.Payload | None, + ] + ] + ] = [ + [("other", RecordKind.DATA, base)], + [("events", RecordKind.DATA, changed_metadata)], + [("events", RecordKind.DATA, changed_data)], + [ + ("events", RecordKind.DATA, base), + ("events", RecordKind.FINISH, None), + ], + [ + ("events", RecordKind.FINISH, None), + ("events", RecordKind.DATA, base), + ], + ] + baseline = fingerprint_logical_records([("events", RecordKind.DATA, base)]) + + assert all(fingerprint_logical_records(variant) != baseline for variant in variants) + assert fingerprint_logical_records(variants[-1]) != fingerprint_logical_records( + variants[-2] + ) + + +def test_length_prefixes_prevent_field_and_record_repartitioning() -> None: + topic_a = canonical_logical_record_frame( + "a", RecordKind.DATA, temporalio.api.common.v1.Payload(data=b"bc") + ) + topic_ab = canonical_logical_record_frame( + "ab", RecordKind.DATA, temporalio.api.common.v1.Payload(data=b"c") + ) + + assert topic_a != topic_ab + assert fingerprint_logical_frames([b"a", b"bc"]) != fingerprint_logical_frames( + [b"ab", b"c"] + ) + + +def test_finish_has_a_canonical_payload_free_frame() -> None: + frame = canonical_logical_record_frame("events", RecordKind.FINISH, None) + + assert frame == b"".join( + ( + lp(b"events"), + lp(b"\x03"), + struct.pack(">Q", 0), + lp(b""), + ) + ) + + +def test_logical_frame_rejects_invalid_payload_shapes() -> None: + with pytest.raises(ValueError, match="DATA.*needs a Payload"): + canonical_logical_record_frame("events", RecordKind.DATA, None) + with pytest.raises(ValueError, match="FINISH.*carries no Payload"): + canonical_logical_record_frame( + "events", RecordKind.FINISH, temporalio.api.common.v1.Payload() + ) + with pytest.raises(ValueError, match="non-empty topic"): + canonical_logical_record_frame( + "", RecordKind.DATA, temporalio.api.common.v1.Payload() + ) diff --git a/tests/contrib/external_workflow_streams/test_output_continuation.py b/tests/contrib/external_workflow_streams/test_output_continuation.py new file mode 100644 index 0000000000..6b87dd2686 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_output_continuation.py @@ -0,0 +1,50 @@ +"""Continue-As-New state for explicitly finished output topics.""" + +from __future__ import annotations + +import pytest + +from temporalio.contrib.external_workflow_streams._annotation import ( + AnnotationDecodeError, +) +from temporalio.contrib.external_workflow_streams._output_continuation import ( + OUTPUT_CONTINUATION_HEADER, + OutputContinuation, + decode_output_continuation, + encode_output_continuation, + read_output_continuation_header, + write_output_continuation_header, +) + + +def state(*topics: str) -> OutputContinuation: + return OutputContinuation(frozenset(topics), "redis-streams", 1) + + +def test_output_continuation_round_trips_stably() -> None: + first = state("status", "events") + second = state("events", "status") + + assert encode_output_continuation(first) == encode_output_continuation(second) + assert decode_output_continuation(encode_output_continuation(first)) == first + + +def test_output_continuation_uses_a_separate_must_understand_header() -> None: + payload = write_output_continuation_header(state("events")) + + assert read_output_continuation_header( + {OUTPUT_CONTINUATION_HEADER: payload} + ) == state("events") + assert read_output_continuation_header({}) is None + + +def test_output_continuation_rejects_unknown_schema_or_encoding() -> None: + payload = write_output_continuation_header(state("events")) + payload.data = b"\x02" + payload.data[1:] + with pytest.raises(AnnotationDecodeError, match="schema version 2"): + read_output_continuation_header({OUTPUT_CONTINUATION_HEADER: payload}) + + payload = write_output_continuation_header(state("events")) + payload.metadata["encoding"] = b"json/plain" + with pytest.raises(AnnotationDecodeError, match="unsupported encoding"): + read_output_continuation_header({OUTPUT_CONTINUATION_HEADER: payload}) diff --git a/tests/contrib/external_workflow_streams/test_output_producer.py b/tests/contrib/external_workflow_streams/test_output_producer.py new file mode 100644 index 0000000000..257f711736 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_output_producer.py @@ -0,0 +1,258 @@ +"""Direct Activity/external producer tests for output streams.""" + +from __future__ import annotations + +import asyncio +import dataclasses +from collections.abc import Sequence +from typing import Any + +import pytest + +import temporalio.api.common.v1 +import temporalio.converter +from temporalio.contrib.external_workflow_streams._backend import ( + AppendConflictError, + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputReadResult, + OutputStage, + OutputStageManifest, + OutputStreamBackend, + OutputStreamRecord, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._output_producer import ( + ExternalOutputStreamProducer, + OutputAppendNotAcknowledgedError, +) +from temporalio.contrib.external_workflow_streams._producer import WorkflowChainKey +from temporalio.contrib.external_workflow_streams._record import ( + BEGINNING, + Cursor, + Offset, + RecordKind, + StreamRecord, +) + + +class FakeOutputBackend(OutputStreamBackend): + guarantees_immutability = True + provider_id = "fake-output" + provider_format_version = 1 + + def __init__(self) -> None: + self.append_calls: list[tuple[StreamKey, StreamRecord]] = [] + self.stored: dict[ + tuple[StreamKey, str, int], tuple[StreamRecord, OutputStreamRecord] + ] = {} + self.fail_before_store_once = False + self.fail_after_store_once = False + self.block_data = False + self.data_append_started = asyncio.Event() + self.release_data_append = asyncio.Event() + self._next_offset = 1 + + async def stage_output( + self, + manifest: OutputStageManifest, + records: Sequence[StagedOutputRecord], + ) -> OutputStage: + raise NotImplementedError + + async def commit_output(self, manifest: OutputStageManifest) -> OutputStage: + raise NotImplementedError + + async def abort_output(self, manifest: OutputStageManifest) -> OutputStage: + raise NotImplementedError + + async def output_stage(self, manifest: OutputStageManifest) -> OutputStage | None: + raise NotImplementedError + + async def append_output( + self, key: StreamKey, record: StreamRecord + ) -> OutputStreamRecord: + assert key.direction is StreamDirection.OUTPUT + self.append_calls.append((key, record)) + if self.fail_before_store_once: + self.fail_before_store_once = False + raise ConnectionError("append outcome unknown before reply") + + identity = (key, record.producer_session_id, record.sequence) + existing = self.stored.get(identity) + if existing is not None: + original, placed = existing + if original != record: + raise AppendConflictError(record.idempotency_key) + return placed + + if self.block_data and record.kind is RecordKind.DATA: + self.data_append_started.set() + await self.release_data_append.wait() + placed = OutputStreamRecord( + kind=record.kind, + payload=record.payload, + offset=Offset(str(self._next_offset)), + ) + self._next_offset += 1 + self.stored[identity] = (record, placed) + if self.fail_after_store_once: + self.fail_after_store_once = False + raise ConnectionError("append committed but acknowledgement was lost") + return placed + + async def read_output_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: Any = None, + ) -> OutputReadResult: + raise NotImplementedError + + async def output_tail(self, key: StreamKey) -> Cursor: + return BEGINNING + + def compare_offsets(self, left: Offset, right: Offset) -> int: + return int(left.token) - int(right.token) + + +CHAIN = WorkflowChainKey("ns", "wf", "first-run") + + +def producer( + backend: FakeOutputBackend, + *, + data_converter: temporalio.converter.DataConverter = temporalio.converter.DataConverter.default, +) -> ExternalOutputStreamProducer: + return ExternalOutputStreamProducer( + backend=backend, + workflow=CHAIN, + data_converter=data_converter, + session_id="activity-session", + ) + + +async def test_publish_uses_output_identity_and_workflow_serialization_context() -> ( + None +): + observed: list[temporalio.converter.SerializationContext | None] = [] + + class RecordingCodec( + temporalio.converter.PayloadCodec, + temporalio.converter.WithSerializationContext, + ): + def __init__( + self, + context: temporalio.converter.SerializationContext | None = None, + ) -> None: + self.context = context + + def with_context( + self, context: temporalio.converter.SerializationContext + ) -> RecordingCodec: + return RecordingCodec(context) + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + observed.append(self.context) + return list(payloads) + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return list(payloads) + + backend = FakeOutputBackend() + output = producer( + backend, + data_converter=temporalio.converter.DataConverter( + payload_codec=RecordingCodec() + ), + ) + + placed = await output.topic("events", type=str).publish("hello") + + key, record = backend.append_calls[0] + assert key.direction is StreamDirection.OUTPUT + assert key.stream_name == "events" + assert record.kind is RecordKind.DATA + assert record.producer_session_id == "activity-session" + assert record.sequence == 0 + assert placed.offset == Offset("1") + assert observed == [ + temporalio.converter.WorkflowSerializationContext( + namespace="ns", workflow_id="wf" + ) + ] + + +async def test_finish_waits_for_preceding_publish_and_closes_all_topic_handles() -> ( + None +): + backend = FakeOutputBackend() + backend.block_data = True + output = producer(backend) + topic = output.topic("events", type=str) + other_handle = output.topic("events", type=str) + + publish_task = asyncio.create_task(topic.publish("first")) + await backend.data_append_started.wait() + finish_task = asyncio.create_task(other_handle.finish_writing()) + backend.release_data_append.set() + await asyncio.gather(publish_task, finish_task) + + records = [record for _, record in backend.append_calls] + assert [record.kind for record in records] == [ + RecordKind.DATA, + RecordKind.FINISH, + ] + assert [record.sequence for record in records] == [0, 1] + assert records[1].payload == b"" + with pytest.raises(RuntimeError, match="already finished"): + await topic.publish("too late") + with pytest.raises(RuntimeError, match="already finished"): + await other_handle.finish_writing() + + +async def test_ambiguous_append_retries_exact_record_without_duplicate() -> None: + backend = FakeOutputBackend() + backend.fail_after_store_once = True + topic = producer(backend).topic("events", type=str) + + with pytest.raises(OutputAppendNotAcknowledgedError) as unknown: + await topic.publish("once") + with pytest.raises(OutputAppendNotAcknowledgedError) as blocked: + await topic.publish("must not draw another sequence") + assert blocked.value.record == unknown.value.record + + recovered = await topic.resolve_append(unknown.value.record) + following = await topic.publish("after recovery") + + assert recovered.offset == Offset("1") + assert following.offset == Offset("2") + assert len(backend.stored) == 2 + assert [record.sequence for _, record in backend.append_calls] == [0, 0, 1] + + +async def test_conflict_while_resolving_is_definitive_not_another_unknown() -> None: + backend = FakeOutputBackend() + backend.fail_before_store_once = True + topic = producer(backend).topic("events", type=str) + + with pytest.raises(OutputAppendNotAcknowledgedError) as unknown: + await topic.publish("original") + conflicting = dataclasses.replace(unknown.value.record, payload=b"different") + await backend.append_output(unknown.value.stream_key, conflicting) + + with pytest.raises(AppendConflictError): + await topic.resolve_append(unknown.value.record) + + # The provider answered definitively, so the failed recovery is no longer + # reported as an unknown operation on every later call. + placed = await topic.publish("next") + assert placed.offset == Offset("2") diff --git a/tests/contrib/external_workflow_streams/test_output_runtime.py b/tests/contrib/external_workflow_streams/test_output_runtime.py new file mode 100644 index 0000000000..51c7857335 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_output_runtime.py @@ -0,0 +1,1200 @@ +"""Workflow-thread batching and staging for external output streams.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence +from datetime import timedelta +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +import temporalio.api.common.v1 +import temporalio.bridge.proto.external_data +import temporalio.bridge.proto.workflow_activation +import temporalio.bridge.proto.workflow_completion +import temporalio.converter +import temporalio.workflow +from temporalio.contrib.external_workflow_streams._annotation import decode_annotation +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._errors import ( + ExternalStreamCapacityError, + StreamStorageError, +) +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._output_api import ( + ExternalOutputStreamOptions, +) +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputReadResult, + OutputStage, + OutputStageManifest, + OutputStageStatus, + OutputStreamBackend, + OutputStreamRecord, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._output_codec import ( + fingerprint_logical_frames, +) +from temporalio.contrib.external_workflow_streams._record import ( + BEGINNING, + Cursor, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._runtime import ( + WorkflowStreamRuntime, +) +from temporalio.worker._workflow import _WorkflowWorker +from temporalio.worker._workflow_instance import _WorkflowInstanceImpl +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + + +class _OutputMemoryBackend(MemoryStreamBackend, OutputStreamBackend): + provider_id = "output-memory" + provider_format_version = 1 + + def __init__(self) -> None: + super().__init__() + self.stages: dict[tuple[str, int], OutputStage] = {} + self.stage_calls = 0 + self.stage_failures = 0 + self.stage_payloads: list[tuple[bytes, ...]] = [] + + async def stage_output( + self, + manifest: OutputStageManifest, + records: Sequence[StagedOutputRecord], + ) -> OutputStage: + self.stage_calls += 1 + self.stage_payloads.append(tuple(record.payload for record in records)) + if self.stage_failures: + self.stage_failures -= 1 + raise OSError("output backend unavailable") + identity = (manifest.stage_token, manifest.sub_batch_id) + existing = self.stages.get(identity) + if existing is not None: + return existing + placed = tuple( + OutputStreamRecord( + record.kind, + record.payload, + Offset(f"{manifest.sub_batch_id + 1}-{record.publish_index}"), + ) + for record in records + ) + stage = OutputStage(manifest, placed, OutputStageStatus.PENDING) + self.stages[identity] = stage + return stage + + async def commit_output(self, manifest: OutputStageManifest) -> OutputStage: + return self._resolve(manifest, OutputStageStatus.COMMITTED) + + async def abort_output(self, manifest: OutputStageManifest) -> OutputStage: + return self._resolve(manifest, OutputStageStatus.ABORTED) + + def _resolve( + self, manifest: OutputStageManifest, status: OutputStageStatus + ) -> OutputStage: + existing = self.stages[(manifest.stage_token, manifest.sub_batch_id)] + resolved = OutputStage(existing.manifest, existing.records, status) + self.stages[(manifest.stage_token, manifest.sub_batch_id)] = resolved + return resolved + + async def output_stage(self, manifest: OutputStageManifest) -> OutputStage | None: + return self.stages.get((manifest.stage_token, manifest.sub_batch_id)) + + async def append_output( + self, key: StreamKey, record: StreamRecord + ) -> OutputStreamRecord: + placed = await self.append(key, record) + assert placed.offset is not None + return OutputStreamRecord(placed.kind, placed.payload, placed.offset) + + async def read_output_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = None, + ) -> OutputReadResult: + del key, after, max_records, block + return OutputReadResult(()) + + async def output_tail(self, key: StreamKey) -> Cursor: + del key + return BEGINNING + + +class _ChangingPayloadCodec(temporalio.converter.PayloadCodec): + """A valid codec whose wire bytes deliberately change on every call.""" + + def __init__(self) -> None: + self.encode_calls = 0 + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + self.encode_calls += 1 + encoded = [] + for payload in payloads: + copied = temporalio.api.common.v1.Payload() + copied.CopyFrom(payload) + copied.data += f":encoding-{self.encode_calls}".encode() + encoded.append(copied) + return encoded + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return list(payloads) + + +async def _notify(run_id: str, wait_id: int, generation: int) -> str: + del run_id, wait_id, generation + return ReadinessResult.ACCEPTED + + +@pytest.fixture +def backend() -> _OutputMemoryBackend: + return _OutputMemoryBackend() + + +@pytest.fixture +async def runtime(backend: _OutputMemoryBackend) -> WorkflowStreamRuntime: + # The manager captures the loop it is built on, which in the Worker is the + # Worker's own. A sync fixture has none to capture. + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=1), + ) + return WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id="run", + namespace="namespace", + workflow_id="workflow", + first_execution_run_id="first-run", + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + ) + + +def _successful_completion( + *variants: str, +) -> temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion: + completion = ( + temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() + ) + completion.successful.SetInParent() + for variant in variants: + getattr(completion.successful.commands.add(), variant).SetInParent() + return completion + + +def _activation(*, finalize_output_latency: bool = False) -> Any: + activation = temporalio.bridge.proto.workflow_activation.WorkflowActivation( + run_id="run", + history_floor_event_id=7, + ) + if finalize_output_latency: + activation.jobs.add().finalize_external_streams.reason = ( + temporalio.bridge.proto.external_data.ParkReason.PARK_REASON_OUTPUT_LATENCY + ) + return activation + + +def _park_activation() -> Any: + activation = _activation() + activation.jobs.add().prepare_external_stream_park.quiescence_generation = 1 + return activation + + +class _ParkManager: + def __init__(self, *, became_ready: bool) -> None: + self.became_ready = became_ready + + async def prepare_park(self, *args: Any, **kwargs: Any) -> bool: + del args, kwargs + return self.became_ready + + +def _worker() -> _WorkflowWorker: + worker = object.__new__(_WorkflowWorker) + worker._pending_external_output_stages = {} + return worker + + +async def _publish_one( + runtime: WorkflowStreamRuntime, + *, + max_records: int = 10, +) -> None: + runtime.begin_activation(7) + await runtime.publish_output( + topic="events", + value="value", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=250), + max_records=max_records, + max_logical_bytes=10_000, + ) + + +async def test_retainable_output_emits_buffered_without_provider_staging( + runtime: WorkflowStreamRuntime, + backend: _OutputMemoryBackend, +) -> None: + await _publish_one(runtime) + completion = _successful_completion( + "workflow_stream_progress", + "workflow_stream_quiescent", + ) + + await _worker()._stage_or_buffer_external_output(_activation(), completion, runtime) + + assert backend.stage_calls == 0 + assert runtime.has_output + assert [ + command.WhichOneof("variant") for command in completion.successful.commands + ] == [ + "workflow_stream_progress", + "workflow_output_stream_buffered", + "workflow_stream_quiescent", + ] + buffered = completion.successful.commands[1].workflow_output_stream_buffered + assert buffered.max_publish_latency.ToTimedelta() == timedelta(milliseconds=250) + + +async def test_output_latency_finalization_stages_and_answers_both_commands( + runtime: WorkflowStreamRuntime, + backend: _OutputMemoryBackend, +) -> None: + await _publish_one(runtime) + worker = _worker() + worker._external_stream_runtimes = {"run": runtime} + initial = _successful_completion("workflow_stream_quiescent") + await worker._stage_or_buffer_external_output(_activation(), initial, runtime) + assert backend.stage_calls == 0 + + activation = _activation(finalize_output_latency=True) + completion = await worker._handle_external_stream_jobs( + activation, + cast(Any, None), + ) + assert completion is not None + await worker._stage_or_buffer_external_output(activation, completion, runtime) + + assert backend.stage_calls == 1 + assert not runtime.has_output + assert [ + command.WhichOneof("variant") for command in completion.successful.commands + ] == [ + "workflow_output_stream_commit", + "external_stream_finalized", + ] + assert len(worker._pending_external_output_stages["run"]) == 1 + + +async def test_confirmed_park_stages_output_in_the_same_completion( + runtime: WorkflowStreamRuntime, + backend: _OutputMemoryBackend, +) -> None: + await _publish_one(runtime) + worker = _worker() + worker._external_stream_runtimes = {"run": runtime} + worker._external_stream_manager = _ParkManager(became_ready=False) + activation = _park_activation() + + completion = await worker._handle_external_stream_jobs( + activation, + cast(Any, None), + ) + assert completion is not None + await worker._stage_or_buffer_external_output(activation, completion, runtime) + + assert backend.stage_calls == 1 + assert not runtime.has_output + assert [ + command.WhichOneof("variant") for command in completion.successful.commands + ] == [ + "workflow_output_stream_commit", + "external_stream_park_result", + ] + assert completion.successful.commands[1].external_stream_park_result.HasField( + "confirmed" + ) + + +async def test_park_that_became_ready_keeps_output_buffered( + runtime: WorkflowStreamRuntime, + backend: _OutputMemoryBackend, +) -> None: + await _publish_one(runtime) + worker = _worker() + worker._external_stream_runtimes = {"run": runtime} + worker._external_stream_manager = _ParkManager(became_ready=True) + activation = _park_activation() + + completion = await worker._handle_external_stream_jobs( + activation, + cast(Any, None), + ) + assert completion is not None + await worker._stage_or_buffer_external_output(activation, completion, runtime) + + assert backend.stage_calls == 0 + assert runtime.has_output + assert [ + command.WhichOneof("variant") for command in completion.successful.commands + ] == [ + "workflow_output_stream_buffered", + "external_stream_park_result", + ] + assert completion.successful.commands[1].external_stream_park_result.HasField( + "became_ready" + ) + buffered = completion.successful.commands[0].workflow_output_stream_buffered + assert buffered.max_publish_latency.ToTimedelta() == timedelta(milliseconds=250) + + +async def test_output_without_quiescent_snapshot_stages_immediately( + runtime: WorkflowStreamRuntime, + backend: _OutputMemoryBackend, +) -> None: + await _publish_one(runtime) + completion = _successful_completion() + + await _worker()._stage_or_buffer_external_output(_activation(), completion, runtime) + + assert backend.stage_calls == 1 + assert [ + command.WhichOneof("variant") for command in completion.successful.commands + ] == ["workflow_output_stream_commit"] + + +async def test_capacity_rollover_bypasses_output_buffering( + runtime: WorkflowStreamRuntime, + backend: _OutputMemoryBackend, +) -> None: + await _publish_one(runtime, max_records=1) + blocked_publish = asyncio.create_task( + runtime.publish_output( + topic="events", + value="second", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=250), + max_records=1, + max_logical_bytes=10_000, + ) + ) + await asyncio.sleep(0) + assert runtime.output_rollover_requested + completion = _successful_completion("workflow_stream_quiescent") + + try: + await _worker()._stage_or_buffer_external_output( + _activation(), completion, runtime + ) + assert backend.stage_calls == 1 + assert [ + command.WhichOneof("variant") for command in completion.successful.commands + ] == [ + "workflow_output_stream_commit", + "workflow_stream_quiescent", + ] + finally: + blocked_publish.cancel() + with pytest.raises(asyncio.CancelledError): + await blocked_publish + + +async def test_stage_uses_one_token_and_pre_codec_logical_manifests( + runtime: WorkflowStreamRuntime, backend: _OutputMemoryBackend +) -> None: + runtime.begin_activation(7) + await runtime.publish_output( + topic="events", + value={"value": 1}, + value_type=dict, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=100), + max_records=10, + max_logical_bytes=10_000, + ) + await runtime.publish_output( + topic="status", + value=None, + value_type=str, + kind=RecordKind.FINISH, + max_publish_latency=timedelta(milliseconds=50), + max_records=10, + max_logical_bytes=10_000, + ) + + staged = await runtime.stage_output(7) + + assert len(staged.topics) == 2 + assert {topic.manifest.stage_token for topic in staged.topics} == { + staged.stage_token + } + assert staged.segment_record_counts == ((1, 1),) + assert staged.topics[1].finished + assert all( + stage.status is OutputStageStatus.PENDING for stage in backend.stages.values() + ) + + +async def test_changed_codec_bytes_keep_logical_retry_identity_and_first_bytes( + backend: _OutputMemoryBackend, +) -> None: + codec = _ChangingPayloadCodec() + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=1), + ) + runtime = WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id="run", + namespace="namespace", + workflow_id="workflow", + first_execution_run_id="first-run", + data_converter=temporalio.converter.DataConverter(payload_codec=codec), + default_idle_timeout=timedelta(seconds=1), + ) + runtime.begin_activation(7) + await runtime.publish_output( + topic="events", + value="same logical value", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=100), + max_records=10, + max_logical_bytes=10_000, + ) + + first = await runtime.stage_output(7) + repeated = await runtime.stage_output(7) + + assert codec.encode_calls == 2 + assert first == repeated + assert first.topics[0].manifest == repeated.topics[0].manifest + assert len(first.topics[0].manifest.fingerprint) == 32 + assert backend.stage_payloads[0] != backend.stage_payloads[1] + stored = backend.stages[(first.stage_token, 0)] + assert ( + tuple(record.payload for record in stored.records) == backend.stage_payloads[0] + ) + assert ( + tuple(record.payload for record in stored.records) != backend.stage_payloads[1] + ) + + +async def test_capacity_publish_waits_for_a_replacement_workflow_task( + runtime: WorkflowStreamRuntime, +) -> None: + options: dict[str, Any] = { + "value_type": str, + "kind": RecordKind.DATA, + "max_publish_latency": timedelta(milliseconds=100), + "max_records": 1, + "max_logical_bytes": 10_000, + } + runtime.begin_activation(7) + await runtime.publish_output(topic="events", value="one", **options) + second = asyncio.create_task( + runtime.publish_output(topic="events", value="two", **options) + ) + await asyncio.sleep(0) + assert runtime.output_rollover_requested + assert not second.done() + + await runtime.stage_output(7) + runtime.output_stage_recorded() + runtime.begin_activation(9) + await second + + staged = await runtime.stage_output(9) + assert staged.topics[0].manifest.record_count == 1 + + +async def test_stage_outage_is_storage_failure_and_retry_reuses_attempt( + runtime: WorkflowStreamRuntime, backend: _OutputMemoryBackend +) -> None: + runtime.begin_activation(7) + await runtime.publish_output( + topic="events", + value="value", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=100), + max_records=10, + max_logical_bytes=10_000, + ) + backend.stage_failures = 1 + + with pytest.raises(StreamStorageError, match="events"): + await runtime.stage_output(7) + first_token = runtime._output_stage_token + + staged = await runtime.stage_output(7) + + assert staged.stage_token == first_token + assert backend.stage_calls == 2 + + +async def test_oversized_marker_manifest_is_rejected_before_external_io( + runtime: WorkflowStreamRuntime, backend: _OutputMemoryBackend +) -> None: + runtime.begin_activation(7) + await runtime.publish_output( + topic="t" * (70 * 1024), + value="value", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=100), + max_records=10, + max_logical_bytes=100 * 1024, + ) + + with pytest.raises(ExternalStreamCapacityError, match="marker budget"): + await runtime.stage_output(7) + + assert backend.stage_calls == 0 + assert runtime._staged_output is None + + +async def test_replay_validates_the_recorded_shared_segment_schedule_without_io( + runtime: WorkflowStreamRuntime, backend: _OutputMemoryBackend +) -> None: + runtime.begin_output_replay( + SimpleNamespace( + schema_version=1, + fingerprint_version=1, + topics=[ + SimpleNamespace( + topic="events", + record_count=1, + logical_byte_count=0, + logical_fingerprint=b"", + finished=False, + ) + ], + segments=[SimpleNamespace(record_counts_by_topic=[1])], + ) + ) + runtime.begin_output_replay_segment() + await runtime.publish_output( + topic="events", + value="value", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=100), + max_records=1, + max_logical_bytes=10_000, + ) + record = runtime._output_records[0] + fingerprint = fingerprint_logical_frames([record.frame]) + expected = runtime._output_replay_manifest.topics[0] + expected.logical_byte_count = fingerprint.logical_byte_count + expected.logical_fingerprint = fingerprint.digest + + runtime.verify_output_replay() + + assert backend.stage_calls == 0 + + +async def test_replay_ignores_live_capacity_and_does_not_leak_live_policy( + runtime: WorkflowStreamRuntime, +) -> None: + expected = SimpleNamespace( + schema_version=1, + fingerprint_version=1, + topics=[ + SimpleNamespace( + topic="events", + record_count=1, + logical_byte_count=0, + logical_fingerprint=b"", + finished=False, + ) + ], + segments=[SimpleNamespace(record_counts_by_topic=[1])], + ) + runtime.begin_output_replay(expected) + runtime.begin_output_replay_segment() + # The recorded value is larger than today's deliberately tiny live limit. + # Replay must validate it instead of moving or rejecting the old boundary. + await runtime.publish_output( + topic="events", + value="recorded before the limit changed", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=1), + max_records=1, + max_logical_bytes=1, + ) + record = runtime._output_records[0] + fingerprint = fingerprint_logical_frames([record.frame]) + expected.topics[0].logical_byte_count = fingerprint.logical_byte_count + expected.topics[0].logical_fingerprint = fingerprint.digest + + assert runtime.output_max_publish_latency is None + runtime.verify_output_replay() + + runtime.begin_activation(11) + await runtime.publish_output( + topic="events", + value="live", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=250), + max_records=10, + max_logical_bytes=10_000, + ) + assert runtime.output_max_publish_latency == timedelta(milliseconds=250) + + +async def test_output_replay_performs_no_io_token_mint_or_live_policy_split( + runtime: WorkflowStreamRuntime, + backend: _OutputMemoryBackend, + monkeypatch: pytest.MonkeyPatch, +) -> None: + expected = SimpleNamespace( + schema_version=1, + fingerprint_version=1, + topics=[ + SimpleNamespace( + topic="events", + record_count=2, + logical_byte_count=0, + logical_fingerprint=b"", + finished=False, + ) + ], + segments=[SimpleNamespace(record_counts_by_topic=[2])], + ) + runtime.begin_output_replay(expected) + runtime.begin_output_replay_segment() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._runtime.secrets.token_urlsafe", + lambda _size: pytest.fail("replay minted an output stage token"), + ) + + # Today's live policy would split after the first record and arm a 1 ms + # flush. Replay must instead reproduce the one recorded two-record segment. + for value in ("one", "two"): + await runtime.publish_output( + topic="events", + value=value, + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=1), + max_records=1, + max_logical_bytes=1, + ) + fingerprint = fingerprint_logical_frames( + record.frame for record in runtime._output_records + ) + expected.topics[0].logical_byte_count = fingerprint.logical_byte_count + expected.topics[0].logical_fingerprint = fingerprint.digest + + assert runtime._output_stage_token is None + assert not runtime.output_rollover_requested + assert runtime.output_max_publish_latency is None + runtime.verify_output_replay() + assert backend.stage_calls == 0 + + +class _SizedPayloadCodec(temporalio.converter.PayloadCodec): + def __init__(self, suffix_size: int) -> None: + self.suffix_size = suffix_size + self.encode_calls = 0 + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + self.encode_calls += 1 + encoded: list[temporalio.api.common.v1.Payload] = [] + for payload in payloads: + copied = temporalio.api.common.v1.Payload() + copied.CopyFrom(payload) + copied.data += b"x" * self.suffix_size + encoded.append(copied) + return encoded + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return list(payloads) + + +def _runtime_with_codec( + backend: _OutputMemoryBackend, codec: temporalio.converter.PayloadCodec +) -> WorkflowStreamRuntime: + return WorkflowStreamRuntime( + manager=StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=1), + ), + backend=backend, + run_id="run", + namespace="namespace", + workflow_id="workflow", + first_execution_run_id="first-run", + data_converter=temporalio.converter.DataConverter(payload_codec=codec), + default_idle_timeout=timedelta(seconds=1), + ) + + +async def test_replay_keeps_recorded_segments_when_codec_wire_size_changes( + backend: _OutputMemoryBackend, +) -> None: + live_codec = _SizedPayloadCodec(1) + live = _runtime_with_codec(backend, live_codec) + for value in ("one", "two"): + live.begin_activation(7) + await live.publish_output( + topic="events", + value=value, + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=100), + max_records=10, + max_logical_bytes=10_000, + ) + staged = await live.stage_output(7) + assert staged.segment_record_counts == ((1,), (1,)) + assert live_codec.encode_calls == 2 + live_wire = backend.stage_payloads[-1] + + replay_codec = _SizedPayloadCodec(100_000) + replay = _runtime_with_codec(backend, replay_codec) + topic = staged.topics[0] + expected = SimpleNamespace( + schema_version=staged.schema_version, + fingerprint_version=staged.fingerprint_version, + topics=[ + SimpleNamespace( + topic=topic.manifest.stream_key.stream_name, + record_count=topic.manifest.record_count, + logical_byte_count=topic.manifest.logical_byte_count, + logical_fingerprint=topic.manifest.fingerprint, + finished=topic.finished, + ) + ], + segments=[ + SimpleNamespace(record_counts_by_topic=list(counts)) + for counts in staged.segment_record_counts + ], + ) + replay.begin_output_replay(expected) + for value in ("one", "two"): + replay.begin_output_replay_segment() + await replay.publish_output( + topic="events", + value=value, + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=1), + max_records=1, + max_logical_bytes=1, + ) + + replay.verify_output_replay() + assert replay_codec.suffix_size > sum(len(payload) for payload in live_wire) + assert replay_codec.encode_calls == 0 + assert backend.stage_calls == 1 + + +async def test_one_oversized_logical_record_is_refused_without_poisoning_batch( + runtime: WorkflowStreamRuntime, + backend: _OutputMemoryBackend, +) -> None: + runtime.begin_activation(7) + with pytest.raises(ExternalStreamCapacityError, match="one record"): + await runtime.publish_output( + topic="events", + value="too large", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=100), + max_records=10, + max_logical_bytes=1, + ) + + assert not runtime.has_output + assert not runtime.output_rollover_requested + assert backend.stage_calls == 0 + await runtime.publish_output( + topic="events", + value="fits the next valid policy", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=100), + max_records=10, + max_logical_bytes=10_000, + ) + assert runtime.has_output + + +async def test_abandon_output_replay_clears_partial_finish_and_expectations( + runtime: WorkflowStreamRuntime, +) -> None: + runtime.begin_output_replay( + SimpleNamespace( + schema_version=1, + fingerprint_version=1, + topics=[], + segments=[], + ) + ) + runtime.begin_output_replay_segment() + await runtime.publish_output( + topic="events", + value=None, + value_type=str, + kind=RecordKind.FINISH, + max_publish_latency=timedelta(milliseconds=1), + max_records=1, + max_logical_bytes=1, + ) + + runtime.abandon_output_replay() + runtime.begin_activation(11) + await runtime.publish_output( + topic="events", + value="live after failed replay", + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(milliseconds=250), + max_records=10, + max_logical_bytes=10_000, + ) + + assert runtime.has_output + assert runtime.output_max_publish_latency == timedelta(milliseconds=250) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("max_publish_latency", timedelta(0), "max_publish_latency"), + ("max_records", 0, "max_records"), + ("max_logical_bytes", 0, "max_logical_bytes"), + ], +) +def test_output_options_validate_direct_construction( + field: str, value: Any, message: str +) -> None: + kwargs = {field: value} + with pytest.raises(ValueError, match=message): + ExternalOutputStreamOptions(**kwargs) + + +async def test_workflow_output_rejects_read_only_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + temporalio.workflow.unsafe, + "is_read_only", + lambda: True, + ) + topic = ExternalOutputStreamOptions().topic("events", type=str) + + with pytest.raises( + temporalio.workflow.ReadOnlyContextError, match="external output" + ): + await topic.publish("not allowed from a query") + + +class _OutputOnlyReplayRuntime: + def __init__(self) -> None: + self.events: list[str] = [] + + def begin_output_replay(self, _manifest: Any) -> None: + self.events.append("begin-output") + + def take_replay_plan(self) -> None: + return None + + def begin_output_replay_segment(self) -> None: + self.events.append("segment") + + def resolve_all_pending(self) -> None: + self.events.append("resolve") + + +class _OutputOnlyReplayDriver: + def __init__(self, runtime: _OutputOnlyReplayRuntime) -> None: + self._external_stream_runtime = runtime + self._pending_output_replay_finish = False + + def _run_once(self, *, check_conditions: bool) -> None: + assert check_conditions + self._external_stream_runtime.events.append("drain") + + +def test_output_only_replay_drives_all_but_the_trailing_segment() -> None: + runtime = _OutputOnlyReplayRuntime() + driver = _OutputOnlyReplayDriver(runtime) + output = SimpleNamespace(segments=[object(), object(), object()]) + job = SimpleNamespace( + output=output, + HasField=lambda field: field == "output", + ) + + apply_replay = cast(Any, _WorkflowInstanceImpl._apply_replay_external_streams) + apply_replay(driver, job) + + assert runtime.events == [ + "begin-output", + "segment", + "resolve", + "drain", + "segment", + "resolve", + "drain", + "segment", + "resolve", + ] + assert driver._pending_output_replay_finish + + +class _CombinedReplayRuntime: + def __init__(self) -> None: + self.events: list[str] = [] + self._plan: Any = SimpleNamespace( + annotation=SimpleNamespace(header=SimpleNamespace(streams={1: object()})), + segments=tuple( + SimpleNamespace(deliveries=((1, index),)) for index in range(3) + ), + committed_boundaries={1: BEGINNING}, + ) + + def begin_output_replay(self, _manifest: Any) -> None: + self.events.append("begin-output") + + def take_replay_plan(self) -> Any: + plan, self._plan = self._plan, None + return plan + + def begin_replay(self, _streams: Any) -> None: + self.events.append("begin-input") + + def begin_output_replay_segment(self) -> None: + self.events.append("output-segment") + + def begin_replay_segment(self, _deliveries: Any) -> None: + self.events.append("input-segment") + + def resolve_all_pending(self) -> None: + self.events.append("resolve") + + def verify_replay_consumed(self) -> None: + self.events.append("verify-input") + + def reposition_after_replay(self, _boundaries: Any) -> None: + self.events.append("reposition-input") + + def verify_output_replay(self) -> None: + self.events.append("verify-output") + + def abandon_output_replay(self) -> None: + self.events.append("abandon-output") + + def end_replay(self) -> None: + self.events.append("end-input") + + +class _CombinedReplayDriver: + def __init__(self, runtime: _CombinedReplayRuntime) -> None: + self._external_stream_runtime = runtime + self._pending_replay_finish: Any = None + self._pending_output_replay_finish = False + + def _run_once(self, *, check_conditions: bool) -> None: + assert check_conditions + self._external_stream_runtime.events.append("drain") + + +def test_input_and_output_share_one_replay_segment_drain_schedule() -> None: + runtime = _CombinedReplayRuntime() + driver = _CombinedReplayDriver(runtime) + output = SimpleNamespace(segments=(object(), object(), object())) + job = SimpleNamespace( + output=output, + HasField=lambda field: field == "output", + ) + apply_replay = cast(Any, _WorkflowInstanceImpl._apply_replay_external_streams) + finish_replay = cast(Any, _WorkflowInstanceImpl._finish_replay_external_streams) + + apply_replay(driver, job) + driver._run_once(check_conditions=True) + finish_replay(driver) + + assert runtime.events == [ + "begin-output", + "begin-input", + "output-segment", + "input-segment", + "resolve", + "drain", + "output-segment", + "input-segment", + "resolve", + "drain", + "output-segment", + "input-segment", + "resolve", + "drain", + "verify-input", + "reposition-input", + "verify-output", + "abandon-output", + "end-input", + ] + + +def test_ambiguous_legacy_combined_schedule_is_rejected_before_drain() -> None: + runtime = _CombinedReplayRuntime() + driver = _CombinedReplayDriver(runtime) + output = SimpleNamespace(segments=(object(), object(), object(), object())) + job = SimpleNamespace(output=output, HasField=lambda field: field == "output") + + apply_replay = cast(Any, _WorkflowInstanceImpl._apply_replay_external_streams) + with pytest.raises( + temporalio.workflow.NondeterminismError, match="incompatible input and output" + ): + apply_replay(driver, job) + + assert runtime.events == ["begin-output"] + + +@pytest.mark.parametrize("subscription_exists", [False, True]) +async def test_combined_marker_preserves_empty_input_activations( + runtime: WorkflowStreamRuntime, subscription_exists: bool +) -> None: + key = runtime.stream_key("inputs") + if subscription_exists: + runtime.begin_activation(1) + runtime.register(wait_id=1, stream_key=key) + runtime.take_observation_delta() + runtime.add_terminal() + + deltas = [] + try: + for index in range(4): + runtime.begin_activation(7) + if index == 1 and not subscription_exists: + runtime.register(wait_id=1, stream_key=key) + if index in (1, 3): + record = StreamRecord( + RecordKind.DATA, b"input", "producer", index + ).placed_at(Offset(f"{index}-0")) + runtime.record_delivery(1, record) + await runtime.publish_output( + topic="events", + value=str(index), + value_type=str, + kind=RecordKind.DATA, + max_publish_latency=timedelta(seconds=1), + max_records=10, + max_logical_bytes=10_000, + ) + delta = runtime.take_observation_delta() + if delta is not None: + deltas.append(delta) + deltas.append(runtime.add_terminal()) + staged = await runtime.stage_output(7) + annotation = decode_annotation(b"".join(deltas)) + + assert [bool(segment.runs) for segment in annotation.segments] == [ + False, + True, + False, + True, + ] + assert staged.segment_record_counts == ((0,), (1,), (0,), (1,)) + finally: + await runtime._manager.shutdown() + + +class _SchedulingStub: + """The two calls the activation's job dispatch makes on the instance.""" + + def __init__(self, *, single_batch: bool) -> None: + self._single_batch_activation = single_batch + self.events: list[str] = [] + + def _apply(self, job: Any) -> None: + self.events.append(f"apply-{job.name}") + + def _run_once(self, *, check_conditions: bool) -> None: + self.events.append(f"drain({check_conditions})") + + +def _job(name: str) -> Any: + return SimpleNamespace(name=name) + + +def _dispatch(stub: _SchedulingStub, job_sets: list[list[Any]], replay: list[Any]): + cast(Any, _WorkflowInstanceImpl._apply_activation_jobs)(stub, job_sets, replay) + + +@pytest.mark.parametrize("single_batch", [False, True]) +def test_a_replay_marker_rides_the_patch_set_drain(single_batch: bool) -> None: + """A marker beside patch jobs alone must not earn a second drain. + + A patch job set drains once, and that drain is the first one that can + publish, so the install belongs in front of it. Behind it the activation + runs two drains where the recorded task ran one, and every + ``wait_condition`` predicate fires an extra time. + """ + stub = _SchedulingStub(single_batch=single_batch) + _dispatch(stub, [[_job("patch")], [], [], []], [_job("marker")]) + + assert stub.events == ["apply-patch", "apply-marker", "drain(False)"] + + +def test_a_marker_still_waits_for_the_signal_set_it_precedes() -> None: + stub = _SchedulingStub(single_batch=False) + _dispatch(stub, [[_job("patch")], [_job("signal")], [], []], [_job("marker")]) + + assert stub.events == [ + "apply-patch", + "drain(False)", + "apply-signal", + "apply-marker", + "drain(True)", + ] + + +def test_a_query_only_activation_answers_after_the_marker() -> None: + stub = _SchedulingStub(single_batch=False) + _dispatch(stub, [[], [], [], [_job("query")]], [_job("marker")]) + + assert stub.events == ["apply-query", "apply-marker", "drain(False)"] + + +@pytest.mark.parametrize("single_batch", [False, True]) +def test_a_marker_alone_drains_once_without_checking_conditions( + single_batch: bool, +) -> None: + stub = _SchedulingStub(single_batch=single_batch) + _dispatch(stub, [[], [], [], []], [_job("marker")]) + + assert stub.events == ["apply-marker", "drain(False)"] diff --git a/tests/contrib/external_workflow_streams/test_output_worker_integration.py b/tests/contrib/external_workflow_streams/test_output_worker_integration.py new file mode 100644 index 0000000000..668697f58f --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_output_worker_integration.py @@ -0,0 +1,1257 @@ +"""Real Worker/Core integration for Workflow-originated external output. + +PYTEST_DONT_REWRITE: the sandboxed Workflow re-imports this module, so pytest's +injected imports must not become part of sandbox validation. +""" + +from __future__ import annotations + +import asyncio +import time +import uuid +from collections.abc import Sequence +from dataclasses import replace +from datetime import timedelta +from typing import Any, cast + +import pytest + +import temporalio.api.enums.v1 +import temporalio.bridge.worker +from temporalio import workflow +from temporalio.client import Client, WorkflowExecutionStatus +from temporalio.contrib.external_workflow_streams import ( + ExternalOutputStreamClient, + WorkflowChainKey, +) +from temporalio.contrib.external_workflow_streams._backend import ( + ParkIntent, + ParkIntentRemoval, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputReadResult, + OutputStage, + OutputStageConflictError, + OutputStageManifest, + OutputStageNotFoundError, + OutputStageResolutionError, + OutputStageStatus, + OutputStreamBackend, + OutputStreamRecord, + PendingOutputBarrier, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.worker import Worker +from tests.contrib.external_workflow_streams.memory_backend import ( + MemoryStreamBackend, +) + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams import ( + external_output_stream, + external_stream, + ) + + +@workflow.defn +class PublishExternalOutputWorkflow: + """Publishes a value without returning it through Temporal.""" + + @workflow.run + async def run(self) -> int: + topic = external_output_stream.topic("events", type=str) + await topic.publish("workflow-output-secret") + await topic.finish() + return 1 + + +@workflow.defn +class FinishOutputAcrossContinueAsNewWorkflow: + """Finishes in one Run and verifies the successor restores that fact.""" + + @workflow.run + async def run(self) -> str: + topic = external_output_stream.topic("events", type=str) + if workflow.info().continued_run_id is None: + await topic.finish() + workflow.continue_as_new() + try: + await topic.publish("must not be staged") + except workflow.NondeterminismError: + return "successor rejected finished topic" + return "successor incorrectly reopened finished topic" + + +@workflow.defn +class PublishOutputWhileInputIsRetainedWorkflow: + """Publishes output, then keeps the Workflow Task open on an input wait.""" + + @workflow.run + async def run(self) -> int: + output = external_output_stream.with_options( + max_publish_latency=timedelta(seconds=1) + ).topic("events", type=str) + input_topic = external_stream.with_options( + idle_timeout=timedelta(seconds=30) + ).topic("release", type=str) + await output.publish("visible-before-input") + await input_topic.subscribe().__aiter__().__anext__() + return 1 + + +@workflow.defn +class PublishAcrossRolloverAndContinueAsNewWorkflow: + """Crosses output capacity rollover, then continues as new.""" + + @workflow.run + async def run(self) -> str: + topic = external_output_stream.with_options(max_records=1).topic( + "events", type=str + ) + if workflow.info().continued_run_id is None: + await topic.publish("before-rollover") + await topic.publish("after-rollover") + workflow.continue_as_new() + await topic.publish("after-continue-as-new") + return "done" + + +@workflow.defn +class ConcurrentTurnOutputWorkflow: + """Returns stable turn IDs that clients correlate with output envelopes.""" + + def __init__(self) -> None: + self._done = False + + @workflow.run + async def run(self) -> str: + await workflow.wait_condition(lambda: self._done) + return "done" + + @workflow.update + async def start_turn(self, turn_id: str) -> str: + await external_output_stream.topic("events", type=str).publish( + f"turn_started:{turn_id}" + ) + return turn_id + + @workflow.update + async def finish(self) -> None: + await external_output_stream.topic("events", type=str).finish() + self._done = True + + +@workflow.defn +class PublishWhileTwoInputsParkWorkflow: + """Keeps two input intents outstanding while output latency wins.""" + + @workflow.run + async def run(self) -> None: + output = external_output_stream.with_options( + max_publish_latency=timedelta(milliseconds=500) + ).topic("events", type=str) + inputs = external_stream.with_options(idle_timeout=timedelta(milliseconds=50)) + first = inputs.topic("first", type=str).subscribe().__aiter__() + second = inputs.topic("second", type=str).subscribe().__aiter__() + await output.publish("latency-wins") + await asyncio.gather(first.__anext__(), second.__anext__()) + + +@workflow.defn +class ThreeRetainedOutputWindowsWorkflow: + """Publishes twice per retained latency window for three windows.""" + + @workflow.run + async def run(self) -> int: + output = external_output_stream.with_options( + max_publish_latency=timedelta(milliseconds=150) + ).topic("events", type=str) + triggers = external_stream.with_options( + idle_timeout=timedelta(seconds=30) + ).topic("triggers", type=str) + windows = 0 + async for trigger in triggers.subscribe(): + if trigger == "stop": + return windows + await output.publish(f"{trigger}:first") + await output.publish(f"{trigger}:second") + windows += 1 + return windows + + +class _OutputMemoryBackend(MemoryStreamBackend, OutputStreamBackend): + """In-memory output provider used to exercise the real Worker/Core path.""" + + provider_id = "output-memory" + provider_format_version = 1 + + def __init__(self) -> None: + super().__init__() + self.stages: dict[tuple[str, int], OutputStage] = {} + self._stage_by_offset: dict[tuple[StreamKey, Offset], tuple[str, int]] = {} + self.read_operations: list[str] = [] + self.commit_attempts = 0 + self.commit_failures_remaining = 0 + self.commit_failed = asyncio.Event() + self.stage_placed = asyncio.Event() + self.stage_release: asyncio.Event | None = None + self.pending_read = asyncio.Event() + self.expected_park_intents = 0 + self.all_park_intents_installed = asyncio.Event() + self.recheck_release: asyncio.Event | None = None + self.removed_park_intents: list[int] = [] + + async def stage_output( + self, + manifest: OutputStageManifest, + records: Sequence[StagedOutputRecord], + ) -> OutputStage: + identity = (manifest.stage_token, manifest.sub_batch_id) + existing = self.stages.get(identity) + if existing is not None: + if existing.manifest != manifest: + raise OutputStageConflictError(manifest) + return existing + if len(records) != manifest.record_count or any( + record.publish_index != index for index, record in enumerate(records) + ): + raise ValueError("staged output indexes do not match the manifest") + + placed: list[OutputStreamRecord] = [] + for record in records: + stored = await self.append( + manifest.stream_key, + StreamRecord( + kind=record.kind, + payload=record.payload, + producer_session_id=( + f"stage:{manifest.stage_token}:{manifest.sub_batch_id}" + ), + sequence=record.publish_index, + ), + ) + assert stored.offset is not None + placed.append( + OutputStreamRecord(stored.kind, stored.payload, stored.offset) + ) + self._stage_by_offset[(manifest.stream_key, stored.offset)] = identity + stage = OutputStage(manifest, tuple(placed), OutputStageStatus.PENDING) + self.stages[identity] = stage + self.stage_placed.set() + if self.stage_release is not None: + await self.stage_release.wait() + return stage + + async def commit_output(self, manifest: OutputStageManifest) -> OutputStage: + self.commit_attempts += 1 + if self.commit_failures_remaining: + self.commit_failures_remaining -= 1 + self.commit_failed.set() + raise OSError("injected post-report output commit failure") + return self._resolve(manifest, OutputStageStatus.COMMITTED) + + async def abort_output(self, manifest: OutputStageManifest) -> OutputStage: + return self._resolve(manifest, OutputStageStatus.ABORTED) + + def _resolve( + self, manifest: OutputStageManifest, status: OutputStageStatus + ) -> OutputStage: + identity = (manifest.stage_token, manifest.sub_batch_id) + existing = self.stages.get(identity) + if existing is None: + raise OutputStageNotFoundError(manifest) + if existing.manifest != manifest: + raise OutputStageConflictError(manifest) + if existing.status is not OutputStageStatus.PENDING: + if existing.status is status: + return existing + raise OutputStageResolutionError( + manifest, + current=existing.status, + requested=status, + ) + resolved = replace(existing, status=status) + self.stages[identity] = resolved + return resolved + + async def output_stage(self, manifest: OutputStageManifest) -> OutputStage | None: + self.read_operations.append("output_stage") + existing = self.stages.get((manifest.stage_token, manifest.sub_batch_id)) + if existing is not None and existing.manifest != manifest: + raise OutputStageConflictError(manifest) + return existing + + async def append_output( + self, key: StreamKey, record: StreamRecord + ) -> OutputStreamRecord: + placed = await self.append(key, record) + assert placed.offset is not None + return OutputStreamRecord(placed.kind, placed.payload, placed.offset) + + async def read_output_after( + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = None, + ) -> OutputReadResult: + del block + self.read_operations.append("read_output_after") + visible: list[OutputStreamRecord] = [] + for record in self._after(key, after): + assert record.offset is not None + identity = self._stage_by_offset.get((key, record.offset)) + if identity is not None: + stage = self.stages[identity] + if stage.status is OutputStageStatus.PENDING: + self.pending_read.set() + return OutputReadResult( + tuple(visible), + PendingOutputBarrier(stage.manifest, stage.records[0].offset), + ) + if stage.status is OutputStageStatus.ABORTED: + continue + visible.append( + OutputStreamRecord(record.kind, record.payload, record.offset) + ) + if len(visible) >= max_records: + break + return OutputReadResult(tuple(visible)) + + async def output_tail(self, key: StreamKey) -> Cursor: + self.read_operations.append("output_tail") + result = await self.read_output_after( + key, BEGINNING, max_records=2**31 - 1, block=None + ) + if not result.records: + return BEGINNING + return AFTER(result.records[-1].offset) + + async def install_park_intent(self, key: StreamKey, intent: ParkIntent) -> None: + await super().install_park_intent(key, intent) + if ( + self.expected_park_intents + and len(self._intents) == self.expected_park_intents + ): + self.all_park_intents_installed.set() + + async def recheck(self, key: StreamKey, wait_id: int) -> bool: + if self.recheck_release is not None: + await self.recheck_release.wait() + return await super().recheck(key, wait_id) + + async def remove_park_intent_if_matches( + self, + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + result = await super().remove_park_intent_if_matches( + key, + wait_id, + run_id=run_id, + park_generation=park_generation, + ) + if result in (ParkIntentRemoval.REMOVED, ParkIntentRemoval.ABSENT): + self.removed_park_intents.append(wait_id) + return result + + +class _UnavailableOutputMemoryBackend(_OutputMemoryBackend): + """Fails one stage, then holds its retry until a test restores storage.""" + + def __init__(self) -> None: + super().__init__() + self.failed_once = False + self.failed_stage_attempted = asyncio.Event() + self.recovery_release = asyncio.Event() + + async def stage_output( + self, + manifest: OutputStageManifest, + records: Sequence[StagedOutputRecord], + ) -> OutputStage: + if not self.failed_once: + self.failed_once = True + self.failed_stage_attempted.set() + raise OSError("injected output staging outage") + await self.recovery_release.wait() + return await super().stage_output(manifest, records) + + +async def _history(handle: Any) -> list[Any]: + return [event async for event in handle.fetch_history_events()] + + +def _output_marker_data(events: Sequence[Any]) -> list[tuple[Any, Any]]: + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + + markers: list[tuple[Any, Any]] = [] + for event in events: + if not event.HasField("marker_recorded_event_attributes"): + continue + attributes = event.marker_recorded_event_attributes + if attributes.marker_name != "core_external_stream": + continue + payloads = attributes.details["external_stream"].payloads + if len(payloads) != 1: + continue + marker = ExternalStreamMarkerData() + marker.ParseFromString(payloads[0].data) + if marker.HasField("output"): + markers.append((event, marker)) + return markers + + +def _output_marker_tokens(events: Sequence[Any]) -> list[str]: + return [marker.output.stage_token for _, marker in _output_marker_data(events)] + + +async def _first_failed_task(handle: Any, *, timeout: float = 30) -> Any: + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + for event in await _history(handle): + if event.HasField("workflow_task_failed_event_attributes"): + return event.workflow_task_failed_event_attributes + await asyncio.sleep(0.1) + raise AssertionError("the Workflow produced no failed Workflow Task") + + +def _failure_types(failure: Any) -> list[str]: + types: list[str] = [] + while True: + types.append(failure.application_failure_info.type) + if not failure.HasField("cause"): + return types + failure = failure.cause + + +async def test_workflow_output_round_trips_through_core_and_client( + client: Client, +) -> None: + """Stage, record compact proof, promote after reporting, and decode.""" + backend = _OutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishExternalOutputWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + PublishExternalOutputWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + assert await asyncio.wait_for(handle.result(), 30) == 1 + + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + assert len(backend.stages) == 1 + staged = next(iter(backend.stages.values())) + + # The server result and Worker's post-report History reconciliation + # race one another. Wait for the healthy-path promotion rather than + # requiring a reader to repair every successfully committed batch. + async def wait_until_committed() -> None: + while next(iter(backend.stages.values())).status is not ( + OutputStageStatus.COMMITTED + ): + await asyncio.sleep(0.01) + + await asyncio.wait_for(wait_until_committed(), 30) + + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + + async def read_all() -> list[str]: + return [ + item.data + async for item in output_client.topic("events", type=str).subscribe() + ] + + assert await asyncio.wait_for(read_all(), 30) == ["workflow-output-secret"] + assert next(iter(backend.stages.values())).status is OutputStageStatus.COMMITTED + + events = [event async for event in handle.fetch_history_events()] + markers = [ + event.marker_recorded_event_attributes + for event in events + if event.HasField("marker_recorded_event_attributes") + and event.marker_recorded_event_attributes.marker_name + == "core_external_stream" + ] + assert len(markers) == 1 + envelope = markers[0].details["external_stream"] + assert len(envelope.payloads) == 1 + + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + + marker = ExternalStreamMarkerData() + marker.ParseFromString(envelope.payloads[0].data) + assert marker.HasField("output") + output = marker.output + assert output.stage_token == staged.manifest.stage_token + assert output.run_id == handle.result_run_id + assert output.provider_id == backend.provider_id + assert output.provider_format_version == backend.provider_format_version + assert output.history_floor_event_id == staged.manifest.history_floor_event_id + assert output.history_floor_event_id > 0 + by_id = {event.event_id: event for event in events} + assert output.history_floor_event_id in by_id + assert by_id[output.history_floor_event_id + 1].HasField( + "workflow_task_scheduled_event_attributes" + ) + assert [ + (topic.topic, topic.record_count, topic.finished) for topic in output.topics + ] == [("events", 2, True)] + assert [ + tuple(segment.record_counts_by_topic) for segment in output.segments + ] == [(2,)] + assert len(output.topics[0].logical_fingerprint) == 32 + + history_bytes = b"".join(event.SerializeToString() for event in events) + assert b"workflow-output-secret" not in history_bytes + assert b"workflow-output-secret" not in envelope.payloads[0].data + + +async def test_output_latency_flushes_a_retained_workflow_task( + client: Client, +) -> None: + """The latency deadline exposes output while the Workflow is still blocked.""" + backend = _OutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + handle = None + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishOutputWhileInputIsRetainedWorkflow], + external_stream_backend=backend, + ): + started_at = time.monotonic() + handle = await client.start_workflow( + PublishOutputWhileInputIsRetainedWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + + try: + # The ordinary activation is retainable, so it must not stage on + # return. Core's one-second output deadline is the first legal + # boundary that can make this record visible. + await asyncio.sleep(0.1) + assert backend.stages == {} + + async def wait_until_committed() -> OutputStage: + while not backend.stages: + await asyncio.sleep(0.01) + stage = next(iter(backend.stages.values())) + while stage.status is not OutputStageStatus.COMMITTED: + await asyncio.sleep(0.01) + stage = next(iter(backend.stages.values())) + return stage + + staged = await asyncio.wait_for(wait_until_committed(), 30) + elapsed = time.monotonic() - started_at + assert elapsed >= 0.75 + assert staged.manifest.record_count == 1 + assert [record.kind for record in staged.records] == [RecordKind.DATA] + + # The output is externally readable even though Workflow code is + # still waiting for its input record. + assert (await handle.describe()).status == WorkflowExecutionStatus.RUNNING + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + subscription = output_client.topic("events", type=str).subscribe() + item = await asyncio.wait_for(subscription.__anext__(), 30) + assert item.data == "visible-before-input" + + codec = StreamPayloadCodec(client.data_converter, str) + await backend.append( + chain.stream_key("release"), + StreamRecord( + RecordKind.DATA, + await codec.encode("release"), + "test", + 0, + ), + ) + assert await asyncio.wait_for(handle.result(), 30) == 1 + finally: + if (await handle.describe()).status == WorkflowExecutionStatus.RUNNING: + await handle.terminate() + + +@pytest.mark.timeout(90) +async def test_three_retained_output_windows_write_three_markers_and_wfts( + client: Client, +) -> None: + """Two publishes coalesce per window; three crossed windows stay distinct.""" + backend = _OutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + handle = None + async with Worker( + client, + task_queue=task_queue, + workflows=[ThreeRetainedOutputWindowsWorkflow], + external_stream_backend=backend, + ): + try: + handle = await client.start_workflow( + ThreeRetainedOutputWindowsWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + codec = StreamPayloadCodec(client.data_converter, str) + trigger_key = chain.stream_key("triggers") + + async def append_trigger(sequence: int, value: str) -> None: + await backend.append( + trigger_key, + StreamRecord( + RecordKind.DATA, + await codec.encode(value), + "latency-window-test", + sequence, + ), + ) + + async def wait_for_output_markers(count: int) -> list[tuple[Any, Any]]: + while True: + markers = _output_marker_data(await _history(handle)) + if len(markers) >= count: + return markers + await asyncio.sleep(0.01) + + for sequence, window in enumerate(("one", "two", "three")): + await append_trigger(sequence, window) + markers = await asyncio.wait_for( + wait_for_output_markers(sequence + 1), 30 + ) + assert len(markers) == sequence + 1 + + assert (await handle.describe()).status == WorkflowExecutionStatus.RUNNING + markers = _output_marker_data(await _history(handle)) + assert len(markers) == 3 + completed_event_ids = { + event.marker_recorded_event_attributes.workflow_task_completed_event_id + for event, _ in markers + } + assert len(completed_event_ids) == 3 + + from temporalio.bridge.proto.external_data import ParkReason + + for _, marker in markers: + assert marker.terminal_boundary == ParkReason.PARK_REASON_OUTPUT_LATENCY + assert len(marker.output.topics) == 1 + assert marker.output.topics[0].record_count == 2 + assert ( + sum( + segment.record_counts_by_topic[0] + for segment in marker.output.segments + ) + == 2 + ) + + events = await _history(handle) + assert completed_event_ids <= { + event.event_id + for event in events + if event.HasField("workflow_task_completed_event_attributes") + } + assert len(backend.stages) == 3 + assert all( + stage.status is OutputStageStatus.COMMITTED + for stage in backend.stages.values() + ) + + await append_trigger(3, "stop") + assert await asyncio.wait_for(handle.result(), 30) == 3 + assert len(_output_marker_data(await _history(handle))) == 3 + finally: + if ( + handle is not None + and (await handle.describe()).status == WorkflowExecutionStatus.RUNNING + ): + await handle.terminate() + + +async def test_finished_output_survives_continue_as_new_without_backend_reads( + client: Client, +) -> None: + backend = _OutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[FinishOutputAcrossContinueAsNewWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + FinishOutputAcrossContinueAsNewWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + + assert await asyncio.wait_for(handle.result(), 30) == ( + "successor rejected finished topic" + ) + + # The first Run staged only FINISH. The successor rejected DATA from its + # reserved continuation header before consulting mutable provider state. + assert len(backend.stages) == 1 + [stage] = backend.stages.values() + assert [record.kind.name for record in stage.records] == ["FINISH"] + assert backend.read_operations == [] + + +async def test_cold_client_repairs_post_report_commit_failure( + client: Client, +) -> None: + """A marker stays authoritative when opportunistic promotion fails once.""" + backend = _OutputMemoryBackend() + backend.commit_failures_remaining = 1 + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + chain: WorkflowChainKey + staged: OutputStage + + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishExternalOutputWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + PublishExternalOutputWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + assert await asyncio.wait_for(handle.result(), 30) == 1 + await asyncio.wait_for(backend.commit_failed.wait(), 30) + + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + assert len(backend.stages) == 1 + staged = next(iter(backend.stages.values())) + assert staged.status is OutputStageStatus.PENDING + assert backend.commit_attempts == 1 + + # The injected commit is reached only after the Worker found the token + # in exact-run History. Assert that proof at the failure boundary too, + # before shutting the Worker down and handing recovery to a new client. + events = [event async for event in handle.fetch_history_events()] + from temporalio.contrib.external_workflow_streams._output_client import ( + _event_has_output_stage_token, + ) + + assert any( + _event_has_output_stage_token(event, staged.manifest.stage_token) + for event in events + ) + + cold_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + + async def read_all() -> list[str]: + return [ + item.data + async for item in cold_client.topic("events", type=str).subscribe() + ] + + assert await asyncio.wait_for(read_all(), 30) == ["workflow-output-secret"] + assert ( + backend.stages[ + (staged.manifest.stage_token, staged.manifest.sub_batch_id) + ].status + is OutputStageStatus.COMMITTED + ) + assert backend.commit_attempts == 2 + + +@pytest.mark.timeout(90) +async def test_reader_waits_at_pending_barrier_until_marker_commit( + client: Client, +) -> None: + """A staged record is physically present but unreadable before its proof.""" + backend = _OutputMemoryBackend() + backend.stage_release = asyncio.Event() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + handle = None + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishExternalOutputWorkflow], + external_stream_backend=backend, + ): + try: + handle = await client.start_workflow( + PublishExternalOutputWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + await asyncio.wait_for(backend.stage_placed.wait(), 30) + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + + async def read_all() -> list[str]: + return [ + item.data + async for item in output_client.topic( + "events", type=str + ).subscribe() + ] + + read_task = asyncio.create_task(read_all()) + await asyncio.wait_for(backend.pending_read.wait(), 30) + await asyncio.sleep(0.1) + assert not read_task.done() + [stage] = backend.stages.values() + assert stage.status is OutputStageStatus.PENDING + assert _output_marker_tokens(await _history(handle)) == [] + + backend.stage_release.set() + assert await asyncio.wait_for(handle.result(), 30) == 1 + assert await asyncio.wait_for(read_task, 30) == ["workflow-output-secret"] + [stage] = backend.stages.values() + assert stage.status is OutputStageStatus.COMMITTED + assert _output_marker_tokens(await _history(handle)) == [ + stage.manifest.stage_token + ] + finally: + backend.stage_release.set() + if ( + handle is not None + and (await handle.describe()).status == WorkflowExecutionStatus.RUNNING + ): + await handle.terminate() + + +@pytest.mark.timeout(90) +async def test_rejected_post_stage_completion_never_exposes_phantom_output( + client: Client, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reject the first report after staging; only its fresh retry is visible.""" + backend = _OutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + original = temporalio.bridge.worker.Worker.complete_workflow_activation + rejected = asyncio.Event() + rejected_tokens: list[str] = [] + + async def reject_first_output_commit( + bridge: temporalio.bridge.worker.Worker, + completion: Any, + ) -> None: + if not rejected.is_set() and completion.HasField("successful"): + commits = [ + command.workflow_output_stream_commit + for command in completion.successful.commands + if command.HasField("workflow_output_stream_commit") + ] + if commits: + rejected_token = commits[0].manifest.stage_token + rejected_tokens.append(rejected_token) + matching = [ + stage + for stage in backend.stages.values() + if stage.manifest.stage_token == rejected_token + ] + assert len(matching) == 1 + assert matching[0].status is OutputStageStatus.PENDING + # Replace the successful report after provider staging. Core + # reports an ordinary failed WFT, then retries the Workflow. + completion.failed.failure.message = ( + "injected rejection after output stage" + ) + completion.failed.failure.application_failure_info.type = ( + "InjectedPostStageFailure" + ) + rejected.set() + await original(bridge, completion) + + monkeypatch.setattr( + temporalio.bridge.worker.Worker, + "complete_workflow_activation", + reject_first_output_commit, + ) + + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishExternalOutputWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + PublishExternalOutputWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + await asyncio.wait_for(rejected.wait(), 30) + assert await asyncio.wait_for(handle.result(), 30) == 1 + + [rejected_token] = rejected_tokens + events = await _history(handle) + assert any( + event.HasField("workflow_task_failed_event_attributes") for event in events + ) + stages = list(backend.stages.values()) + assert len(stages) == 2 + [abandoned] = [ + stage for stage in stages if stage.manifest.stage_token == rejected_token + ] + assert abandoned.status is OutputStageStatus.ABORTED + + marker_tokens = _output_marker_tokens(events) + assert len(marker_tokens) == 1 + assert marker_tokens != [rejected_token] + + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + assert [ + item.data + async for item in output_client.topic("events", type=str).subscribe() + ] == ["workflow-output-secret"] + committed = [ + stage + for stage in backend.stages.values() + if stage.status is OutputStageStatus.COMMITTED + ] + assert len(committed) == 1 + assert marker_tokens == [committed[0].manifest.stage_token] + + +@pytest.mark.timeout(90) +async def test_concurrent_updates_preserve_their_own_turn_ids( + client: Client, +) -> None: + backend = _OutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[ConcurrentTurnOutputWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + ConcurrentTurnOutputWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + topic = output_client.topic("events", type=str) + boundary = await topic.tail() + first_id = f"turn-{uuid.uuid4()}" + second_id = f"turn-{uuid.uuid4()}" + + returned = await asyncio.gather( + handle.execute_update(ConcurrentTurnOutputWorkflow.start_turn, first_id), + handle.execute_update(ConcurrentTurnOutputWorkflow.start_turn, second_id), + ) + assert returned == [first_id, second_id] + await handle.execute_update(ConcurrentTurnOutputWorkflow.finish) + assert await asyncio.wait_for(handle.result(), 30) == "done" + + async def scan_for(own_id: str) -> list[str]: + seen: list[str] = [] + async for item in topic.subscribe(after=boundary): + seen.append(item.data) + if item.data == f"turn_started:{own_id}": + return seen + raise AssertionError(f"no output for {own_id}") + + first_seen, second_seen = await asyncio.gather( + scan_for(first_id), scan_for(second_id) + ) + assert first_seen[-1] == f"turn_started:{first_id}" + assert second_seen[-1] == f"turn_started:{second_id}" + # Both subscribers use the same boundary; one necessarily sees the + # other concurrent turn first, but still correlates its own envelope. + assert sorted((len(first_seen), len(second_seen))) == [1, 2] + + +@pytest.mark.timeout(90) +async def test_stage_outage_blocks_wft_and_reports_external_storage_cause( + client: Client, +) -> None: + backend = _UnavailableOutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishExternalOutputWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + PublishExternalOutputWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + result_task = asyncio.create_task(handle.result()) + await asyncio.wait_for(backend.failed_stage_attempted.wait(), 30) + await asyncio.sleep(0.05) + assert not result_task.done() + assert (await handle.describe()).status == WorkflowExecutionStatus.RUNNING + assert backend.stages == {} + + backend.recovery_release.set() + assert await asyncio.wait_for(result_task, 30) == 1 + failed = await _first_failed_task(handle) + assert ( + failed.cause + == temporalio.api.enums.v1.WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_EXTERNAL_STORAGE_FAILURE + ) + assert "StreamStorageError" in _failure_types(failed.failure) + assert len(_output_marker_tokens(await _history(handle))) == 1 + stages = list(backend.stages.values()) + assert len(stages) == 1 + assert stages[0].status is OutputStageStatus.COMMITTED + + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + workflow_id, + description.raw_description.workflow_execution_info.first_run_id, + ) + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + assert [ + item.data + async for item in output_client.topic("events", type=str).subscribe() + ] == ["workflow-output-secret"] + + +@pytest.mark.timeout(90) +async def test_cursor_resume_crosses_rollover_and_continue_as_new( + client: Client, + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend = _OutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + original = temporalio.bridge.worker.Worker.complete_workflow_activation + rollover_requests: list[bool] = [] + + async def capture_rollover_request( + bridge: temporalio.bridge.worker.Worker, + completion: Any, + ) -> None: + if completion.HasField("successful"): + rollover_requests.extend( + command.workflow_output_stream_commit.request_rollover + for command in completion.successful.commands + if command.HasField("workflow_output_stream_commit") + ) + await original(bridge, completion) + + monkeypatch.setattr( + temporalio.bridge.worker.Worker, + "complete_workflow_activation", + capture_rollover_request, + ) + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishAcrossRolloverAndContinueAsNewWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + PublishAcrossRolloverAndContinueAsNewWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + description = await handle.describe() + first_run_id = description.raw_description.workflow_execution_info.first_run_id + + async def wait_for_replacement_stage() -> None: + while len(backend.stages) < 2: + await asyncio.sleep(0.01) + + try: + await asyncio.wait_for(wait_for_replacement_stage(), 10) + except asyncio.TimeoutError: + events = await _history(handle) + statuses = [stage.status.name for stage in backend.stages.values()] + marker_tokens = _output_marker_tokens(events) + status = (await handle.describe()).status + await handle.terminate() + pytest.fail( + "output capacity did not create its replacement Workflow Task: " + f"requests={rollover_requests}, stages={statuses}, " + f"markers={marker_tokens}, " + f"status={status.name if status is not None else 'NONE'}" + ) + assert await asyncio.wait_for(handle.result(), 30) == "done" + assert rollover_requests[:2] == [True, False] + + chain = WorkflowChainKey(client.namespace, workflow_id, first_run_id) + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + topic = output_client.topic("events", type=str) + initial = topic.subscribe() + first = await asyncio.wait_for(initial.__anext__(), 30) + await cast(Any, initial).aclose() + assert first.data == "before-rollover" + + resumed = topic.subscribe(after=first.offset) + second = await asyncio.wait_for(resumed.__anext__(), 30) + third = await asyncio.wait_for(resumed.__anext__(), 30) + await cast(Any, resumed).aclose() + assert [second.data, third.data] == [ + "after-rollover", + "after-continue-as-new", + ] + assert len({first.offset, second.offset, third.offset}) == 3 + + stages_by_run: dict[str, int] = {} + for stage in backend.stages.values(): + stages_by_run[stage.manifest.run_id] = ( + stages_by_run.get(stage.manifest.run_id, 0) + 1 + ) + assert stage.status is OutputStageStatus.COMMITTED + assert stages_by_run[first_run_id] == 2 + successor_runs = { + run_id: count + for run_id, count in stages_by_run.items() + if run_id != first_run_id + } + assert list(successor_runs.values()) == [1] + + +@pytest.mark.timeout(90) +async def test_output_deadline_wins_park_race_and_removes_every_intent( + client: Client, +) -> None: + backend = _OutputMemoryBackend() + backend.expected_park_intents = 2 + backend.recheck_release = asyncio.Event() + task_queue = f"tq-{uuid.uuid4()}" + workflow_id = f"wf-{uuid.uuid4()}" + handle = None + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishWhileTwoInputsParkWorkflow], + external_stream_backend=backend, + ): + try: + handle = await client.start_workflow( + PublishWhileTwoInputsParkWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + await asyncio.wait_for(backend.all_park_intents_installed.wait(), 30) + assert len(backend._intents) == 2 + + # Keep PrepareExternalStreamPark outstanding until after the + # already-armed output deadline expires. Core must discard the + # losing park result and roll back both provider intents. + await asyncio.sleep(0.75) + backend.recheck_release.set() + + async def wait_for_flush_and_rollback() -> None: + while True: + committed = any( + stage.status is OutputStageStatus.COMMITTED + for stage in backend.stages.values() + ) + if committed and not backend._intents: + return + await asyncio.sleep(0.01) + + await asyncio.wait_for(wait_for_flush_and_rollback(), 30) + assert set(backend.removed_park_intents) == {1, 2} + assert len(backend.stages) == 1 + assert (await handle.describe()).status == WorkflowExecutionStatus.RUNNING + assert len(_output_marker_tokens(await _history(handle))) == 1 + finally: + backend.recheck_release.set() + if ( + handle is not None + and (await handle.describe()).status == WorkflowExecutionStatus.RUNNING + ): + await handle.terminate() diff --git a/tests/contrib/external_workflow_streams/test_parking_conformance.py b/tests/contrib/external_workflow_streams/test_parking_conformance.py new file mode 100644 index 0000000000..ad563d4914 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_parking_conformance.py @@ -0,0 +1,289 @@ +"""P2b — the parking conformance suite, and proof that it fails a broken stub.""" + +from __future__ import annotations + +import uuid +from datetime import timedelta + +import pytest + +from temporalio.contrib.external_workflow_streams._backend import ( + ParkIntent, + StreamBackend, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._record import BEGINNING +from tests.contrib.external_workflow_streams import conformance +from tests.contrib.external_workflow_streams.conformance import ( + PARKING_CONFORMANCE_CHECKS, + Check, +) +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + + +@pytest.fixture +def stream_key() -> StreamKey: + return StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +# --- the reference backend conforms ----------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("check", PARKING_CONFORMANCE_CHECKS, ids=lambda c: c.__name__) +async def test_reference_backend_conforms( + check: Check, backend: MemoryStreamBackend, stream_key: StreamKey +) -> None: + await check(backend, stream_key) + + +def test_the_parking_suite_is_not_silently_empty() -> None: + assert len(PARKING_CONFORMANCE_CHECKS) >= 10 + assert len({c.__name__ for c in PARKING_CONFORMANCE_CHECKS}) == len( + PARKING_CONFORMANCE_CHECKS + ) + + +class ObserveOnlyBackend(MemoryStreamBackend): + """Cannot lease, and says so. Every producer signals idempotently.""" + + supports_leased_claims = False + + async def claim_park_generation( + self, + key: StreamKey, + wait_id: int, + park_generation: int, + *, + claimant: str, + lease: timedelta, + ) -> bool: + return True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("check", PARKING_CONFORMANCE_CHECKS, ids=lambda c: c.__name__) +async def test_an_observe_only_backend_conforms( + check: Check, stream_key: StreamKey +) -> None: + """The suite must accept the fallback it explicitly permits. + + A provider that cannot lease is not broken -- it is a different, weaker + contract the design allows for, and a suite that rejected it would push + implementers toward an unleased claim, which *is* broken. + """ + await check(ObserveOnlyBackend(), stream_key) + + +# --- deliberately broken backends ------------------------------------------- + + +class StreamKeyedIntentBackend(MemoryStreamBackend): + """Keys park intents by stream alone, ignoring the wait id. + + The realistic mistake: thinking of a park as a property of the stream rather + than of the subscription. + """ + + async def install_park_intent(self, key: StreamKey, intent: ParkIntent) -> None: + self._intents[(key, 0)] = intent + + async def park_intent(self, key: StreamKey, wait_id: int) -> ParkIntent | None: + return self._intents.get((key, 0)) + + +class NonExpiringClaimBackend(MemoryStreamBackend): + """Claims are exclusive forever. Nothing recovers a crashed producer's.""" + + async def claim_park_generation( + self, + key: StreamKey, + wait_id: int, + park_generation: int, + *, + claimant: str, + lease: timedelta, + ) -> bool: + held = self._claims.get((key, wait_id)) + if held is not None and held[0] != claimant and held[1] == park_generation: + return False + self._claims[(key, wait_id)] = (claimant, park_generation, float("inf")) + return True + + +class LastIntentOnlyBackend(MemoryStreamBackend): + """Enumerates only the most recently installed intent. + + A plausible implementation for a provider that tracks "the" park per stream, + and it looks correct until a Workflow subscribes to one stream twice: the + subscription it omits stays parked on records already sitting in the stream, + and nothing ever wakes it because no producer knows it exists. + """ + + async def parked_wait_ids(self, key: StreamKey) -> list[int]: + matching = [wait_id for (stored, wait_id) in self._intents if stored == key] + return matching[-1:] + + +class BlindRecheckBackend(MemoryStreamBackend): + """Rechecks against the stream's state when the intent was installed.""" + + async def recheck(self, key: StreamKey, wait_id: int) -> bool: + return False + + +class GenerationOutlivesRemovalBackend(MemoryStreamBackend): + """Answers the generation from a remembered value beside the intent. + + The plausible mistake: reading "the current park generation" as a fact about + the subscription's history rather than as "is a park outstanding right now". + It passes every other parking check -- the intent really is removed, and + really does stop being enumerable -- while the one call every wake path + actually asks keeps naming a park that is over. + """ + + def __init__(self) -> None: + super().__init__() + self._generations: dict[tuple[StreamKey, int], int] = {} + + async def install_park_intent(self, key: StreamKey, intent: ParkIntent) -> None: + await super().install_park_intent(key, intent) + self._generations[(key, intent.wait_id)] = intent.park_generation + + async def current_park_generation(self, key: StreamKey, wait_id: int) -> int | None: + return self._generations.get((key, wait_id)) + + +class DelimiterJoiningBackend(MemoryStreamBackend): + """Renders an identity by joining its fields, the way a first draft does. + + The natural implementation for any store with a flat keyspace, and the one + the Redis provider originally had. It looks correct for every identity that + contains no delimiter, which is every identity anyone writes by hand -- so + the collision needs a Workflow ID or a topic name carrying a `:`, which is + an ordinary character in both. + """ + + def _collapse(self, key: StreamKey) -> StreamKey: + joined = ":".join( + ( + key.namespace, + key.workflow_id, + key.first_execution_run_id, + key.stream_name, + ) + ) + # The remaining fields are placeholders: what matters is that two + # distinct identities collapse onto one, which is exactly what a + # delimiter-joined physical key does. They stay non-empty because + # StreamKey rejects empty components. + return StreamKey(joined, "-", "-", "-") + + async def append(self, key, record): # type: ignore[no-untyped-def] + return await super().append(self._collapse(key), record) + + async def read_after(self, key, after, *, max_records, block=None): # type: ignore[no-untyped-def] + return await super().read_after( + self._collapse(key), after, max_records=max_records, block=block + ) + + async def read_range(self, key, first, last): # type: ignore[no-untyped-def] + return await super().read_range(self._collapse(key), first, last) + + async def install_park_intent(self, key, intent): # type: ignore[no-untyped-def] + return await super().install_park_intent(self._collapse(key), intent) + + async def park_intent(self, key, wait_id): # type: ignore[no-untyped-def] + return await super().park_intent(self._collapse(key), wait_id) + + async def current_park_generation(self, key, wait_id): # type: ignore[no-untyped-def] + return await super().current_park_generation(self._collapse(key), wait_id) + + async def parked_wait_ids(self, key): # type: ignore[no-untyped-def] + return await super().parked_wait_ids(self._collapse(key)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("broken", "check", "reason"), + [ + pytest.param( + StreamKeyedIntentBackend, + conformance.check_intents_are_keyed_by_stream_and_wait_id, + r"keyed \(stream key, wait_id\)", + id="intents-keyed-by-stream-alone", + ), + pytest.param( + NonExpiringClaimBackend, + conformance.check_an_expired_claim_is_taken_over, + "expired claim was not taken over", + id="claim-never-expires", + ), + pytest.param( + DelimiterJoiningBackend, + conformance.check_distinct_identities_never_share_storage, + "share one physical stream", + id="identity-fields-joined-without-escaping", + ), + pytest.param( + LastIntentOnlyBackend, + conformance.check_every_parked_subscription_is_enumerable, + "expected both parked subscriptions", + id="enumerates-only-the-last-intent", + ), + pytest.param( + BlindRecheckBackend, + conformance.check_recheck_sees_an_append_past_the_cursor, + "append/park race is not closed", + id="recheck-cannot-see-a-late-append", + ), + pytest.param( + GenerationOutlivesRemovalBackend, + conformance.check_a_removed_intent_reports_no_generation, + "still reports a generation", + id="generation-outlives-its-intent", + ), + ], +) +async def test_a_broken_backend_fails_for_the_right_reason( + broken: type[StreamBackend], check: Check, reason: str, stream_key: StreamKey +) -> None: + with pytest.raises(AssertionError, match=reason): + await check(broken(), stream_key) + + +# --- the intent value type -------------------------------------------------- + + +def test_a_park_generation_of_zero_is_refused() -> None: + """0 is the reserved unparked-wake sentinel and can never be a real park.""" + with pytest.raises(ValueError, match="starts at 1"): + ParkIntent(1, BEGINNING, park_generation=0, run_id="run-a") + + +@pytest.mark.asyncio +async def test_two_same_stream_subscriptions_are_visible_separately( + backend: MemoryStreamBackend, stream_key: StreamKey +) -> None: + """The end state P21 asserts, checked here at the provider level. + + Inspecting both in the backend is what distinguishes "two subscriptions + parked independently" from "one overwrote the other and happens to look + parked". + """ + for wait_id in (1, 2): + await backend.install_park_intent( + stream_key, + ParkIntent(wait_id, BEGINNING, park_generation=5, run_id="run-a"), + ) + + assert await backend.current_park_generation(stream_key, 1) == 5 + assert await backend.current_park_generation(stream_key, 2) == 5 + assert (await backend.park_intent(stream_key, 1)) != ( + await backend.park_intent(stream_key, 2) + ) diff --git a/tests/contrib/external_workflow_streams/test_producer.py b/tests/contrib/external_workflow_streams/test_producer.py new file mode 100644 index 0000000000..6cea72eed3 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_producer.py @@ -0,0 +1,987 @@ +"""P6a/P6 — producer binding, and append plus write fence. + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import uuid + +import pytest + +import temporalio.converter +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.contrib.external_workflow_streams._backend import ( + AppendConflictError, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._producer import ( + AppendNotAcknowledgedError, + ChainKeyMismatchError, + ExternalStreamProducer, + PrecedingWriteFailedError, + WorkflowChainKey, + _default_session_id, +) +from temporalio.contrib.external_workflow_streams._record import ( + BEGINNING, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.testing import ActivityEnvironment +from temporalio.worker import Worker +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + + +@workflow.defn +class ChainWorkflow: + """Runs long enough for a producer to describe it.""" + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: False) + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +# --- the chain key ---------------------------------------------------------- + + +@pytest.mark.parametrize( + "missing", ["namespace", "workflow_id", "first_execution_run_id"] +) +def test_every_part_of_the_chain_key_is_required(missing: str) -> None: + parts = { + "namespace": "ns", + "workflow_id": "wf", + "first_execution_run_id": "run-1", + } + parts[missing] = "" + + with pytest.raises(ValueError, match=missing): + WorkflowChainKey(**parts) + + +def test_the_stream_name_appears_only_in_topic() -> None: + """No two arguments can disagree about the stream name. + + ``connect()`` takes the chain key and ``topic(name)`` completes it, so + there is exactly one place the name is written. + """ + key = WorkflowChainKey("ns", "wf", "run-1") + + assert key.stream_key("tokens") == StreamKey("ns", "wf", "run-1", "tokens") + assert key.stream_key("tokens") != key.stream_key("tool-events") + assert not hasattr(key, "stream_name") + + +# --- session IDs ------------------------------------------------------------ + + +def test_a_plain_process_must_supply_a_session_id() -> None: + """A random default would look correct until the first retry. + + Idempotency is on ``(session_id, sequence)``, so a fresh session per attempt + means every record appended twice -- and the failure shows up as duplicate + delivery to the Workflow, a long way from here. + """ + with pytest.raises(ValueError, match="no default outside an Activity"): + _default_session_id() + + +async def test_an_activity_derives_a_session_id_stable_across_attempts() -> None: + env = ActivityEnvironment() + base = dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="publish-tokens", + workflow_run_id="run-abc", + ) + + seen = [] + for attempt in (1, 2, 3): + env.info = dataclasses.replace(base, attempt=attempt) + + @activity.defn + async def capture() -> str: + return _default_session_id() + + seen.append(await env.run(capture)) + + assert len(set(seen)) == 1, ( + f"the session id must not vary by attempt, got {seen}; a retried attempt " + "must reuse its predecessor's idempotency keys" + ) + assert "run-abc" in seen[0] and "publish-tokens" in seen[0] + + +# --- binding and verification ----------------------------------------------- + + +async def test_a_wrong_first_execution_run_id_fails_loudly( + client: Client, backend: MemoryStreamBackend +) -> None: + task_queue = f"tq-{uuid.uuid4()}" + async with Worker(client, task_queue=task_queue, workflows=[ChainWorkflow]): + handle = await client.start_workflow( + ChainWorkflow.run, id=f"wf-{uuid.uuid4()}", task_queue=task_queue + ) + try: + with pytest.raises(ChainKeyMismatchError, match="first execution Run ID"): + await ExternalStreamProducer.connect( + backend=backend, + workflow=WorkflowChainKey( + client.namespace, handle.id, "definitely-not-the-run-id" + ), + client=client, + ) + finally: + await handle.terminate() + + +async def test_a_wrong_namespace_fails_before_any_server_call( + client: Client, backend: MemoryStreamBackend +) -> None: + with pytest.raises(ChainKeyMismatchError, match="namespace"): + await ExternalStreamProducer.connect( + backend=backend, + workflow=WorkflowChainKey("some-other-namespace", "wf", "run-1"), + client=client, + session_id="s", + ) + + +async def test_an_activity_and_a_plain_process_publish_under_one_verified_key( + client: Client, backend: MemoryStreamBackend +) -> None: + """Two very different producers, one stream, one verified chain key. + + The Activity gets its session id for free from its own identity; the plain + process supplies one. Neither can derive the chain key -- ``activity.Info`` + has no first execution Run ID -- so both are handed it by the Workflow. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker(client, task_queue=task_queue, workflows=[ChainWorkflow]): + handle = await client.start_workflow( + ChainWorkflow.run, id=f"wf-{uuid.uuid4()}", task_queue=task_queue + ) + try: + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + ) + + @activity.defn + async def publish_from_activity() -> str: + producer = await ExternalStreamProducer.connect( + backend=backend, workflow=chain, client=client + ) + tokens = producer.topic("tokens", type=str) + await tokens.publish("from-activity", wake=False) + await tokens.finish_writing(wake=False) + return producer.session_id + + env = ActivityEnvironment() + env.info = dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="publish-tokens", + workflow_run_id=description.run_id, + ) + activity_session = await env.run(publish_from_activity) + assert activity_session.startswith("activity:"), ( + "an Activity must derive its session id from its own identity" + ) + + # And a plain, non-Temporal-hosted process against the same key. + plain = await ExternalStreamProducer.connect( + backend=backend, + workflow=chain, + client=client, + session_id="standalone-script", + ) + await plain.topic("tokens", type=str).publish("from-script", wake=False) + + records = backend.all_records(chain.stream_key("tokens")) + assert [r.kind for r in records] == [ + RecordKind.DATA, + RecordKind.WRITE_FENCE, + RecordKind.DATA, + ] + assert {r.producer_session_id for r in records} == { + activity_session, + "standalone-script", + } + finally: + await handle.terminate() + + +# --- append and the fence, without a server --------------------------------- + + +def _offline_producer(backend: MemoryStreamBackend, session: str = "s"): # type: ignore[no-untyped-def] + """A producer built directly, skipping the verification a bind performs. + + Verification is P6a's and is covered above against a real server; these + cases are about what append and the fence do once a producer is bound. + """ + import temporalio.converter + + return ExternalStreamProducer( + backend=backend, + workflow=WorkflowChainKey("ns", "wf", "run-1"), + data_converter=temporalio.converter.DataConverter.default, + session_id=session, + ) + + +def _placed_offsets(records: list[StreamRecord]) -> list[Offset]: + offsets: list[Offset] = [] + for record in records: + assert record.offset is not None + offsets.append(record.offset) + return offsets + + +async def test_records_are_appended_in_order(backend: MemoryStreamBackend) -> None: + tokens = _offline_producer(backend).topic("tokens", type=str) + + for value in ["a", "b", "c"]: + await tokens.publish(value, wake=False) + + records = backend.all_records(StreamKey("ns", "wf", "run-1", "tokens")) + assert [r.sequence for r in records] == [0, 1, 2] + assert backend.strictly_increasing(_placed_offsets(records)) + + +async def test_a_fence_takes_its_place_in_the_sequence( + backend: MemoryStreamBackend, +) -> None: + tokens = _offline_producer(backend).topic("tokens", type=str) + + await tokens.publish("a", wake=False) + await tokens.finish_writing(wake=False) + await tokens.publish("b", wake=False) + + records = backend.all_records(StreamKey("ns", "wf", "run-1", "tokens")) + assert [r.kind for r in records] == [ + RecordKind.DATA, + RecordKind.WRITE_FENCE, + RecordKind.DATA, + ] + + +async def test_a_retried_attempt_appends_no_duplicate( + backend: MemoryStreamBackend, +) -> None: + """The whole reason the session id must be stable across attempts.""" + key = StreamKey("ns", "wf", "run-1", "tokens") + + first = _offline_producer(backend, "attempt-stable").topic("tokens", type=str) + await first.publish("a", wake=False) + await first.publish("b", wake=False) + + # The Activity is retried: a fresh producer, the *same* session id, and the + # same values published again from the top. + retried = _offline_producer(backend, "attempt-stable").topic("tokens", type=str) + await retried.publish("a", wake=False) + await retried.publish("b", wake=False) + + assert len(backend.all_records(key)) == 2 + + +class GatedPayloadCodec(temporalio.converter.PayloadCodec): + """A codec whose ``encode`` completes only when the test releases that value. + + Stands in for what a real one does: an external payload store or a KMS round + trip, which is arbitrary I/O and completes in whatever order the service + answers. That order is not stable across Activity attempts, which is the whole + point -- so the test drives it explicitly instead of racing it. + """ + + def __init__(self) -> None: + self.arrived: dict[str, asyncio.Event] = {} + self.gates: dict[str, asyncio.Event] = {} + + def _for(self, marker: str) -> tuple[asyncio.Event, asyncio.Event]: + self.arrived.setdefault(marker, asyncio.Event()) + self.gates.setdefault(marker, asyncio.Event()) + return self.arrived[marker], self.gates[marker] + + async def wait_inside(self, marker: str) -> None: + """Blocks until this value's encode has started.""" + arrived, _ = self._for(marker) + await asyncio.wait_for(arrived.wait(), 2) + + def release(self, marker: str) -> None: + _, gate = self._for(marker) + gate.set() + + async def encode(self, payloads): # type: ignore[no-untyped-def] + # The default converter renders a `str` as JSON, so the value is legible + # in the payload's bytes and needs no side channel to identify it. + marker = payloads[0].data.decode() + arrived, gate = self._for(marker) + arrived.set() + await gate.wait() + return list(payloads) + + async def decode(self, payloads): # type: ignore[no-untyped-def] + return list(payloads) + + +def _gated_producer( + backend: MemoryStreamBackend, codec: GatedPayloadCodec, session: str +): # type: ignore[no-untyped-def] + import temporalio.converter + + return ExternalStreamProducer( + backend=backend, + workflow=WorkflowChainKey("ns", "wf", "run-1"), + data_converter=dataclasses.replace( + temporalio.converter.DataConverter.default, payload_codec=codec + ), + session_id=session, + ) + + +async def test_concurrent_publishes_take_their_sequence_in_invocation_order( + backend: MemoryStreamBackend, +) -> None: + """Idempotency may not depend on the order a payload codec answers in. + + ``(session_id, sequence)`` is the idempotency key and a retried Activity + reuses the session id on purpose, so the sequence a call draws has to be a + property of *the call*. Drawing it after awaiting the encode makes it a + property of the encode's completion order instead -- and a codec is allowed to + do real I/O, so that order is not stable across attempts. Two concurrent + publishes then exchange sequence numbers whenever the store answers the other + way round, the backend sees each stable key reused with different bytes, and + the retry raises ``AppendConflictError`` on both calls: a valid concurrent + Activity made permanently non-retryable by timing alone. + + The two attempts here are byte-identical in what they do and differ only in + which encode finishes first. + """ + key = StreamKey("ns", "wf", "run-1", "tokens") + + codec = GatedPayloadCodec() + topic = _gated_producer(backend, codec, "attempt-stable").topic("tokens", type=str) + first = asyncio.ensure_future(topic.publish("a", wake=False)) + second = asyncio.ensure_future(topic.publish("b", wake=False)) + await codec.wait_inside('"a"') + await codec.wait_inside('"b"') + codec.release('"a"') + codec.release('"b"') + offsets = [await first, await second] + + # The Activity is retried: a fresh producer, the *same* session id, the same + # calls in the same order -- and the store answers in the other order. + retry_codec = GatedPayloadCodec() + retried = _gated_producer(backend, retry_codec, "attempt-stable").topic( + "tokens", type=str + ) + first_again = asyncio.ensure_future(retried.publish("a", wake=False)) + second_again = asyncio.ensure_future(retried.publish("b", wake=False)) + await retry_codec.wait_inside('"a"') + await retry_codec.wait_inside('"b"') + retry_codec.release('"b"') + retry_codec.release('"a"') + + assert [await first_again, await second_again] == offsets, ( + "the retry appended the same values at different offsets, so the two " + "attempts disagree about what landed where" + ) + records = backend.all_records(key) + assert [r.sequence for r in records] == [0, 1] + assert len(records) == 2, "the retry was not idempotent; it appended duplicates" + + +async def test_reordered_encodes_cannot_duplicate_across_two_topics( + backend: MemoryStreamBackend, +) -> None: + """The same defect on the other side of the deduplication boundary. + + Deduplication is scoped by stream key, so two publishes to *different* topics + that exchange sequence numbers do not collide -- they each land under a key + the other stream has never seen, and the retry appends a second record + instead of raising. The conflict is the loud failure mode; this one is silent. + """ + tokens_key = StreamKey("ns", "wf", "run-1", "tokens") + events_key = StreamKey("ns", "wf", "run-1", "tool-events") + + codec = GatedPayloadCodec() + producer = _gated_producer(backend, codec, "attempt-stable") + tokens = producer.topic("tokens", type=str) + events = producer.topic("tool-events", type=str) + first = asyncio.ensure_future(tokens.publish("a", wake=False)) + second = asyncio.ensure_future(events.publish("b", wake=False)) + await codec.wait_inside('"a"') + await codec.wait_inside('"b"') + codec.release('"a"') + codec.release('"b"') + await first + await second + + retry_codec = GatedPayloadCodec() + retried = _gated_producer(backend, retry_codec, "attempt-stable") + retried_tokens = retried.topic("tokens", type=str) + retried_events = retried.topic("tool-events", type=str) + first_again = asyncio.ensure_future(retried_tokens.publish("a", wake=False)) + second_again = asyncio.ensure_future(retried_events.publish("b", wake=False)) + await retry_codec.wait_inside('"a"') + await retry_codec.wait_inside('"b"') + retry_codec.release('"b"') + retry_codec.release('"a"') + await first_again + await second_again + + assert len(backend.all_records(tokens_key)) == 1 + assert len(backend.all_records(events_key)) == 1, ( + "reversing the encode order appended a duplicate on a topic where the " + "swapped key could not collide, so nothing raised" + ) + + +# --- the fence's ordering claim --------------------------------------------- + + +TOKENS = StreamKey("ns", "wf", "run-1", "tokens") +EVENTS = StreamKey("ns", "wf", "run-1", "tool-events") + + +async def _let_every_ready_task_run() -> None: + """Runs every task that can run without waiting for anything. + + What makes "the fence has not appended" an assertion rather than a race. + Nothing here sleeps in wall-clock terms: an unordered ``finish_writing()`` + reaches the backend without awaiting anything that yields, so one turn of + the loop is enough for it to have finished, and the extra turns only make + that margin obvious. + """ + for _ in range(10): + await asyncio.sleep(0) + + +class GatedFailingCodec(GatedPayloadCodec): + """A gated codec whose release makes the encode *fail* rather than finish. + + The payload store rejecting a value, at the point where a concurrent fence + is already waiting behind the publish. + """ + + async def encode(self, payloads): # type: ignore[no-untyped-def] + await super().encode(payloads) + raise RuntimeError("the payload store rejected the value") + + +class CommittingThenFailingBackend(MemoryStreamBackend): + """Commits an append and then loses the answer, on cue. + + The unknown outcome the producer cannot see through: the record is durable + and its offset reached nobody, so it may not be treated as absent -- and a + fence appended in front of it would claim durability for a write whose place + in the stream is not yet fixed (ADR-038). + """ + + def __init__(self) -> None: + super().__init__() + self.committed = asyncio.Event() + self.lose_the_answer = asyncio.Event() + self.losing = True + + async def append(self, key, record): # type: ignore[no-untyped-def] + placed = await super().append(key, record) + if self.losing: + self.losing = False + self.committed.set() + await self.lose_the_answer.wait() + raise ConnectionError("connection reset") + return placed + + +async def test_a_fence_waits_for_a_publish_still_inside_its_codec( + backend: MemoryStreamBackend, +) -> None: + """The fence's claim is about invocation order, so it must hold under one. + + ``publish()`` draws its sequence before awaiting the codec -- which is what + keeps idempotency keys stable across attempts -- so an earlier publish can + still be encoding when ``finish_writing()`` is called. A fence appended there + says every preceding write in this session is in the stream while the first + one has not been sent yet, and a consumer that drains through it may park on + exactly that. The unsafe shape is this test's: a ``wake=False`` publish + batched behind a fence that carries the batch's only wake, where the wake is + spent before the data exists. + """ + codec = GatedPayloadCodec() + tokens = _gated_producer(backend, codec, "batch").topic("tokens", type=str) + + publishing = asyncio.ensure_future(tokens.publish("a", wake=False)) + await codec.wait_inside('"a"') + fencing = asyncio.ensure_future(tokens.finish_writing(wake=False)) + await _let_every_ready_task_run() + + assert not fencing.done(), ( + "the fence completed while the publish was still encoding" + ) + assert backend.all_records(TOKENS) == [], ( + "the fence reached the backend ahead of a publish invoked before it, so " + "its durability claim was false when a consumer could act on it" + ) + + codec.release('"a"') + await publishing + await fencing + + records = backend.all_records(TOKENS) + assert [(r.sequence, r.kind) for r in records] == [ + (0, RecordKind.DATA), + (1, RecordKind.WRITE_FENCE), + ], "backend order and invocation order disagree" + assert backend.strictly_increasing(_placed_offsets(records)) + + +async def test_two_handles_for_one_topic_share_the_fence_order( + backend: MemoryStreamBackend, +) -> None: + """``topic()`` returns a fresh handle per call; the stream is still one. + + So the order cannot live on the handle. A producer that publishes through + one handle and fences through another -- the shape a helper that takes a + topic name rather than a handle produces -- is the same stream and the same + claim. + """ + codec = GatedPayloadCodec() + producer = _gated_producer(backend, codec, "batch") + writing = producer.topic("tokens", type=str) + finishing = producer.topic("tokens", type=str) + assert writing is not finishing + + publishing = asyncio.ensure_future(writing.publish("a", wake=False)) + await codec.wait_inside('"a"') + fencing = asyncio.ensure_future(finishing.finish_writing(wake=False)) + await _let_every_ready_task_run() + + assert not fencing.done() and backend.all_records(TOKENS) == [], ( + "the second handle fenced without waiting for the first handle's publish" + ) + + codec.release('"a"') + await publishing + await fencing + + assert [r.kind for r in backend.all_records(TOKENS)] == [ + RecordKind.DATA, + RecordKind.WRITE_FENCE, + ] + + +async def test_a_fence_does_not_wait_for_another_stream( + backend: MemoryStreamBackend, +) -> None: + """The claim is per stream, so the ordering must be too. + + One connection serves several topics, and a publish blocked in a codec on + one of them says nothing about a fence on another -- stalling it there would + make an unrelated slow payload store hold up a finished topic. + """ + codec = GatedPayloadCodec() + producer = _gated_producer(backend, codec, "batch") + tokens = producer.topic("tokens", type=str) + events = producer.topic("tool-events", type=str) + + publishing = asyncio.ensure_future(tokens.publish("a", wake=False)) + await codec.wait_inside('"a"') + + await asyncio.wait_for(events.finish_writing(wake=False), 2) + + assert [r.kind for r in backend.all_records(EVENTS)] == [RecordKind.WRITE_FENCE] + assert backend.all_records(TOKENS) == [] + + codec.release('"a"') + await publishing + assert [r.kind for r in backend.all_records(TOKENS)] == [RecordKind.DATA] + + +async def test_a_fence_will_not_overtake_an_append_of_unknown_outcome() -> None: + """An append with no answer may still be in front of the fence. + + The record is durable and its offset reached nobody, so the one thing that + cannot be assumed is that it is absent. The fence is refused with *that* + operation's error rather than its own, which is the refusal that already + governs the stream while an append is unsettled: settle it with + ``resolve_append()``, then fence. + """ + backend = CommittingThenFailingBackend() + tokens = _offline_producer(backend, "batch").topic("tokens", type=str) + + publishing = asyncio.ensure_future(tokens.publish("a", wake=False)) + await asyncio.wait_for(backend.committed.wait(), 2) + fencing = asyncio.ensure_future(tokens.finish_writing(wake=False)) + await _let_every_ready_task_run() + + assert not fencing.done() + assert [r.sequence for r in backend.all_records(TOKENS)] == [0], ( + "the fixture must actually have committed, or this asserts nothing" + ) + + backend.lose_the_answer.set() + with pytest.raises(AppendNotAcknowledgedError) as lost: + await publishing + with pytest.raises(AppendNotAcknowledgedError) as refused: + await fencing + + assert refused.value.record.idempotency_key == lost.value.record.idempotency_key, ( + "the fence must report the unsettled operation's recovery, not invent one " + "for a fence that was never appended" + ) + assert [r.kind for r in backend.all_records(TOKENS)] == [RecordKind.DATA], ( + "a fence was appended in front of a record whose position was not settled" + ) + + await tokens.resolve_append(lost.value.record, wake=False) + await tokens.finish_writing(wake=False) + + records = backend.all_records(TOKENS) + assert [(r.sequence, r.kind) for r in records] == [ + (0, RecordKind.DATA), + (2, RecordKind.WRITE_FENCE), + ], "the settled record must be in the stream once, ahead of the fence" + + +async def test_a_fence_refuses_to_stand_in_for_a_failed_publish( + backend: MemoryStreamBackend, +) -> None: + """A failed earlier write is propagated, not passed over. + + The caller asked for that write before it asked for the fence, and a fence + appended over the gap tells a consumer the batch is complete when it is + short a record. Nothing is appended for the refused fence, so once the + failure has been dealt with -- by republishing the value or by accepting the + batch without it -- ``finish_writing()`` again appends one. + """ + codec = GatedFailingCodec() + tokens = _gated_producer(backend, codec, "batch").topic("tokens", type=str) + + publishing = asyncio.ensure_future(tokens.publish("a", wake=False)) + await codec.wait_inside('"a"') + fencing = asyncio.ensure_future(tokens.finish_writing(wake=False)) + await _let_every_ready_task_run() + codec.release('"a"') + + with pytest.raises(RuntimeError): + await publishing + with pytest.raises(PrecedingWriteFailedError) as caught: + await fencing + + assert caught.value.sequence == 0 and caught.value.stream_key == TOKENS + assert isinstance(caught.value.__cause__, RuntimeError), ( + "the fence must carry what actually failed, not describe it" + ) + assert backend.all_records(TOKENS) == [] + + await tokens.finish_writing(wake=False) + assert [(r.sequence, r.kind) for r in backend.all_records(TOKENS)] == [ + (2, RecordKind.WRITE_FENCE) + ], "the failed write is no longer outstanding, so a fence must go in now" + + +async def test_a_cancelled_fence_does_not_refuse_a_later_fence( + backend: MemoryStreamBackend, +) -> None: + """A failed *fence* is not a failed write, so it is not propagated as one. + + Two concurrent fences make independent claims about the publishes each was + invoked after, and neither is inside the other's claim. Holding them in one + append order made a fence that never reached the backend -- cancelled here, + but a refusal does the same -- look to a later fence exactly like a publish + whose record went missing, so a fence with every preceding write durable + behind it was refused with ``PrecedingWriteFailedError``. That error is + documented as reporting a failed ``publish()``, and nothing else. + """ + codec = GatedPayloadCodec() + tokens = _gated_producer(backend, codec, "batch").topic("tokens", type=str) + + publishing = asyncio.ensure_future(tokens.publish("a", wake=False)) + await codec.wait_inside('"a"') + abandoned = asyncio.ensure_future(tokens.finish_writing(wake=False)) + await _let_every_ready_task_run() + fencing = asyncio.ensure_future(tokens.finish_writing(wake=False)) + await _let_every_ready_task_run() + + abandoned.cancel() + with pytest.raises(asyncio.CancelledError): + await abandoned + assert not fencing.done(), ( + "the second fence appended while the publish it covers was still encoding" + ) + + codec.release('"a"') + await publishing + await fencing + + assert [(r.sequence, r.kind) for r in backend.all_records(TOKENS)] == [ + (0, RecordKind.DATA), + (2, RecordKind.WRITE_FENCE), + ], ( + "the surviving fence must append behind the data, and the cancelled one " + "must leave nothing in the stream" + ) + + +class RefusingRecoveryBackend(MemoryStreamBackend): + """Loses the answer to a data append, then refuses its recovery. + + Both halves are contract behaviour rather than a broken store: a backend + commits before it answers, so the answer can be lost, and + ``AppendConflictError`` is the one definite refusal the contract defines -- + the key it names holds different bytes, so the record being recovered did + not land and never will. Together they are the case where "no longer + unresolved" and "durable" are not the same thing, which is reachable from a + session-id collision or a retry that reused the sequence for other bytes. + """ + + def __init__(self) -> None: + super().__init__() + self.reached = asyncio.Event() + self.answer = asyncio.Event() + self.data_appends = 0 + + async def append(self, key, record): # type: ignore[no-untyped-def] + if record.kind is not RecordKind.DATA: + return await super().append(key, record) + self.data_appends += 1 + if self.data_appends == 1: + self.reached.set() + await self.answer.wait() + raise ConnectionError("connection reset") + # Refused without suspending, so nothing runs between the unresolved + # entry disappearing and the waiting fence reading the outcome. That is + # the window in which a stale "unknown" looked like durability. + raise AppendConflictError(record.idempotency_key) + + +async def test_a_fence_is_refused_when_recovery_proves_the_write_absent() -> None: + """A resolved append is not a durable one, so the fence may not infer it. + + An append whose answer was lost is unknown, not failed -- and the fence + waits rather than refusing outright. What ends the wait is + ``resolve_append()``, and it ends it *two* ways: the record is durable, or + the backend refuses the key and the record demonstrably never landed. + Deciding between them by whether the record is still in the producer's + unresolved set cannot work, because both outcomes remove it, and the fence + then appended over a hole while claiming the batch complete. + """ + backend = RefusingRecoveryBackend() + tokens = _offline_producer(backend, "batch").topic("tokens", type=str) + + async def publish_then_recover() -> None: + with pytest.raises(AppendNotAcknowledgedError) as lost: + await tokens.publish("a", wake=False) + with pytest.raises(AppendConflictError): + await tokens.resolve_append(lost.value.record, wake=False) + + publishing = asyncio.ensure_future(publish_then_recover()) + await asyncio.wait_for(backend.reached.wait(), 2) + fencing = asyncio.ensure_future(tokens.finish_writing(wake=False)) + await _let_every_ready_task_run() + + assert not fencing.done(), "the fence overtook an append of unknown outcome" + + backend.answer.set() + await publishing + with pytest.raises(PrecedingWriteFailedError) as refused: + await fencing + + assert refused.value.sequence == 0 and refused.value.stream_key == TOKENS + assert isinstance(refused.value.__cause__, AppendConflictError), ( + "the fence must carry the refusal recovery learned, not the unknown " + "outcome the interrupted call reported" + ) + assert backend.all_records(TOKENS) == [], ( + "a fence was appended claiming a write that recovery proved absent" + ) + + +async def test_a_fence_goes_in_once_recovery_makes_the_write_durable() -> None: + """The other half of the same decision, and the one that must not refuse. + + A recovery that finds the record already there resolves the unknown outcome + to *durable*, and the write the fence was waiting for is now in the stream + ahead of it. The fence has to be released, not refused: reporting + ``PrecedingWriteFailedError`` for a record that landed sends the caller to + republish a value the stream already holds. + """ + backend = CommittingThenFailingBackend() + tokens = _offline_producer(backend, "batch").topic("tokens", type=str) + + async def publish_then_recover() -> None: + with pytest.raises(AppendNotAcknowledgedError) as lost: + await tokens.publish("a", wake=False) + # The record did commit, so re-appending byte-identical content is the + # no-op the recovery depends on -- and it returns without suspending, so + # the fence's next turn is the first one after the resolution. + await tokens.resolve_append(lost.value.record, wake=False) + + publishing = asyncio.ensure_future(publish_then_recover()) + await asyncio.wait_for(backend.committed.wait(), 2) + fencing = asyncio.ensure_future(tokens.finish_writing(wake=False)) + await _let_every_ready_task_run() + + assert not fencing.done() + + backend.lose_the_answer.set() + await publishing + await fencing + + records = backend.all_records(TOKENS) + assert [(r.sequence, r.kind) for r in records] == [ + (0, RecordKind.DATA), + (1, RecordKind.WRITE_FENCE), + ], "the recovered record must be in the stream once, ahead of the fence" + + +class LosesOneAnswerThenRefusesIt(MemoryStreamBackend): + """Loses sequence 0's answer, refuses its recovery, and holds sequence 1. + + Two data writes precede one fence, and only the *second* is still in flight + when the first is recovered -- which is the schedule that decides whether the + fence reads an outcome or a memory of one. + """ + + def __init__(self) -> None: + super().__init__() + self.reached = asyncio.Event() + self.answer = asyncio.Event() + self.release_second = asyncio.Event() + + async def append(self, key, record): # type: ignore[no-untyped-def] + if record.kind is RecordKind.DATA and record.sequence == 0: + if not self.reached.is_set(): + self.reached.set() + await self.answer.wait() + raise ConnectionError("connection reset") + raise AppendConflictError(record.idempotency_key) + if record.kind is RecordKind.DATA and record.sequence == 1: + await self.release_second.wait() + return await super().append(key, record) + + +async def test_a_resolution_reached_while_the_fence_waits_on_a_later_write() -> None: + """An outcome read at wait time can still change; one read after cannot. + + The fence waits on each earlier publish in turn, so it can pass the first + one -- reading "unknown", which is not yet a failure -- and then sit on the + second while ``resolve_append()`` proves the first absent. An outcome + captured on the way past is the stale one, and stale here means permissive: + the fence skips a write recovery has refused and appends over the hole. + """ + backend = LosesOneAnswerThenRefusesIt() + tokens = _offline_producer(backend, "batch").topic("tokens", type=str) + + first = asyncio.ensure_future(tokens.publish("a", wake=False)) + await asyncio.wait_for(backend.reached.wait(), 2) + second = asyncio.ensure_future(tokens.publish("b", wake=False)) + await _let_every_ready_task_run() + fencing = asyncio.ensure_future(tokens.finish_writing(wake=False)) + await _let_every_ready_task_run() + + assert not fencing.done() + + backend.answer.set() + with pytest.raises(AppendNotAcknowledgedError) as lost: + await first + # Lets the fence take its turn on the *first* write and park on the second, + # which is the state this case is about. + await _let_every_ready_task_run() + assert not fencing.done(), "the fence must still be held by the second write" + + with pytest.raises(AppendConflictError): + await tokens.resolve_append(lost.value.record, wake=False) + + backend.release_second.set() + await second + + with pytest.raises(PrecedingWriteFailedError) as refused: + await fencing + + assert refused.value.sequence == 0, ( + "the fence must report the write recovery refused, not the one that landed" + ) + assert isinstance(refused.value.__cause__, AppendConflictError) + assert [(r.sequence, r.kind) for r in backend.all_records(TOKENS)] == [ + (1, RecordKind.DATA) + ], "only the second write may be in the stream, and no fence behind it" + + +async def test_republishing_different_content_under_one_key_is_an_error( + backend: MemoryStreamBackend, +) -> None: + """Silently accepting it would rewrite what a consumer may have delivered.""" + first = _offline_producer(backend, "same-session").topic("tokens", type=str) + await first.publish("a", wake=False) + + retried = _offline_producer(backend, "same-session").topic("tokens", type=str) + with pytest.raises(AppendConflictError): + await retried.publish("changed", wake=False) + + +async def test_sequences_are_per_connection_not_per_topic( + backend: MemoryStreamBackend, +) -> None: + """Two topics' first records must not collide on ``(session, 0)``.""" + producer = _offline_producer(backend) + tokens = producer.topic("tokens", type=str) + events = producer.topic("tool-events", type=str) + + await tokens.publish("a", wake=False) + await events.publish("b", wake=False) + + (token_record,) = backend.all_records(StreamKey("ns", "wf", "run-1", "tokens")) + (event_record,) = backend.all_records(StreamKey("ns", "wf", "run-1", "tool-events")) + assert token_record.sequence != event_record.sequence + + +async def test_a_topic_needs_a_name(backend: MemoryStreamBackend) -> None: + with pytest.raises(ValueError, match="non-empty name"): + _offline_producer(backend).topic("") + + +async def test_the_producer_handle_has_no_subscribe( + backend: MemoryStreamBackend, +) -> None: + """The two sides are mirror images, not one type used twice.""" + tokens = _offline_producer(backend).topic("tokens", type=str) + + assert not hasattr(tokens, "subscribe") + assert hasattr(tokens, "publish") + assert hasattr(tokens, "finish_writing") + + +async def test_a_standalone_read_back_sees_what_was_published( + backend: MemoryStreamBackend, +) -> None: + """The P6 criterion, without a Temporal server in the picture.""" + tokens = _offline_producer(backend).topic("tokens", type=str) + for value in ["a", "b", "c"]: + await tokens.publish(value, wake=False) + await tokens.finish_writing(wake=False) + + key = StreamKey("ns", "wf", "run-1", "tokens") + read_back = await backend.read_after(key, BEGINNING, max_records=100, block=None) + + import temporalio.converter + from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec + + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + values = [await codec.decode(r.payload) for r in read_back if not r.is_control] + assert values == ["a", "b", "c"] diff --git a/tests/contrib/external_workflow_streams/test_record.py b/tests/contrib/external_workflow_streams/test_record.py new file mode 100644 index 0000000000..680cd4632b --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_record.py @@ -0,0 +1,197 @@ +"""P1 — record model, offsets, cursor boundaries, control records.""" + +from __future__ import annotations + +import pytest + +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + IdempotencyKey, + Offset, + RecordKind, + StreamRecord, +) + + +def redis_like_compare(left: Offset, right: Offset) -> int: + """The ``(ms, seq)`` numeric ordering, used here to stand in for a provider.""" + + def parts(offset: Offset) -> tuple[int, int]: + ms, _, seq = offset.token.partition("-") + return int(ms), int(seq or 0) + + return (parts(left) > parts(right)) - (parts(left) < parts(right)) + + +# --- offsets ---------------------------------------------------------------- + + +def test_offset_round_trips() -> None: + offset = Offset("1700000000000-3") + assert Offset.deserialize(offset.serialize()) == offset + + +def test_offset_is_hashable_and_equatable() -> None: + assert Offset("1-0") == Offset("1-0") + assert len({Offset("1-0"), Offset("1-0"), Offset("1-1")}) == 2 + + +def test_offset_rejects_an_empty_token() -> None: + with pytest.raises(ValueError, match="may not be empty"): + Offset("") + + +def test_offset_does_not_expose_lexical_ordering() -> None: + """The failure this prevents is invisible until a millisecond width change. + + ``"9-0" < "10-0"`` is false lexically and true numerically, so an offset + that supported ``<`` would silently order wrongly for one provider in ten. + """ + with pytest.raises(TypeError): + _ = Offset("9-0") < Offset("10-0") # type: ignore[operator] + + assert redis_like_compare(Offset("9-0"), Offset("10-0")) < 0 + + +# --- cursors ---------------------------------------------------------------- + + +def test_cursor_forms_round_trip() -> None: + for cursor in (BEGINNING, AFTER(Offset("1700000000000-0"))): + assert Cursor.deserialize(cursor.serialize()) == cursor + + +def test_beginning_is_distinguishable_from_a_provider_beginning_sentinel() -> None: + """Redis spells its beginning ``0-0``; that is an offset, not ``BEGINNING``.""" + assert BEGINNING != AFTER(Offset("0-0")) + assert BEGINNING.serialize() != AFTER(Offset("0-0")).serialize() + assert BEGINNING.is_beginning + assert not AFTER(Offset("0-0")).is_beginning + + +def test_cursor_deserialize_rejects_junk() -> None: + with pytest.raises(ValueError, match="not a serialized cursor"): + Cursor.deserialize("1700000000000-0") + + +def test_after_excludes_up_to_and_including_its_offset() -> None: + cursor = AFTER(Offset("100-0")) + + assert cursor.excludes(Offset("99-9"), redis_like_compare) + assert cursor.excludes(Offset("100-0"), redis_like_compare) + assert not cursor.excludes(Offset("100-1"), redis_like_compare) + + +def test_beginning_excludes_nothing() -> None: + assert not BEGINNING.excludes(Offset("0-1"), redis_like_compare) + + +def test_after_names_a_boundary_past_the_tail() -> None: + """A consumer parked at the tail names the last record, never the next one. + + Nothing here requires the record after ``100-0`` to exist -- which is the + whole point of a boundary cursor. + """ + assert str(AFTER(Offset("100-0"))) == "AFTER(100-0)" + + +# --- records ---------------------------------------------------------------- + + +def test_record_round_trips_through_fields() -> None: + record = StreamRecord( + kind=RecordKind.DATA, + payload=b"\x00\xffhello", + producer_session_id="session-a", + sequence=7, + ).placed_at(Offset("100-0")) + + assert StreamRecord.from_fields(Offset("100-0"), record.to_fields()) == record + + +def test_record_round_trips_from_bytes_keyed_fields() -> None: + """A provider that did not decode its responses must still round-trip.""" + record = StreamRecord(RecordKind.DATA, b"x", "session-a", 0).placed_at( + Offset("100-0") + ) + raw = {key.encode(): value for key, value in record.to_fields().items()} + + assert StreamRecord.from_fields(Offset("100-0"), raw) == record + + +def test_fence_round_trips_and_carries_no_payload() -> None: + fence = StreamRecord(RecordKind.WRITE_FENCE, b"", "session-a", 8).placed_at( + Offset("101-0") + ) + + assert fence.is_control + assert StreamRecord.from_fields(Offset("101-0"), fence.to_fields()) == fence + + +def test_a_fence_with_a_payload_is_rejected() -> None: + with pytest.raises(ValueError, match="carries no payload"): + StreamRecord(RecordKind.WRITE_FENCE, b"nope", "session-a", 0) + + +def test_from_fields_names_what_is_missing() -> None: + with pytest.raises(ValueError, match="missing field"): + StreamRecord.from_fields(Offset("1-0"), {"__tes_kind": b"1"}) + + +def test_data_and_control_share_one_offset_sequence() -> None: + """The fence sits between data records and advances the cursor like them.""" + placed = [ + StreamRecord(RecordKind.DATA, b"a", "s", 0).placed_at(Offset("100-0")), + StreamRecord(RecordKind.DATA, b"b", "s", 1).placed_at(Offset("100-1")), + StreamRecord(RecordKind.WRITE_FENCE, b"", "s", 2).placed_at(Offset("100-2")), + StreamRecord(RecordKind.DATA, b"c", "other", 0).placed_at(Offset("100-3")), + ] + + offsets = [r.offset for r in placed] + assert all( + redis_like_compare(a, b) < 0 # type: ignore[arg-type] + for a, b in zip(offsets, offsets[1:]) + ) + assert [r.is_control for r in placed] == [False, False, True, False] + + +def test_record_offset_is_absent_until_the_provider_places_it() -> None: + unplaced = StreamRecord(RecordKind.DATA, b"a", "s", 0) + + assert unplaced.offset is None + assert unplaced.placed_at(Offset("1-0")).offset == Offset("1-0") + + +def test_to_fields_excludes_the_offset() -> None: + """Two appends under one idempotency key must be byte-comparable. + + Including the offset would make every re-append look like different bytes, + turning the idempotent retry path into a spurious conflict. + """ + record = StreamRecord(RecordKind.DATA, b"a", "s", 0) + + assert record.to_fields() == record.placed_at(Offset("1-0")).to_fields() + + +# --- idempotency keys ------------------------------------------------------- + + +def test_idempotency_key_comes_from_session_and_sequence() -> None: + record = StreamRecord(RecordKind.DATA, b"a", "session-a", 4) + + assert record.idempotency_key == IdempotencyKey("session-a", 4) + + +def test_idempotency_key_rejects_a_missing_session() -> None: + with pytest.raises(ValueError, match="session id may not be empty"): + IdempotencyKey("", 0) + + +@pytest.mark.parametrize("bad", [-1, -100]) +def test_negative_sequences_are_rejected(bad: int) -> None: + with pytest.raises(ValueError, match="non-negative"): + IdempotencyKey("s", bad) + with pytest.raises(ValueError, match="non-negative"): + StreamRecord(RecordKind.DATA, b"a", "s", bad) diff --git a/tests/contrib/external_workflow_streams/test_redis_backend.py b/tests/contrib/external_workflow_streams/test_redis_backend.py new file mode 100644 index 0000000000..e1e0bcb792 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_redis_backend.py @@ -0,0 +1,480 @@ +"""P3/P3b — the Redis provider, against the conformance suite unmodified. + +"Unmodified" is the point: if a check had to be relaxed for Redis, it would be +encoding this provider's behaviour rather than the contract, and the next +provider would inherit the exemption. +""" + +from __future__ import annotations + +import uuid +from collections.abc import AsyncGenerator +from datetime import timedelta + +import pytest +import pytest_asyncio + +from temporalio.contrib.external_workflow_streams._backend import ParkIntent, StreamKey +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._redis import RedisStreamBackend +from tests.contrib.external_workflow_streams.conformance import ( + CONFORMANCE_CHECKS, + PARKING_CONFORMANCE_CHECKS, + Check, +) +from tests.contrib.external_workflow_streams.conftest import ( + KEY_NAMESPACE, + redis_url, +) + + +@pytest_asyncio.fixture # type: ignore[reportUntypedFunctionDecorator] +async def redis_backend( + redis_worker_id: str, +) -> AsyncGenerator[RedisStreamBackend, None]: + """A backend whose keys all sit under this test's own prefix. + + The prefix is the isolation: one Redis serves the whole suite, and two + xdist workers running this module must not see each other's streams. + """ + pytest.importorskip("redis.asyncio", reason="redis is not installed") + backend = RedisStreamBackend( + url=redis_url(), + key_prefix=f"{KEY_NAMESPACE}:{redis_worker_id}:{uuid.uuid4().hex}", + ) + try: + await backend._client.ping() + except Exception as err: + await backend.aclose() + pytest.skip(f"Redis is not reachable at {redis_url()}: {err}") + + try: + yield backend + finally: + keys = [ + k async for k in backend._client.scan_iter(match=f"{backend._key_prefix}*") + ] + if keys: + await backend._client.delete(*keys) + await backend.aclose() + + +@pytest.fixture +def stream_key() -> StreamKey: + return StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + + +# --- the conformance suites, unmodified ------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("check", CONFORMANCE_CHECKS, ids=lambda c: c.__name__) +async def test_redis_passes_the_core_conformance_suite( + check: Check, redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + await check(redis_backend, stream_key) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("check", PARKING_CONFORMANCE_CHECKS, ids=lambda c: c.__name__) +async def test_redis_passes_the_parking_conformance_suite( + check: Check, redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + await check(redis_backend, stream_key) + + +# --- Redis specifics the suite cannot state --------------------------------- + + +@pytest.mark.asyncio +async def test_offsets_are_real_redis_ids( + redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + placed = await redis_backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"a", "s", 0) + ) + + assert placed.offset is not None + ms, _, seq = placed.offset.token.partition("-") + assert ms.isdigit() and seq.isdigit(), ( + f"a Redis offset is -, got {placed.offset.token!r}" + ) + + +@pytest.mark.asyncio +async def test_xrange_and_xread_disagree_exactly_as_the_contract_says( + redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + """The reason replay must not use XREAD, asserted against a real Redis. + + XRANGE includes the id it is given; XREAD starts strictly after it. A + provider that reached for XREAD to serve the replay read would drop the + first record of every recorded range -- and nothing before the first replay + would notice. + """ + first = await redis_backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"a", "s", 0) + ) + second = await redis_backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"b", "s", 1) + ) + assert first.offset is not None and second.offset is not None + + inclusive = await redis_backend.read_range(stream_key, first.offset, second.offset) + exclusive = await redis_backend.read_after( + stream_key, AFTER(first.offset), max_records=10, block=None + ) + + assert [r.payload for r in inclusive] == [b"a", b"b"] + assert [r.payload for r in exclusive] == [b"b"] + + +@pytest.mark.asyncio +async def test_beginning_reads_from_the_zero_sentinel( + redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + """`BEGINNING` is a boundary; `0-0` is how Redis spells that boundary. + + They are not the same thing, which is why the cursor type keeps them apart + -- but the provider has to know the translation. + """ + await redis_backend.append(stream_key, StreamRecord(RecordKind.DATA, b"a", "s", 0)) + + from_beginning = await redis_backend.read_after( + stream_key, BEGINNING, max_records=10, block=None + ) + assert [r.payload for r in from_beginning] == [b"a"] + + +@pytest.mark.asyncio +async def test_binary_payloads_survive_the_round_trip( + redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + """A decoding client would corrupt these, so the provider must not use one.""" + payload = bytes(range(256)) + placed = await redis_backend.append( + stream_key, StreamRecord(RecordKind.DATA, payload, "s", 0) + ) + + assert placed.offset is not None + (got,) = await redis_backend.read_range(stream_key, placed.offset, placed.offset) + assert got.payload == payload + + +@pytest.mark.asyncio +async def test_an_idempotent_reappend_writes_nothing_to_the_stream( + redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + """Not merely "returns the same offset" -- no second entry may exist. + + Asserted at the Redis level because a provider could return the original + offset while still having XADDed a duplicate, which replay would then find + inside a recorded range and count as an integrity failure. + """ + record = StreamRecord(RecordKind.DATA, b"payload", "retry", 0) + await redis_backend.append(stream_key, record) + await redis_backend.append(stream_key, record) + + length = await redis_backend._client.xlen(redis_backend.stream_key(stream_key)) + assert length == 1 + + +@pytest.mark.asyncio +async def test_a_blocking_read_returns_empty_rather_than_hanging( + redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + """`XREAD BLOCK 0` blocks forever, so a zero timeout must omit BLOCK.""" + got = await redis_backend.read_after( + stream_key, BEGINNING, max_records=10, block=timedelta(0) + ) + assert got == [] + + +@pytest.mark.asyncio +async def test_offsets_compare_numerically_not_lexically( + redis_backend: RedisStreamBackend, +) -> None: + """Real ids cross the width boundary; the comparator must not care.""" + assert redis_backend.compare_offsets(Offset("9-0"), Offset("10-0")) < 0 + assert ( + redis_backend.compare_offsets( + Offset("1700000000000-1"), Offset("1700000000000-2") + ) + < 0 + ) + assert redis_backend.compare_offsets(Offset("100-0"), Offset("100-0")) == 0 + + +# --- retention loss against real Redis --------------------------------------- + + +@pytest.mark.asyncio +async def test_a_trimmed_range_is_integrity_loss_not_a_storage_failure( + redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + """`XTRIM MAXLEN` is how a recorded range actually disappears in production. + + Not a hypothetical: a stream with a retention policy trims its own head + while a Workflow is still parked against it. That must read as integrity + loss -- an operator has to restore or terminate -- and never as a transient + storage failure, which would clear on its own and so would be retried + forever against data that is not coming back. + """ + from temporalio.contrib.external_workflow_streams._annotation import Run + from temporalio.contrib.external_workflow_streams._errors import ( + StreamIntegrityError, + ) + from temporalio.contrib.external_workflow_streams._replay import validate_run + + placed = [] + for i in range(5): + placed.append( + await redis_backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"x", "producer", i) + ) + ) + run = Run(1, placed[0].offset, placed[-1].offset, 5) # type: ignore[arg-type] + + # The head of the recorded range is trimmed away, exactly as a retention + # policy would do it. + await redis_backend._client.xtrim( + redis_backend.stream_key(stream_key), maxlen=2, approximate=False + ) + survivors = await redis_backend.read_range( + stream_key, + placed[0].offset, # type: ignore[arg-type] + placed[-1].offset, # type: ignore[arg-type] + ) + + with pytest.raises(StreamIntegrityError) as caught: + validate_run(run, survivors, redis_backend) + + assert "missing from the stream" in str(caught.value) + + +@pytest.mark.asyncio +async def test_a_deleted_write_fence_is_integrity_loss( + redis_backend: RedisStreamBackend, stream_key: StreamKey +) -> None: + """A control record occupies an offset inside a run, so losing one is loss. + + It is never yielded to Workflow code, which is exactly why it could be + dismissed as harmless -- but the run's count includes it and its position is + recorded, so a range missing it no longer reads back as written. Letting it + pass would also shift every later control position by one. + """ + from temporalio.contrib.external_workflow_streams._annotation import Run + from temporalio.contrib.external_workflow_streams._errors import ( + StreamIntegrityError, + ) + from temporalio.contrib.external_workflow_streams._replay import validate_run + + first = await redis_backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"a", "producer", 0) + ) + fence = await redis_backend.append( + stream_key, StreamRecord(RecordKind.WRITE_FENCE, b"", "producer", 1) + ) + last = await redis_backend.append( + stream_key, StreamRecord(RecordKind.DATA, b"b", "producer", 2) + ) + run = Run(1, first.offset, last.offset, 3, control_positions=(1,)) # type: ignore[arg-type] + + await redis_backend.delete_for_test(stream_key, fence.offset) # type: ignore[arg-type] + survivors = await redis_backend.read_range( + stream_key, + first.offset, # type: ignore[arg-type] + last.offset, # type: ignore[arg-type] + ) + + with pytest.raises(StreamIntegrityError) as caught: + validate_run(run, survivors, redis_backend) + + assert "contains 2 record(s)" in str(caught.value), ( + f"a deleted fence must fail the count check, got: {caught.value}" + ) + + +# --- key injectivity --------------------------------------------------------- + + +def _colliding_identities() -> tuple[StreamKey, StreamKey]: + """Two distinct identities a delimiter-joined key layout renders identically. + + Nothing here is exotic: a Workflow ID and a stream name are user-chosen + strings, and `:` is an ordinary character in both. + """ + first_run, second_run = uuid.uuid4().hex, uuid.uuid4().hex + return ( + StreamKey("ns", "wf", first_run, f"{second_run}:tokens"), + StreamKey("ns", f"wf:{first_run}", second_run, "tokens"), + ) + + +@pytest.mark.asyncio +async def test_distinct_identities_never_share_one_physical_key( + redis_backend: RedisStreamBackend, +) -> None: + """Every derivation, not only the stream: they all come from `stream_key`. + + Two `StreamKey`s that are not equal are two streams, and nothing they own -- + records, idempotency hashes, park intents, claims -- may land in one place. + Two Workflows sharing one data structure is a cross-Workflow leak, and it + reads as ordinary corruption rather than as a key-layout bug. + """ + first, second = _colliding_identities() + assert first != second + + for name, derive in ( + ("stream", redis_backend.stream_key), + ("idempotency", redis_backend._idempotency_key), + ("park intent", lambda k: redis_backend._intent_key(k, 1)), + ("claim", lambda k: redis_backend._claim_key(k, 1)), + ): + assert derive(first) != derive(second), ( + f"the {name} key is not injective: {first} and {second} both " + f"render as {derive(first)!r}" + ) + + +@pytest.mark.asyncio +async def test_colliding_identities_hold_isolated_records( + redis_backend: RedisStreamBackend, +) -> None: + """The consequence, asserted through the public operations rather than keys.""" + first, second = _colliding_identities() + + await redis_backend.append( + first, StreamRecord(RecordKind.DATA, b"first", "producer-first", 0) + ) + await redis_backend.append( + second, StreamRecord(RecordKind.DATA, b"second", "producer-second", 0) + ) + + from_first = await redis_backend.read_after( + first, BEGINNING, max_records=10, block=None + ) + from_second = await redis_backend.read_after( + second, BEGINNING, max_records=10, block=None + ) + + assert [r.payload for r in from_first] == [b"first"] + assert [r.payload for r in from_second] == [b"second"] + + +@pytest.mark.asyncio +async def test_colliding_identities_hold_isolated_park_state( + redis_backend: RedisStreamBackend, +) -> None: + """An intent and a claim installed on one must be invisible on the other. + + A shared claim key is worse than a shared stream: the second Workflow's + producer reads a claim it never made, concludes the wake is someone else's, + and stays silent -- so the Run is never woken at all. + """ + first, second = _colliding_identities() + + await redis_backend.install_park_intent( + first, ParkIntent(wait_id=1, cursor=BEGINNING, park_generation=3, run_id="r") + ) + + assert await redis_backend.park_intent(second, 1) is None + assert await redis_backend.current_park_generation(second, 1) is None + assert await redis_backend.parked_wait_ids(second) == [] + assert await redis_backend.parked_wait_ids(first) == [1] + + lease = timedelta(seconds=30) + assert await redis_backend.claim_park_generation( + first, 1, 3, claimant="producer-first", lease=lease + ) + assert await redis_backend.claim_park_generation( + second, 1, 3, claimant="producer-second", lease=lease + ), "a claim on one identity's generation must say nothing about another's" + + +@pytest.mark.parametrize( + ("metacharacter", "impostor_name"), + [ + # Each impostor name is a *different* stream whose rendering, read as a + # `SCAN MATCH` glob, matches the victim's park keys. The lengths are + # chosen so the suffix a naive implementation slices off still parses as + # a wait id -- otherwise the leak hides behind an incidental filter. + ("*", "token*"), + ("?", "token?"), + ("[", "toke[n]s"), + ], + ids=["star", "question", "bracket"], +) +@pytest.mark.asyncio +async def test_glob_metacharacters_cannot_enumerate_another_streams_intents( + redis_backend: RedisStreamBackend, + metacharacter: str, + impostor_name: str, +) -> None: + """`parked_wait_ids` must answer about one stream, whatever the name spells. + + The key goes into a `SCAN MATCH` pattern, so an unescaped `*`, `?` or `[` in + a user-chosen stream name turns the enumeration into a wildcard over the + keyspace -- and the producer then wakes, claims and rechecks waits that + belong to a different Workflow. + """ + assert metacharacter in impostor_name + run_id = uuid.uuid4().hex + victim = StreamKey("ns", "wf", run_id, "tokens") + impostor = StreamKey("ns", "wf", run_id, impostor_name) + + for wait_id in (101, 202): + await redis_backend.install_park_intent( + victim, + ParkIntent( + wait_id=wait_id, cursor=BEGINNING, park_generation=1, run_id="r" + ), + ) + await redis_backend.install_park_intent( + impostor, ParkIntent(wait_id=7, cursor=BEGINNING, park_generation=1, run_id="r") + ) + + assert await redis_backend.parked_wait_ids(impostor) == [7] + assert await redis_backend.parked_wait_ids(victim) == [101, 202] + + +@pytest.mark.asyncio +async def test_a_glob_metacharacter_in_the_key_prefix_does_not_widen_the_scan( + redis_backend: RedisStreamBackend, +) -> None: + """The prefix is the operator's, not the identity's, so encoding cannot fix it. + + `key_prefix` separates deployments sharing one Redis and is passed through + verbatim -- which is what makes it readable and what makes it the one place + a `SCAN MATCH` pattern can still be widened. The two prefixes here are the + same length so a leaked key's suffix still parses as a wait id; a shorter + impostor would be hidden by that filter rather than by the escaping. + """ + base = redis_backend._key_prefix # cleaned up by the fixture's scan + victim_backend = RedisStreamBackend( + client=redis_backend._client, key_prefix=f"{base}:px" + ) + impostor_backend = RedisStreamBackend( + client=redis_backend._client, key_prefix=f"{base}:p*" + ) + key = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + + for wait_id in (101, 202): + await victim_backend.install_park_intent( + key, + ParkIntent( + wait_id=wait_id, cursor=BEGINNING, park_generation=1, run_id="r" + ), + ) + await impostor_backend.install_park_intent( + key, ParkIntent(wait_id=7, cursor=BEGINNING, park_generation=1, run_id="r") + ) + + assert await impostor_backend.parked_wait_ids(key) == [7] + assert await victim_backend.parked_wait_ids(key) == [101, 202] diff --git a/tests/contrib/external_workflow_streams/test_redis_fixture.py b/tests/contrib/external_workflow_streams/test_redis_fixture.py new file mode 100644 index 0000000000..afe0c0299e --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_redis_fixture.py @@ -0,0 +1,62 @@ +"""X2 — the Redis fixture itself, exercised through XADD/XREAD.""" + +from __future__ import annotations + +import pytest + +from tests.contrib.external_workflow_streams.conftest import ( + KEY_NAMESPACE, + RedisKeyspace, +) + + +@pytest.mark.asyncio +async def test_xadd_and_xread_through_the_fixture( + redis_keyspace: RedisKeyspace, +) -> None: + stream = redis_keyspace.key("tokens") + + first = await redis_keyspace.client.xadd(stream, {"payload": "a"}) # type: ignore[attr-defined] + second = await redis_keyspace.client.xadd(stream, {"payload": "b"}) # type: ignore[attr-defined] + + # XRANGE is inclusive of both endpoints -- this is the read replay uses. + entries = await redis_keyspace.client.xrange(stream, first, second) # type: ignore[attr-defined] + assert [fields["payload"] for _, fields in entries] == ["a", "b"] + + # XREAD is exclusive of the supplied id -- this is the read watching uses. + after_first = await redis_keyspace.client.xread({stream: first}) # type: ignore[attr-defined] + assert [fields["payload"] for _, fields in after_first[0][1]] == ["b"] + + +@pytest.mark.asyncio +async def test_keys_are_namespaced_to_this_test(redis_keyspace: RedisKeyspace) -> None: + assert redis_keyspace.key("tokens").startswith(f"{KEY_NAMESPACE}:") + assert redis_keyspace.prefix.endswith(":") + + +@pytest.mark.asyncio +async def test_two_keyspaces_do_not_collide( + redis_keyspace: RedisKeyspace, redis_client: object +) -> None: + """A second keyspace over the same server sees none of the first's keys. + + This is the property xdist parallelism depends on; asserting it here means + a regression shows up as a fixture failure rather than as a flaky suite. + """ + other = RedisKeyspace(client=redis_client, prefix=f"{KEY_NAMESPACE}:other:x:") + try: + await redis_keyspace.client.xadd(redis_keyspace.key("s"), {"v": "1"}) # type: ignore[attr-defined] + assert await redis_client.exists(other.key("s")) == 0 # type: ignore[attr-defined] + finally: + await other.cleanup() + + +@pytest.mark.asyncio +async def test_cleanup_removes_every_key( + redis_keyspace: RedisKeyspace, redis_client: object +) -> None: + stream = redis_keyspace.key("tokens") + await redis_keyspace.client.xadd(stream, {"payload": "a"}) # type: ignore[attr-defined] + + assert await redis_keyspace.cleanup() == 1 + assert await redis_client.exists(stream) == 0 # type: ignore[attr-defined] diff --git a/tests/contrib/external_workflow_streams/test_redis_output.py b/tests/contrib/external_workflow_streams/test_redis_output.py new file mode 100644 index 0000000000..9b0036a2f3 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_redis_output.py @@ -0,0 +1,284 @@ +"""Redis output staging, terminal transitions, and read-barrier behavior.""" + +from __future__ import annotations + +import dataclasses +import uuid +from collections.abc import AsyncGenerator + +import pytest +import pytest_asyncio + +from temporalio.contrib.external_workflow_streams._backend import ( + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._errors import StreamIntegrityError +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputStageConflictError, + OutputStageManifest, + OutputStageResolutionError, + OutputStageStatus, + StagedOutputRecord, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._redis import RedisStreamBackend +from tests.contrib.external_workflow_streams.conftest import ( + KEY_NAMESPACE, + redis_url, +) + + +@pytest_asyncio.fixture # type: ignore[reportUntypedFunctionDecorator] +async def output_backend( + redis_worker_id: str, +) -> AsyncGenerator[RedisStreamBackend, None]: + pytest.importorskip("redis.asyncio", reason="redis is not installed") + backend = RedisStreamBackend( + url=redis_url(), + key_prefix=f"{KEY_NAMESPACE}:{redis_worker_id}:{uuid.uuid4().hex}", + ) + try: + await backend._client.ping() + except Exception as err: + await backend.aclose() + pytest.skip(f"Redis is not reachable at {redis_url()}: {err}") + + try: + yield backend + finally: + keys = [ + key + async for key in backend._client.scan_iter(match=f"{backend._key_prefix}*") + ] + if keys: + await backend._client.delete(*keys) + await backend.aclose() + + +@pytest.fixture +def output_key() -> StreamKey: + return StreamKey( + "ns", + "wf", + uuid.uuid4().hex, + "events", + StreamDirection.OUTPUT, + ) + + +def manifest( + key: StreamKey, + *, + token: str = "stage-1", + sub_batch_id: int = 0, + fingerprint: bytes = b"f" * 32, + record_count: int = 2, +) -> OutputStageManifest: + return OutputStageManifest( + stream_key=key, + provider_id="redis-streams", + provider_format_version=1, + stage_token=token, + run_id="run-1", + history_floor_event_id=10, + sub_batch_id=sub_batch_id, + fingerprint_version=1, + fingerprint=fingerprint, + record_count=record_count, + logical_byte_count=20, + ) + + +def staged(*payloads: bytes) -> tuple[StagedOutputRecord, ...]: + return tuple( + StagedOutputRecord(index, RecordKind.DATA, payload) + for index, payload in enumerate(payloads) + ) + + +def direct(payload: bytes, *, sequence: int = 0) -> StreamRecord: + return StreamRecord( + RecordKind.DATA, + payload, + producer_session_id="activity", + sequence=sequence, + ) + + +@pytest.mark.parametrize( + ("terminal", "status"), + [ + ("commit", OutputStageStatus.COMMITTED), + ("abort", OutputStageStatus.ABORTED), + ], +) +async def test_exact_stage_retry_returns_actual_status_after_terminal( + output_backend: RedisStreamBackend, + output_key: StreamKey, + terminal: str, + status: OutputStageStatus, +) -> None: + batch = manifest(output_key) + records = staged(b"one", b"two") + + first = await output_backend.stage_output(batch, records) + repeated_pending = await output_backend.stage_output(batch, records) + resolved = await getattr(output_backend, f"{terminal}_output")(batch) + repeated_resolution = await getattr(output_backend, f"{terminal}_output")(batch) + repeated_terminal = await output_backend.stage_output( + batch, + # Encoded bytes may vary under a randomized PayloadCodec. The immutable + # logical manifest makes this the same operation, so Redis must retain + # and return the first successfully staged bytes. + staged(b"random-one", b"random-two"), + ) + + assert first.status is OutputStageStatus.PENDING + assert repeated_pending == first + assert resolved.status is status + assert repeated_resolution == resolved + assert repeated_terminal.status is status + assert repeated_terminal.records == first.records + assert [record.payload for record in repeated_terminal.records] == [ + b"one", + b"two", + ] + assert await output_backend.output_stage(batch) == repeated_terminal + + +async def test_reusing_stage_identity_with_another_manifest_conflicts_atomically( + output_backend: RedisStreamBackend, + output_key: StreamKey, +) -> None: + original = manifest(output_key) + first = await output_backend.stage_output(original, staged(b"one", b"two")) + conflicting = dataclasses.replace(original, fingerprint=b"x" * 32) + + with pytest.raises(OutputStageConflictError): + await output_backend.stage_output( + conflicting, staged(b"other-one", b"other-two") + ) + + assert await output_backend.output_stage(original) == first + entries = await output_backend._client.xrange(output_backend.stream_key(output_key)) + assert entries is not None + assert len(entries) == 2 + + +async def test_pending_stage_blocks_later_direct_output_until_commit( + output_backend: RedisStreamBackend, + output_key: StreamKey, +) -> None: + batch = manifest(output_key, record_count=1) + pending = await output_backend.stage_output(batch, staged(b"workflow")) + later = await output_backend.append_output(output_key, direct(b"activity")) + + blocked = await output_backend.read_output_after( + output_key, BEGINNING, max_records=10, block=None + ) + assert blocked.records == () + assert blocked.pending is not None + assert blocked.pending.manifest == batch + assert blocked.pending.offset == pending.records[0].offset + assert await output_backend.output_tail(output_key) == BEGINNING + + await output_backend.commit_output(batch) + visible = await output_backend.read_output_after( + output_key, BEGINNING, max_records=10, block=None + ) + assert [record.payload for record in visible.records] == [ + b"workflow", + b"activity", + ] + assert visible.records[-1].offset == later.offset + assert visible.pending is None + + +async def test_abort_skips_stage_and_cannot_be_reversed( + output_backend: RedisStreamBackend, + output_key: StreamKey, +) -> None: + batch = manifest(output_key, record_count=1) + staged_batch = await output_backend.stage_output(batch, staged(b"discarded")) + later = await output_backend.append_output(output_key, direct(b"visible")) + + aborted = await output_backend.abort_output(batch) + repeated = await output_backend.abort_output(batch) + with pytest.raises(OutputStageResolutionError): + await output_backend.commit_output(batch) + + visible = await output_backend.read_output_after( + output_key, BEGINNING, max_records=10, block=None + ) + assert aborted.status is OutputStageStatus.ABORTED + assert repeated == aborted + assert aborted.records == staged_batch.records + assert [record.payload for record in visible.records] == [b"visible"] + assert await output_backend.output_tail(output_key) == AFTER(later.offset) + + +async def test_missing_stage_metadata_is_an_integrity_failure( + output_backend: RedisStreamBackend, + output_key: StreamKey, +) -> None: + batch = manifest(output_key, record_count=1) + await output_backend.stage_output(batch, staged(b"workflow")) + stage_id = f"{len(batch.stage_token)}:{batch.stage_token}:{batch.sub_batch_id}" + await output_backend._client.hdel( + output_backend._output_status_key(output_key), stage_id + ) + + with pytest.raises(StreamIntegrityError, match="no stage status"): + await output_backend.read_output_after( + output_key, BEGINNING, max_records=10, block=None + ) + with pytest.raises(StreamIntegrityError, match="missing its status"): + await output_backend.output_stage(batch) + + +async def test_missing_staged_record_is_an_integrity_failure( + output_backend: RedisStreamBackend, + output_key: StreamKey, +) -> None: + batch = manifest(output_key, record_count=1) + stage = await output_backend.stage_output(batch, staged(b"workflow")) + await output_backend._client.xdel( + output_backend.stream_key(output_key), stage.records[0].offset.serialize() + ) + + with pytest.raises(StreamIntegrityError, match="record.*missing"): + await output_backend.output_stage(batch) + + +async def test_input_and_output_with_same_user_identity_are_physically_isolated( + output_backend: RedisStreamBackend, + output_key: StreamKey, +) -> None: + input_key = dataclasses.replace(output_key, direction=StreamDirection.INPUT) + input_record = direct(b"input") + output_record = direct(b"output") + + placed_input = await output_backend.append(input_key, input_record) + placed_output = await output_backend.append_output(output_key, output_record) + + input_records = await output_backend.read_after( + input_key, BEGINNING, max_records=10, block=None + ) + output_records = await output_backend.read_output_after( + output_key, BEGINNING, max_records=10, block=None + ) + assert output_backend.stream_key(input_key) != output_backend.stream_key(output_key) + assert [record.payload for record in input_records] == [b"input"] + assert placed_input.offset == input_records[0].offset + assert [record.payload for record in output_records.records] == [b"output"] + assert placed_output.offset == output_records.records[0].offset + with pytest.raises(ValueError, match="OUTPUT"): + await output_backend.append_output( + input_key, direct(b"wrong-direction", sequence=1) + ) diff --git a/tests/contrib/external_workflow_streams/test_registry.py b/tests/contrib/external_workflow_streams/test_registry.py new file mode 100644 index 0000000000..d18d658b4d --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_registry.py @@ -0,0 +1,228 @@ +"""P17 — validation of the Worker's stream backend.""" + +from __future__ import annotations + +import uuid + +import pytest + +from temporalio import workflow +from temporalio.client import Client +from temporalio.contrib import external_workflow_streams +from temporalio.contrib.external_workflow_streams import ( + RedisStreamBackend, + StreamBackend, + external_stream, +) +from temporalio.contrib.external_workflow_streams._api import ( + external_stream as private_external_stream, +) +from temporalio.contrib.external_workflow_streams._backend import ( + _validate_backend, +) +from temporalio.contrib.external_workflow_streams._redis import ( + RedisStreamBackend as PrivateRedisStreamBackend, +) +from temporalio.worker import Worker +from temporalio.worker.workflow_sandbox._restrictions import SandboxRestrictions +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + + +# These tests exercise Worker construction, not the sandbox. Keeping this +# unsandboxed avoids re-importing this pytest assertion-rewritten module while +# the constructor validates its registered workflows. +@workflow.defn(sandboxed=False) +class NoOpWorkflow: + """Workers need at least one registered workflow to be constructible.""" + + @workflow.run + async def run(self) -> None: + pass + + +class UndeclaredBackend(MemoryStreamBackend): + """Never considered whether its records can change.""" + + guarantees_immutability = None + + +class DeniedBackend(MemoryStreamBackend): + """Considered it and cannot make the guarantee.""" + + guarantees_immutability = False + + +class AnonymousBackend(MemoryStreamBackend): + """Conforming, but nameless in the annotation header.""" + + provider_id = "" + + +# --- public package --------------------------------------------------------- + + +def test_supported_entry_points_are_public() -> None: + assert external_stream is private_external_stream + assert RedisStreamBackend is PrivateRedisStreamBackend + assert issubclass(RedisStreamBackend, StreamBackend) + assert set(external_workflow_streams.__all__) == { + "AFTER", + "BEGINNING", + "AppendConflictError", + "AppendNotAcknowledgedError", + "ChainKeyMismatchError", + "ConcurrentStreamConsumerError", + "Cursor", + "ExternalStreamCapacityError", + "ExternalOutputStreamClient", + "ExternalOutputStreamClientTopic", + "ExternalOutputStreamItem", + "ExternalOutputStreamOptions", + "ExternalOutputStreamProducer", + "ExternalOutputStreamProducerTopic", + "ExternalOutputStreamTopic", + "ExternalStreamOptions", + "ExternalStreamProducer", + "ExternalStreamProducerTopic", + "ExternalStreamSubscription", + "ExternalStreamTopic", + "IdempotencyKey", + "Offset", + "OffsetComparator", + "OutputAppendNotAcknowledgedError", + "OutputReadResult", + "OutputStage", + "OutputStageConflictError", + "OutputStageManifest", + "OutputStageNotFoundError", + "OutputStageResolutionError", + "OutputStageStatus", + "OutputStreamBackend", + "OutputStreamRecord", + "ParkIntent", + "ParkIntentRemoval", + "PrecedingWriteFailedError", + "RecordKind", + "RedisStreamBackend", + "StartAtTail", + "StreamBackend", + "StreamDecodeError", + "StreamError", + "StreamIntegrityError", + "StreamDirection", + "StreamKey", + "StreamPayloadCodec", + "StreamRecord", + "StreamStorageError", + "PendingOutputBarrier", + "StagedOutputRecord", + "WakeNotAcknowledgedError", + "WakeRequest", + "WorkflowChainKey", + "external_stream", + "external_output_stream", + "merge", + } + + +# --- backend validation ----------------------------------------------------- + + +def test_a_conforming_backend_is_accepted() -> None: + backend = MemoryStreamBackend() + + assert _validate_backend(backend) is backend + + +@pytest.mark.parametrize( + "backend_type", [UndeclaredBackend, DeniedBackend], ids=["undeclared", "denied"] +) +def test_a_backend_without_the_guarantee_is_rejected( + backend_type: type[MemoryStreamBackend], +) -> None: + """One message for "forgot" and "cannot": both break the same thing. + + Replay validates presence, count, order, and control positions only, and + that is sufficient *because* the bytes cannot change. A provider that + cannot promise it has not satisfied the contract, however it got there. + """ + with pytest.raises(ValueError) as caught: + _validate_backend(backend_type()) + + message = str(caught.value) + assert "guarantees_immutability" in message + assert "cannot change" in message + + +def test_a_backend_without_a_provider_id_is_rejected() -> None: + with pytest.raises(ValueError, match="no provider_id"): + _validate_backend(AnonymousBackend()) + + +def test_a_non_backend_is_rejected() -> None: + with pytest.raises(TypeError, match="not a StreamBackend"): + _validate_backend(object()) + + +# --- through Worker construction -------------------------------------------- + + +async def test_a_conforming_backend_registers_on_a_worker(client: Client) -> None: + async with Worker( + client, + task_queue=f"tq-{uuid.uuid4()}", + workflows=[NoOpWorkflow], + external_stream_backend=MemoryStreamBackend(), + ) as worker: + assert isinstance(worker._external_stream_backend, MemoryStreamBackend) + + +async def test_a_backend_without_the_guarantee_fails_worker_construction( + client: Client, +) -> None: + """Loudly, at construction -- not quietly, at replay. + + By replay time a cursor has already been committed against data the + provider could have changed, and there is nothing left to do about it. + """ + with pytest.raises(ValueError, match="guarantees_immutability"): + Worker( + client, + task_queue=f"tq-{uuid.uuid4()}", + workflows=[NoOpWorkflow], + external_stream_backend=DeniedBackend(), + ) + + +async def test_no_backend_is_the_default(client: Client) -> None: + async with Worker( + client, task_queue=f"tq-{uuid.uuid4()}", workflows=[NoOpWorkflow] + ) as worker: + assert worker._external_stream_backend is None + + +# --- the sandbox ------------------------------------------------------------ + + +def test_the_sandbox_refuses_a_direct_provider_import() -> None: + """Workflow code never constructs the configured backend. + + A provider reached from inside the sandbox would be a second, unregistered + instance -- its own connection, no watcher owning it, and none of the + registration checks applied to it. + """ + matcher = SandboxRestrictions.invalid_module_members_default + contrib = matcher.children["temporalio"].children["contrib"] + streams = contrib.children["external_workflow_streams"] + + assert streams.children["_redis"].match_self + assert "RedisStreamBackend" in streams.access + + from temporalio.worker.workflow_sandbox._restrictions import RestrictionContext + + context = RestrictionContext() + context.is_runtime = True + assert streams.children["_redis"].match_access(context) + assert "external_stream_backend=... instead" in ( + streams.children["_redis"].leaf_message or "" + ) diff --git a/tests/contrib/external_workflow_streams/test_replay.py b/tests/contrib/external_workflow_streams/test_replay.py new file mode 100644 index 0000000000..d0d15490b6 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_replay.py @@ -0,0 +1,2189 @@ +"""P13 — the replay read path and its four range checks. + +The checks are sufficient *because* every provider guarantees a record's bytes +cannot change, so the only damage replay has to detect is a record that is no +longer there. Each test below deletes a record in a different position, and each +must fail a different one of the four -- which is what makes the set complete +rather than merely plausible. +""" + +# pyright: reportUnusedImport=false +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest +import pytest_asyncio + +import temporalio.api.common.v1 +import temporalio.converter +import temporalio.workflow +from temporalio.contrib.external_workflow_streams._annotation import ( + Annotation, + AnnotationHeader, + Run, + Segment, + SegmentEndReason, + StreamBinding, + decode_annotation, + encode_annotation, +) +from temporalio.contrib.external_workflow_streams._api import ( + _install_runtime, + external_stream, + merge, +) +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._errors import ( + StreamDecodeError, + StreamIntegrityError, + StreamStorageError, + classify_read_failure, +) +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._replay import ( + ReplayPlan, + validate_run, +) +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend +from tests.contrib.external_workflow_streams.test_replay_end_to_end import publish + +RUN_ID = "run-1" + + +async def _notify( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] +) -> str: + return ReadinessResult.ACCEPTED + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +@pytest.fixture +def key() -> StreamKey: + return StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + + +@pytest_asyncio.fixture # type: ignore[reportUntypedFunctionDecorator] +async def manager(backend: MemoryStreamBackend): + mgr = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + yield mgr + await mgr.shutdown() + + +async def append_five( + backend: MemoryStreamBackend, key: StreamKey +) -> list[StreamRecord]: + """Five data records, so first / middle / last deletions are all distinct.""" + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + placed = [] + for i in range(5): + placed.append( + await backend.append( + key, + StreamRecord( + RecordKind.DATA, await codec.encode(f"v{i}"), "producer", i + ), + ) + ) + return placed + + +def placed_offset(record: StreamRecord) -> Offset: + """Return the provider-assigned offset of a record already appended here.""" + assert record.offset is not None + return record.offset + + +def binding( + key: StreamKey, + start_cursor: Cursor = BEGINNING, + *, + provider_id: str = MemoryStreamBackend.provider_id, + provider_format_version: int = MemoryStreamBackend.provider_format_version, +) -> StreamBinding: + """A binding carrying the configured backend's provider identity.""" + return StreamBinding( + stream_key=key, + start_cursor=start_cursor, + provider_id=provider_id, + provider_format_version=provider_format_version, + ) + + +def annotation_for( + key: StreamKey, placed: list[StreamRecord], control_positions: tuple[int, ...] = () +) -> bytes: + """One marker recording all of ``placed`` as a single run.""" + first, last = placed[0].offset, placed[-1].offset + assert first is not None and last is not None + return encode_annotation( + Annotation( + header=AnnotationHeader(streams={1: binding(key)}), + segments=( + Segment( + runs=( + Run( + wait_id=1, + first_offset=first, + last_offset=last, + count=len(placed), + control_positions=control_positions, + ), + ), + end_reason=SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(last)}, + ) + ) + + +# --- the happy path ---------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_an_intact_range_replays_in_recorded_order( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + placed = await append_five(backend, key) + + plan = await manager.prepare_replay(RUN_ID, annotation_for(key, placed)) + + assert plan.total_records == 5 + (segment,) = plan.segments + assert [r.offset for _, r in segment.deliveries] == [r.offset for r in placed] + assert all(wait_id == 1 for wait_id, _ in segment.deliveries) + + +@pytest.mark.asyncio +async def test_replay_performs_no_live_waiting( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """It reads recorded ranges; it never asks what comes next. + + A live watch here would block on an empty stream and, worse, could deliver a + record the original run never saw. + """ + placed = await append_five(backend, key) + + await manager.prepare_replay(RUN_ID, annotation_for(key, placed)) + + # `read_range` is the inclusive replay read; `read_after` is the exclusive + # live watch, and it must not have been used at all. + assert backend.range_reads, "replay must read the recorded range" + assert len(backend.range_reads) == 1, ( + "one read per run, not one per record -- replay I/O is a function of " + f"the consumed range, got {len(backend.range_reads)} reads for 5 records" + ) + + +@pytest.mark.asyncio +async def test_one_segment_per_recorded_activation( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """Segments stay apart so replay reproduces the same number of drains. + + Collapsing them would reproduce the record order while changing when + ``wait_condition`` predicates fire. + """ + placed = await append_five(backend, key) + first, second = placed[0].offset, placed[-1].offset + assert first is not None and second is not None + + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(key)}), + segments=( + Segment( + (Run(1, first, placed[1].offset, 2),), # type: ignore[arg-type] + SegmentEndReason.BATCH_LIMIT, + ), + Segment((), SegmentEndReason.NO_DATA_AVAILABLE), + Segment( + (Run(1, placed[2].offset, second, 3),), # type: ignore[arg-type] + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(second)}, + ) + ) + + plan = await manager.prepare_replay(RUN_ID, annotation) + + assert [len(s.deliveries) for s in plan.segments] == [2, 0, 3], ( + "the empty segment is a drain that happened and must survive replay" + ) + + +# --- integrity: a deletion in each position ---------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("position", [0, 2, 4], ids=["first", "middle", "last"]) +async def test_deleting_a_record_fails_as_integrity_loss( + manager: StreamSubscriptionManager, + backend: MemoryStreamBackend, + key: StreamKey, + position: int, +) -> None: + """And never resolves to an alternate stream result. + + Substituting a later record for a deleted one would hand Workflow code a + different history than the one its commands were derived from, and the + divergence would surface much later as an unrelated nondeterminism error. + """ + placed = await append_five(backend, key) + annotation = annotation_for(key, placed) + target = placed[position].offset + assert target is not None + await backend.delete_for_test(key, target) + + with pytest.raises(StreamIntegrityError) as caught: + await manager.prepare_replay(RUN_ID, annotation) + + assert str(target) in str(caught.value) or "contains" in str(caught.value) + + +@pytest.mark.asyncio +async def test_each_deletion_position_fails_a_different_check( + backend: MemoryStreamBackend, key: StreamKey +) -> None: + """What makes the set of four complete rather than merely plausible. + + If a middle deletion and an endpoint deletion tripped the same check, one of + the four would be redundant and some other damage would be going undetected. + """ + placed = await append_five(backend, key) + run = Run(1, placed[0].offset, placed[-1].offset, 5) # type: ignore[arg-type] + + messages = {} + for name, position in (("first", 0), ("middle", 2), ("last", 4)): + surviving = [r for i, r in enumerate(placed) if i != position] + with pytest.raises(StreamIntegrityError) as caught: + validate_run(run, surviving, backend) + messages[name] = str(caught.value) + + assert "is missing from the stream" in messages["first"] + assert "is missing from the stream" in messages["last"] + assert "contains 4 record(s)" in messages["middle"] + assert messages["first"] != messages["last"] + + +@pytest.mark.asyncio +async def test_a_wrong_control_position_fails_validation( + backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A control record swapped for a data record would otherwise be yielded. + + Workflow code never saw the control record the first time, and delivering a + data record in its place would put a value in front of it that the original + run never produced. + """ + placed = await append_five(backend, key) + run = Run(1, placed[0].offset, placed[-1].offset, 5, control_positions=(2,)) # type: ignore[arg-type] + + with pytest.raises(StreamIntegrityError, match="control records at"): + validate_run(run, placed, backend) + + +@pytest.mark.asyncio +async def test_out_of_order_records_fail_validation( + backend: MemoryStreamBackend, key: StreamKey +) -> None: + placed = await append_five(backend, key) + reordered = [placed[0], placed[2], placed[1], placed[3], placed[4]] + run = Run(1, placed[0].offset, placed[-1].offset, 5) # type: ignore[arg-type] + + with pytest.raises(StreamIntegrityError, match="strictly increasing"): + validate_run(run, reordered, backend) + + +@pytest.mark.asyncio +async def test_a_marker_naming_an_unknown_wait_is_nondeterminism_not_integrity_loss( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """Row four of the taxonomy, and the row it must not be confused with. + + Nothing is wrong with the backend here: the recorded ranges are exactly + where they were written, and the Workflow code changed underneath them. + Reporting it as integrity loss would send an operator to repair a backend + that is fine, when the fix is to version the Workflow code -- the same + mistake in the opposite direction from calling an outage integrity loss. + + Reported rather than skipped either way, since skipping would silently + deliver a different stream result. + """ + placed = await append_five(backend, key) + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({}), + segments=( + Segment( + (Run(7, placed[0].offset, placed[-1].offset, 5),), # type: ignore[arg-type] + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={}, + ) + ) + + with pytest.raises( + temporalio.workflow.NondeterminismError, match="did not create" + ) as caught: + await manager.prepare_replay(RUN_ID, annotation) + + assert not isinstance(caught.value, StreamIntegrityError), ( + "this row must not be reachable through the storage-failure taxonomy" + ) + assert "workflow.patched" in str(caught.value), ( + "the message must name the remedy, which is versioning the Workflow " + "code rather than touching the backend" + ) + + +# --- storage failure is not integrity loss ----------------------------------- + + +class UnreachableBackend(MemoryStreamBackend): + """Stands in for a backend that is down rather than damaged.""" + + async def read_range(self, key, first, last): # type: ignore[no-untyped-def] + raise ConnectionError("connection refused") + + +@pytest.mark.asyncio +async def test_an_unreachable_backend_fails_as_transient_storage( + key: StreamKey, +) -> None: + """Distinguished from integrity loss by error type, never by retry behavior. + + The server retries a Workflow Task failure regardless of cause. An outage + clears on its own; integrity loss does not -- and an operator sent to repair + a backend that was merely unreachable would find nothing wrong with it. + """ + reachable = MemoryStreamBackend() + placed = await append_five(reachable, key) + annotation = annotation_for(key, placed) + + unreachable = UnreachableBackend() + manager = StreamSubscriptionManager( + backend=unreachable, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + try: + with pytest.raises(StreamStorageError, match="connection refused"): + await manager.prepare_replay(RUN_ID, annotation) + finally: + await manager.shutdown() + + +# --- decode failure is not integrity loss ------------------------------------ + + +@pytest.mark.asyncio +async def test_an_intact_but_undecodable_record_is_a_decode_error( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """The range validated, so the bytes are the bytes that were written. + + That makes this a configuration error on the *consumer* -- its converter or + codec does not match the producer's -- and reporting it as stream integrity + loss would send an operator to restore a backend that was never damaged. + """ + placed = [] + for i in range(3): + placed.append( + await backend.append( + key, + # Not a serialized Payload at all: intact bytes, undecodable. + StreamRecord(RecordKind.DATA, b"\\xff\\xfe raw", "producer", i), + ) + ) + annotation = annotation_for(key, placed) + + # The range itself validates -- nothing is missing. + plan = await manager.prepare_replay(RUN_ID, annotation) + assert plan.total_records == 3 + + # It is only decoding that fails, and only then is it classified. + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + _, record = plan.segments[0].deliveries[0] + try: + await codec.decode(record.payload) + raise AssertionError("expected the record to be undecodable") + except AssertionError: + raise + except Exception as err: + classified = classify_read_failure(range_validated=True, cause=err) + + assert isinstance(classified, StreamDecodeError) + assert not isinstance(classified, StreamIntegrityError) + + +class _FailingRetrieveDriver(temporalio.converter.StorageDriver): + """Stores payloads in memory and refuses to give them back once armed. + + Stands in for an external payload store that is briefly unreachable. It + raises the exception such a driver's client would raise -- a bare + ``ConnectionError`` -- rather than anything this module defines, which is the + whole point: the converter does not wrap it, so if no layer labels it the + record arrives at the consumer as an unclassified failure. + """ + + def __init__(self) -> None: + self._stored: dict[str, bytes] = {} + self.fail = False + self.retrieves = 0 + + def name(self) -> str: + return "failing-retrieve" + + async def store(self, context, payloads): # type: ignore[no-untyped-def] + claims = [] + for index, payload in enumerate(payloads): + key = f"k{len(self._stored) + index}" + self._stored[key] = payload.SerializeToString() + claims.append( + temporalio.converter.StorageDriverClaim(claim_data={"k": key}) + ) + return claims + + async def retrieve(self, context, claims): # type: ignore[no-untyped-def] + self.retrieves += 1 + if self.fail: + raise ConnectionError("the external payload store is unreachable") + out = [] + for claim in claims: + payload = temporalio.api.common.v1.Payload() + payload.ParseFromString(self._stored[claim.claim_data["k"]]) + out.append(payload) + return out + + +def _extstore_converter( + driver: _FailingRetrieveDriver, +) -> temporalio.converter.DataConverter: + return temporalio.converter.DataConverter( + external_storage=temporalio.converter.ExternalStorage( + drivers=[driver], payload_size_threshold=1 + ) + ) + + +@pytest.mark.parametrize("path", ["live", "replay"]) +@pytest.mark.asyncio +async def test_an_unreachable_payload_store_is_a_storage_failure_not_a_decode_one( + backend: MemoryStreamBackend, key: StreamKey, path: str +) -> None: + """Row one, on both delivery paths, whichever way the payload is fetched. + + A record's bytes can be a *reference*: with external storage configured, what + the stream holds is a claim and the value does not exist until the payload + store hands it over. A store that cannot be reached is therefore the same + condition as a stream backend that cannot be reached -- transient, clearing on + its own, with nothing for an operator to change. + + Left unlabelled it is not reported that way. The driver raises whatever its + client raises, the converter does not wrap it, and the consumer's + classification rule is "the range validated, so the bytes are the bytes that + were written" -- which turns it into a decode failure and tells an operator to + align a converter that was never wrong. Both paths are checked because they + prepare in different places: the watcher's loop for live delivery, the replay + read for a recorded range. + """ + driver = _FailingRetrieveDriver() + converter = _extstore_converter(driver) + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + data_converter=converter, + watch_block=timedelta(milliseconds=10), + ) + try: + codec = StreamPayloadCodec(converter, str) + placed = await backend.append( + key, StreamRecord(RecordKind.DATA, await codec.encode("alpha"), "p", 0) + ) + assert driver._stored, ( + "the payload was not externalized, so nothing here depends on the " + "payload store at all" + ) + + # The manager is what prepares, so the converter under test is the one it + # holds; the runtime here only needs to make the subscription. + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + + driver.fail = True + if path == "live": + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + await _until( + lambda: subscription.buffered == 1, + 5, + "the watcher never buffered the record", + ) + prepared = manager.drain(RUN_ID, 1)[0] + else: + plan = await manager.prepare_replay(RUN_ID, annotation_for(key, [placed])) + _, prepared = plan.segments[0].deliveries[0] + + carried = getattr(prepared, "prepare_error", None) + assert carried is not None, ( + f"the {path} path prepared the record without noticing that its " + "payload could not be fetched" + ) + assert isinstance(carried, StreamStorageError), ( + f"the {path} path carried {type(carried).__name__}, so the delivery " + "that raises it will classify a storage outage as a decode failure" + ) + assert not isinstance(carried, StreamDecodeError) + + # And the classification rule leaves it alone rather than relabelling it: + # a storage failure is about reaching the store at all, so "the range + # validated" says nothing about it. + assert classify_read_failure(range_validated=True, cause=carried) is carried + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_replay_plan_is_consumed_once( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A second delivery would replay the same records twice.""" + placed = await append_five(backend, key) + await manager.prepare_replay(RUN_ID, annotation_for(key, placed)) + + assert manager.take_replay_plan(RUN_ID) is not None + assert manager.take_replay_plan(RUN_ID) is None + + +@pytest.mark.asyncio +async def test_eviction_discards_an_unconsumed_plan( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A plan outliving its Run would be delivered to whatever came next.""" + placed = await append_five(backend, key) + await manager.prepare_replay(RUN_ID, annotation_for(key, placed)) + + await manager.evict_run(RUN_ID) + + assert manager.take_replay_plan(RUN_ID) is None + + +# --- the delivery driver (ADR-018) ------------------------------------------- + + +class DriverStub: + """The two things the replay driver is allowed to touch. + + Calling the real ``_apply_replay_external_streams`` against this asserts what + the driver does rather than what a Workflow happens to observe -- a Workflow + that never used ``wait_condition`` would pass either way. + """ + + def __init__(self, runtime) -> None: # type: ignore[no-untyped-def] + self._external_stream_runtime = runtime + self._pending_replay_finish: ReplayPlan | None = None + self.drains: list[int] = [] + + def _finish_replay_external_streams(self) -> None: + """The real one. A plan with no segments closes from inside the job. + + Present because a stub stands in for the instance, and the job calls this + on itself when there is no segment for the activation's drain to serve. + """ + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + _WorkflowInstanceImpl._finish_replay_external_streams(self) # type: ignore[arg-type] + + def _run_once(self, *, check_conditions: bool) -> None: + assert check_conditions, ( + "a replay drain must check conditions -- otherwise `wait_condition` " + "predicates fire fewer times than they did live" + ) + self.drains.append(len(self.drains)) + + +def make_runtime(manager, backend): # type: ignore[no-untyped-def] + from temporalio.contrib.external_workflow_streams._runtime import ( + WorkflowStreamRuntime, + ) + + return WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id=uuid.uuid4().hex, + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + ) + + +class ConsumingStub(DriverStub): + """A stub that also takes what the segment holds, as Workflow code does. + + Every record in a run was handed to Workflow code in the activation that run + was recorded in, so a stub that drains nothing is not a stand-in for a + Workflow -- it is a stand-in for a Workflow that stopped subscribing, which + the driver now reports as nondeterminism. Tests that only care about how + many drains happened use this; tests that assert *what* each drain saw drain + for themselves. + + It drains only the waits the Run **registered**, for the same reason: a + Workflow has no handle to a subscription it never made, so a stub that + drains by whatever the segment holds reports a removed ``subscribe()`` as + consumed and hides the misrouting that goes with it. + """ + + def __init__(self, runtime) -> None: # type: ignore[no-untyped-def] + super().__init__(runtime) + self._runtime = runtime + + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + # Only the waits this Run actually registered. Draining whatever the + # segment happens to hold is something no Workflow can do -- it has no + # handle to a subscription it never made -- and a stub that does it + # hides exactly the failure the driver exists to catch: a removed + # `subscribe()` whose records are then taken by nobody's subscription + # and reported as consumed. + registered = set(self._runtime.subscriptions()) + for wait_id in sorted({w for w, _ in self._runtime._replay_ready or []}): + if wait_id in registered: + self._runtime.drain(wait_id) + + +def drive_with(stub): # type: ignore[no-untyped-def] + """Runs the whole activation shape around one replay job, not just `_apply`. + + The trailing drain is neither optional nor this helper's invention: the replay + job lands in the non-query job set, so the activation runs exactly one + `_run_once` for that set whatever `_apply` did with the job. Driving `_apply` + alone is what let the driver perform one drain more than the marker records + without any test noticing -- ADR-018 requires *k* drains for *k* segments and + it was performing *k + 1* -- so every test here drives the activation's drain + too, and the close that the driver now defers until after it. + """ + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + try: + _WorkflowInstanceImpl._apply_replay_external_streams(stub, object()) # type: ignore[arg-type] + stub._run_once(check_conditions=True) + finally: + _WorkflowInstanceImpl._finish_replay_external_streams(stub) # type: ignore[arg-type] + return stub + + +def drive(runtime) -> DriverStub: # type: ignore[no-untyped-def] + return drive_with(ConsumingStub(runtime)) + + +@pytest.mark.asyncio +async def test_the_driver_drains_once_per_recorded_segment( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """Three recorded activations become three drains, not one. + + Delivering all five records in a single drain would reproduce the record + order while changing how many times the event loop ran, and a + ``wait_condition`` predicate would then see a different sequence of states + than it did live -- a nondeterminism that only shows up on replay. + """ + placed = await append_five(backend, key) + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(key)}), + segments=( + Segment( + (Run(1, placed_offset(placed[0]), placed_offset(placed[1]), 2),), + SegmentEndReason.BATCH_LIMIT, + ), # type: ignore[arg-type] + Segment( + (Run(1, placed_offset(placed[2]), placed_offset(placed[2]), 1),), + SegmentEndReason.BATCH_LIMIT, + ), # type: ignore[arg-type] + Segment( + (Run(1, placed_offset(placed[3]), placed_offset(placed[4]), 2),), + SegmentEndReason.NO_DATA_AVAILABLE, + ), # type: ignore[arg-type] + ), + terminal={1: AFTER(placed[4].offset)}, # type: ignore[arg-type] + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + # The `subscribe()` call the marker recorded. A stub stands in for Workflow + # code, so it has to make it: a replay that ends with a recorded binding + # never recreated is a removed subscription whatever is driving the drains. + runtime.register(wait_id=1, stream_key=key) + + stub = drive(runtime) + + assert len(stub.drains) == 3, ( + f"expected one drain per recorded segment, got {len(stub.drains)}" + ) + + +@pytest.mark.asyncio +async def test_a_drain_sees_only_its_own_segments_records( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """Records may not run ahead of the drain they were recorded in. + + Buffering all of them and letting the first drain take everything would put + values in front of Workflow code before the activation that produced them. + """ + placed = await append_five(backend, key) + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(key)}), + segments=( + Segment( + (Run(1, placed_offset(placed[0]), placed_offset(placed[1]), 2),), + SegmentEndReason.BATCH_LIMIT, + ), # type: ignore[arg-type] + Segment( + (Run(1, placed_offset(placed[2]), placed_offset(placed[4]), 3),), + SegmentEndReason.NO_DATA_AVAILABLE, + ), # type: ignore[arg-type] + ), + terminal={1: AFTER(placed[4].offset)}, # type: ignore[arg-type] + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + + seen_per_drain: list[list[Offset]] = [] + + class DrainingStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + seen_per_drain.append([r.offset for r in runtime.drain(1)]) # type: ignore[misc] + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + drive_with(DrainingStub(runtime)) + + assert seen_per_drain == [ + [placed[0].offset, placed[1].offset], + [placed[2].offset, placed[3].offset, placed[4].offset], + ] + + +@pytest.mark.asyncio +async def test_the_driver_reads_nothing_and_ends_replay_mode( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """Delivery is memory-only, and the Run goes back to live drains afterwards. + + Leaving replay mode latched would make every later live drain read from an + empty recorded buffer, and the Workflow would block forever on records that + had already arrived. + """ + placed = await append_five(backend, key) + await manager.prepare_replay(RUN_ID, annotation_for(key, placed)) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + before = len(backend.range_reads) + + drive(runtime) + + assert len(backend.range_reads) == before, "the delivering pass must read nothing" + assert runtime._replay_ready is None, "replay mode must not outlive the job" + + +@pytest.mark.asyncio +async def test_a_replayed_drain_never_reaches_the_live_buffer( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A watcher may have buffered past where the marker stops. + + That buffer is speculative -- reading a record is not consuming it -- so + delivering it would hand Workflow code records this Workflow Task never saw. + Replay would then be the thing introducing the nondeterminism it exists to + prevent, which is why the recorded segment is the *only* thing a drain can + see while one is being delivered. + """ + placed = await append_five(backend, key) + # The marker committed the first two records only. + await manager.prepare_replay(RUN_ID, annotation_for(key, placed[:2])) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + subscription._append(placed[2:], subscription._prefetch_epoch) + + seen: list[list[Offset]] = [] + + class DrainingStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + seen.append([r.offset for r in runtime.drain(1)]) # type: ignore[misc] + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + drive_with(DrainingStub(runtime)) + + assert seen == [[placed[0].offset, placed[1].offset]] + # And the buffered-past records were never handed to that drain. Closing the + # replay retracts them rather than leaving them in front of the next one: + # reading is not consuming, so the speculative buffer is discarded and the + # watcher re-reads from the boundary the marker committed. Withheld either + # way -- what changed is that the retraction is now certain by the time the + # replay returns rather than posted onto the manager's loop and raced. + assert subscription.committed_cursor == AFTER(placed_offset(placed[1])) + assert subscription.prefetch_cursor == AFTER(placed_offset(placed[1])), ( + "the watcher must resume reading at the marker's boundary, which is " + "what re-reads the records this retraction dropped" + ) + assert runtime.drain(1) == [], ( + "records read past the marker must not survive the replay in the " + "buffer, or the first live drain hands over what it never saw recorded" + ) + + +@pytest.mark.asyncio +async def test_two_markers_reassemble_in_workflow_task_order( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A batch split by a rollover replays as one continuous consumption. + + The byte-budget high-water mark rolls the Workflow Task over rather than + growing the marker, so one logical batch ends up in *two* markers -- and + replay reaches them as two separate ``ReplayExternalStreams`` jobs, one per + replayed Workflow Task. Core's own test proves the two annotations + concatenate at the byte level; what only this side can show is that feeding + them through the manager in Workflow Task order reproduces the original + consumption exactly. + + Two failures this rules out, both silent: + + - a seam that loses or repeats a record, which is what an exclusive read or + an off-by-one at the split would produce -- the second marker resumes at + the first's terminal, so the record *at* that boundary must appear exactly + once across the pair; + - a second marker delivered against the first one's segmentation, which + would put records in front of Workflow code in a drain that never saw + them. + """ + placed = await append_five(backend, key) + split = 1 # The first marker stops after `placed[1]`. + + first_marker = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(key)}), + segments=( + Segment( + (Run(1, placed[0].offset, placed[0].offset, 1),), # type: ignore[arg-type] + SegmentEndReason.BATCH_LIMIT, + ), + Segment( + (Run(1, placed[1].offset, placed[1].offset, 1),), # type: ignore[arg-type] + # The annotation passed its high-water mark here, so this + # batch continues in the following marker. + SegmentEndReason.BUDGET_ROLLOVER, + ), + ), + terminal={1: AFTER(placed[split].offset)}, # type: ignore[arg-type] + ) + ) + # The replacement Workflow Task's annotation begins where the first one's + # terminal left the subscription: a rollover preserves the cursor rather + # than restarting it. + second_marker = encode_annotation( + Annotation( + header=AnnotationHeader( + {1: binding(key, AFTER(placed[split].offset))} # type: ignore[arg-type] + ), + segments=( + Segment( + (Run(1, placed[2].offset, placed[4].offset, 3),), # type: ignore[arg-type] + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(placed[4].offset)}, # type: ignore[arg-type] + ) + ) + + runtime = make_runtime(manager, backend) + # One subscription across both markers, as the rollover preserves it. + runtime.register(wait_id=1, stream_key=key) + per_marker: list[list[list[Offset]]] = [] + + for annotation in (first_marker, second_marker): + await manager.prepare_replay(RUN_ID, annotation) + drains: list[list[Offset]] = [] + + class DrainingStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + drains.append([r.offset for r in runtime.drain(1)]) # type: ignore[misc] + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + drive_with(DrainingStub(runtime)) + per_marker.append(drains) + + assert per_marker == [ + [[placed[0].offset], [placed[1].offset]], + [[placed[2].offset, placed[3].offset, placed[4].offset]], + ], ( + "each marker must deliver its own segments and nothing else; a marker " + "reassembled against the wrong segmentation puts records in a drain " + f"that never saw them, got {per_marker}" + ) + + reassembled = [ + offset for drains in per_marker for drain in drains for offset in drain + ] + assert reassembled == [r.offset for r in placed], ( + "the two markers must reassemble into the original consumption, in " + f"Workflow Task order and with nothing lost or repeated at the seam: " + f"{reassembled}" + ) + + +@pytest.mark.asyncio +async def test_one_segment_delivers_each_waits_own_records( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A segment can carry runs from several subscriptions at once. + + Two subscriptions are two independent cursors, so handing one's records to + the other is a different stream result rather than merely a different order. + """ + other = StreamKey( + key.namespace, key.workflow_id, key.first_execution_run_id, "tool-events" + ) + placed = await append_five(backend, key) + other_placed = await append_five(backend, other) + + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(key), 2: binding(other)}), + segments=( + Segment( + ( + Run(1, placed[0].offset, placed[4].offset, 5), # type: ignore[arg-type] + Run(2, other_placed[0].offset, other_placed[4].offset, 5), # type: ignore[arg-type] + ), + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(placed[4].offset), 2: AFTER(other_placed[4].offset)}, # type: ignore[arg-type] + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + runtime.register(wait_id=2, stream_key=other) + + drained: dict[int, list[Offset]] = {} + + class DrainingStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + for wait_id in (1, 2): + drained[wait_id] = [r.offset for r in runtime.drain(wait_id)] # type: ignore[misc] + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + drive_with(DrainingStub(runtime)) + + assert drained[1] == [r.offset for r in placed] + assert drained[2] == [r.offset for r in other_placed] + + +# --- what the live buffer must look like afterwards --------------------------- + + +async def _until(predicate, timeout: float, message: str) -> None: # type: ignore[no-untyped-def] + """Polls rather than sleeping a fixed time, so a slow watcher is not a flake.""" + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if predicate(): + return + await asyncio.sleep(0.01) + raise AssertionError(message) + + +@pytest.mark.asyncio +async def test_live_delivery_after_a_replay_resumes_past_the_marker( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A replayed marker's records must not be handed over a second time, live. + + Replay delivers from the annotation, but the manager's watcher has been + reading the *same* records from the subscription's start cursor into the + live buffer the whole time -- nothing out there knows the marker exists. So + the first live drain after a replay re-delivers everything the marker + recorded: observed end-to-end as a Workflow that received + ``['alpha', 'alpha', 'beta', 'gamma']``. + + Repositioning to the marker's terminal is what makes live delivery resume + after the recorded range rather than in front of it. + """ + placed = await append_five(backend, key) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + await _until( + lambda: subscription.buffered == 5, + 5, + "the watcher never buffered the records, so nothing would be " + "re-delivered either way and this proves nothing", + ) + + # A marker recording only the first two of the five, so "resumed past it" + # and "delivered nothing at all" are distinguishable. + await manager.prepare_replay(RUN_ID, annotation_for(key, placed[:2])) + drive(runtime) + # The reposition is hopped onto this loop, exactly as the Workflow thread + # would hop it. + await asyncio.sleep(0) + + await _until( + lambda: subscription.buffered == 3, + 5, + "the buffer did not refill from the marker's boundary", + ) + assert [r.offset for r in manager.drain(RUN_ID, 1)] == [ + r.offset for r in placed[2:] + ], ( + "the live buffer still holds records the replayed marker already " + "delivered, so the Workflow receives them twice" + ) + assert subscription.committed_cursor == AFTER(placed_offset(placed[1])), ( + "the marker's boundary was never committed, so a later reset would " + "send the subscription back to the start cursor" + ) + + +@pytest.mark.asyncio +async def test_the_first_live_drain_after_a_replay_needs_no_loop_turn( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """Repositioning has to have *happened*, not merely been scheduled. + + The drain that follows replay is on the Workflow thread, and so is the replay + itself. A reposition posted to the manager's loop with `call_soon_threadsafe` + and then left to arrive whenever the loop next runs is ordered by nothing at + all: if Workflow code drains first -- which one ordinary scheduler pass is + enough to do -- the live buffer still holds every record the marker just + delivered and hands them over a second time. + + So this test deliberately gives the manager's loop **no** opportunity to run + between the replay and the drain: no `await`, no `sleep(0)`, nothing. That is + the whole point. A version of this test with a yield in the middle passes + against a posted reposition too, and proves only that the loop happened to + win. + """ + placed = await append_five(backend, key) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + await _until( + lambda: subscription.buffered == 5, + 5, + "the watcher never buffered the records, so nothing would be " + "re-delivered either way and this proves nothing", + ) + + # A marker covering the first two of the five only. + await manager.prepare_replay(RUN_ID, annotation_for(key, placed[:2])) + drive(runtime) + + # No yield here on purpose. Everything below runs in the same synchronous + # stretch of Workflow-thread work that the replay just finished in. + assert subscription.committed_cursor == AFTER(placed_offset(placed[1])), ( + "the marker's boundary must be committed by the time the replay returns, " + "not once some later loop turn gets round to it" + ) + assert manager.drain(RUN_ID, 1) == [], ( + "the drain immediately after a replay still saw the marker-covered " + "records in the live buffer, so the Workflow receives them twice" + ) + assert subscription.prefetch_cursor == AFTER(placed_offset(placed[1])), ( + "the watcher must have been moved to re-read from the marker's boundary" + ) + + # And the records past the marker are not lost: the watcher re-reads them + # from the boundary that was committed. + await _until( + lambda: subscription.buffered == 3, + 5, + "the buffer never refilled from the marker's boundary", + ) + assert [r.offset for r in manager.drain(RUN_ID, 1)] == [ + r.offset for r in placed[2:] + ] + + +@pytest.mark.asyncio +async def test_a_failing_activation_reports_its_own_error_not_the_replays( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """Closing a replay may not overwrite the error that stopped it. + + The close runs ``verify_replay_consumed``, which raises whenever a recorded + delivery is still armed -- and after a drain fails, one always is: the drain + that would have taken it is the one that failed. An exception raised while + another is propagating **replaces** it, so running the close from a bare + ``finally`` turned every failing replay activation into a nondeterminism error + blaming a ``subscribe()`` call nobody had touched. + + Two things are lost that way, not one. The diagnosis, and the *classification*: + a ``FailureError`` out of Workflow code fails the Workflow, while a + nondeterminism error fails the Workflow Task and is retried forever. + + Replay mode is still left -- a Run stuck in it has every later drain return + nothing -- so what the abandon path skips is only the checks and the cursor + move, and the cursor move has to be skipped anyway: an activation that failed + committed nothing. + """ + placed = await append_five(backend, key) + await manager.prepare_replay(RUN_ID, annotation_for(key, placed)) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + # A replay left mid-flight, exactly as a drain that raised leaves it: the + # segment armed, its deliveries untaken, the close still owed. + stub = DriverStub(runtime) + runtime.begin_replay(decode_annotation(annotation_for(key, placed)).header.streams) + plan = runtime.take_replay_plan() + assert plan is not None + runtime.begin_replay_segment(list(plan.segments[0].deliveries)) + stub._pending_replay_finish = plan + + # Closing from here is what a bare `finally` did, and it raises: the recorded + # deliveries are still armed, because the drain that would have taken them is + # the one that failed. Asserted so that what follows is a contrast and not a + # coincidence. + with pytest.raises(temporalio.workflow.NondeterminismError): + _WorkflowInstanceImpl._finish_replay_external_streams(stub) # type: ignore[arg-type] + + # Abandoning does not. Re-arm and take the other path. + runtime.begin_replay_segment(list(plan.segments[0].deliveries)) + stub._pending_replay_finish = plan + _WorkflowInstanceImpl._abandon_replay_external_streams(stub) # type: ignore[arg-type] + + assert runtime._replay_ready is None and runtime._replay_bindings is None, ( + "replay mode was left set, so every later drain on this Run returns nothing" + ) + assert stub._pending_replay_finish is None + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + assert subscription.committed_cursor == BEGINNING, ( + "an activation that failed committed nothing, so the cursor may not have " + "moved to the marker's boundary" + ) + + +@pytest.mark.asyncio +async def test_a_marker_with_no_segments_closes_before_the_activations_own_drain( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A marker that recorded no activation of its own defers nothing. + + A Workflow Task that subscribed to a quiet stream and blocked writes a marker + with a header and a terminal and **no segments** -- and its terminal names the + cursor nothing was ever delivered past. There is no recorded segment for the + activation's own drain to serve, so that drain is a *live* one: records that + arrived while the Run was evicted are in the buffer and it hands them over. + + Repositioning after that drain retracts what it just delivered. The cursor goes + back to the marker's boundary, the buffer is cleared, and the watcher re-reads + and re-delivers records Workflow code already has. So a plan with no segments + closes inside the job, before the drain, and only a plan with segments defers. + + This is not reachable by inspection alone: the drain only has something to + deliver if the watcher buffered first, so the defect it guards against + presents as an occasional end-to-end failure and passes in isolation. + """ + placed = await append_five(backend, key) + # The marker of a Workflow Task that saw nothing: header and terminal only, + # with the terminal at the start cursor. + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(key)}), + segments=(), + terminal={1: BEGINNING}, + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + await _until( + lambda: subscription.buffered == 5, + 5, + "the watcher never buffered anything, so the activation's drain has " + "nothing to deliver and this proves nothing", + ) + + taken: list[Offset] = [] + + class LiveDrainingStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + taken.extend(r.offset for r in runtime.drain(1)) # type: ignore[misc] + + drive_with(LiveDrainingStub(runtime)) + + assert taken == [], ( + "the activation's drain handed over buffered records and the replay's " + "close then retracted the cursor past them, so the watcher re-reads and " + f"the Workflow receives every one of them twice: {taken}" + ) + assert subscription.committed_cursor == BEGINNING, ( + "the marker committed nothing, so its boundary is the start cursor" + ) + assert subscription.prefetch_cursor == BEGINNING, ( + "the watcher must be re-reading from the boundary the marker committed" + ) + + # And the records are not lost -- they arrive on the activation the re-read + # announces, which is the ordinary live path. + await _until( + lambda: subscription.buffered == 5, + 5, + "the buffer never refilled, so the records the retraction dropped are " + "gone rather than merely deferred", + ) + assert [r.offset for r in manager.drain(RUN_ID, 1)] == [r.offset for r in placed] + + +@pytest.mark.asyncio +async def test_a_read_in_flight_across_a_reposition_cannot_be_appended( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """Reposition and append must be atomic against the prefetch epoch. + + Making the reposition synchronous on the Workflow thread is only half a fix. + The watcher captures the epoch, reads, *awaits* a user codec, and appends -- so + a check made before the append can pass and then have the reposition land in + between, putting the retracted records straight back into the buffer and + re-advancing ``prefetch_cursor`` past them. Under the reposition's own lock, + the append is either cleared by it or rejected by it, with no third + interleaving. + + The watcher is held **between its read and its append**, which is the window + that matters and the one an epoch check placed outside the lock cannot cover. + """ + placed = await append_five(backend, key) + reached_prepare = asyncio.Event() + release_prepare = asyncio.Event() + real_prepare = manager._prepare + + async def paused_prepare(records): # type: ignore[no-untyped-def] + reached_prepare.set() + await release_prepare.wait() + return await real_prepare(records) + + manager._prepare = paused_prepare # type: ignore[assignment] + + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + + await asyncio.wait_for(reached_prepare.wait(), 5) + assert subscription.buffered == 0, "the watcher got past its append already" + + # The Workflow thread's half, while that read is still in flight. + manager.reposition_to_committed(RUN_ID, {1: AFTER(placed_offset(placed[1]))}) + assert subscription.committed_cursor == AFTER(placed_offset(placed[1])) + + release_prepare.set() + # Give the watcher every chance to append what it read from the old position. + for _ in range(20): + await asyncio.sleep(0.01) + if subscription.buffered: + break + + assert all(r.offset != placed[0].offset for r in list(subscription._buffer)), ( + "a read that started before the reposition was appended after it, so the " + "records the marker already accounts for are back in front of the Workflow" + ) + assert subscription.prefetch_cursor != BEGINNING, ( + "the watcher never resumed from the new boundary" + ) + + # And the contract that makes the above hold whatever the interleaving, which a + # single-threaded loop cannot demonstrate on its own: the epoch is compared + # under the same lock the reposition takes, so an append carrying a retracted + # epoch is *refused* rather than merely unlikely. In production the two run on + # different threads, where "there is no await between the check and the append" + # is not an argument. + with subscription._lock: + stale_epoch = subscription._prefetch_epoch - 1 + assert subscription._append(list(placed), stale_epoch) is False, ( + "an append from a retracted epoch was accepted, so a watcher that read " + "before the reposition can undo it" + ) + assert all(r.offset != placed[0].offset for r in list(subscription._buffer)) + + +@pytest.mark.asyncio +async def test_a_replay_activation_never_delivers_a_record_the_marker_omits( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """A record the annotation does not name may not reach Workflow code. + + The dangerous shape is a *coalesced* readiness job: a replayed Run registers a + live watcher while it replays, the stream already holds records, and Core can + resolve a stream wait in the same activation that carries the replay job. If + the activation's own drain reaches the live buffer, Workflow code receives a + record no marker records -- observed downstream as a replay that either + diverges or fails as nondeterminism. + + The record here is *poison*: it is in the buffer and in no run of the + annotation. Whatever the drains do, it must not come out. This drives the + activation shape rather than a real ``activate()``, so what it covers is the + Python-side invariant; the end-to-end case is + ``test_replay_end_to_end.py``'s parked-and-evicted test. + """ + placed = await append_five(backend, key) + # The marker records the first two records, across two segments. + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(key)}), + segments=( + Segment( + (Run(1, placed[0].offset, placed[0].offset, 1),), # type: ignore[arg-type] + SegmentEndReason.BATCH_LIMIT, + ), + Segment( + (Run(1, placed[1].offset, placed[1].offset, 1),), # type: ignore[arg-type] + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(placed[1].offset)}, # type: ignore[arg-type] + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + + subscription = manager.subscription(RUN_ID, 1) + assert subscription is not None + await _until( + lambda: subscription.buffered > 0, + 5, + "the watcher never filled the live buffer, so there is no poison record " + "for a stray drain to find", + ) + + taken: list[Offset] = [] + + class ResolvingStub(DriverStub): + """Resolves pending waits on every drain, as a coalesced readiness job does.""" + + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + runtime.resolve_all_pending() + taken.extend(r.offset for r in runtime.drain(1)) # type: ignore[misc] + + stub = drive_with(ResolvingStub(runtime)) + + assert len(stub.drains) == 2, ( + f"two recorded segments must produce two drains, got {len(stub.drains)}" + ) + assert taken == [placed[0].offset, placed[1].offset], ( + f"the replay activation delivered something the marker does not name: {taken}" + ) + # A delta is expected -- the `subscribe()` this Run replayed is itself + # replay-visible and puts the stream in the next annotation's header. What must + # not be in it is a *run*: every record this activation handed over came from + # the marker, and re-recording those would ask Core to write a second marker + # for observations already in History. + delta = runtime.take_observation_delta() + assert delta is not None + recorded = decode_annotation(delta) + assert all(not segment.runs for segment in recorded.segments), ( + "the replay activation recorded runs of its own, so a record it delivered " + "came from the live buffer rather than from the marker" + ) + + +@pytest.mark.asyncio +async def test_a_marker_drains_once_per_segment_including_the_activations_own( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend, key: StreamKey +) -> None: + """*k* segments produce *k* drains for the whole activation, not *k + 1*. + + The driver is not the only thing that drains. The replay job lands in the + non-query job set, so the activation runs one `_run_once` for that set after + every job in it has been applied -- meaning a driver that drained all *k* + segments itself produced one drain more than the marker records. ADR-018 + requires exactly *k*: under `_single_batch_activation` each live stream + activation drove exactly one condition-checking drain, so an extra one + evaluates `wait_condition` predicates at a point that did not exist live. + + Counted across the activation rather than inside `_apply`, because inside + `_apply` the count was always right and the defect was never visible there. + An empty segment is included: it records an activation that drained and found + nothing, and it must cost a drain like any other. + """ + placed = await append_five(backend, key) + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(key)}), + segments=( + Segment( + (Run(1, placed[0].offset, placed[1].offset, 2),), # type: ignore[arg-type] + SegmentEndReason.BATCH_LIMIT, + ), + # The empty one: an activation that ran a drain and found nothing. + Segment((), SegmentEndReason.NO_DATA_AVAILABLE), + Segment( + (Run(1, placed[2].offset, placed[4].offset, 3),), # type: ignore[arg-type] + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(placed[4].offset)}, # type: ignore[arg-type] + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=key) + + predicate_evaluations = 0 + + class ConditionCountingStub(ConsumingStub): + def _run_once(self, *, check_conditions: bool) -> None: + nonlocal predicate_evaluations + super()._run_once(check_conditions=check_conditions) + if check_conditions: + # Stands in for a `wait_condition` predicate that stays false: + # what it counts is how many times the loop asked. + predicate_evaluations += 1 + + stub = drive_with(ConditionCountingStub(runtime)) + + assert len(stub.drains) == 3, ( + f"three recorded segments must produce three drains across the whole " + f"activation, got {len(stub.drains)}" + ) + assert predicate_evaluations == 3, ( + f"a still-false wait_condition predicate must be evaluated once per " + f"recorded segment, got {predicate_evaluations}" + ) + + +# --- the annotation must match the subscriptions the Workflow makes ---------- +# +# Delivery joins a recorded run to a subscription by `wait_id`, and an integer +# is not an identity. Everything below is row four of the failure taxonomy -- +# nondeterminism, remedied by versioning the Workflow -- and specifically *not* +# `StreamIntegrityError`, which would send an operator to repair a backend that +# holds exactly the bytes it was given. + + +async def _one_record(backend: MemoryStreamBackend, key: StreamKey) -> StreamRecord: + await publish(backend, key, [f"{key.stream_name}-value"]) + return backend.all_records(key)[-1] + + +def _single_run_annotation(bindings, runs) -> bytes: # type: ignore[no-untyped-def] + """One segment holding one run per wait, and a terminal that closes it.""" + return encode_annotation( + Annotation( + header=AnnotationHeader(bindings), + segments=(Segment(tuple(runs), SegmentEndReason.NO_DATA_AVAILABLE),), + terminal={run.wait_id: AFTER(run.last_offset) for run in runs}, + ) + ) + + +@pytest.mark.asyncio +async def test_a_wait_rebound_to_another_stream_is_nondeterminism( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """Moving wait 1 from one stream to another must fail, not deliver. + + This is the silent failure the whole check exists for: with the join done on + ``wait_id`` alone, the left stream's recorded bytes are handed to Workflow + code through the right stream's subscription. Nothing anywhere reports it -- + the range validates, because the records really are where the marker says -- + and the Workflow proceeds on data it never received. + + The taxonomy puts "annotation does not match the subscriptions made" in the + nondeterminism row precisely so the remedy is versioning the Workflow rather + than repairing a healthy backend. + """ + left = StreamKey("ns", "wf", uuid.uuid4().hex, "left") + right = StreamKey("ns", "wf", uuid.uuid4().hex, "right") + recorded = await _one_record(backend, left) + await _one_record(backend, right) + + await manager.prepare_replay( + RUN_ID, + _single_run_annotation( + {1: binding(left)}, + [Run(1, recorded.offset, recorded.offset, 1)], # type: ignore[arg-type] + ), + ) + runtime = make_runtime(manager, backend) + # The Workflow's `subscribe()` calls are unchanged in number and order, so + # wait 1 is still wait 1 -- it just reads a different stream now. + runtime.register(wait_id=1, stream_key=right) + + with pytest.raises(temporalio.workflow.NondeterminismError) as caught: + drive(runtime) + + assert not isinstance(caught.value, StreamIntegrityError), ( + "the backend holds exactly what it was given; this must not be " + "reachable through the storage-failure taxonomy" + ) + assert "workflow.patched" in str(caught.value) + assert runtime.drain(1) == [], ( + "the recorded record must never reach the rebound subscription" + ) + + +@pytest.mark.asyncio +async def test_a_rebinding_made_during_the_replay_activation_is_caught_too( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """The other order, which is the ordinary one for a Run's first marker. + + An activation carrying both ``InitializeWorkflow`` and + ``ReplayExternalStreams`` applies every job before any Workflow code runs, + so the ``subscribe()`` call happens *inside* the replay driver's first + drain. Checking only what was registered beforehand would find nothing to + check on exactly the activation where the first marker is replayed. + """ + left = StreamKey("ns", "wf", uuid.uuid4().hex, "left") + right = StreamKey("ns", "wf", uuid.uuid4().hex, "right") + recorded = await _one_record(backend, left) + await _one_record(backend, right) + + await manager.prepare_replay( + RUN_ID, + _single_run_annotation( + {1: binding(left)}, + [Run(1, recorded.offset, recorded.offset, 1)], # type: ignore[arg-type] + ), + ) + runtime = make_runtime(manager, backend) + + class LateRegisteringStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + runtime.register(wait_id=1, stream_key=right) + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + with pytest.raises(temporalio.workflow.NondeterminismError, match="workflow"): + drive_with(LateRegisteringStub(runtime)) + + +@pytest.mark.asyncio +async def test_a_removed_subscription_leaves_recorded_deliveries_unconsumed( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """Records the marker recorded may not be quietly dropped. + + Deleting a ``subscribe()`` call leaves its wait unregistered, so nothing + drains it and the prepared deliveries are discarded when the segment is + replaced. The Workflow then reaches its next command having consumed less + than History says it consumed -- which surfaces later, somewhere else, as an + unrelated command mismatch. + """ + left = StreamKey("ns", "wf", uuid.uuid4().hex, "left") + right = StreamKey("ns", "wf", uuid.uuid4().hex, "right") + left_record = await _one_record(backend, left) + right_record = await _one_record(backend, right) + + await manager.prepare_replay( + RUN_ID, + _single_run_annotation( + {1: binding(left), 2: binding(right)}, + [ + Run(1, left_record.offset, left_record.offset, 1), # type: ignore[arg-type] + Run(2, right_record.offset, right_record.offset, 1), # type: ignore[arg-type] + ], + ), + ) + runtime = make_runtime(manager, backend) + # Wait 2's `subscribe()` call is gone; wait 1 is untouched. + runtime.register(wait_id=1, stream_key=left) + + class OnlyWaitOneStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + runtime.drain(1) + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + with pytest.raises(temporalio.workflow.NondeterminismError) as caught: + drive_with(OnlyWaitOneStub(runtime)) + + assert "[2]" in str(caught.value), ( + f"the error must name the wait whose records went undelivered: {caught.value}" + ) + + +@pytest.mark.asyncio +async def test_a_removed_subscription_that_recorded_nothing_is_nondeterminism( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """A binding with no runs is still a subscription the Workflow must make. + + The first observation of a wait carries its provider, stream key, and start + cursor even when nothing was ever delivered, so a ``subscribe()`` on a + stream that stayed quiet for that Workflow Task records exactly this: a + binding and not one run. Every delivery-side check is then vacuous -- there + are no records left over to notice -- and the only thing that says the wait + existed at all is the marker's own header. + + Removing the *last* ``subscribe()`` renumbers nothing, so no other wait's + binding disagrees either. Without a check that every recorded binding was + recreated, the removal replays clean and the Workflow reaches its next + command having subscribed to one stream fewer than History says it did. + """ + left = StreamKey("ns", "wf", uuid.uuid4().hex, "left") + quiet = StreamKey("ns", "wf", uuid.uuid4().hex, "quiet") + left_record = await _one_record(backend, left) + + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(left), 2: binding(quiet)}), + segments=( + Segment( + (Run(1, left_record.offset, left_record.offset, 1),), # type: ignore[arg-type] + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + # Wait 2 delivered nothing, so the terminal names only wait 1. + terminal={1: AFTER(left_record.offset)}, # type: ignore[arg-type] + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + # Wait 2's `subscribe()` call is gone. Wait 1 is untouched and consumes + # exactly what the marker recorded for it. + runtime.register(wait_id=1, stream_key=left) + + with pytest.raises(temporalio.workflow.NondeterminismError) as caught: + drive(runtime) + + assert not isinstance(caught.value, StreamIntegrityError), ( + "the backend holds exactly what it was given; this must not be " + "reachable through the storage-failure taxonomy" + ) + assert "[2]" in str(caught.value), ( + f"the error must name the wait the marker recorded and the code no " + f"longer creates: {caught.value}" + ) + assert "workflow.patched" in str(caught.value), ( + "the remedy is versioning the Workflow code, exactly as for a removed timer" + ) + + +@pytest.mark.asyncio +async def test_a_removed_middle_subscription_to_the_same_stream_is_nondeterminism( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """The renumbering case no binding comparison can see. + + Three waits on one stream and one backend, with the last of them quiet for + this Workflow Task. Deleting the *middle* ``subscribe()`` renumbers what was + wait 3 down to wait 2 -- and because all three name the same stream and the + same backend, the binding check compares equal for every wait that still + exists. The delivery-side check is quiet too: the records the marker + recorded for wait 2 are taken by the subscription that used to be wait 3, so + nothing is left over. + + That is the whole failure: the marker's records reach a *different* + subscription than the one that consumed them live, with its own cursor and + its own later reads, and every check that looks at what the code did agrees + with what the code did. Only the wait the marker recorded and the code no + longer creates gives it away. + """ + key = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + placed = await append_five(backend, key) + + annotation = encode_annotation( + Annotation( + header=AnnotationHeader( + {1: binding(key), 2: binding(key), 3: binding(key)} + ), + segments=( + Segment( + ( + Run(1, placed[0].offset, placed[0].offset, 1), # type: ignore[arg-type] + Run(2, placed[1].offset, placed[1].offset, 1), # type: ignore[arg-type] + ), + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + # Wait 3 was subscribed and quiet: a binding, no run, no terminal + # entry. + terminal={ + 1: AFTER(placed[0].offset), # type: ignore[arg-type] + 2: AFTER(placed[1].offset), # type: ignore[arg-type] + }, + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + # The middle `subscribe()` is gone, so the Workflow now makes two waits + # where History records three -- and the survivor that was wait 3 is + # registered as wait 2. + runtime.register(wait_id=1, stream_key=key) + runtime.register(wait_id=2, stream_key=key) + + with pytest.raises(temporalio.workflow.NondeterminismError) as caught: + drive(runtime) + + assert "[3]" in str(caught.value), ( + f"the error must name the wait whose subscribe() call went missing, " + f"which is the only trace the marker leaves of it: {caught.value}" + ) + assert "workflow.patched" in str(caught.value) + + +@pytest.mark.asyncio +async def test_a_subscription_the_marker_never_bound_is_not_a_removal( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """The check is one-directional, and that is not an oversight. + + Replay does not stop at the Workflow Task the marker covers: the Workflow + runs forward, and the ``subscribe()`` calls it makes past that point belong + to the *next* marker's header rather than to this one. Requiring every + registered wait to appear in the marker would report every one of them as + nondeterminism, and would turn adding a ``subscribe()`` at the end -- a + supported change, as adding a timer at the end is -- into a Workflow that + can never replay. + """ + left = StreamKey("ns", "wf", uuid.uuid4().hex, "left") + added = StreamKey("ns", "wf", uuid.uuid4().hex, "added") + left_record = await _one_record(backend, left) + await _one_record(backend, added) + + await manager.prepare_replay( + RUN_ID, + _single_run_annotation( + {1: binding(left)}, + [Run(1, left_record.offset, left_record.offset, 1)], # type: ignore[arg-type] + ), + ) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=left) + # The new `subscribe()`, reached because replay keeps running after the + # marker's own Workflow Task. Nothing in this marker binds it, and nothing + # should. + runtime.register(wait_id=2, stream_key=added) + + stub = drive(runtime) + + assert len(stub.drains) == 1, ( + "the marker's one segment must still replay; a wait the marker never " + "bound is not a removed subscription" + ) + + +@pytest.mark.asyncio +async def test_a_subscription_made_during_the_replay_drive_is_not_missing( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """Which is why the check runs once at the end, not before each segment. + + An activation carrying both ``InitializeWorkflow`` and + ``ReplayExternalStreams`` applies every job before any Workflow code runs, + so on a Run's first marker the ``subscribe()`` call happens *inside* the + driver's first drain -- after the segment it belongs to has already been + made current. A binding check placed on the per-segment path would fail + every such replay, which is every Workflow that consumes a stream from its + first Workflow Task. + """ + key = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + recorded = await _one_record(backend, key) + + await manager.prepare_replay( + RUN_ID, + _single_run_annotation( + {1: binding(key)}, + [Run(1, recorded.offset, recorded.offset, 1)], # type: ignore[arg-type] + ), + ) + runtime = make_runtime(manager, backend) + + taken: list[Offset] = [] + + class LateSubscribingStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + if 1 not in runtime.subscriptions(): + runtime.register(wait_id=1, stream_key=key) + taken.extend(r.offset for r in runtime.drain(1)) # type: ignore[misc] + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + drive_with(LateSubscribingStub(runtime)) + + assert taken == [recorded.offset], ( + f"the subscription made inside the drive must satisfy the marker's " + f"binding and take its record: {taken}" + ) + + +@pytest.mark.asyncio +async def test_a_removed_subscriptions_records_are_taken_by_nobody( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """Nothing may consume on behalf of a subscription that does not exist. + + When the removed ``subscribe()`` did record records, the delivery check is + what reports it -- the records reach the end of the replay untaken, because + a Workflow holds no handle to a wait it never made and so cannot drain it. + Consuming them anyway, as a driver that drained by whatever the segment + holds would, reports the replay clean on that count and leaves the removal + to be caught, if at all, by something else. + """ + left = StreamKey("ns", "wf", uuid.uuid4().hex, "left") + right = StreamKey("ns", "wf", uuid.uuid4().hex, "right") + left_record = await _one_record(backend, left) + right_record = await _one_record(backend, right) + + await manager.prepare_replay( + RUN_ID, + _single_run_annotation( + {1: binding(left), 2: binding(right)}, + [ + Run(1, left_record.offset, left_record.offset, 1), # type: ignore[arg-type] + Run(2, right_record.offset, right_record.offset, 1), # type: ignore[arg-type] + ], + ), + ) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=left) + + with pytest.raises( + temporalio.workflow.NondeterminismError, match="never took" + ) as caught: + drive(runtime) + + assert "[2]" in str(caught.value), ( + f"the error must name the wait whose recorded records nobody took: " + f"{caught.value}" + ) + + +# --- provider binding --------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_backend_declaring_another_provider_is_refused_before_any_read( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """A differently configured implementation is a deployment problem. + + Neither nondeterminism -- the Workflow is unchanged -- nor integrity loss: + the store that wrote the records is simply not the one configured here. It + is refused before the read, so an implementation that cannot + interpret those offsets never gets to try. + """ + left = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + record = await _one_record(backend, left) + + with pytest.raises(StreamStorageError, match="declares"): + await manager.prepare_replay( + RUN_ID, + _single_run_annotation( + {1: binding(left, provider_id="redis-streams")}, + [Run(1, record.offset, record.offset, 1)], # type: ignore[arg-type] + ), + ) + + assert backend.range_reads == [], ( + "the recorded range must not be read through an implementation that " + "did not write it" + ) + + +@pytest.mark.asyncio +async def test_an_unreadable_provider_format_version_is_refused_before_any_read( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """The recorded format version is checked, not merely stored. + + A provider that keeps its id while changing how it lays records out is the + case a matching id cannot catch: the offsets read back mean something else, + and every one of replay's four range checks would be applied to the wrong + interpretation of the bytes. + """ + left = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + record = await _one_record(backend, left) + + with pytest.raises(StreamStorageError, match="format version"): + await manager.prepare_replay( + RUN_ID, + _single_run_annotation( + { + 1: binding( + left, + provider_format_version=( + MemoryStreamBackend.provider_format_version + 1 + ), + ) + }, + [Run(1, record.offset, record.offset, 1)], # type: ignore[arg-type] + ), + ) + + assert backend.range_reads == [] + + +# --- a wait registered after the header, replayed ---------------------------- + + +class FakeInstance: + """Stands in for the Workflow object the per-Run subscription state hangs off.""" + + +@pytest.mark.asyncio +async def test_a_wait_registered_mid_annotation_replays( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """The whole round trip for a subscription created after the first delta. + + `register` accepts one at any activation of a retained Workflow Task, long + after the header frame went to Core -- and Core appends deltas rather than + rewriting them. A wait bound nowhere reaches the marker as a run and a + terminal entry alone, and `prepare_replay` then has no stream key and no + backend for it, so replay of *unchanged* code fails as a wait "this + Workflow did not create". + """ + from temporalio.contrib.external_workflow_streams._runtime import ( + WorkflowStreamRuntime, + ) + + chain = uuid.uuid4().hex + first_key = StreamKey("ns", "wf", chain, "tokens") + late_key = StreamKey("ns", "wf", chain, "tool-events") + first_record = (await append_five(backend, first_key))[0] + late_record = (await append_five(backend, late_key))[0] + + recorder = WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id="recording-run", + namespace="ns", + workflow_id="wf", + first_execution_run_id=chain, + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + ) + recorder.register(wait_id=1, stream_key=first_key) + recorder.record_delivery(1, first_record) + deltas = [recorder.take_observation_delta()] + # The header is fixed from here on: Core already holds those bytes. + recorder.register(wait_id=2, stream_key=late_key) + recorder.record_delivery(2, late_record) + deltas.append(recorder.take_observation_delta()) + annotation = b"".join(d for d in deltas if d is not None) + recorder.add_terminal() + + assert set(decode_annotation(annotation).header.streams) == {1, 2} + + plan = await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=first_key) + runtime.register(wait_id=2, stream_key=late_key) + + drained: dict[int, list[Offset]] = {1: [], 2: []} + + class DrainingStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + for wait_id in (1, 2): + drained[wait_id].extend(r.offset for r in runtime.drain(wait_id)) # type: ignore[misc] + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + drive_with(DrainingStub(runtime)) + + assert drained == {1: [first_record.offset], 2: [late_record.offset]} + assert plan.committed_boundaries == { + 1: AFTER(first_record.offset), # type: ignore[arg-type] + 2: AFTER(late_record.offset), # type: ignore[arg-type] + } + + +# --- a segment's global order is the order Workflow code receives in --------- + + +@pytest.mark.asyncio +async def test_a_segment_replays_in_its_recorded_cross_stream_order( + manager: StreamSubscriptionManager, + backend: MemoryStreamBackend, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A segment recorded as (wait 2, wait 1) must not replay as (wait 1, wait 2). + + The order is reachable live and `merge` is how: a pass asks in `wait_id` + order, finds wait 1's buffer empty and wait 2's holding a record, and wait + 1's record lands from the manager's loop before the next pass. That is one + activation, so it is one segment, and the segment records (2, 1). + + On replay `merge` asks in the same `wait_id` order -- but every record of + the segment is already in hand. A drain that searched the segment for its + own wait would answer the first ask with wait 1's record and reverse the + two, which is a different sequence of values reaching Workflow code and so + a different sequence of commands. Taking only from the front is what makes + the empty answer replay as empty. + """ + chain = uuid.uuid4().hex + left = StreamKey("ns", "wf", chain, "left") + right = StreamKey("ns", "wf", chain, "right") + left_record = (await append_five(backend, left))[0] + right_record = (await append_five(backend, right))[0] + + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(left), 2: binding(right)}), + segments=( + Segment( + ( + Run(2, right_record.offset, right_record.offset, 1), # type: ignore[arg-type] + Run(1, left_record.offset, left_record.offset, 1), # type: ignore[arg-type] + ), + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={ + 1: AFTER(left_record.offset), # type: ignore[arg-type] + 2: AFTER(right_record.offset), # type: ignore[arg-type] + }, + ) + ) + plan = await manager.prepare_replay(RUN_ID, annotation) + assert [wait_id for wait_id, _ in plan.segments[0].deliveries] == [2, 1] + + runtime = make_runtime(manager, backend) + instance = FakeInstance() + monkeypatch.setattr( + "temporalio.contrib.external_workflow_streams._api._run_holder", + lambda: instance, + ) + _install_runtime(instance, runtime) + + runtime.begin_replay(plan.annotation.header.streams) + try: + first = external_stream.topic("left", type=str).subscribe() + second = external_stream.topic("right", type=str).subscribe() + assert (first.wait_id, second.wait_id) == (1, 2) + + runtime.begin_replay_segment(list(plan.segments[0].deliveries)) + merged = merge(first, second) + seen = [await merged.__anext__() for _ in range(2)] + await merged.aclose() + runtime.verify_replay_consumed() + finally: + runtime.end_replay() + + assert [(subscription.wait_id, value) for subscription, value in seen] == [ + (2, "v0"), + (1, "v0"), + ], "replay reordered a segment that recorded wait 2's record before wait 1's" + + +@pytest.mark.asyncio +async def test_a_replayed_drain_stops_at_another_waits_record( + manager: StreamSubscriptionManager, backend: MemoryStreamBackend +) -> None: + """The same rule at the drain, where a wait appears twice in one segment. + + Live, the second batch was not in this wait's buffer when the first drain + ran -- the record between them belongs to a drain that had not happened + yet. Handing both over at once would collapse two of the segment's drains + into one and let the values interleave differently. + """ + chain = uuid.uuid4().hex + left = StreamKey("ns", "wf", chain, "left") + right = StreamKey("ns", "wf", chain, "right") + left_records = await append_five(backend, left) + right_records = await append_five(backend, right) + + annotation = encode_annotation( + Annotation( + header=AnnotationHeader({1: binding(left), 2: binding(right)}), + segments=( + Segment( + ( + Run(1, left_records[0].offset, left_records[0].offset, 1), # type: ignore[arg-type] + Run(2, right_records[0].offset, right_records[0].offset, 1), # type: ignore[arg-type] + Run(1, left_records[1].offset, left_records[1].offset, 1), # type: ignore[arg-type] + ), + SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={ + 1: AFTER(left_records[1].offset), # type: ignore[arg-type] + 2: AFTER(right_records[0].offset), # type: ignore[arg-type] + }, + ) + ) + await manager.prepare_replay(RUN_ID, annotation) + runtime = make_runtime(manager, backend) + runtime.register(wait_id=1, stream_key=left) + runtime.register(wait_id=2, stream_key=right) + + taken: list[tuple[int, Offset]] = [] + + class InterleavingStub(DriverStub): + def _run_once(self, *, check_conditions: bool) -> None: + super()._run_once(check_conditions=check_conditions) + for wait_id in (1, 2, 1): + taken.extend((wait_id, r.offset) for r in runtime.drain(wait_id)) # type: ignore[misc] + + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + drive_with(InterleavingStub(runtime)) + + assert taken == [ + (1, left_records[0].offset), + (2, right_records[0].offset), + (1, left_records[1].offset), + ] diff --git a/tests/contrib/external_workflow_streams/test_replay_end_to_end.py b/tests/contrib/external_workflow_streams/test_replay_end_to_end.py new file mode 100644 index 0000000000..279b9a5dba --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_replay_end_to_end.py @@ -0,0 +1,1098 @@ +"""Replay through the real path: a live run, then its own history replayed. + +The unit tests drive the replay driver directly, which proves the mechanism but +not that a history a Worker actually produced can be fed back through it. These +run a Workflow against a live server, fetch the history it wrote, and replay it +with the real ``Replayer`` -- the same tool a user reaches for to check new code +against old histories. + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +# pyright: reportMissingParameterType=false +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest + +import temporalio.api.enums.v1 +import temporalio.converter +from temporalio import workflow +from temporalio.client import Client +from temporalio.contrib.external_workflow_streams._annotation import decode_annotation +from temporalio.contrib.external_workflow_streams._backend import ( + DEFAULT_WATCH_BLOCK, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._record import ( + BEGINNING, + Cursor, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.worker import Replayer, Worker +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend +from tests.contrib.external_workflow_streams.test_output_worker_integration import ( + _OutputMemoryBackend, +) + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams import external_output_stream + from temporalio.contrib.external_workflow_streams._api import external_stream + from tests.contrib.external_workflow_streams import observations + + +EXTERNAL_STREAM_MARKER = "core_external_stream" +"""The marker name Core writes a replay annotation under.""" + + +# Unsandboxed on purpose: the observation sink below has to be the *same* +# object in Workflow code as in the test. A sandboxed Workflow re-imports every +# module it touches, including a passed-through one, so it would write its +# observations into a copy and the comparison would silently compare nothing. +# Nothing here depends on the sandbox; the sandbox itself is covered elsewhere. +@workflow.defn(sandboxed=False) +class ConditionWorkflow: + """Consumes records while a ``wait_condition`` watches the same state. + + The condition is the point. It is evaluated once per event-loop drain, so it + is a direct probe of activation segmentation: if replay collapses several + recorded segments into one, the predicate sees a different sequence of + states than it did live even though the records arrive in the same order. + """ + + def __init__(self) -> None: + self._seen: list[str] = [] + self._states_observed: list[int] = [] + + @workflow.run + async def run(self, expected: int) -> list[int]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + + async def watch() -> None: + # Records what the predicate saw on every evaluation. Returning this + # rather than the records is what makes a collapsed replay visible: + # the record order would look identical either way. + await workflow.wait_condition(self._observe) + + watcher = asyncio.ensure_future(watch()) + async for token in tokens.subscribe(): + self._seen.append(token) + if len(self._seen) >= expected: + break + await watcher + # Recorded outside the Workflow as well as returned: the return value is + # only visible for the live run, and the whole question here is whether + # the *replay* saw the same thing. + observations.record(workflow.info().run_id, self._states_observed) + return self._states_observed + + def _observe(self) -> bool: + self._states_observed.append(len(self._seen)) + return len(self._seen) >= 2 + + +@workflow.defn(sandboxed=False) +class CombinedInputOutputConditionWorkflow: + """Records the drain schedule while input and output share one retained WFT.""" + + def __init__(self) -> None: + self._seen: list[str] = [] + self._states_observed: list[int] = [] + + @workflow.run + async def run(self, expected: int) -> list[int]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + output = external_output_stream.with_options( + max_publish_latency=timedelta(seconds=30) + ).topic("events", type=str) + + async def watch() -> None: + await workflow.wait_condition(self._observe) + + watcher = asyncio.ensure_future(watch()) + async for token in tokens.subscribe(): + self._seen.append(token) + await output.publish(f"observed:{token}") + if len(self._seen) >= expected: + break + await watcher + observations.record(workflow.info().run_id, self._states_observed) + return self._states_observed + + def _observe(self) -> bool: + self._states_observed.append(len(self._seen)) + return len(self._seen) >= 2 + + +@workflow.defn +class EmptyStreamWorkflow: + """Subscribes to a stream nothing is ever written to, then gives up. + + The case an implicit "start wherever" would leave unrecorded: the marker has + to carry an explicit start cursor even though no record was ever delivered, + or replay has no boundary to reproduce. + """ + + @workflow.run + async def run(self) -> str: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=1)).topic( + "tokens", type=str + ) + subscription = tokens.subscribe() + iterator = subscription.__aiter__() + try: + await asyncio.wait_for(iterator.__anext__(), 3) + except asyncio.TimeoutError: + return "nothing arrived" + return "unexpected record" + + +@workflow.defn(name="QuietSubscriptionWorkflow") +class QuietSubscriptionWorkflow: + """Creates replay-visible stream state without reading a record.""" + + @workflow.run + async def run(self) -> str: + external_stream.topic("tokens", type=str).subscribe() + return "done" + + +@workflow.defn(name="QuietSubscriptionWorkflow") +class QuietSubscriptionRemovedWorkflow: + """The next version of ``QuietSubscriptionWorkflow``, with no stream call.""" + + @workflow.run + async def run(self) -> str: + return "done" + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +#: Per-stream sequence, so successive publishes do not collide. `(session_id, +#: sequence)` is the idempotency key: restarting the count re-uses a key with +#: different content, which the backend contract rejects outright -- correctly, +#: and it is the test that is wrong when it happens. +_sequences: dict[StreamKey, int] = {} + + +async def publish(backend: MemoryStreamBackend, key: StreamKey, values: list[str]): + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + start = _sequences.get(key, 0) + for i, value in enumerate(values, start=start): + await backend.append( + key, + StreamRecord(RecordKind.DATA, await codec.encode(value), "producer", i), + ) + _sequences[key] = start + len(values) + + +async def stream_key_for(client: Client, handle, name: str) -> StreamKey: # type: ignore[no-untyped-def] + description = await handle.describe() + return StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + name, + ) + + +async def test_replaying_a_stream_history_reproduces_the_same_observations( + client: Client, backend: MemoryStreamBackend +) -> None: + """The end-to-end form of ADR-018, through the tool users actually run. + + A collapsed replay would deliver the same records in the same order and + still be wrong: the predicate would fire a different number of times, and + any Workflow whose control flow depends on a condition would diverge. + + So "replay did not fail" is not the assertion. Replay is only obliged to + match the *commands* in History, and this Workflow's commands say nothing + about how many times its predicate ran -- a replay that delivered both + records in one drain would produce the same completion and pass a + failure-only check. What is compared instead is the sequence the predicate + itself observed, live against replayed, for a marker that spans several + activations. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[ConditionWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + ConditionWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + key = await stream_key_for(client, handle, "tokens") + await asyncio.sleep(1) + # Two separate arrivals, so the marker spans more than one activation. + await publish(backend, key, ["alpha"]) + await asyncio.sleep(0.5) + await publish(backend, key, ["beta"]) + + live = await asyncio.wait_for(handle.result(), 60) + history = await handle.fetch_history() + run_id = handle.first_execution_run_id or handle.result_run_id + + markers = [ + e + for e in history.events + if e.HasField("marker_recorded_event_attributes") + and e.marker_recorded_event_attributes.marker_name == EXTERNAL_STREAM_MARKER + ] + assert markers, "no stream marker was written, so there is nothing to replay" + + replayer = Replayer( + workflows=[ConditionWorkflow], + external_stream_backend=backend, + ) + result = await replayer.replay_workflow(history) + + assert result.replay_failure is None, ( + f"replaying the history the Worker just wrote failed: {result.replay_failure}" + ) + assert live, "the live run observed nothing, so this proves nothing" + assert len(live) > 1, ( + "the predicate ran only once live, so a collapsed replay would look " + f"identical and this proves nothing: {live}" + ) + + assert run_id is not None + runs = observations.executions(run_id) + assert len(runs) == 2, ( + "expected exactly two executions to be recorded -- the live run and the " + f"replay -- got {len(runs)}" + ) + live_observed, replayed_observed = runs + assert live_observed == live, ( + "the live run recorded something other than it returned" + ) + assert replayed_observed == live_observed, ( + "the replayed run's predicate saw a different sequence of states than " + "the live one did. The records arrive in the same order either way, so " + "this is the segmentation itself diverging: recorded activation " + f"boundaries were collapsed or re-cut. live={live_observed} " + f"replayed={replayed_observed}" + ) + + +async def test_live_and_replay_share_one_input_output_drain_schedule( + client: Client, +) -> None: + """A real shared marker reproduces one drain sequence, not two drivers.""" + backend = _OutputMemoryBackend() + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[CombinedInputOutputConditionWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + CombinedInputOutputConditionWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + key = await stream_key_for(client, handle, "tokens") + await asyncio.sleep(0.5) + await publish(backend, key, ["alpha"]) + # The first readiness activation must finish and retain the same WFT + # before the second arrival, so the marker records two live drains. + await asyncio.sleep(0.25) + await publish(backend, key, ["beta"]) + + live = await asyncio.wait_for(handle.result(), 60) + history = await handle.fetch_history() + run_id = handle.first_execution_run_id or handle.result_run_id + + markers = stream_markers(history.events) + assert len(markers) == 1 + marker_event = markers[0] + + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + + marker = ExternalStreamMarkerData() + marker.ParseFromString( + marker_event.marker_recorded_event_attributes.details["external_stream"] + .payloads[0] + .data + ) + assert marker.HasField("output") + assert marker.waits + annotation = decode_annotation(marker.replay_annotation) + assert len(annotation.segments) >= 3 + assert len(marker.output.segments) == len(annotation.segments) + assert ( + sum(sum(segment.record_counts_by_topic) for segment in marker.output.segments) + == 2 + ) + assert sum(bool(segment.runs) for segment in annotation.segments) == 2 + + result = await Replayer( + workflows=[CombinedInputOutputConditionWorkflow], + external_stream_backend=backend, + ).replay_workflow(history) + assert result.replay_failure is None, ( + f"combined input/output replay failed: {result.replay_failure}" + ) + assert len(live) > 1 + assert run_id is not None + runs = observations.executions(run_id) + assert len(runs) == 2 + live_observed, replayed_observed = runs + assert live_observed == live + assert replayed_observed == live_observed, ( + "the shared input/output replay driver changed the live drain schedule: " + f"live={live_observed}, replay={replayed_observed}" + ) + assert len(backend.stages) == 1, "replay must not stage output again" + + +async def test_a_history_with_stream_markers_needs_its_backend_to_replay( + client: Client, backend: MemoryStreamBackend +) -> None: + """Replay re-reads the recorded ranges, so it needs the provider. + + Without the option there is no way to supply one, and the replay fails the + way a Worker with no backend would -- correct, but a configuration error + rather than a finding about the history. Asserted so the option cannot + quietly stop being threaded through. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[ConditionWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + ConditionWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + key = await stream_key_for(client, handle, "tokens") + await asyncio.sleep(1) + await publish(backend, key, ["alpha", "beta"]) + await asyncio.wait_for(handle.result(), 60) + history = await handle.fetch_history() + + without = await Replayer(workflows=[ConditionWorkflow]).replay_workflow( + history, raise_on_replay_failure=False + ) + + assert without.replay_failure is not None + assert "external_stream_backend" in str(without.replay_failure), ( + "the failure must name the missing option rather than surface as an " + f"attribute error: {without.replay_failure}" + ) + + +async def test_a_no_backend_replayer_validates_a_removed_quiet_subscription( + client: Client, backend: MemoryStreamBackend +) -> None: + """History, not current Worker configuration, decides whether replay runs. + + A quiet subscription still writes its binding into the marker. The next + Workflow version removes the only ``subscribe()`` call while leaving its + ordinary Temporal command sequence unchanged, and the Replayer deliberately + has no backend configured. It must still decode the marker and apply the reverse + binding check rather than accepting the history because no live stream + runtime happened to be configured. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[QuietSubscriptionWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + QuietSubscriptionWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + assert await asyncio.wait_for(handle.result(), 30) == "done" + history = await handle.fetch_history() + + markers = [ + event + for event in history.events + if event.HasField("marker_recorded_event_attributes") + and event.marker_recorded_event_attributes.marker_name == EXTERNAL_STREAM_MARKER + ] + assert markers, "the quiet subscription produced no marker to validate" + + result = await Replayer( + workflows=[QuietSubscriptionRemovedWorkflow] + ).replay_workflow(history, raise_on_replay_failure=False) + + assert result.replay_failure is not None, ( + "removing the recorded subscription replayed successfully when the " + "Replayer had no backend configured" + ) + assert "never created" in str(result.replay_failure), ( + "the marker's reverse binding check did not report the removed quiet " + f"subscription: {result.replay_failure}" + ) + + +async def test_an_empty_stream_replays_from_its_recorded_boundary( + client: Client, backend: MemoryStreamBackend +) -> None: + """A subscription that never received a record still has a boundary. + + This is the case the explicit start cursor exists for: the marker records + where the subscription began even though nothing was delivered, so replay + reproduces an empty observation rather than resolving a position from + whatever the stream holds by then. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[EmptyStreamWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + EmptyStreamWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + assert await asyncio.wait_for(handle.result(), 60) == "nothing arrived" + history = await handle.fetch_history() + key = await stream_key_for(client, handle, "tokens") + + # Records land *after* the Run finished. Replay must not see them: the + # marker's boundary is where the subscription was, not where the stream got + # to afterwards. + await publish(backend, key, ["late-one", "late-two"]) + + result = await Replayer( + workflows=[EmptyStreamWorkflow], + external_stream_backend=backend, + ).replay_workflow(history, raise_on_replay_failure=False) + + assert result.replay_failure is None, ( + "replay resolved a position from live stream state rather than from the " + f"recorded boundary: {result.replay_failure}" + ) + + +#: ``run_id -> [whether each execution of the run body began while replaying]``. +#: +#: A Run that was evicted and came back executes its body from the top a second +#: time, replaying; one that merely stayed cached never does. Written from +#: Workflow code, which is why the Workflow below is unsandboxed: a sandboxed +#: one would write into its own copy of this module and the count would stay at +#: one however many times the Run was rebuilt. +STARTS: dict[str, list[bool]] = {} + + +@workflow.defn(sandboxed=False) +class EmptyStreamParkWorkflow: + """Subscribes to an empty stream, parks on it, and consumes what arrives later. + + Nothing else is pending -- no timer, no other command -- so the first + Workflow Task is retained until the idle timeout parks it. Both halves of + that matter here: the park is what writes the empty marker, and it is also + what leaves the Run resumable *only* by a wake Signal, so the records + published while it is evicted cannot be delivered by a watcher that happened + to survive. + """ + + def __init__(self) -> None: + self._seen: list[str] = [] + + @workflow.run + async def run(self, expected: int) -> list[str]: + STARTS.setdefault(workflow.info().run_id, []).append( + workflow.unsafe.is_replaying() + ) + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=1)).topic( + "tokens", type=str + ) + async for token in tokens.subscribe(): + self._seen.append(token) + if len(self._seen) >= expected: + break + # Recorded outside the Workflow as well as returned: the return value + # exists only for the live run, and what the *replays* observed is half + # the question here. + observations.record(workflow.info().run_id, self._seen) + return self._seen + + +@workflow.defn +class CacheFillerWorkflow: + """Occupies the one cache slot, which is what evicts the Run under test.""" + + @workflow.run + async def run(self) -> str: + return "done" + + +class RecordedRangesOnlyBackend(MemoryStreamBackend): + """Serves recorded ranges faithfully and poisons every live read. + + Replay is obliged to read the ranges its markers name and nothing else, so + those ranges read back exactly as written while the *live* view of the same + stream answers with records no marker ever recorded. Any position or + delivery replay took from live state then shows up as a record the live Run + never saw, rather than as nothing -- which is the difference that matters + for an empty marker, whose live-resolved position would otherwise hand the + Workflow the very records it was about to receive anyway and look correct. + + The poison is served once. A live view that answered forever would fill the + buffer and say nothing more about it than the first answer already does. + + A watcher still runs during replay -- the Run goes on living once the last + marker is exhausted, and something has to be watching for that -- so + ``read_after`` is answered rather than refused. Writes and the park + handshake are recorded rather than refused for a reason worth keeping: an + exception raised inside a watcher is caught by the manager's own retry loop + and logged, so a refusal there would be asserted into a log line nobody + reads. + """ + + def __init__(self, source: MemoryStreamBackend, poison: list[StreamRecord]) -> None: + super().__init__() + # The same storage the live Run wrote, so the recorded ranges -- and + # only those -- still read back through `read_range`. + self._records = source._records + self._by_key = source._by_key + self._poison = poison + self._poison_served = False + #: Every provider call replay made that was not a recorded-range read. + self.live_calls: list[str] = [] + + async def read_after( # type: ignore[override] + self, + key: StreamKey, + after: Cursor, + *, + max_records: int, + block: timedelta | None = DEFAULT_WATCH_BLOCK, + ) -> list[StreamRecord]: + self.live_calls.append("read_after") + if not self._poison_served: + self._poison_served = True + return list(self._poison[:max_records]) + if block is not None: + # Blocked rather than returned empty immediately: a watcher whose + # read returns instantly spins, and the busy loop would be this + # test's own doing. + await asyncio.sleep(block.total_seconds()) + return [] + + async def append(self, key: StreamKey, record: StreamRecord) -> StreamRecord: + self.live_calls.append("append") + raise AssertionError("replay must never append to the stream") + + async def install_park_intent(self, key, intent) -> None: # type: ignore[no-untyped-def] + self.live_calls.append("install_park_intent") + + async def recheck(self, key: StreamKey, wait_id: int) -> bool: + self.live_calls.append("recheck") + return False + + +def stream_markers(events) -> list: # type: ignore[no-untyped-def] + return [ + e + for e in events + if e.HasField("marker_recorded_event_attributes") + and e.marker_recorded_event_attributes.marker_name == EXTERNAL_STREAM_MARKER + ] + + +def marker_annotation(marker_event): # type: ignore[no-untyped-def] + """The decoded annotation a marker carries.""" + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + + data = ExternalStreamMarkerData() + data.ParseFromString( + marker_event.marker_recorded_event_attributes.details["external_stream"] + .payloads[0] + .data + ) + return decode_annotation(data.replay_annotation) + + +async def wait_for_markers( + handle, count: int, message: str, timeout: float = 30 +) -> None: # type: ignore[no-untyped-def] + """Polls until History holds ``count`` stream markers. + + Polled rather than slept: a marker is written when the Workflow Task ends, + and waiting a fixed time instead would make every assertion after it a + statement about how fast this machine is. + """ + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if ( + len(stream_markers([e async for e in handle.fetch_history_events()])) + >= count + ): + return + await asyncio.sleep(0.3) + raise AssertionError(message) + + +async def wait_until(predicate, message: str, timeout: float = 15) -> None: # type: ignore[no-untyped-def] + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if predicate(): + return + await asyncio.sleep(0.1) + raise AssertionError(message) + + +async def _stall_diagnosis(client, handle, backend, key, run_id) -> str: # type: ignore[no-untyped-def] + """What the Run, its History, and the stream look like when the resume stalls. + + The failure is a 45-second timeout, and a timeout on its own cannot tell + "no Workflow Task was ever created" from "one was created and delivered + nothing" -- which are different defects in different components. Each line + below separates a pair of them. + """ + lines: list[str] = [] + try: + events = [event async for event in handle.fetch_history_events()] + kinds: dict[str, int] = {} + for event in events: + kinds[event.WhichOneof("attributes") or "?"] = ( + kinds.get(event.WhichOneof("attributes") or "?", 0) + 1 + ) + lines.append(f"history: {len(events)} event(s), {kinds}") + # A Workflow Task after the wake is the difference between "the wake never + # produced one" and "it did, and the Run delivered nothing". + started = [ + i + for i, e in enumerate(events) + if e.HasField("workflow_task_started_event_attributes") + ] + lines.append(f"workflow tasks started at event indices: {started}") + failures = [ + e.workflow_task_failed_event_attributes + for e in events + if e.HasField("workflow_task_failed_event_attributes") + ] + lines.append( + f"workflow task failures: {[(f.cause, f.failure.message[:200]) for f in failures]}" + ) + lines.append(f"stream markers in history: {len(stream_markers(events))}") + except Exception as err: # noqa: BLE001 -- diagnosis must not mask the failure + lines.append(f"history unavailable: {err!r}") + try: + description = await handle.describe() + lines.append(f"workflow status: {description.status}") + except Exception as err: # noqa: BLE001 + lines.append(f"describe failed: {err!r}") + try: + records = backend.all_records(key) + lines.append( + f"stream holds {len(records)} record(s) at " + f"{[str(r.offset) for r in records]}" + ) + parked = await backend.parked_wait_ids(key) + lines.append(f"parked wait ids: {parked}") + for wait_id in parked: + intent = await backend.park_intent(key, wait_id) + lines.append(f" wait {wait_id} intent: {intent}") + except Exception as err: # noqa: BLE001 + lines.append(f"backend state unavailable: {err!r}") + lines.append(f"workflow body starts (is_replaying per start): {STARTS.get(run_id)}") + lines.append(f"observations for this Run: {observations.executions(run_id)}") + # The readiness handshake, which is the only thing that can turn a buffered + # record into a Workflow Task. Its answers separate "the watcher never read + # the records" from "it read them and Core had nowhere to put them". + lines.append(f"readiness calls: {READINESS_TRACE}") + for manager, subscription in _live_subscriptions(): + lines.append( + f"subscription {subscription.run_id}/{subscription.wait_id}: " + f"buffered={subscription.buffered} " + f"committed={subscription.committed_cursor} " + f"delivery={subscription.delivery_cursor} " + f"prefetch={subscription.prefetch_cursor} " + f"generation={subscription.current_wait_generation()} " + f"wakes_owed={subscription.wakes_owed} " + f"cancelled={subscription._cancelled} " + f"watcher_done={None if subscription._watcher is None else subscription._watcher.done()}" + ) + del manager + return "Stall diagnosis:\n " + "\n ".join(lines) + + +#: Every readiness call the Worker's manager made, with what Core answered. +READINESS_TRACE: list[tuple[str, int, int, object]] = [] + +#: Managers alive in this process, so a stalled test can read their state. +LIVE_MANAGERS: list[object] = [] + + +def _live_subscriptions(): # type: ignore[no-untyped-def] + """Every subscription still registered on any manager in this process.""" + found = [] + for manager in LIVE_MANAGERS: + for run_subscriptions in getattr(manager, "_runs", {}).values(): + for subscription in run_subscriptions.values(): + found.append((manager, subscription)) + return found + + +@pytest.fixture +def trace_readiness(monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def] + """Records what Core answered for every readiness call, and every manager. + + A stall in this file is always "a buffered record never became a Workflow + Task", and the readiness answers are where that happens. Without them the + failure cannot distinguish a watcher that never read from a Worker that read + and was told there was nowhere to deliver. + """ + from temporalio.contrib.external_workflow_streams._manager import ( + StreamSubscriptionManager, + ) + + READINESS_TRACE.clear() + LIVE_MANAGERS.clear() + + original_init = StreamSubscriptionManager.__init__ + original_notify = StreamSubscriptionManager._notify_ready_with_retries + + def spy_init(self, *args, **kwargs): # type: ignore[no-untyped-def] + original_init(self, *args, **kwargs) + LIVE_MANAGERS.append(self) + + async def spy_notify(self, subscription): # type: ignore[no-untyped-def] + result = await original_notify(self, subscription) + READINESS_TRACE.append( + ( + subscription.run_id[-8:], + subscription.wait_id, + subscription.current_wait_generation(), + result, + ) + ) + return result + + monkeypatch.setattr(StreamSubscriptionManager, "__init__", spy_init) + monkeypatch.setattr( + StreamSubscriptionManager, "_notify_ready_with_retries", spy_notify + ) + yield + LIVE_MANAGERS.clear() + + +async def test_an_empty_stream_parked_and_evicted_replays_from_the_recorded_cursor( + client: Client, + backend: MemoryStreamBackend, + monkeypatch: pytest.MonkeyPatch, + trace_readiness: None, # pyright: ignore[reportUnusedParameter] +) -> None: + """The empty boundary, carried across a park, an eviction, and two replays. + + Four things happen in order, and each is asserted rather than assumed -- + which is the whole difficulty of this case, because a test that quietly + skipped the park or the eviction would still pass on a Run that simply sat + in cache: + + 1. **Subscribe to an empty stream.** Nothing is ever published before the + eviction, so the subscription's entire first Workflow Task observes + nothing. + 2. **Park.** Asserted in the backend, where the park handshake installs its + intent, and not by waiting out the idle timeout and hoping: the intent's + own cursor is the empty boundary, ``BEGINNING``. + 3. **Evict.** One cache slot, so the filler Workflow pushes the Run out; the + manager's eviction of the Run is observed directly, and History is + checked at that boundary before a later wake can legitimately race the + terminal command. + 4. **Replay.** The records are published while the Run is *gone*, so the + cursor the resumed Run starts from cannot have come from anything still + running on this Worker. It starts where the marker says -- ``BEGINNING``, + the boundary the empty subscription recorded -- so it receives both, and + a start resolved from live backend state would have started at the tail + and received neither. + + Then the marker itself is decoded, and the finished history is replayed + twice: once against the real backend with records appended *after* the Run + finished, which replay must not see, and once against a backend whose only + working operation is the recorded-range read, which nothing but the marker's + own ranges can satisfy. + """ + from temporalio.contrib.external_workflow_streams._manager import ( + StreamSubscriptionManager, + ) + from temporalio.contrib.external_workflow_streams._wake import ( + UNPARKED_WAKE_GENERATION, + WakeRequest, + send_wake_signal, + ) + + #: Which Runs the Worker **finished** tearing down. `RemoveFromCache` is the + #: only thing that reaches this while a Worker is running, so it is the + #: eviction itself rather than a symptom of one. + #: + #: Recorded *after* the teardown is awaited, not before. Before, this said only + #: that teardown had **started**: the Run's watcher was still alive, and the + #: records this test publishes next could be read into a buffer that was about + #: to be discarded -- their readiness reported by a watcher with nothing left to + #: deliver it to. What the test needs to know is that nothing is watching any + #: more, which is the *post*-condition of this call. + evicted: list[str] = [] + evict_run = StreamSubscriptionManager.evict_run + + async def spy_on_eviction(self, run_id: str) -> None: # type: ignore[no-untyped-def] + await evict_run(self, run_id) + evicted.append(run_id) + + monkeypatch.setattr(StreamSubscriptionManager, "evict_run", spy_on_eviction) + + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[EmptyStreamParkWorkflow, CacheFillerWorkflow], + external_stream_backend=backend, + # One slot, so running anything else evicts the Run under test. + max_cached_workflows=1, + max_concurrent_workflow_tasks=2, + ): + handle = await client.start_workflow( + EmptyStreamParkWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + key = await stream_key_for(client, handle, "tokens") + run_id = key.first_execution_run_id + + # --- 1 and 2: an empty subscription, parked ------------------------ + await wait_for_markers( + handle, + 1, + "the Run never ended a Workflow Task, so it cannot have parked on " + "the empty stream", + ) + parked = await backend.parked_wait_ids(key) + assert parked, ( + "no park intent was installed, so the Run never actually parked -- " + "it ended its Workflow Task some other way and the eviction below " + "would be evicting a Run in the wrong state" + ) + intent = await backend.park_intent(key, parked[0]) + assert intent is not None + assert intent.cursor == BEGINNING, ( + "the subscription parked at " + f"{intent.cursor} rather than at the beginning of the stream, so " + "the boundary it recorded is not the empty one this case is about" + ) + assert not backend.all_records(key), ( + "the stream was not empty while the Run was subscribed to it, so " + "this is no longer the empty-boundary case" + ) + + # --- 3: eviction --------------------------------------------------- + await client.execute_workflow( + CacheFillerWorkflow.run, + id=f"filler-{uuid.uuid4()}", + task_queue=task_queue, + ) + await wait_until( + lambda: run_id in evicted, + "the filler Workflow did not evict the Run: it is still cached, so " + "what follows would be a live resume rather than a replay", + ) + history_at_eviction = await handle.fetch_history() + assert not [ + event + for event in history_at_eviction.events + if event.HasField("workflow_task_failed_event_attributes") + ], ( + "a Workflow Task failed before the explicit cache eviction completed, " + "so the next execution would not prove that eviction caused the replay" + ) + assert STARTS.get(run_id) == [False], ( + "the Workflow body restarted before the explicit cache eviction boundary; " + f"the next replay would be ambiguous: {STARTS.get(run_id)}" + ) + + # --- 4: published while the Run is gone, then woken ---------------- + # Nothing on this Worker is watching for these: the Run is parked and + # evicted, its watcher torn down with it. The wake Signal is the only + # thing that can create a Workflow Task, and it carries no records. + await publish(backend, key, ["alpha", "beta"]) + # The unparked generation rather than the one the intent names, which is + # not the shape it looks like. A wake that names a park generation is + # identified *by* that generation -- its request ID deliberately ignores + # the sender, so every producer racing to wake one park is deduplicated + # to a single Workflow Task. That is correct for the producer's own wake + # and fatal for the Worker's follow-up: this Run's records land in the + # buffer just after the replayed Workflow Task closes, the Worker is + # answered `NoOpenWorkflowTask`, and the wake it then owes re-derives the + # very request ID the wake above already used. The server deduplicates + # it, no Workflow Task is created, and the Run waits forever on records + # it is already holding. Generation 0 keeps that follow-up wake + # distinguishable, which leaves this test about the cursor rather than + # about the wake protocol (ADR-023). + await send_wake_signal( + client, + WakeRequest( + namespace=client.namespace, + workflow_id=handle.id, + first_execution_run_id=run_id, + stream_name="tokens", + wait_id=intent.wait_id, + park_generation=UNPARKED_WAKE_GENERATION, + sender_identity=f"test-{uuid.uuid4()}", + wake_counter=1, + ), + ) + + try: + live = await asyncio.wait_for(handle.result(), 45) + except asyncio.TimeoutError: + # The same finding as the assertion below, which a Run that never + # finishes would otherwise report as a bare timeout -- reported with + # enough state to tell the two ways it can happen apart, because + # "it timed out" sends the next reader back to square one. + raise AssertionError( + "the resumed Run never received the records published while it " + "was evicted. They were in the stream before it came back and " + "its marker's boundary is the beginning of that stream, so a " + "Run starting where the marker says receives both; one that " + "resolved its position from live backend state starts at the " + "tail and waits for records that are already behind it.\n\n" + + await _stall_diagnosis(client, handle, backend, key, run_id) + ) from None + history = await handle.fetch_history() + + assert live == ["alpha", "beta"], ( + "the resumed Run did not start from the boundary its empty marker " + "recorded. Both records were published before it came back, so a Run " + "starting at BEGINNING sees both; one that resolved its position from " + f"live backend state starts at the tail and sees neither. Got {live}" + ) + + starts = STARTS.get(run_id, []) + assert len(starts) >= 2, ( + "the Workflow body ran only once, so the Run was never rebuilt and " + "nothing was replayed -- the eviction did not take effect" + ) + assert starts[0] is False and starts[1] is True, ( + "the second execution of the Workflow body did not begin in replay, so " + f"the Run came back some way other than by replaying its history: {starts}" + ) + unexpected_wft_failures = [ + event + for event in history.events + if event.HasField("workflow_task_failed_event_attributes") + and event.workflow_task_failed_event_attributes.cause + != temporalio.api.enums.v1.WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND + ] + assert not unexpected_wft_failures, ( + "the resumed Run had a Workflow Task failure other than the expected " + "external-wake/terminal-command race: " + f"{unexpected_wft_failures}" + ) + + # --- the recorded empty boundary ----------------------------------------- + markers = stream_markers(history.events) + assert len(markers) >= 2, ( + "expected a marker for the parked Workflow Task and one for the " + f"Workflow Task that consumed, got {len(markers)}" + ) + empty = marker_annotation(markers[0]) + binding = empty.header.streams.get(intent.wait_id) + assert binding is not None, ( + "the empty subscription's marker records no stream in its header, so " + "replay of it has nothing to start from" + ) + assert binding.stream_key == key + assert binding.start_cursor == BEGINNING, ( + "the marker for a subscription that received nothing must still carry " + "an explicit start cursor, or replay resolves a position from whatever " + f"the stream holds by then; got {binding.start_cursor}" + ) + assert not [run for segment in empty.segments for run in segment.runs], ( + "the marker written for the empty subscription records deliveries, so " + "it is not the empty boundary" + ) + assert empty.terminal == {intent.wait_id: BEGINNING}, ( + "the parked Workflow Task's terminal must say the subscription stopped " + f"at the beginning of the stream; got {empty.terminal}" + ) + + # --- replay 1: the real backend, with records appended afterwards -------- + # They land after the Run finished. Replay must not see them: the marker's + # boundary is where the subscription was, not where the stream got to. + await publish(backend, key, ["late-one", "late-two"]) + against_live = await Replayer( + workflows=[EmptyStreamParkWorkflow], + external_stream_backend=backend, + ).replay_workflow(history, raise_on_replay_failure=False) + assert against_live.replay_failure is None, ( + f"replaying the history the Worker wrote failed: {against_live.replay_failure}" + ) + + # --- replay 2: recorded ranges and nothing else -------------------------- + # The late records are the poison: they are in the stream, they are in no + # recorded range, and the live Run never saw one. A replay that took a + # position or a delivery from live state hands them to Workflow code, and + # the observation below is then not the one the live Run recorded. + late = backend.all_records(key)[-2:] + recorded_only = RecordedRangesOnlyBackend(backend, late) + against_recorded = await Replayer( + workflows=[EmptyStreamParkWorkflow], + external_stream_backend=recorded_only, + ).replay_workflow(history, raise_on_replay_failure=False) + assert against_recorded.replay_failure is None, ( + "replay could not reproduce the Run from the ranges its markers name, " + "so something in it was reading the stream live: " + f"{against_recorded.replay_failure}" + ) + assert [call for call in recorded_only.live_calls if call != "read_after"] == [], ( + "replay wrote to the stream or ran the park handshake, neither of which " + f"it may do: {recorded_only.live_calls}" + ) + expected_ranges: list[tuple[Offset, Offset]] = [ + (run.first_offset, run.last_offset) + for marker in markers + for segment in marker_annotation(marker).segments + for run in segment.runs + ] + assert recorded_only.range_reads == expected_ranges, ( + "replay read ranges the markers do not name -- the empty marker names " + "none at all, so anything read for it came from live stream state: " + f"{recorded_only.range_reads} against {expected_ranges}" + ) + + runs = observations.executions(run_id) + assert len(runs) >= 3, ( + "expected at least the live Run and both explicit replays to record what " + f"they observed, got {len(runs)}" + ) + assert all(run == live for run in runs), ( + "a Workflow Task execution observed something other than the completed " + "Run did. An UnhandledCommand retry may add an execution, but records were " + "appended after the Run finished and every replay could reach them " + f"through the same provider: {runs}" + ) diff --git a/tests/contrib/external_workflow_streams/test_rollover_integration.py b/tests/contrib/external_workflow_streams/test_rollover_integration.py new file mode 100644 index 0000000000..f68a2d1c8d --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_rollover_integration.py @@ -0,0 +1,623 @@ +"""Workflow Task rollover, against a live server. + +A retained Workflow Task is bounded by the server's Workflow Task timeout, and a +continuously fed stream whose inter-record gaps stay below the idle timeout never +reaches the idle parking path at all. Without rollover such a task does not +merely stay open too long -- it *fails*, and everything consumed on it is lost +with it. + +The Core tests prove the deadline exists. What only a live server can show is +that it fires *while a stream is being fed*: that a task genuinely held open by +arriving records is handed on before the server times it out, that the +subscription survives that hand-off, and that inputs queued behind the retained +task -- Signals especially -- are released no later than the deadline. + +Every test here stops short of the Workflow Task timeout on purpose. A retained +task that reaches it is not merely a failed assertion: the server schedules a +replacement, Core receives a Workflow Task for a run it still holds one for, and +the resulting ``dbg_panic`` takes the whole workflow-processing thread with it. + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.client import Client +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.service import RPCError, RPCStatusCode +from temporalio.worker import Worker +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend +from tests.contrib.external_workflow_streams.test_worker_integration import publish + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams._api import external_stream + +TASK_TIMEOUT = timedelta(seconds=10) +"""The bound a retained task must stay inside, and what the deadline derives +from. Long enough that the rollover deadline is unambiguous, short enough that +these tests do not have to run for a minute.""" + +ROLLOVER_FRACTION = 0.8 +"""Core arms the rollover deadline at this fraction of the Workflow Task +timeout, the same one the local-activity heartbeat uses.""" + +ROLLOVER_DEADLINE = TASK_TIMEOUT * ROLLOVER_FRACTION + +FEED_GAP_SECONDS = 0.3 +"""Well below the idle timeout, so the runtime never parks and the only thing +that can release the task is rollover.""" + +WAKE_SIGNAL_NAME = "__temporal_external_stream_wake" + +ROLLOVER_DELIVERY_TOLERANCE = timedelta(milliseconds=500) +"""How far past the deadline a released input may land and still count as bounded by it. + +The deadline is what Core aims the replacement Workflow Task at, not an instant +the server can hit. What a test can measure is the gap between two +``WorkflowTaskStarted`` event times, and between those sit Core's completion +RPC, the server scheduling the replacement, and a poll picking it up. Measured +overheads here are 16-65ms, so half a second is most of an order of magnitude of +headroom. + +It is deliberately far short of ``TASK_TIMEOUT - ROLLOVER_DEADLINE``, which is +two seconds: an input released because the *server* timed the retained task out +lands at about 10s and still fails the assertion, and telling that apart from a +rollover is the only thing the assertion is for. +""" + +POST_ROLLOVER_PAUSE = timedelta(seconds=10) +"""How long the post-rollover Workflow Task boundary is held open for. + +Long enough for an append and the wake it owes to be observed in History +without waiting on anything else, short enough that the test finishes. +""" + + +@workflow.defn +class SteadyStreamWorkflow: + """Consumes records until it has the number it was asked for. + + Nothing else it does can end a Workflow Task: no timers, no activities, and + the gaps the test feeds are far under the idle timeout, so retention is only + ever released by rollover. + """ + + def __init__(self) -> None: + self._seen = 0 + + @workflow.run + async def run(self, expected: int) -> int: + tokens = external_stream.topic("tokens", type=str) + async for _ in tokens.subscribe(): + self._seen += 1 + if self._seen >= expected: + break + return self._seen + + @workflow.query + def seen(self) -> int: + return self._seen + + +@workflow.defn +class SignalledStreamWorkflow: + """Consumes a stream until a Signal arrives, then reports when it did. + + The Signal is sent while the Workflow Task is retained, so the server has to + hold it until that task completes. What it returns is *Workflow* time, taken + the moment the handler ran, which is what makes "no later than the rollover + deadline" measurable from the outside rather than inferred from wall clock. + """ + + def __init__(self) -> None: + self._signalled: float | None = None + self._started: float | None = None + + @workflow.run + async def run(self) -> float: + tokens = external_stream.topic("tokens", type=str) + self._started = workflow.now().timestamp() + subscription = tokens.subscribe() + + async def consume() -> None: + async for _ in subscription: + pass + + consumer = asyncio.ensure_future(consume()) + await workflow.wait_condition(lambda: self._signalled is not None) + consumer.cancel() + assert self._signalled is not None and self._started is not None + return self._signalled - self._started + + @workflow.signal + def poke(self) -> None: + self._signalled = workflow.now().timestamp() + + +@workflow.defn +class RolloverThenPauseWorkflow: + """Consumes across a rollover, then pauses on a Timer and consumes once more. + + The pause is what makes the post-rollover window addressable. A completion + carrying a server-bound command cannot ask for retention, so the Workflow + Task ends there with the subscription still active and still unparked -- + which is the state a rollover leaves behind, held still. The rollover's own + window is not addressable from outside the process: Core completes a + rollover with ``force_new_wft``, and the server has a replacement Workflow + Task started within a millisecond of it. + """ + + def __init__(self) -> None: + self._seen = 0 + + @workflow.run + async def run(self, before_pause: int) -> int: + tokens = external_stream.topic("tokens", type=str) + subscription = tokens.subscribe() + iterator = subscription.__aiter__() + + while self._seen < before_pause: + await iterator.__anext__() + self._seen += 1 + + # A real Timer: the completion that carries it cannot be retained. + await asyncio.sleep(POST_ROLLOVER_PAUSE.total_seconds()) + + # Whatever was appended into that window. That it arrives at all proves + # only that it was not lost -- the Timer creates a Workflow Task of its + # own when it fires, and that task would find a buffered record anyway. + # What proves the wake is the Signal in History, which the test asserts. + await iterator.__anext__() + self._seen += 1 + return self._seen + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + backend = MemoryStreamBackend() + # These cases count the reserved Signal itself, so the transport is pinned + # to it rather than left to step down from a server that has more. + backend.wake_transport = "signal" + return backend + + +async def stream_key_for(client: Client, handle: Any) -> StreamKey: + description = await handle.describe() + return StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + + +async def history(handle: Any) -> list[Any]: + return [e async for e in handle.fetch_history_events()] + + +def completed_tasks(events: list[Any]) -> list[Any]: + return [e for e in events if e.HasField("workflow_task_completed_event_attributes")] + + +def timed_out_tasks(events: list[Any]) -> list[Any]: + return [e for e in events if e.HasField("workflow_task_timed_out_event_attributes")] + + +async def stop_if_running(handle: Any) -> None: + """Terminates the Workflow, unless it has already finished on its own. + + Every test here terminates in a ``finally``, because a Workflow left running + keeps a Worker's Run alive past the test that owns it. A test whose whole + point is that the Workflow *finishes* has nothing left to terminate, though, + and the server answers that with ``NOT_FOUND`` -- an error about the + cleanup, raised after the assertions have all passed, which reads exactly + like the feature having failed. Only that one status is swallowed: anything + else is a real failure of the teardown and is left to surface. + """ + try: + await handle.terminate() + except RPCError as err: + if err.status is not RPCStatusCode.NOT_FOUND: + raise + + +def timers_started(events: list[Any]) -> list[Any]: + return [e for e in events if e.HasField("timer_started_event_attributes")] + + +def held_for(events: list[Any], completed: Any) -> timedelta: + """How long the Workflow Task that ``completed`` closes was held open.""" + started = { + e.event_id: e + for e in events + if e.HasField("workflow_task_started_event_attributes") + } + start = started[completed.workflow_task_completed_event_attributes.started_event_id] + return completed.event_time.ToDatetime() - start.event_time.ToDatetime() + + +def rollover_completion(events: list[Any]) -> Any | None: + """The first Workflow Task completion that can only be a rollover. + + Identified by how long the task it closes was held: an idle park closes one + an idle timeout after its last record, and a command-carrying completion + closes one as soon as the activation returns. Only rollover holds a task + open for the whole deadline, so a completion at the deadline is the + mechanism itself rather than a task that happened to take a while. + + The window is the deadline itself, less :data:`ROLLOVER_DELIVERY_TOLERANCE` + and up to the Workflow Task timeout. The same margin applies in this + direction for the same reason: what is measured is the distance between two + server event times, and Core arms its deadline from neither of them. + """ + for completed in completed_tasks(events): + held = held_for(events, completed) + if ROLLOVER_DEADLINE - ROLLOVER_DELIVERY_TOLERANCE <= held < TASK_TIMEOUT: + return completed + return None + + +def wake_signals(events: list[Any]) -> list[Any]: + """The reserved wake Signals in a Workflow's own History. + + The mechanism itself, rather than a record turning up -- which after a + rollover proves nothing, since the server has already scheduled a + replacement task that could have picked the record up on its own. + """ + return [ + e + for e in events + if e.HasField("workflow_execution_signaled_event_attributes") + and e.workflow_execution_signaled_event_attributes.signal_name + == WAKE_SIGNAL_NAME + ] + + +async def wait_for_history( + handle: Any, + predicate: Any, + *, + timeout: float, + message: str, +) -> list[Any]: + """Polls the Workflow's History until ``predicate`` holds, or fails saying why.""" + deadline = asyncio.get_running_loop().time() + timeout + while True: + events = await history(handle) + if predicate(events): + return events + if asyncio.get_running_loop().time() >= deadline: + raise AssertionError(message) + await asyncio.sleep(0.2) + + +async def feed( + backend: MemoryStreamBackend, + key: StreamKey, + session: str, + *, + count: int, + stop: asyncio.Event | None = None, +) -> None: + """Appends one record at a time, with gaps well under the idle timeout. + + One session with a continuous sequence, because ``(session_id, sequence)`` + is the append idempotency key: restarting the numbering re-uses a key with + different content, the backend rejects it, and the symptom is a Workflow + that appears to hang far from the cause. + """ + for i in range(count): + if stop is not None and stop.is_set(): + return + await publish(backend, key, [f"t{i}"], session=session) + await asyncio.sleep(FEED_GAP_SECONDS) + + +# --- case 19 ------------------------------------------------------------------ + + +@pytest.mark.timeout(180) +async def test_a_continuously_fed_stream_survives_a_rollover( + client: Client, backend: MemoryStreamBackend +) -> None: + """The case rollover exists for, fed for longer than one task may live. + + Three things have to hold at once, and each fails differently: + + - the deadline has to fire *while records are arriving*, which is the only + state in which the idle timer never will; + - the replacement task has to inherit the subscription and its cursor, so + the count the Workflow ends with is exactly what was published; + - no task may run past the Workflow Task timeout, asserted against the + server's own event timestamps rather than inferred from the Workflow + finishing. + + The check is made *before* the timeout could be reached, so a broken + rollover fails here rather than as a timed-out task and the Core panic that + follows one. + """ + total = 30 # 30 * 0.3s = 9s of feeding, past a deadline at 8s. + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[SteadyStreamWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + SteadyStreamWorkflow.run, + total, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=TASK_TIMEOUT, + ) + key = await stream_key_for(client, handle) + stop = asyncio.Event() + feeder = asyncio.ensure_future( + feed(backend, key, f"steady-{uuid.uuid4()}", count=total, stop=stop) + ) + try: + # A margin past the deadline, and still short of the timeout: by now + # a working rollover has completed at least one task. + await asyncio.sleep(ROLLOVER_DEADLINE.total_seconds() + 1) + events = await history(handle) + completed = completed_tasks(events) + assert completed, ( + "the stream was fed continuously for longer than the rollover " + f"deadline of {ROLLOVER_DEADLINE} and not one Workflow Task has " + "completed, so the retained task is on its way to being timed " + "out by the server -- which is the failure rollover exists to " + "prevent" + ) + + result = await asyncio.wait_for(handle.result(), 90) + assert result == total, ( + "the replacement task did not inherit the subscription's " + f"cursor: {result} of {total} records were consumed" + ) + + events = await history(handle) + assert not timed_out_tasks(events), ( + "a Workflow Task was held past the server's timeout, so " + "rollover did not bound it" + ) + for event in completed_tasks(events): + held = held_for(events, event) + assert held < TASK_TIMEOUT, ( + f"a Workflow Task was held for {held}, past the " + f"{TASK_TIMEOUT} it must stay inside" + ) + assert rollover_completion(events) is not None, ( + "no Workflow Task was held to the rollover deadline, so the " + f"{total} records were consumed without one -- and this test " + "asserts nothing about rollover unless one fired" + ) + finally: + stop.set() + feeder.cancel() + await stop_if_running(handle) + + +# --- case 21 ------------------------------------------------------------------ + + +@pytest.mark.timeout(180) +async def test_a_signal_into_a_retained_task_lands_by_the_rollover_deadline( + client: Client, backend: MemoryStreamBackend +) -> None: + """Retention delays Signals, and rollover is the only thing bounding that. + + While a Workflow Task is retained the server cannot start another one, so + Signals, Updates, and non-legacy Queries queue behind it -- and a stream can + hold a task open far longer than an outstanding local activity ever would. + Retention latency for those inputs is therefore bounded by the rollover + deadline and by nothing else. + + Both halves are asserted, because either alone is satisfiable by accident: + that the task really was retained when the Signal was sent -- no Workflow + Task had completed, so the Signal had nothing to be delivered on -- and that + the Workflow saw it within the deadline, measured in *Workflow* time from + the Run's own start. + + The second half carries :data:`ROLLOVER_DELIVERY_TOLERANCE`, because what it + measures is the distance between two ``WorkflowTaskStarted`` event times and + the deadline can only be the moment Core *decides* to hand the task on. A + completion RPC, the server scheduling the replacement, and a poll picking it + up all land between the two, so an exact bound is unreachable by + construction rather than merely flaky. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[SignalledStreamWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + SignalledStreamWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=TASK_TIMEOUT, + ) + started_at = asyncio.get_running_loop().time() + key = await stream_key_for(client, handle) + stop = asyncio.Event() + feeder = asyncio.ensure_future( + feed(backend, key, f"signalled-{uuid.uuid4()}", count=40, stop=stop) + ) + try: + # Long enough for the stream to be driving the Workflow, far short + # of the deadline. + await asyncio.sleep(2) + assert not completed_tasks(await history(handle)), ( + "a Workflow Task had already completed, so the Signal below " + "would not be sent into a retained one and this proves nothing" + ) + + await handle.signal(SignalledStreamWorkflow.poke) + # Waited out only as far as the deadline and its tolerance, never as + # far as the Workflow Task timeout: a retained task that reaches that + # is a panicking Core, not a failed assertion. + bound = (ROLLOVER_DEADLINE + ROLLOVER_DELIVERY_TOLERANCE).total_seconds() + waited = asyncio.get_running_loop().time() - started_at + try: + elapsed = await asyncio.wait_for(handle.result(), bound + 0.5 - waited) + except asyncio.TimeoutError: + raise AssertionError( + "the Signal had still not been delivered by the rollover " + f"deadline of {ROLLOVER_DEADLINE}: the Workflow Task is " + "still retained, and nothing but that deadline bounds how " + "long an input queues behind it" + ) from None + + assert elapsed <= bound, ( + f"the Signal was delivered {elapsed:.3f}s into the Run, past the " + f"rollover deadline of {ROLLOVER_DEADLINE} and the " + f"{ROLLOVER_DELIVERY_TOLERANCE} allowed for getting the " + "replacement task started. Retention latency is bounded by that " + "deadline and by nothing else" + ) + events = await history(handle) + assert not timed_out_tasks(events), ( + "the Signal was released by the server timing the retained task " + "out rather than by a rollover" + ) + finally: + stop.set() + feeder.cancel() + await stop_if_running(handle) + + +# --- case 23 ------------------------------------------------------------------ + + +@pytest.mark.timeout(180) +async def test_an_append_after_a_rollover_completion_wakes_the_subscription( + client: Client, backend: MemoryStreamBackend +) -> None: + """A subscription that has been through a rollover is still woken by a Signal. + + A rollover ends a Workflow Task with the subscription active and unparked: + there is no retained task to notify locally and no park generation for a + producer to observe, so the consumer's own watcher is what covers the + window -- it observes ``NoOpenWorkflowTask`` and sends the reserved wake + Signal itself. + + That window cannot be appended into directly. Core completes a rollover with + ``force_new_wft``, so the server starts the replacement Workflow Task within + a millisecond, and an append made from a test lands on the replacement task + instead: readiness is answered ``Accepted``, the record is delivered live, + and no wake is owed or sent -- which is correct behaviour and not the + mechanism this case is about. The Workflow therefore *holds* the same state + open on the far side of the rollover with a Timer, whose completion is + server-bound and equally cannot be retained, and the append goes in there. + + Both halves are asserted, since either alone is satisfiable without the + other: that a rollover really happened, by a Workflow Task held to the + deadline and closed inside the timeout; and that the append was answered by + a wake, by ``__temporal_external_stream_wake`` reaching the Workflow's own + History *after* the completion that opened the window. The record arriving + proves nothing on its own -- the Timer creates a Workflow Task when it + fires, which would have picked the record up anyway. + """ + before_pause = 30 # 30 * 0.3s = 9s of feeding, past a deadline at 8s. + task_queue = f"tq-{uuid.uuid4()}" + session = f"rollover-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[RolloverThenPauseWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + RolloverThenPauseWorkflow.run, + before_pause, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=TASK_TIMEOUT, + ) + key = await stream_key_for(client, handle) + stop = asyncio.Event() + feeder = asyncio.ensure_future( + feed(backend, key, session, count=before_pause, stop=stop) + ) + try: + # The Timer is started only once every fed record has been consumed, + # which takes longer than the deadline -- so its appearance is also + # the point past which the rollover has already happened. + events = await wait_for_history( + handle, + timers_started, + timeout=60, + message=( + f"the Workflow never consumed the {before_pause} records fed " + "to it and reached its pause, so there is no unparked " + "Workflow Task boundary to append into" + ), + ) + stop.set() + feeder.cancel() + + rollover = rollover_completion(events) + assert rollover is not None, ( + "no Workflow Task was held to the rollover deadline of " + f"{ROLLOVER_DEADLINE}, so the subscription this appends to has " + "not been through a rollover and the case is untested" + ) + assert not timed_out_tasks(events), ( + "the task was released by the server's timeout rather than by a " + "rollover" + ) + pause = timers_started(events)[0] + assert pause.event_id > rollover.event_id, ( + "the pause began before the rollover, so the window appended " + "into below is not a post-rollover one" + ) + + # Counted by name, and by the *same* name the check below uses: a + # baseline that counted every Signal would be compared against a + # total that counts only wake Signals, and any unrelated Signal in + # this Workflow's History would then make the comparison meaningless + # in whichever direction it happened to fall. + before = len(wake_signals(events)) + + await publish(backend, key, ["after-rollover"], session=session) + + def woken(events: list[Any]) -> bool: + wakes = wake_signals(events) + # Position as well as count: a Signal sent while a Workflow Task + # was open reaches History only when that task completes, so + # counting alone would accept one that was owed long before this + # append and merely landed late. + return len(wakes) > before and wakes[-1].event_id > pause.event_id + + await wait_for_history( + handle, + woken, + timeout=POST_ROLLOVER_PAUSE.total_seconds(), + message=( + "no wake Signal followed the append: the Workflow Task that " + "carried the pause left the subscription active and " + "unparked, so there was no retained task to notify and no " + "park generation to observe, and the watcher owes one" + ), + ) + + assert await asyncio.wait_for(handle.result(), 60) == before_pause + 1, ( + "the record appended into the unparked window never reached the " + "Workflow, so the wake Signal did not carry the subscription " + "over the boundary" + ) + finally: + stop.set() + feeder.cancel() + await stop_if_running(handle) diff --git a/tests/contrib/external_workflow_streams/test_runtime.py b/tests/contrib/external_workflow_streams/test_runtime.py new file mode 100644 index 0000000000..021637b192 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_runtime.py @@ -0,0 +1,1386 @@ +"""P10a/P10b — the observation delta and the quiescent snapshot. + +Driven directly against the runtime handle: what is under test is what gets +*recorded*, and threading it through a real Worker would obscure that behind +scheduling. +""" + +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest +import pytest_asyncio + +import temporalio.converter +from temporalio.contrib.external_workflow_streams._annotation import ( + ROLLOVER_HIGH_WATER, + AnnotationHeader, + SegmentEndReason, + StreamBinding, + decode_annotation, + encode_header, + encode_terminal, +) +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._errors import ( + ExternalStreamCapacityError, +) +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._runtime import ( + _RUN_COST_FLOOR, + WorkflowStreamRuntime, +) +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +RUN_ID = "run-1" + + +async def _notify( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] +) -> str: + return ReadinessResult.ACCEPTED + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +@pytest_asyncio.fixture # type: ignore[reportUntypedFunctionDecorator] +async def runtime(backend: MemoryStreamBackend): + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + yield WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id=uuid.uuid4().hex, + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + ) + # Watchers outlive the runtime unless the manager is told to stop, and a + # leaked watcher keeps polling a backend nothing is reading any more. + await manager.shutdown() + + +def subscribe( + runtime: WorkflowStreamRuntime, + wait_id: int, + name: str = "tokens", + idle_timeout: timedelta | None = None, +) -> StreamKey: + key = runtime.stream_key(name) + runtime.register( + wait_id=wait_id, + stream_key=key, + idle_timeout=idle_timeout, + ) + return key + + +def data(offset: str, session: str = "s", seq: int = 0) -> StreamRecord: + return StreamRecord(RecordKind.DATA, b"x", session, seq).placed_at(Offset(offset)) + + +def fence(offset: str, session: str = "s", seq: int = 0) -> StreamRecord: + return StreamRecord(RecordKind.WRITE_FENCE, b"", session, seq).placed_at( + Offset(offset) + ) + + +def annotation_of(runtime: WorkflowStreamRuntime) -> object: + """The annotation Core would hold, decoded. + + Core accumulates deltas by byte concatenation, so this joins them the same + way rather than reaching for anything the runtime keeps internally. + """ + parts = [] + delta = runtime.take_observation_delta() + if delta is not None: + parts.append(delta) + parts.append(runtime.add_terminal()) + return decode_annotation(b"".join(parts)) + + +# --- emission is not conditional on records --------------------------------- + + +async def test_a_first_subscription_to_an_empty_stream_still_emits( + runtime: WorkflowStreamRuntime, +) -> None: + """Without this, replay of an empty stream has nowhere to begin. + + The binding carries the stream key, an explicit start cursor, and the + configured provider identity. None is derivable from mutable backend state. + """ + key = subscribe(runtime, 1) + + decoded = annotation_of(runtime) + + binding = decoded.header.streams[1] # type: ignore[attr-defined] + assert binding.provider_id == "memory" + assert decoded.header.streams[1].stream_key == key # type: ignore[attr-defined] + assert decoded.header.streams[1].start_cursor == BEGINNING # type: ignore[attr-defined] + assert decoded.terminal == {1: BEGINNING} # type: ignore[attr-defined] + + +async def test_an_activation_that_drained_nothing_emits_an_empty_segment( + runtime: WorkflowStreamRuntime, +) -> None: + """The drain happened, so replay must reproduce it. + + Reproducing only the record order while changing how many drains occurred + would change when ``wait_condition`` predicates fire. + """ + subscribe(runtime, 1) + + decoded = annotation_of(runtime) + + assert len(decoded.segments) == 1 # type: ignore[attr-defined] + assert decoded.segments[0].runs == () # type: ignore[attr-defined] + + +async def test_an_activation_that_observed_nothing_at_all_emits_nothing( + runtime: WorkflowStreamRuntime, +) -> None: + """The one case where a completion legitimately carries no progress.""" + assert runtime.take_observation_delta() is None + + +# --- runs ------------------------------------------------------------------- + + +async def test_consecutive_deliveries_from_one_stream_collapse_into_one_run( + runtime: WorkflowStreamRuntime, +) -> None: + """The claim the whole design rests on: no field in a run is per-record.""" + subscribe(runtime, 1) + for i in range(1, 1001): + runtime.record_delivery(1, data(f"{i}-0")) + + decoded = annotation_of(runtime) + + (run,) = decoded.segments[0].runs # type: ignore[attr-defined] + assert run.count == 1000 + assert run.first_offset == Offset("1-0") + assert run.last_offset == Offset("1000-0") + + +async def test_alternating_streams_produce_one_run_per_delivery( + runtime: WorkflowStreamRuntime, +) -> None: + """The honest worst case, recorded as such rather than assumed away. + + Per-stream ranges alone are insufficient: concurrent coroutines could + otherwise observe a different order than the one that actually happened. + """ + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + for i, wait_id in enumerate([1, 2, 1, 2], start=1): + runtime.record_delivery(wait_id, data(f"{i}-0")) + + decoded = annotation_of(runtime) + + runs = decoded.segments[0].runs # type: ignore[attr-defined] + assert [r.wait_id for r in runs] == [1, 2, 1, 2] + assert all(r.count == 1 for r in runs) + + +async def test_a_run_resumes_after_the_other_stream_interrupts( + runtime: WorkflowStreamRuntime, +) -> None: + """Maximal, not merged: the interruption is itself part of the schedule.""" + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + runtime.record_delivery(1, data("1-0")) + runtime.record_delivery(1, data("2-0")) + runtime.record_delivery(2, data("3-0")) + runtime.record_delivery(1, data("4-0")) + + runs = annotation_of(runtime).segments[0].runs # type: ignore[attr-defined] + + assert [(r.wait_id, r.count) for r in runs] == [(1, 2), (2, 1), (1, 1)] + + +# --- control records -------------------------------------------------------- + + +async def test_control_records_are_counted_and_their_positions_recorded( + runtime: WorkflowStreamRuntime, +) -> None: + """They occupy offsets, so replay's range read will find them. + + Leaving them out of ``count`` would make every such range read find more + records than the marker claims and fail as integrity loss -- for a stream + that is perfectly intact. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + runtime.record_delivery(1, fence("2-0")) + runtime.record_delivery(1, data("3-0")) + + (run,) = annotation_of(runtime).segments[0].runs # type: ignore[attr-defined] + + assert run.count == 3 + assert run.control_positions == (1,) + assert run.last_offset == Offset("3-0") + + +async def test_control_positions_are_sparse(runtime: WorkflowStreamRuntime) -> None: + """One entry per fence, not one kind tag per record.""" + subscribe(runtime, 1) + for i in range(1, 101): + runtime.record_delivery(1, data(f"{i}-0")) + runtime.record_delivery(1, fence("101-0")) + + (run,) = annotation_of(runtime).segments[0].runs # type: ignore[attr-defined] + + assert run.count == 101 + assert run.control_positions == (100,) + + +# --- the terminal ----------------------------------------------------------- + + +async def test_the_terminal_is_where_delivery_stopped( + runtime: WorkflowStreamRuntime, +) -> None: + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + runtime.record_delivery(1, data("5-0")) + + decoded = annotation_of(runtime) + + assert decoded.terminal == { # type: ignore[attr-defined] + 1: AFTER(Offset("5-0")), + 2: BEGINNING, + } + + +async def test_the_terminal_reads_no_backend( + runtime: WorkflowStreamRuntime, backend: MemoryStreamBackend +) -> None: + """Finalization performs no backend I/O (ADR-010). + + The boundary is where this Workflow Task's deliveries stopped, already + fixed. Refreshing it against the stream could name a position replay must + not reproduce. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("5-0")) + reads_before = len(backend.range_reads) + + runtime.add_terminal() + + assert len(backend.range_reads) == reads_before + + +# --- deltas concatenate ----------------------------------------------------- + + +async def test_deltas_across_activations_concatenate_into_one_annotation( + runtime: WorkflowStreamRuntime, +) -> None: + """What Core holds is the byte concatenation, so that is what is decoded.""" + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + first = runtime.take_observation_delta() + + runtime.record_delivery(1, data("2-0")) + second = runtime.take_observation_delta() + + terminal = runtime.add_terminal() + + assert first is not None and second is not None + decoded = decode_annotation(first + second + terminal) + assert len(decoded.segments) == 2, "one segment per activation, not one per task" + assert [r.count for s in decoded.segments for r in s.runs] == [1, 1] + assert decoded.terminal == {1: AFTER(Offset("2-0"))} + + +async def test_terminating_an_already_closed_annotation_adds_nothing( + runtime: WorkflowStreamRuntime, +) -> None: + """A second terminal for one Workflow Task must not start a second annotation. + + Core can ask for a terminal for a boundary it decided -- a rollover + deadline, a shutdown -- on a task whose last completion had already closed + the annotation. Creating a fresh header and terminal for that would append a + complete second annotation to the one Core is about to write, and the marker + would decode as far as the first terminal and fail on the frame after it. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + delta = runtime.take_observation_delta() + assert delta is not None + + closed = delta + runtime.add_terminal() + again = runtime.add_terminal() + + assert again == b"", "the annotation was already closed; there was nothing to add" + decoded = decode_annotation(closed + again) + assert decoded.terminal == {1: AFTER(Offset("1-0"))} + + +async def test_a_terminal_after_new_observations_opens_a_fresh_annotation( + runtime: WorkflowStreamRuntime, +) -> None: + """The suppression above is about a *closed* annotation, not about the Run. + + Anything observed after the close begins the next annotation, and that one + needs its own header and its own terminal like any other. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + first = runtime.take_observation_delta() + assert first is not None + first += runtime.add_terminal() + + runtime.record_delivery(1, data("2-0")) + second = runtime.take_observation_delta() + assert second is not None + second += runtime.add_terminal() + + assert decode_annotation(first).terminal == {1: AFTER(Offset("1-0"))} + assert decode_annotation(second).terminal == {1: AFTER(Offset("2-0"))} + + +async def test_a_new_annotation_starts_from_the_current_cursors( + runtime: WorkflowStreamRuntime, +) -> None: + """The next Workflow Task continues where this one stopped. + + Restarting the header at the original start cursor would make replay of the + second marker re-deliver everything the first one already recorded. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("9-0")) + runtime.take_observation_delta() + runtime.add_terminal() + + runtime.start_new_annotation() + runtime.record_delivery(1, data("10-0")) + decoded = annotation_of(runtime) + + assert decoded.header.streams[1].start_cursor == AFTER(Offset("9-0")) # type: ignore[attr-defined] + + +# --- a subscription created after the header went out ----------------------- + + +async def test_a_wait_registered_after_the_first_delta_reaches_the_header( + runtime: WorkflowStreamRuntime, +) -> None: + """`register` accepts a subscription at any activation of a retained task. + + The header frame is emitted with the first delta and Core appends rather + than rewrites, so a wait that joined later cannot be added to it in place. + Without a binding of its own that wait reaches the marker as a run and a + terminal entry with no stream key, no backend, and no start cursor, and + replay of *unchanged* code fails as a wait "this Workflow did not create". + """ + subscribe(runtime, 1, "tokens") + runtime.record_delivery(1, data("1-0")) + first = runtime.take_observation_delta() + assert first is not None + assert set(decode_annotation(first).header.streams) == {1}, ( + "the first delta's header is what this test is about; if it already " + "carried wait 2 the fix under test is not being exercised" + ) + + key = runtime.stream_key("tokens") + runtime.register(wait_id=2, stream_key=key) + runtime.record_delivery(2, data("2-0")) + second = runtime.take_observation_delta() + assert second is not None + terminal = runtime.add_terminal() + + decoded = decode_annotation(first + second + terminal) + + assert set(decoded.header.streams) == {1, 2}, ( + "wait 2 has a run and a terminal entry but no binding, so replay has " + "no stream key, no backend, and no start cursor for it" + ) + assert decoded.header.streams[2].stream_key == key + assert decoded.terminal == {1: AFTER(Offset("1-0")), 2: AFTER(Offset("2-0"))} + + +async def test_a_late_wait_is_bound_before_the_segment_that_records_it( + runtime: WorkflowStreamRuntime, +) -> None: + """Order within the delta, not merely presence somewhere in the marker. + + A decoder that met the run first would have no binding to attach it to, and + a marker truncated at a rollover boundary would carry the run without the + binding at all. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + first = runtime.take_observation_delta() + assert first is not None + + subscribe(runtime, 2) + runtime.record_delivery(2, data("2-0")) + second = runtime.take_observation_delta() + assert second is not None + + # 0x04 is the bindings frame tag. + assert second[0] == 0x04, ( + "the delta must open with the binding for the wait it goes on to " + f"record a run for, got {second.hex()}" + ) + + +async def test_a_late_wait_starts_at_its_own_cursor( + runtime: WorkflowStreamRuntime, +) -> None: + """Not wherever the waits already in the header have reached. + + The annotation-wide position belongs to the waits that were there when it + opened. Recording it for a wait that joined afterwards would start replay + of that wait past records it in fact received. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("9-0")) + first = runtime.take_observation_delta() + assert first is not None + + subscribe(runtime, 2) + runtime.record_delivery(2, data("3-0")) + second = runtime.take_observation_delta() + assert second is not None + + decoded = decode_annotation(first + second + runtime.add_terminal()) + + assert decoded.header.streams[2].start_cursor == BEGINNING + assert decoded.header.streams[1].start_cursor == BEGINNING + + +async def test_a_wait_is_bound_once_per_annotation( + runtime: WorkflowStreamRuntime, +) -> None: + """Three activations, one binding each for the two waits that exist. + + A binding re-emitted every activation would grow the marker with the + activation count and be rejected on decode as a wait bound twice. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + parts = [runtime.take_observation_delta()] + + subscribe(runtime, 2) + runtime.record_delivery(2, data("2-0")) + parts.append(runtime.take_observation_delta()) + + runtime.record_delivery(2, data("3-0")) + parts.append(runtime.take_observation_delta()) + + joined = b"".join(p for p in parts if p is not None) + + # A re-emitted binding is not merely wasteful: the decoder refuses a wait + # bound twice, so this decoding at all is the assertion. + decoded = decode_annotation(joined + runtime.add_terminal()) + assert set(decoded.header.streams) == {1, 2} + + +async def test_the_next_annotation_binds_every_wait_in_its_own_header( + runtime: WorkflowStreamRuntime, +) -> None: + """A late binding belongs to the annotation it was emitted into. + + The next Workflow Task writes a header from scratch, so the wait that + joined late must appear in *that* header rather than being remembered as + already announced -- a second marker missing it is the same defect one + Workflow Task later. + """ + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + runtime.take_observation_delta() + subscribe(runtime, 2) + runtime.record_delivery(2, data("2-0")) + runtime.take_observation_delta() + runtime.add_terminal() + + runtime.start_new_annotation() + runtime.record_delivery(2, data("3-0")) + decoded = annotation_of(runtime) + + assert set(decoded.header.streams) == {1, 2} # type: ignore[attr-defined] + assert decoded.header.streams[2].start_cursor == AFTER(Offset("2-0")) # type: ignore[attr-defined] + + +# --- quiescence (P10a) ------------------------------------------------------ + + +async def test_the_quiescent_snapshot_is_the_complete_blocked_set( + runtime: WorkflowStreamRuntime, +) -> None: + """A partial set would be worse than none. + + It would let one idle stream park a Workflow Task another stream is still + driving, which is the whole reason the timeout is a property of the set. + """ + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + subscribe(runtime, 3, name="more") + + snapshot = runtime.quiescent_snapshot() + + assert snapshot is not None + assert [w.wait_id for w in snapshot] == [1, 2, 3] + + +async def test_nothing_blocked_asks_for_no_retention( + runtime: WorkflowStreamRuntime, +) -> None: + subscribe(runtime, 1) + runtime.note_blocked(1, False) + + assert runtime.quiescent_snapshot() is None + + +async def test_a_fenced_subscription_reports_immediately_parkable( + runtime: WorkflowStreamRuntime, +) -> None: + """One fenced stream does not park the task; Core decides that from the set.""" + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + runtime.record_delivery(1, fence("1-0")) + + snapshot = runtime.quiescent_snapshot() + + assert snapshot is not None + assert [(w.wait_id, w.immediately_parkable) for w in snapshot] == [ + (1, True), + (2, False), + ] + + +async def test_a_later_record_clears_the_fence(runtime: WorkflowStreamRuntime) -> None: + """A fence asserts nothing about other producers; a later record just wakes.""" + subscribe(runtime, 1) + runtime.record_delivery(1, fence("1-0")) + runtime.record_delivery(1, data("2-0")) + + snapshot = runtime.quiescent_snapshot() + + assert snapshot is not None + assert snapshot[0].immediately_parkable is False + + +async def test_differing_idle_timeouts_reduce_to_their_min( + runtime: WorkflowStreamRuntime, +) -> None: + """The set shares one timer, so the configured values must reduce to one. + + ``min`` in ``wait_id`` order over the blocked set and nothing else, so the + result is deterministic and reproduces on replay. + """ + subscribe(runtime, 1, idle_timeout=timedelta(seconds=5)) + subscribe(runtime, 2, name="b", idle_timeout=timedelta(seconds=2)) + subscribe(runtime, 3, name="c", idle_timeout=timedelta(seconds=9)) + + assert runtime.effective_idle_timeout() == timedelta(seconds=2) + + +async def test_only_blocked_subscriptions_contribute_to_the_reduction( + runtime: WorkflowStreamRuntime, +) -> None: + """The inputs are the quiescent set's values and nothing else.""" + subscribe(runtime, 1, idle_timeout=timedelta(seconds=5)) + subscribe(runtime, 2, name="b", idle_timeout=timedelta(milliseconds=1)) + runtime.note_blocked(2, False) + + assert runtime.effective_idle_timeout() == timedelta(seconds=5) + + +async def test_re_entering_the_blocked_state_bumps_the_generation( + runtime: WorkflowStreamRuntime, +) -> None: + """This is what makes a readiness notification for the old block stale.""" + subscribe(runtime, 1) + snapshot = runtime.quiescent_snapshot() + assert snapshot is not None and snapshot[0].generation == 0 + + runtime.note_blocked(1, False) + runtime.note_blocked(1, True) + + snapshot = runtime.quiescent_snapshot() + assert snapshot is not None and snapshot[0].generation == 1 + + +async def test_readiness_after_a_re_block_names_the_current_generation( + backend: MemoryStreamBackend, +) -> None: + """Bumping the generation is only half of it; the manager has to be told. + + Core compares the generation a readiness notification carries against the + one lang put in the quiescent snapshot. The manager is what makes that call + and only the runtime knows when a wait re-blocks, so a manager left to + itself reports 0 for the life of the subscription. Every notification after + the first block is then answered ``Stale``, which the manager treats as + "re-probe later" -- but the watcher's prefetch cursor is already past the + record and no second notification is coming, so an append after a confirmed + park never wakes the Workflow at all. + """ + reported: list[int] = [] + notified = asyncio.Event() + + async def notify( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, + ) -> str: + reported.append(generation) + notified.set() + return ReadinessResult.ACCEPTED + + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id=uuid.uuid4().hex, + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + ) + try: + key = subscribe(runtime, 1) + # One resolved block, so the wait is on its second one -- the state + # every subscription is in for every record after its first. + runtime.note_blocked(1, False) + runtime.note_blocked(1, True) + + await backend.append(key, StreamRecord(RecordKind.DATA, b"x", "s", 0)) + await asyncio.wait_for(notified.wait(), 5) + + snapshot = runtime.quiescent_snapshot() + assert snapshot is not None + assert reported == [snapshot[0].generation] == [1], ( + "the generation reported to Core is not the one lang put in the " + f"quiescent snapshot, so Core answers Stale: reported={reported} " + f"snapshot={snapshot[0].generation}" + ) + finally: + await manager.shutdown() + + +async def test_a_closed_wait_can_never_re_enter_the_blocked_set( + runtime: WorkflowStreamRuntime, +) -> None: + """Closing keeps the wait's state, so the state has to refuse to block again. + + The subscription is deliberately not forgotten -- its binding is what a + replay of unchanged code reads, and its cursor is what stops a + Continue-As-New successor restarting the stream from the beginning -- which + leaves it reachable by ``wait_id`` long after the coroutine that was + awaiting it is gone. A wait that could re-enter the blocked set from there + would be named by the quiescent snapshot, and Core would go back to + retaining -- and eventually parking -- the Workflow Task for nobody. + """ + subscribe(runtime, 1) + runtime.note_blocked(1, False) + runtime.unsubscribe(1) + + runtime.note_blocked(1, True) + + assert runtime.quiescent_snapshot() is None, ( + "a closed wait re-entered the quiescent set, so Core is asked to hold " + "the Workflow Task open for a wait nothing is awaiting" + ) + # And its binding is still there to be read, which is the whole reason + # closing does not simply drop the state. + assert set(annotation_of(runtime).header.streams) == {1} # type: ignore[attr-defined] + + +# --- the byte budget -------------------------------------------------------- + + +async def test_rollover_is_requested_before_the_budget_is_reached( + backend: MemoryStreamBackend, +) -> None: + """The runtime asks Core to roll the task over rather than grow the marker.""" + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id="first", + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + max_annotation_bytes=2048, + ) + try: + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + + assert not runtime.request_rollover + + delivered = 0 + while not runtime.request_rollover: + # Alternating, so every delivery is its own run -- the one workload + # that cannot be range-compressed and therefore the one that reaches + # the cap. + for wait_id in (1, 2): + delivered += 1 + runtime.record_delivery(wait_id, data(f"{delivered}-0")) + runtime.take_observation_delta() + + assert delivered > 0 + finally: + await manager.shutdown() + + +def _budget_runtime( + backend: MemoryStreamBackend, + manager: StreamSubscriptionManager, + *, + max_annotation_bytes: int, +) -> WorkflowStreamRuntime: + return WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id="first", + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + max_annotation_bytes=max_annotation_bytes, + ) + + +async def test_the_segment_that_crosses_the_mark_asks_for_rollover_itself( + backend: MemoryStreamBackend, +) -> None: + """The crossing segment's own completion carries the request, not the next one. + + ``take_observation_delta`` is what *closes* the activation's segment, so a + rollover flag sampled before it describes the annotation as it stood one + activation ago. The segment that crossed the high-water mark then went out + with ``request_rollover = false``, and the activation after it was free to add + another frame -- and to overflow -- before Core had ever been asked to roll + over. One activation of delay is the whole margin the high-water mark exists + to provide. + """ + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + from tests.contrib.external_workflow_streams.test_delivery_budget import ( + _CompletionStub, + ) + + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = _budget_runtime(backend, manager, max_annotation_bytes=2048) + try: + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + + # Fill to just short of the mark, one flushed activation at a time, so + # that the *next* activation is the one that crosses it. + accumulated: list[bytes] = [] + delivered = 0 + high_water = int(2048 * ROLLOVER_HIGH_WATER) + while True: + for wait_id in (1, 2): + delivered += 1 + runtime.record_delivery(wait_id, data(f"{delivered}-0")) + delta = runtime.take_observation_delta() + if delta is not None: + accumulated.append(delta) + assert runtime._accumulator is not None + # Stops with a gap the next activation's segment is bigger than, so + # that segment is unambiguously the frame that crosses the mark. + if runtime._accumulator.size >= high_water - 400: + break + assert not runtime.request_rollover, ( + "the mark was crossed by a flushed activation, so nothing here " + "is testing the activation that crosses it" + ) + + # One more activation, whose closing segment is what crosses the mark -- + # and which is therefore the completion that has to carry the request. + for _ in range(15): + for wait_id in (1, 2): + delivered += 1 + runtime.record_delivery(wait_id, data(f"{delivered}-0")) + assert not runtime.request_rollover, ( + "the crossing segment is still open, so the accumulator cannot know " + "about it yet -- which is the entire reason the flag must be read " + "after the delta is taken and not before" + ) + + stub = _CompletionStub(runtime) + _WorkflowInstanceImpl._emit_external_stream_commands(stub) # type: ignore[arg-type] + + commands = stub._current_completion.successful.commands + progress = commands[0].workflow_stream_progress + assert progress.request_rollover, ( + "the completion that carried the crossing segment did not ask for the " + "rollover, so an entire further activation may append to an " + "annotation Core has not been told to close" + ) + accumulated.append(progress.observation_delta) + annotation = decode_annotation(b"".join(accumulated)) + assert annotation.terminal is not None, ( + "Core issues no finalization job for a rollover it was asked for, so " + "the terminal has to ride this same delta" + ) + assert annotation.segments[-1].end_reason in ( + SegmentEndReason.BUDGET_ROLLOVER, + SegmentEndReason.NO_DATA_AVAILABLE, + ) + finally: + await manager.shutdown() + + +async def test_a_frame_larger_than_the_slack_rolls_over_instead_of_raising( + backend: MemoryStreamBackend, +) -> None: + """An indivisible frame bigger than the remaining budget is not a failure. + + The high-water mark is a *fraction* of the budget, and a segment frame can be + larger than the fraction that is left: a run costs two provider-supplied + offset strings, and nothing bounds their length from this side. Checking at + encode time and raising is what ADR-007 rejects -- the encoding that + overflowed overflows again on the retry, so the Workflow Task fails forever + with no marker, no terminal, and no rollover ever asked for. + + The runtime instead stops handing records over while the annotation can still + record them, and asks for the rollover on that boundary. Long offsets are what + make one activation's segment expensive here; that is exactly the shape the + old check-and-fail path could not survive. + """ + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = _budget_runtime(backend, manager, max_annotation_bytes=1024) + long = "o" * 300 + try: + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + + # Each record is its own run, and each run costs two 300-byte offsets -- + # far more than the 25% of 1024 bytes that the high-water margin leaves. + delivered = 0 + while runtime.delivery_budget_remaining() > 0 and delivered < 50: + wait_id = 1 + (delivered % 2) + delivered += 1 + runtime.record_delivery(wait_id, data(f"{long}{delivered}-0")) + + assert delivered < 50, ( + "the annotation budget never stopped delivery, so this test is not " + "exercising the boundary it exists for" + ) + assert runtime.request_rollover, ( + "the runtime stopped delivering because the annotation could record " + "no more, and said nothing about it -- Core is never asked to roll " + "the task over and the next activation overflows" + ) + assert runtime.annotation_budget_exhausted + assert runtime.delivery_budget_exhausted(), ( + "records left buffered by the annotation budget need their readiness " + "re-reported exactly as the record cap's do" + ) + + # And the annotation still closes, which is what raising would have + # prevented: a marker with no terminal is durable and undecodable past + # the frame after it. + delta = runtime.take_observation_delta() + assert delta is not None + terminal = runtime.add_terminal() + annotation = decode_annotation(delta + terminal) + assert annotation.terminal is not None + assert annotation.segments[-1].end_reason == SegmentEndReason.BUDGET_ROLLOVER, ( + "a segment cut by the byte budget has to say so: the batch continues " + "in the following marker, which no other end reason implies" + ) + finally: + await manager.shutdown() + + +async def test_an_activation_the_annotation_budget_stopped_is_not_wedged_by_it( + backend: MemoryStreamBackend, +) -> None: + """Stopping delivery obliges the same completion to ask for the rollover. + + These two are one mechanism and each is useless alone. The annotation budget + stops delivery so a segment can never overflow; the rollover is what gives the + next Workflow Task a fresh annotation to deliver into. Stop without asking and + the Workflow is **wedged**: the next activation begins against the same full + annotation, is handed a delivery budget of zero, delivers nothing, and so + observes nothing -- and a rollover condition that depended on having observed + something would then never become true again. + + That is not hypothetical. The flag is read after `take_observation_delta`, + which it must be, since that call is what closes the crossing segment -- and + that call also clears the observed flag. The frame here is deliberately larger + than the slack the *fractional* high-water mark leaves but small enough that + the mark itself is not crossed, so the mark cannot cover for it. + """ + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + from tests.contrib.external_workflow_streams.test_delivery_budget import ( + _CompletionStub, + ) + + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = _budget_runtime(backend, manager, max_annotation_bytes=1024) + long = "o" * 300 + try: + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + + delivered = 0 + while runtime.delivery_budget_remaining() > 0 and delivered < 50: + wait_id = 1 + (delivered % 2) + delivered += 1 + runtime.record_delivery(wait_id, data(f"{long}{delivered}-0")) + + assert delivered < 50, "the annotation budget never stopped delivery" + assert runtime._accumulator is None or not ( + runtime._accumulator.request_rollover + ), ( + "the accumulator's own high-water mark is asking for this rollover, " + "so it would mask the condition under test -- the frame has to be " + "larger than the slack the fractional mark leaves without reaching " + "the mark itself" + ) + + stub = _CompletionStub(runtime) + _WorkflowInstanceImpl._emit_external_stream_commands(stub) # type: ignore[arg-type] + + commands = stub._current_completion.successful.commands + assert commands[0].workflow_stream_progress.request_rollover, ( + "delivery stopped for the annotation budget and the completion asked " + "for no rollover, so the next Workflow Task inherits a full " + "annotation and can never deliver again" + ) + assert ( + decode_annotation( + commands[0].workflow_stream_progress.observation_delta + ).terminal + is not None + ) + + # And the next Workflow Task can in fact deliver: the annotation was + # closed and a fresh one begins from its own header. + assert runtime.delivery_budget_remaining() > 0, ( + "the Workflow Task after the rollover is still unable to take a " + "single record" + ) + assert not runtime.annotation_budget_exhausted + finally: + await manager.shutdown() + + +async def test_the_first_record_of_an_activation_is_priced_from_a_measurement( + backend: MemoryStreamBackend, +) -> None: + """A per-segment maximum prices every activation's first record at the floor. + + ``close_segment`` empties the open segment at the end of each activation, so a + price taken from *that* segment's runs is back to the bare floor every time -- + and a real run costs two provider-chosen offset strings, which can be many + times the floor. Delivery then goes ahead on a price that is wrong by a factor, + the closing segment no longer fits, and what surfaces is a byte-budget error + that fails the Workflow. + + The measurement therefore belongs to the *annotation*, not the segment: the + largest run encoded since the header went out is what the next record costs + until a larger one appears. + """ + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = _budget_runtime(backend, manager, max_annotation_bytes=8192) + long = "9" * 300 + try: + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + + # One activation's worth of long-offset runs, then the flush that closes + # its segment. The price has to survive that flush: `close_segment` empties + # the open segment, and a price read from it is back to the floor. + for i in range(4): + runtime.record_delivery(1 + (i % 2), data(f"{long}{i}-0")) + measured = runtime._max_run_bytes + assert measured > _RUN_COST_FLOOR, ( + "the long offsets did not even cost more than the floor, so this test " + "cannot tell a measurement from a guess" + ) + + runtime.take_observation_delta() + + assert runtime._run_sizes == [], "the open segment must have been emptied" + assert runtime._max_run_bytes == measured, ( + "the price fell back to the floor when the segment was closed, so the " + "first record of the next activation is priced at " + f"{_RUN_COST_FLOOR} instead of {measured} -- and delivering on that " + "price hands over a record the closing segment cannot record" + ) + + # And a fresh annotation *does* start from the floor again, which is right: + # it has measured nothing, and the measurement is a property of the + # annotation rather than of the Run. + runtime.add_terminal() + assert runtime._max_run_bytes == 0 + + # Driven to the end, nothing escapes and every annotation closes: the + # arithmetic never hands over a record it cannot then record. + total = 0 + for _ in range(40): + while runtime.delivery_budget_remaining() > 0 and total < 400: + total += 1 + runtime.record_delivery(1 + (total % 2), data(f"{long}{total}-0")) + delta = runtime.take_observation_delta() + if delta is not None and runtime.request_rollover: + assert decode_annotation(delta + runtime.add_terminal()).terminal + assert total > 0, "nothing was ever delivered, so this asserts nothing" + finally: + await manager.shutdown() + + +async def test_a_run_too_large_for_any_annotation_says_so_and_fails_the_workflow( + backend: MemoryStreamBackend, +) -> None: + """The one boundary no rollover can move, reported as itself. + + A record is priced before its offsets are seen, so a provider whose offsets are + far longer than anything measured can make one run cost more than the whole + budget has left -- and a fresh annotation has to carry that same run, so + rolling over changes nothing. That is a genuine capacity limit, and the two + things that matter are how it is *reported*: not as an internal byte-budget + error, which names nothing an operator can act on, and not as a Workflow Task + failure, which the server retries on an encoding that cannot succeed (ADR-007). + """ + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = _budget_runtime(backend, manager, max_annotation_bytes=2048) + try: + subscribe(runtime, 1) + with pytest.raises(ExternalStreamCapacityError) as caught: + for i in range(10): + runtime.record_delivery(1, data(f"{'9' * 5000}{i}-0")) + + assert "offsets" in str(caught.value), ( + f"the error must name what an operator can change: {caught.value}" + ) + assert caught.value.non_retryable, ( + "a retryable capacity limit is a Workflow Task the server retries " + "against an encoding that can never fit" + ) + finally: + await manager.shutdown() + + +async def test_a_subscription_set_too_large_to_record_is_refused_at_subscribe( + backend: MemoryStreamBackend, +) -> None: + """A header that cannot fit is rejected where the Workflow can still act on it. + + A header is one indivisible frame and a rollover writes a fresh one, so an + oversized header is not a rollover problem -- every annotation would be the + same size. Discovered while encoding a completion it fails the Workflow Task, + the server retries Workflow Task failures regardless of cause, and the retry + encodes the identical bytes: a Workflow stuck permanently with nothing durable + to say why. + + Raised from the ``subscribe()`` call instead, it is deterministic, reproduces + under replay, and names what to change. + """ + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = _budget_runtime(backend, manager, max_annotation_bytes=512) + try: + subscribe(runtime, 1) + with pytest.raises(ExternalStreamCapacityError) as caught: + subscribe(runtime, 2, name="s" * 600) + + assert "512" in str(caught.value) + assert "2" in str(caught.value), "the error must name the refused wait" + # Refused *before* it became replay-visible: a wait left half-registered + # would reach the next header as a binding nothing is watching. + assert set(runtime.subscriptions()) == {1} + assert manager.subscription(RUN_ID, 2) is None + + # And the Workflow Task that survives the refusal still completes: the + # annotation for the subscription it does hold encodes and closes. + runtime.record_delivery(1, data("1-0")) + annotation = annotation_of(runtime) + assert set(annotation.header.streams) == {1} # type: ignore[attr-defined] + assert annotation.terminal is not None # type: ignore[attr-defined] + finally: + await manager.shutdown() + + +async def test_the_capacity_floor_covers_everything_an_empty_annotation_carries( + backend: MemoryStreamBackend, +) -> None: + """Clearing header-plus-terminal is not clearing the floor. + + Every annotation also carries at least one **segment frame** -- an activation + that drained and observed nothing still encodes one, and the empty segment is + meaningful (ADR-018) -- and the **spill margin** a mispriced record overruns + into. A check that priced only the header and the terminal accepted a + subscription set that cleared it by a byte or two and then could not encode its + very first completion, which relocates the failure rather than preventing it. + """ + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + try: + # One wait whose stream name makes its header large enough that the + # remaining segment and spill reserve cross the budget. + # The budget below fits both with room to spare -- and does not fit the + # 3-byte segment frame and the 64-byte margin behind them. + name = "s" * 200 + runtime = _budget_runtime(backend, manager, max_annotation_bytes=292) + header_and_terminal = len( + encode_header( + AnnotationHeader( + { + 1: StreamBinding( + stream_key=runtime.stream_key(name), + start_cursor=BEGINNING, + provider_id=MemoryStreamBackend.provider_id, + provider_format_version=( + MemoryStreamBackend.provider_format_version + ), + ) + } + ) + ) + ) + len(encode_terminal({1: BEGINNING})) + assert header_and_terminal <= 292, ( + "this set clears header-plus-terminal, which is the whole point: a " + "floor priced on those two alone accepts it" + ) + + with pytest.raises(ExternalStreamCapacityError) as caught: + runtime.register(wait_id=1, stream_key=runtime.stream_key(name)) + assert "margin" in str(caught.value), ( + f"the error must say what the floor covers: {caught.value}" + ) + assert set(runtime.subscriptions()) == set() + finally: + await manager.shutdown() + + +# --- the completion path closes every annotation it ends -------------------- + + +def _progress_delta(stub: object) -> bytes | None: + """The observation delta a completion is carrying, if any.""" + for command in stub._current_completion.successful.commands: # type: ignore[attr-defined] + if command.HasField("workflow_stream_progress"): + return command.workflow_stream_progress.observation_delta + return None + + +async def test_a_command_producing_completion_closes_its_annotation( + runtime: WorkflowStreamRuntime, +) -> None: + """Two Workflow Tasks in a row, each ended by the Workflow's own command. + + Core writes and clears a marker on each of them, so each delta has to be a + whole annotation: its own header, and its own terminal. Carrying the header + across the first would leave the second starting at whatever frame came + first, and the leading byte of an annotation is read as its schema version + -- so the failure is reported as an unsupported version rather than as + anything to do with framing. + """ + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + from tests.contrib.external_workflow_streams.test_delivery_budget import ( + _CompletionStub, + ) + + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + first = _CompletionStub(runtime) + first._add_command() # a timer, an activity: server-bound, so the task ends + _WorkflowInstanceImpl._emit_external_stream_commands(first) # type: ignore[arg-type] + + runtime.record_delivery(1, data("2-0")) + second = _CompletionStub(runtime) + second._add_command() + _WorkflowInstanceImpl._emit_external_stream_commands(second) # type: ignore[arg-type] + + deltas = {"first": _progress_delta(first), "second": _progress_delta(second)} + # Decoded before anything is asserted about them: it is the *second* delta + # that loses its header, and asserting per delta as each is decoded would + # stop on the first one's own shortcoming instead. + decoded = {} + for name, delta in deltas.items(): + assert delta is not None, f"the {name} completion reported no progress" + decoded[name] = decode_annotation(delta) + + for name, annotation in decoded.items(): + assert annotation.header.streams, f"the {name} marker records no stream" + assert annotation.terminal is not None, ( + f"the {name} marker has no terminal, so nothing in it says where " + "that Workflow Task's deliveries stopped" + ) + + assert decoded["second"].terminal == {1: AFTER(Offset("2-0"))} + + +async def test_a_rollover_request_closes_the_annotation_it_splits( + backend: MemoryStreamBackend, +) -> None: + """A budget rollover ends the task too, and needs no finalization round trip. + + Core takes ``request_rollover`` as authoritative over the retention the same + completion asks for, writes the marker, and forces a replacement task -- + without asking for a terminal, because the progress command carrying the + request is supposed to have carried one. A completion that asked for the + split without closing the annotation would produce a marker with no terminal + *and* leave the next one headerless. + """ + from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + from tests.contrib.external_workflow_streams.test_delivery_budget import ( + _CompletionStub, + ) + + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + runtime = WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id="first", + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + max_annotation_bytes=2048, + ) + try: + subscribe(runtime, 1) + subscribe(runtime, 2, name="tool-events") + + # Accumulated the way Core accumulates, by byte append: the marker + # carries every delta of the Workflow Task, not just its last one. + accumulated: list[bytes] = [] + delivered = 0 + while not runtime.request_rollover: + # Alternating, so every delivery is its own run -- the one workload + # that cannot be range-compressed and therefore the one that reaches + # the cap. + for wait_id in (1, 2): + delivered += 1 + runtime.record_delivery(wait_id, data(f"{delivered}-0")) + earlier = runtime.take_observation_delta() + if earlier is not None: + accumulated.append(earlier) + + # The Workflow is still blocked on both streams, so this completion asks + # for retention as well -- and the rollover wins over it. + stub = _CompletionStub(runtime) + _WorkflowInstanceImpl._emit_external_stream_commands(stub) # type: ignore[arg-type] + + delta = _progress_delta(stub) + assert delta is not None, "the completion carried no progress to roll over" + accumulated.append(delta) + commands = stub._current_completion.successful.commands + assert commands[0].workflow_stream_progress.request_rollover, ( + "the completion did not ask Core to roll the task over" + ) + assert decode_annotation(b"".join(accumulated)).terminal is not None, ( + "the annotation Core is about to write has no terminal, and the " + "rollover path asks for none" + ) + + # And the next annotation begins from a header of its own. + runtime.record_delivery(1, data("9999-0")) + following = runtime.take_observation_delta() + assert following is not None + assert decode_annotation(following).header.streams + finally: + await manager.shutdown() + + +async def test_a_fence_ends_the_segment_as_fence_reached( + runtime: WorkflowStreamRuntime, +) -> None: + """How replay knows the batch stopped at a fence rather than running dry.""" + subscribe(runtime, 1) + runtime.record_delivery(1, fence("1-0")) + + decoded = annotation_of(runtime) + + assert decoded.segments[0].end_reason == SegmentEndReason.FENCE_REACHED # type: ignore[attr-defined] + + +async def test_running_dry_ends_the_segment_as_no_data_available( + runtime: WorkflowStreamRuntime, +) -> None: + subscribe(runtime, 1) + runtime.record_delivery(1, data("1-0")) + + decoded = annotation_of(runtime) + + assert decoded.segments[0].end_reason == SegmentEndReason.NO_DATA_AVAILABLE # type: ignore[attr-defined] + + +async def test_the_stream_key_comes_from_the_chain_not_the_run( + runtime: WorkflowStreamRuntime, +) -> None: + """The stream spans the whole Continue-As-New chain.""" + key = runtime.stream_key("tokens") + + assert key.workflow_id == "wf" + assert key.first_execution_run_id != RUN_ID + assert key.stream_name == "tokens" diff --git a/tests/contrib/external_workflow_streams/test_runtime_only_jobs.py b/tests/contrib/external_workflow_streams/test_runtime_only_jobs.py new file mode 100644 index 0000000000..51ead11e8a --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_runtime_only_jobs.py @@ -0,0 +1,346 @@ +"""P19 — the jobs answered outside the synchronous Workflow thread. + +Two of the four stream activation jobs are *themselves* backend operations, and +a third has to be prepared by one. Routing them through ``activate()`` would put +a multi-second transaction inside a call running under a 2-second deadlock +timeout: it would fail the Workflow Task for a perfectly healthy backend, and get +worse the more records replay had to validate. +""" + +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest + +import temporalio.converter +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._record import ( + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._runtime import WorkflowStreamRuntime +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend +from tests.contrib.external_workflow_streams.test_replay import ( + annotation_for, + append_five, +) + +RUN_ID = "run-1" +DEADLOCK_TIMEOUT_SECONDS = 2 +"""What `_WorkflowWorker` gives an activation before declaring a deadlock.""" + + +async def _notify( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] +) -> str: + return ReadinessResult.ACCEPTED + + +class HostileBackend(MemoryStreamBackend): + """Raises on every provider method. + + Registering one of these is how "this path performs no backend I/O" is + asserted rather than trusted: if any layer reached for the provider, the + call would explode instead of quietly succeeding. + """ + + def _refuse(self, name: str): # type: ignore[no-untyped-def] + raise AssertionError(f"{name} must not be called on this path") + + async def append(self, key, record): # type: ignore[no-untyped-def] + self._refuse("append") + + async def read_range(self, key, first, last): # type: ignore[no-untyped-def] + self._refuse("read_range") + + async def read_after(self, key, after, *, max_records, block=None): # type: ignore[no-untyped-def] + self._refuse("read_after") + + async def install_park_intent(self, key, intent): # type: ignore[no-untyped-def] + self._refuse("install_park_intent") + + async def recheck(self, key, wait_id): # type: ignore[no-untyped-def] + self._refuse("recheck") + + +class SlowParkBackend(MemoryStreamBackend): + """A park handshake that takes longer than the deadlock timeout.""" + + async def install_park_intent(self, key, intent): # type: ignore[no-untyped-def] + await asyncio.sleep(DEADLOCK_TIMEOUT_SECONDS + 0.5) + return await super().install_park_intent(key, intent) + + +class SlowRangeBackend(MemoryStreamBackend): + """A recorded-range read that takes longer than the deadlock timeout. + + Not a contrived latency: an inclusive read over a whole recorded batch is + the operation that gets *slower* the more records the marker committed, so + this is the shape a large, healthy replay actually has. + """ + + async def read_range(self, key, first, last): # type: ignore[no-untyped-def] + await asyncio.sleep(DEADLOCK_TIMEOUT_SECONDS + 0.5) + return await super().read_range(key, first, last) + + +def make_runtime(backend, manager): # type: ignore[no-untyped-def] + return WorkflowStreamRuntime( + manager=manager, + backend=backend, + run_id=RUN_ID, + namespace="ns", + workflow_id="wf", + first_execution_run_id=uuid.uuid4().hex, + data_converter=temporalio.converter.DataConverter.default, + default_idle_timeout=timedelta(seconds=1), + ) + + +def make_manager(backend): # type: ignore[no-untyped-def] + return StreamSubscriptionManager( + backend=backend, + notify_ready=_notify, + watch_block=timedelta(milliseconds=10), + ) + + +# --- finalization performs no backend I/O (ADR-010) -------------------------- + + +@pytest.mark.asyncio +async def test_finalization_touches_no_provider_at_all() -> None: + """Asserted against a provider that raises, rather than trusted. + + The boundary is not "wherever the stream is now"; it is where this Workflow + Task's deliveries stopped, which was fixed the moment the last activation + returned. Refreshing it against the backend would be actively wrong -- it + could name a position replay must not reproduce -- so the only correct + number of provider calls here is zero. + """ + backend = HostileBackend() + manager = make_manager(backend) + runtime = make_runtime(backend, manager) + try: + # Register without going through the manager's watcher, which would call + # `read_after` -- what is under test is the finalization path only. + runtime._subscriptions.clear() + runtime.register( + wait_id=1, + stream_key=runtime.stream_key("tokens"), + ) + + terminal = runtime.add_terminal() + + assert terminal, "finalization must produce a terminal" + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_finalization_reports_where_delivery_stopped_not_where_the_stream_is() -> ( + None +): + """A record arriving mid-finalization changes nothing about the terminal. + + It belongs to the *next* Workflow Task and reaches Core through the normal + readiness path -- or, if none is open by then, through the wake Signal. + """ + backend = MemoryStreamBackend() + manager = make_manager(backend) + runtime = make_runtime(backend, manager) + try: + key = runtime.stream_key("tokens") + runtime.register(wait_id=1, stream_key=key) + runtime.record_delivery( + 1, + StreamRecord(RecordKind.DATA, b"a", "s", 0).placed_at(Offset("5-0")), + ) + + # Something lands in the stream while finalization is being answered. + await backend.append(key, StreamRecord(RecordKind.DATA, b"later", "s", 9)) + + assert runtime.blocked_snapshot()[1].offset == Offset("5-0"), ( + "the terminal must name where deliveries stopped, not where the " + "stream has since got to" + ) + finally: + await manager.shutdown() + + +# --- the park handshake outlives the deadlock timeout ------------------------ + + +@pytest.mark.asyncio +async def test_a_park_slower_than_the_deadlock_timeout_is_still_answered() -> None: + """It is answered off the Workflow thread, so its duration is irrelevant. + + Were this routed through `activate()`, a healthy-but-slow backend would fail + the Workflow Task -- and it would get worse under exactly the conditions + parking exists for. + """ + backend = SlowParkBackend() + manager = make_manager(backend) + runtime = make_runtime(backend, manager) + try: + key = runtime.stream_key("tokens") + runtime.register(wait_id=1, stream_key=key) + + started = asyncio.get_running_loop().time() + became_ready = await asyncio.wait_for( + manager.prepare_park(RUN_ID, 1, runtime.blocked_snapshot()), 30 + ) + elapsed = asyncio.get_running_loop().time() - started + + assert elapsed > DEADLOCK_TIMEOUT_SECONDS, ( + "this test is only meaningful if the handshake really did outlast " + f"the deadlock timeout, took {elapsed:.2f}s" + ) + assert became_ready is False, "nothing was appended, so nothing became ready" + assert await backend.park_intent(key, 1) is not None + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_a_recheck_that_finds_records_abandons_the_whole_park() -> None: + """All-or-nothing across the set, and every intent installed comes back out. + + A park confirmed for a set with a ready member would lose that member's + record until a producer happened to signal it awake. + """ + backend = MemoryStreamBackend() + manager = make_manager(backend) + runtime = make_runtime(backend, manager) + try: + first = runtime.stream_key("tokens") + second = runtime.stream_key("tool-events") + runtime.register(wait_id=1, stream_key=first) + runtime.register(wait_id=2, stream_key=second) + + # A record on *one* of the two streams. + await backend.append(first, StreamRecord(RecordKind.DATA, b"a", "s", 0)) + + became_ready = await manager.prepare_park(RUN_ID, 1, runtime.blocked_snapshot()) + + assert became_ready is True + assert await backend.park_intent(first, 1) is None + assert await backend.park_intent(second, 2) is None, ( + "the other subscription's intent must be removed too -- parking is " + "all-or-nothing across the complete set" + ) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_intents_are_installed_before_anything_is_rechecked() -> None: + """This ordering is what closes the append/park race. + + A producer appends *before* it observes the park generation, so an append is + either seen by the recheck or paired with a wake Signal. Rechecking before + every intent was installed would leave a window where it is neither. + """ + backend = MemoryStreamBackend() + order: list[str] = [] + + class RecordingBackend(MemoryStreamBackend): + async def install_park_intent(self, key, intent): # type: ignore[no-untyped-def] + order.append(f"install:{intent.wait_id}") + return await super().install_park_intent(key, intent) + + async def recheck(self, key, wait_id): # type: ignore[no-untyped-def] + order.append(f"recheck:{wait_id}") + return await super().recheck(key, wait_id) + + backend = RecordingBackend() + manager = make_manager(backend) + runtime = make_runtime(backend, manager) + try: + for wait_id, name in ((1, "tokens"), (2, "tool-events")): + runtime.register( + wait_id=wait_id, + stream_key=runtime.stream_key(name), + ) + + await manager.prepare_park(RUN_ID, 1, runtime.blocked_snapshot()) + + installs = [i for i, step in enumerate(order) if step.startswith("install")] + rechecks = [i for i, step in enumerate(order) if step.startswith("recheck")] + assert installs and rechecks + assert max(installs) < min(rechecks), ( + f"every intent must be installed before any recheck, got {order}" + ) + finally: + await manager.shutdown() + + +# --- replay's recorded reads outlive the deadlock timeout too ---------------- + + +@pytest.mark.asyncio +async def test_a_replay_read_slower_than_the_deadlock_timeout_is_still_answered() -> ( + None +): + """The reason the replay job is partitioned rather than answered in `_apply`. + + Routing it through ``activate()`` would put the recorded range reads inside + a call running under a 2-second deadlock timeout -- failing the Workflow + Task for a perfectly healthy backend, and getting worse the more records + replay had to validate. Out here the duration is simply irrelevant. + """ + backend = SlowRangeBackend() + manager = make_manager(backend) + key = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + try: + placed = await append_five(backend, key) + + started = asyncio.get_running_loop().time() + plan = await asyncio.wait_for( + manager.prepare_replay(RUN_ID, annotation_for(key, placed)), 30 + ) + elapsed = asyncio.get_running_loop().time() - started + + assert elapsed > DEADLOCK_TIMEOUT_SECONDS, ( + "this test is only meaningful if the read really did outlast the " + f"deadlock timeout, took {elapsed:.2f}s" + ) + assert plan.total_records == len(placed) + finally: + await manager.shutdown() + + +@pytest.mark.asyncio +async def test_the_delivering_activation_reads_nothing_itself() -> None: + """Everything a replay activation needs is in memory before it starts. + + Preparation and delivery are two different moments, and the second one must + perform no I/O at all -- which is asserted here by taking the prepared plan + from a provider that refuses every call. + """ + backend = MemoryStreamBackend() + manager = make_manager(backend) + key = StreamKey("ns", "wf", uuid.uuid4().hex, "tokens") + try: + placed = await append_five(backend, key) + await manager.prepare_replay(RUN_ID, annotation_for(key, placed)) + + # From here on the provider is hostile. Delivery still has to work. + manager._backend = HostileBackend() + + plan = manager.take_replay_plan(RUN_ID) + + assert plan is not None + assert plan.total_records == len(placed) + finally: + await manager.shutdown() diff --git a/tests/contrib/external_workflow_streams/test_serialization_context.py b/tests/contrib/external_workflow_streams/test_serialization_context.py new file mode 100644 index 0000000000..4b1e768920 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_serialization_context.py @@ -0,0 +1,746 @@ +"""A stream record is converted with the same context as every other payload. + +A ``DataConverter`` may carry a codec or a payload converter that derives what +it does from *which Workflow* the payload belongs to -- an encryption key per +Workflow ID is the reason ``WithSerializationContext`` exists at all. The Worker +already binds a :py:class:`~temporalio.converter.WorkflowSerializationContext` +around every payload an activation carries, on both halves: ``decode_activation`` +on the Worker's loop and the Workflow instance's own converters on the Workflow +thread. + +A stream record is a payload like any other and has to arrive the same way. If +it does not, a per-Workflow key decrypts nothing: the record fails to decode, +the error travels with it, and the Workflow Task is retried forever against a +record that will never decode -- reported as a converter mismatch when the +configuration is in fact correct. + +The producer is half of the same statement, and the more dangerous half. Both +sides being context-*free* is self-consistent and works; fixing only the +consumer would break every deployment that works today, because the producer +would encrypt with no context while the consumer decrypts with a +Workflow-derived key. These tests therefore pin both sides to the same context. + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +# pyright: reportUnusedImport=false +from __future__ import annotations + +import asyncio +import uuid +from collections.abc import Sequence +from datetime import timedelta + +import pytest + +import temporalio.api.common.v1 +import temporalio.converter +from temporalio import workflow +from temporalio.client import Client +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._producer import ( + ExternalStreamProducer, + WorkflowChainKey, +) +from temporalio.contrib.external_workflow_streams._record import ( + BEGINNING, + RecordKind, + StreamRecord, +) +from temporalio.converter import ( + DataConverter, + DefaultPayloadConverter, + PayloadCodec, + SerializationContext, + WithSerializationContext, + WorkflowSerializationContext, +) +from temporalio.worker import Replayer, Worker +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams._api import external_stream + + +#: Marks the one payload these tests care about. Every other payload an +#: activation carries -- the Workflow's own argument, its result -- travels +#: through the same converter, and the Worker's handling of those is not what is +#: under test here. +_SENTINEL = b"context-probe" + +#: Contexts the codec saw, in the order it saw them. Module-level because the +#: converter is constructed by the Worker, not by the test. +_codec_contexts: list[SerializationContext | None] = [] + +#: Contexts the payload converter saw while converting a stream record. Written +#: from the Workflow thread, which is the half a codec-only probe cannot reach. +_converter_contexts: list[SerializationContext | None] = [] + +#: The same two observations, from the chain-keyed probes an offline replay is +#: driven through. Kept apart from the pair above so the replay assertions can +#: be exact rather than a suffix of everything the process has ever converted. +_chain_codec_contexts: list[SerializationContext | None] = [] +_chain_converter_contexts: list[SerializationContext | None] = [] + + +class ContextRequiredCodec(PayloadCodec, WithSerializationContext): + """A codec that cannot decode a stream record without a context. + + Deliberately *fails* rather than merely recording: a codec keyed on the + Workflow ID has nothing to decrypt with when it is handed no context, and a + probe that succeeded either way would pass against a converter that was + never bound at all. + + Only the sentinel payload is treated specially, so the Worker's own + ``decode_activation`` work -- which is separately context-bound already -- + contributes no observations of its own. + """ + + def __init__(self, context: SerializationContext | None = None) -> None: + self.context = context + + def with_context(self, context: SerializationContext) -> ContextRequiredCodec: + return ContextRequiredCodec(context) + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return list(payloads) + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + for payload in payloads: + if _SENTINEL not in payload.data: + continue + # Recorded before the refusal, so the assertion below can name the + # context that was actually supplied instead of only reporting that + # nothing worked. + _codec_contexts.append(self.context) + if self.context is None: + raise RuntimeError( + "a stream record was decoded with no serialization context" + ) + return list(payloads) + + +class ContextRequiredPayloadConverter( + DefaultPayloadConverter, WithSerializationContext +): + """The same refusal, on the synchronous half that runs on the Workflow thread. + + The codec probe cannot reach here: ``prepare()`` runs the codec on the + Worker's loop and ``convert()`` runs the payload converter inside + ``activate()``, and the two halves are handed converters from two different + places. A fix applied to only one of them leaves the other context-free. + """ + + def __init__(self, context: SerializationContext | None = None) -> None: + super().__init__() + self.context = context + + def with_context( + self, context: SerializationContext + ) -> ContextRequiredPayloadConverter: + return ContextRequiredPayloadConverter(context) + + def from_payloads( + self, + payloads: Sequence[temporalio.api.common.v1.Payload], + type_hints: list[type] | None = None, + ) -> list[object]: + for payload in payloads: + if _SENTINEL not in payload.data: + continue + _converter_contexts.append(self.context) + if self.context is None: + raise RuntimeError( + "a stream record was converted with no serialization context" + ) + return super().from_payloads(payloads, type_hints) + + +def _chain_tag(context: SerializationContext | None) -> bytes: + """The Workflow identity a chain-keyed probe writes into the bytes. + + Both the namespace and the Workflow id, because an offline ``Replayer`` + keeps the Workflow id and substitutes only the namespace: a probe keyed on + the id alone reports success against a converter that never saw the + recorded namespace at all. + """ + assert isinstance(context, WorkflowSerializationContext), ( + f"a stream record's payload was handled with no Workflow context: {context}" + ) + return f"|{context.namespace}|{context.workflow_id}".encode() + + +def _chain_apply( + payloads: Sequence[temporalio.api.common.v1.Payload], + context: SerializationContext | None, + seen: list[SerializationContext | None], + encoding: bool, +) -> list[temporalio.api.common.v1.Payload]: + """Appends or strips the tag, refusing bytes written under another context. + + A probe that only *recorded* the context would pass against a converter + bound to the wrong Workflow, because a wrong context still decodes -- to a + different value, silently. Tagging makes the mismatch the failure it would + be for a real per-Workflow key. + + Only the sentinel payload is touched, so the Workflow's own argument and + result -- which an offline replay legitimately handles under the replay + harness's own namespace -- travel through untouched. + """ + out: list[temporalio.api.common.v1.Payload] = [] + for payload in payloads: + if _SENTINEL not in payload.data: + out.append(payload) + continue + if not encoding: + seen.append(context) + tag = _chain_tag(context) + copied = temporalio.api.common.v1.Payload() + copied.CopyFrom(payload) + if encoding: + copied.data = payload.data + tag + else: + if not payload.data.endswith(tag): + raise RuntimeError( + "a stream record written under one Workflow context was " + f"read back under {context}" + ) + copied.data = payload.data[: -len(tag)] + out.append(copied) + return out + + +class ChainKeyedCodec(PayloadCodec, WithSerializationContext): + """The asynchronous half, keyed on the Workflow the record belongs to.""" + + def __init__(self, context: SerializationContext | None = None) -> None: + self.context = context + + def with_context(self, context: SerializationContext) -> ChainKeyedCodec: + return ChainKeyedCodec(context) + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return _chain_apply(payloads, self.context, _chain_codec_contexts, True) + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return _chain_apply(payloads, self.context, _chain_codec_contexts, False) + + +class ChainKeyedPayloadConverter(DefaultPayloadConverter, WithSerializationContext): + """The synchronous half, keyed the same way and run on the Workflow thread. + + The half an offline ``Replayer`` gets wrong on its own: the manager binds + the codec to the stream key the marker recorded, while the converter that + runs inside ``activate()`` belongs to a runtime the harness built under its + own namespace. + """ + + def __init__(self, context: SerializationContext | None = None) -> None: + super().__init__() + self.context = context + + def with_context(self, context: SerializationContext) -> ChainKeyedPayloadConverter: + return ChainKeyedPayloadConverter(context) + + def to_payloads( + self, values: Sequence[object] + ) -> list[temporalio.api.common.v1.Payload]: + return _chain_apply( + super().to_payloads(values), self.context, _chain_converter_contexts, True + ) + + def from_payloads( + self, + payloads: Sequence[temporalio.api.common.v1.Payload], + type_hints: list[type] | None = None, + ) -> list[object]: + return super().from_payloads( + _chain_apply(payloads, self.context, _chain_converter_contexts, False), + type_hints, + ) + + +@workflow.defn +class CountProbeTokensWorkflow: + """Consumes records and returns only how many, never the values.""" + + @workflow.run + async def run(self, expected: int) -> int: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + + seen = 0 + async for _ in tokens.subscribe(): + seen += 1 + if seen >= expected: + break + return seen + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +async def _await_observation(seen: list[SerializationContext | None]) -> None: + """Waits for the probe to fire, so a missing context is not read as a hang. + + A context-free decode raises, the error travels with the record, and the + Workflow Task is retried forever -- so waiting on the Workflow's *result* + would turn a wrong context into a timeout minutes later. The context is + observable before any of that, which is what makes the assertion sharp. + """ + for _ in range(200): + if seen: + return + await asyncio.sleep(0.1) + + +async def _publish_probe_record( + backend: MemoryStreamBackend, key: StreamKey, converter: DataConverter +) -> None: + from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec + + codec = StreamPayloadCodec(converter, str) + await backend.append( + key, + StreamRecord(RecordKind.DATA, await codec.encode(_SENTINEL.decode()), "p", 0), + ) + + +async def _stream_key(client: Client, handle) -> StreamKey: # type: ignore[no-untyped-def] + description = await handle.describe() + return StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + + +# --- the consumer ----------------------------------------------------------- + + +async def test_a_stream_records_codec_runs_with_the_consuming_workflows_context( + client: Client, backend: MemoryStreamBackend +) -> None: + """The asynchronous half, on the Worker's loop. + + The manager prepares records for every Run on the Worker, so it cannot hold + a bound converter of its own; the context has to come from the record's own + subscription. Getting it from the manager's construction instead would + decode one Workflow's records under another Workflow's key. + """ + _codec_contexts.clear() + config = client.config() + config["data_converter"] = DataConverter(payload_codec=ContextRequiredCodec()) + probe_client = Client(**config) + + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + probe_client, + task_queue=task_queue, + workflows=[CountProbeTokensWorkflow], + external_stream_backend=backend, + ): + handle = await probe_client.start_workflow( + CountProbeTokensWorkflow.run, + 1, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + key = await _stream_key(client, handle) + await _publish_probe_record(backend, key, DataConverter.default) + await _await_observation(_codec_contexts) + + assert _codec_contexts == [ + WorkflowSerializationContext( + namespace=client.namespace, workflow_id=handle.id + ) + ], ( + "the stream record's codec ran with the wrong serialization " + "context; a codec keyed on the Workflow decrypts nothing here, " + "while every other payload in the same activation is bound" + ) + assert await asyncio.wait_for(handle.result(), 30) == 1 + + +async def test_a_stream_records_conversion_runs_with_the_consuming_workflows_context( + client: Client, backend: MemoryStreamBackend +) -> None: + """The synchronous half, inside ``activate()``. + + ``convert()`` runs on the Workflow thread through the runtime's converter, + which is a different object from the manager's. The Workflow's own argument + is converted with a bound converter on this very thread, so a record that is + not is inconsistent with the activation it arrived in. + """ + _converter_contexts.clear() + config = client.config() + config["data_converter"] = DataConverter( + payload_converter_class=ContextRequiredPayloadConverter + ) + probe_client = Client(**config) + + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + probe_client, + task_queue=task_queue, + workflows=[CountProbeTokensWorkflow], + external_stream_backend=backend, + ): + handle = await probe_client.start_workflow( + CountProbeTokensWorkflow.run, + 1, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + key = await _stream_key(client, handle) + await _publish_probe_record(backend, key, DataConverter.default) + await _await_observation(_converter_contexts) + + assert _converter_contexts[:1] == [ + WorkflowSerializationContext( + namespace=client.namespace, workflow_id=handle.id + ) + ], ( + "the stream record was converted on the Workflow thread with the " + "wrong serialization context, although the Workflow's own argument " + "was converted with the right one on the same thread" + ) + assert await asyncio.wait_for(handle.result(), 30) == 1 + + +async def test_a_replayed_record_is_prepared_with_the_recorded_streams_context( + backend: MemoryStreamBackend, +) -> None: + """Replay prepares from the annotation, and must reach the same context. + + A replayed record never passes a subscription: the Workflow has not run far + enough to have called ``subscribe()`` again, so the live path's source for + the context does not exist yet. The annotation's own header carries the + stream key, which is why it records it -- and a replay prepared without it + would decode nothing on exactly the Runs that a Worker restart produces, + while the live path worked. + """ + from temporalio.contrib.external_workflow_streams._annotation import ( + Annotation, + AnnotationHeader, + Run, + Segment, + SegmentEndReason, + StreamBinding, + encode_annotation, + ) + from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec + from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, + ) + from temporalio.contrib.external_workflow_streams._record import AFTER + + _codec_contexts.clear() + key = StreamKey("ns", "wf-replayed", uuid.uuid4().hex, "tokens") + placed = await backend.append( + key, + StreamRecord( + RecordKind.DATA, + await StreamPayloadCodec(DataConverter.default, str).encode( + _SENTINEL.decode() + ), + "p", + 0, + ), + ) + assert placed.offset is not None + + async def notify( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] + ) -> str: + return ReadinessResult.ACCEPTED + + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=notify, + data_converter=DataConverter(payload_codec=ContextRequiredCodec()), + watch_block=timedelta(milliseconds=10), + ) + try: + await manager.prepare_replay( + "run-1", + encode_annotation( + Annotation( + header=AnnotationHeader( + streams={ + 1: StreamBinding( + stream_key=key, + start_cursor=BEGINNING, + provider_id=MemoryStreamBackend.provider_id, + provider_format_version=( + MemoryStreamBackend.provider_format_version + ), + ) + } + ), + segments=( + Segment( + runs=( + Run( + wait_id=1, + first_offset=placed.offset, + last_offset=placed.offset, + count=1, + ), + ), + end_reason=SegmentEndReason.NO_DATA_AVAILABLE, + ), + ), + terminal={1: AFTER(placed.offset)}, + ) + ), + ) + finally: + await manager.shutdown() + + assert _codec_contexts == [ + WorkflowSerializationContext(namespace="ns", workflow_id="wf-replayed") + ], ( + "the replayed record was prepared with the wrong serialization " + "context, so a Worker restart would fail to decode records the live " + "path decodes without complaint" + ) + + +async def test_an_offline_replay_converts_a_record_under_the_recorded_context( + client: Client, backend: MemoryStreamBackend +) -> None: + """Both halves of one record, through the real ``Replayer``. + + A ``Replayer`` runs under its own ``ReplayNamespace`` -- deliberately, and + :meth:`_verify_binding` accommodates it by comparing only the stream name + from a recorded binding. The record's *conversion* has to make the same + accommodation from the other side: the manager prepares a replayed record + under the stream key the marker recorded, so a converter left on the + harness's namespace would see two different Workflow identities while + decoding one payload -- and a namespace-keyed converter then refuses a + history that is entirely valid. + + Live execution cannot show this. There the recorded key and the runtime's + own identity are the same Workflow, so both halves agree whether or not + anything binds them separately. Only a harness that supplies its own + namespace pulls them apart, which is why this runs the history back through + the tool a user would. + """ + assert client.namespace != "ReplayNamespace", ( + "the Replayer's default namespace matches the live one, so this test " + "cannot tell a bound converter from an unbound one" + ) + _chain_codec_contexts.clear() + _chain_converter_contexts.clear() + converter = DataConverter( + payload_converter_class=ChainKeyedPayloadConverter, + payload_codec=ChainKeyedCodec(), + ) + config = client.config() + config["data_converter"] = converter + probe_client = Client(**config) + + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + probe_client, + task_queue=task_queue, + workflows=[CountProbeTokensWorkflow], + external_stream_backend=backend, + ): + handle = await probe_client.start_workflow( + CountProbeTokensWorkflow.run, + 1, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + key = await _stream_key(client, handle) + # Written under the *producing* Workflow's context, which is what the + # marker goes on to record and therefore what replay has to reproduce. + await _publish_probe_record( + backend, + key, + converter.with_context( + WorkflowSerializationContext( + namespace=key.namespace, workflow_id=key.workflow_id + ) + ), + ) + assert await asyncio.wait_for(handle.result(), 30) == 1 + history = await handle.fetch_history() + + assert _chain_codec_contexts and _chain_converter_contexts, ( + "the live run decoded no stream record, so there is nothing recorded " + "for the replay to disagree with" + ) + # Only the replay's observations are asserted on; the live run's are the + # subject of the two tests above. + _chain_codec_contexts.clear() + _chain_converter_contexts.clear() + + result = await Replayer( + workflows=[CountProbeTokensWorkflow], + data_converter=converter, + external_stream_backend=backend, + ).replay_workflow(history, raise_on_replay_failure=False) + + assert result.replay_failure is None, ( + "replaying a valid history failed because the record was converted " + "under the replay harness's namespace rather than the one the marker " + f"recorded: {result.replay_failure}" + ) + recorded = WorkflowSerializationContext( + namespace=client.namespace, workflow_id=handle.id + ) + assert _chain_codec_contexts == [recorded], ( + "the asynchronous half of the replayed record's decoding did not run " + f"with the recorded context: {_chain_codec_contexts}" + ) + assert _chain_converter_contexts == [recorded], ( + "the synchronous half ran with a different context than the " + "asynchronous half of the same record; one payload was decoded under " + f"two Workflow identities: {_chain_converter_contexts}" + ) + + +# --- the producer ----------------------------------------------------------- + + +async def test_the_producer_encodes_with_the_consuming_workflows_context( + backend: MemoryStreamBackend, +) -> None: + """The half that must not be forgotten. + + Producer and consumer share one ``DataConverter``, so they must also share + one context. Two context-*free* sides are self-consistent and work; one side + bound and the other not is a mismatch that appears only on the far side, as + a decode failure long after the append reported success. + + The chain key is the only Workflow identity a producer has, and it is the + right one: it names the whole Continue-As-New chain, which is what the + stream spans and what ``workflow_id`` means on the consumer too. + """ + recorded: list[SerializationContext | None] = [] + + class RecordingCodec(PayloadCodec, WithSerializationContext): + def __init__(self, context: SerializationContext | None = None) -> None: + self.context = context + + def with_context(self, context: SerializationContext) -> RecordingCodec: + return RecordingCodec(context) + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + recorded.append(self.context) + return list(payloads) + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return list(payloads) + + producer = ExternalStreamProducer( + backend=backend, + workflow=WorkflowChainKey("ns", "wf", "run-1"), + data_converter=DataConverter(payload_codec=RecordingCodec()), + session_id="s", + ) + await producer.topic("tokens", type=str).publish("a", wake=False) + + assert recorded == [ + WorkflowSerializationContext(namespace="ns", workflow_id="wf") + ], ( + "the producer encoded the record with a different serialization " + "context than the consuming Worker decodes it with; the two sides " + "share one converter and must share one context" + ) + + +async def test_a_producers_record_decodes_on_a_context_bound_consumer( + backend: MemoryStreamBackend, +) -> None: + """The round trip, with a context that actually changes the bytes. + + A codec that transforms differently per Workflow is the only probe that can + tell "both sides bound to the same context" from "both sides bound to + nothing": the first survives this, and so does today's context-free pair, + but a fix applied to one side alone does not. + """ + + class PerWorkflowCodec(PayloadCodec, WithSerializationContext): + """XORs with a key derived from the Workflow ID, as a real one would.""" + + def __init__(self, context: SerializationContext | None = None) -> None: + self.context = context + + def with_context(self, context: SerializationContext) -> PerWorkflowCodec: + return PerWorkflowCodec(context) + + def _mask(self) -> int: + assert isinstance(self.context, WorkflowSerializationContext), ( + "this codec has no key without a Workflow context" + ) + return sum(self.context.workflow_id.encode()) % 251 or 7 + + def _apply( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + mask = self._mask() + out: list[temporalio.api.common.v1.Payload] = [] + for payload in payloads: + copied = temporalio.api.common.v1.Payload() + copied.CopyFrom(payload) + copied.data = bytes(b ^ mask for b in payload.data) + out.append(copied) + return out + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return self._apply(payloads) + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return self._apply(payloads) + + from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec + + converter = DataConverter(payload_codec=PerWorkflowCodec()) + producer = ExternalStreamProducer( + backend=backend, + workflow=WorkflowChainKey("ns", "wf", "run-1"), + data_converter=converter, + session_id="s", + ) + await producer.topic("tokens", type=str).publish("a", wake=False) + + key = StreamKey("ns", "wf", "run-1", "tokens") + records = await backend.read_after(key, BEGINNING, max_records=10, block=None) + data = [r for r in records if not r.is_control] + assert len(data) == 1 + + consumer = StreamPayloadCodec( + converter.with_context( + WorkflowSerializationContext(namespace="ns", workflow_id="wf") + ), + str, + ) + assert await consumer.decode(data[0].payload) == "a" diff --git a/tests/contrib/external_workflow_streams/test_shutdown_sweep.py b/tests/contrib/external_workflow_streams/test_shutdown_sweep.py new file mode 100644 index 0000000000..b735f32df2 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_shutdown_sweep.py @@ -0,0 +1,1250 @@ +"""P20 — the Worker shutdown wake sweep. + +Two obligations Core cannot discharge. Teardown ordering, so a finalization in +flight is always answered before the Run's state disappears; and the sweep +itself, because an idle cached Run gets no eviction activation at shutdown at +all -- and that is exactly the Run that most needs one, since its records are +buffered in a process about to exit. +""" + +# pyright: reportMissingParameterType=false +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from datetime import timedelta + +import pytest +import pytest_asyncio + +from temporalio.contrib.external_workflow_streams._backend import ( + ParkIntentRemoval, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._manager import ( + SHUTDOWN_WAKE_ATTEMPTS, + ReadinessResult, + RunStatus, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._record import ( + BEGINNING, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._wake import ( + WakeRequest, + wake_request_id, +) +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +RUN_ID = "run-1" + + +class Harness: + """A manager wired to recording stand-ins for Core and the Signal path.""" + + def __init__( + self, + backend: MemoryStreamBackend, + status: str, + *, + wake_fails: bool = False, + readiness: str = ReadinessResult.ACCEPTED, + ) -> None: + self.backend = backend + self.status = status + self.wake_fails = wake_fails + self.readiness = readiness + self.probes: list[str] = [] + self.wakes: list[tuple[str, int]] = [] + self.metric: list[str] = [] + self.manager = StreamSubscriptionManager( + backend=backend, + notify_ready=self._notify, + send_wake=self._wake, + run_status=self._probe, + shutdown_wake_failed_metric=self._on_metric, + watch_block=timedelta(milliseconds=10), + ) + + async def _notify( + self, + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] + ) -> str: + return self.readiness + + async def _probe(self, run_id: str) -> str: + self.probes.append(run_id) + return self.status + + async def _wake(self, subscription) -> None: # type: ignore[no-untyped-def] + if self.wake_fails: + raise ConnectionError("service unavailable") + self.wakes.append((subscription.run_id, subscription.wait_id)) + + def _on_metric(self, subscription) -> None: # type: ignore[no-untyped-def] + self.metric.append(subscription.stream_key.stream_name) + + def register(self, wait_id: int = 1, stream_name: str = "tokens") -> StreamKey: + key = StreamKey("ns", "wf", "first-run", stream_name) + self.manager.register( + run_id=RUN_ID, + wait_id=wait_id, + stream_key=key, + start_cursor=BEGINNING, + ) + return key + + +def _swept_request_id(harness: Harness, subscription) -> str: # type: ignore[no-untyped-def] + """What the Worker's real sender derives for one swept subscription. + + Built from the manager's own sender identity rather than a literal, because + that identity is exactly what has to be fixed across one sweep's retries and + distinct between two Workers. + """ + key = subscription.stream_key + return wake_request_id( + WakeRequest( + namespace=key.namespace, + workflow_id=key.workflow_id, + first_execution_run_id=key.first_execution_run_id, + stream_name=key.stream_name, + wait_id=subscription.wait_id, + park_generation=0, + sender_identity=harness.manager.wake_sender_identity, + wake_counter=subscription.wake_counter, + ) + ) + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +@pytest_asyncio.fixture # type: ignore[reportUntypedFunctionDecorator] +async def harness_factory(backend: MemoryStreamBackend): + made: list[Harness] = [] + + def make(status: str, **kwargs) -> Harness: # type: ignore[no-untyped-def] + harness = Harness(backend, status, **kwargs) + made.append(harness) + return harness + + yield make + for harness in made: + if not harness.manager._shutting_down: + await harness.manager.shutdown() + + +# --- the probe ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_sweep_probes_rather_than_asserting_readiness( + harness_factory, +) -> None: + """Readiness means "a record is buffered" and would be a lie here. + + Probing with it would manufacture a spurious Workflow Task on the way out of + a Worker that is shutting down -- for a Run that may have nothing waiting at + all. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + harness.register() + + await harness.manager.shutdown() + + assert harness.probes == [RUN_ID], "the sweep must use the read-only probe" + + +@pytest.mark.asyncio +async def test_a_run_with_no_subscriptions_is_not_probed(harness_factory) -> None: + """Nothing is owed for a Run holding nothing.""" + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + + await harness.manager.shutdown() + + assert harness.probes == [] + + +# --- the four states ---------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_shutdown_in_the_no_open_task_window_sends_an_unparked_wake( + harness_factory, +) -> None: + """The window the whole sweep exists for. + + The Run is cached with no open Workflow Task, its records are buffered in a + process about to exit, and nothing else will ever tell the Workflow they + arrived. Waiting for an unrelated Workflow event is not a plan. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + harness.register() + + await harness.manager.shutdown() + + assert harness.wakes == [(RUN_ID, 1)] + assert harness.metric == [] + + +@pytest.mark.asyncio +async def test_shutdown_with_a_workflow_task_open_sends_nothing( + harness_factory, +) -> None: + """Core owns that transition; a wake here would race it. + + The result would be a second Workflow Task for a Run already being attended + to, which is waste at best and a duplicate resolve at worst. + """ + harness = harness_factory(RunStatus.WFT_OPEN) + harness.register() + + await harness.manager.shutdown() + + assert harness.wakes == [], "a Run with an open task must be left to Core" + + +@pytest.mark.asyncio +async def test_shutdown_on_a_parked_run_sends_nothing(harness_factory) -> None: + """A producer's next append wakes it through the ordinary path.""" + harness = harness_factory(RunStatus.PARKED) + harness.register() + + await harness.manager.shutdown() + + assert harness.wakes == [] + + +@pytest.mark.asyncio +async def test_a_run_core_no_longer_knows_still_gets_its_wake( + harness_factory, +) -> None: + """Evicted and cached-with-no-task differ in what happens next, not in what + is owed now. + + The Run is gone from *this* Worker, but the Workflow still exists and its + records are still in the stream; another Worker will pick it up from the + marker. + """ + harness = harness_factory(RunStatus.RUN_NOT_FOUND) + harness.register() + + await harness.manager.shutdown() + + assert harness.wakes == [(RUN_ID, 1)] + + +@pytest.mark.asyncio +async def test_every_subscription_of_a_swept_run_is_woken(harness_factory) -> None: + """Each is an independent wait with its own park intent. + + Waking one and not the other would leave the second waiting on records that + were already in its buffer. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + harness.register(wait_id=1, stream_name="a") + harness.register(wait_id=2, stream_name="b") + + await harness.manager.shutdown() + + assert sorted(harness.wakes) == [(RUN_ID, 1), (RUN_ID, 2)] + + +# --- failure is reported, never dropped --------------------------------------- + + +@pytest.mark.asyncio +async def test_an_unacknowledged_wake_surfaces_on_the_metric( + harness_factory, +) -> None: + """A dropped wake is silent by nature. + + The Workflow simply waits, and nothing distinguishes that from a producer + having nothing to say -- so it must be counted, not merely logged. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK, wake_fails=True) + harness.register() + + await harness.manager.shutdown() + + assert harness.metric == ["tokens"] + assert harness.manager.shutdown_wake_failures == 1 + assert harness.wakes == [], "a failed wake must never be counted as delivered" + + +@pytest.mark.asyncio +async def test_a_wake_with_no_sender_configured_is_reported_not_ignored( + backend: MemoryStreamBackend, +) -> None: + """Silently skipping it would report a clean shutdown that lost a record.""" + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=lambda *_: _accepted(), + run_status=lambda _: _no_open_task(), + watch_block=timedelta(milliseconds=10), + ) + manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=StreamKey("ns", "wf", "first-run", "tokens"), + start_cursor=BEGINNING, + ) + + await manager.shutdown() + + assert manager.shutdown_wake_failures == 1 + + +async def _accepted() -> str: + return ReadinessResult.ACCEPTED + + +async def _no_open_task() -> str: + return RunStatus.NO_OPEN_WORKFLOW_TASK + + +@pytest.mark.asyncio +async def test_a_probe_failure_does_not_stop_the_shutdown( + backend: MemoryStreamBackend, +) -> None: + """Shutdown is never blocked by a server that has stopped answering.""" + + async def failing_probe( + run_id: str, # pyright: ignore[reportUnusedParameter] + ) -> str: + raise ConnectionError("service unavailable") + + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=lambda *_: _accepted(), + run_status=failing_probe, + watch_block=timedelta(milliseconds=10), + ) + manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=StreamKey("ns", "wf", "first-run", "tokens"), + start_cursor=BEGINNING, + ) + + await asyncio.wait_for(manager.shutdown(), 5) + + assert manager._runs == {} + + +@pytest.mark.asyncio +async def test_shutdown_is_never_blocked_past_the_grace_period( + backend: MemoryStreamBackend, +) -> None: + """A wake that has not been acknowledged by now is better reported than + waited on indefinitely. + + Without the bound, a Worker that could not reach the server would hang on + the way out -- turning a recoverable delivery problem into an unrecoverable + process one. + """ + + async def hanging_probe( + run_id: str, # pyright: ignore[reportUnusedParameter] + ) -> str: + await asyncio.sleep(60) + return RunStatus.NO_OPEN_WORKFLOW_TASK + + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=lambda *_: _accepted(), + run_status=hanging_probe, + watch_block=timedelta(milliseconds=10), + ) + manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=StreamKey("ns", "wf", "first-run", "tokens"), + start_cursor=BEGINNING, + ) + + started = asyncio.get_running_loop().time() + await manager.shutdown(grace=timedelta(milliseconds=200)) + elapsed = asyncio.get_running_loop().time() - started + + assert elapsed < 5, f"shutdown waited {elapsed:.1f}s past its grace period" + assert manager._runs == {} + + +@pytest.mark.asyncio +async def test_a_grace_period_expiry_counts_every_wake_it_abandons( + backend: MemoryStreamBackend, +) -> None: + """A Worker may give up on a wake; it may not do so quietly. + + The grace period cancels the sweep wherever it happens to be, and + ``_send_owed_wake`` re-raises ``CancelledError`` by design -- so the failure + accounting that sits after it was never reached, and neither was any + subscription later in the serial loop. Shutdown then reported + ``shutdown_wake_failures == 0`` for a Worker that had just abandoned every + one of its handoffs, which is precisely the silence this counter exists to + break: a dropped wake looks exactly like a producer with nothing to say. + """ + hanging = asyncio.Event() + + class HangingWakeHarness(Harness): + async def _wake(self, subscription) -> None: # type: ignore[no-untyped-def] + hanging.set() + await asyncio.sleep(60) + + harness = HangingWakeHarness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + for wait_id in (1, 2): + harness.manager.register( + run_id=RUN_ID, + wait_id=wait_id, + stream_key=StreamKey("ns", "wf", "first-run", f"tokens-{wait_id}"), + start_cursor=BEGINNING, + ) + + started = asyncio.get_running_loop().time() + await harness.manager.shutdown(grace=timedelta(milliseconds=200)) + elapsed = asyncio.get_running_loop().time() - started + + assert elapsed < 5, f"shutdown waited {elapsed:.1f}s past its grace period" + assert hanging.is_set(), "the wake never got as far as hanging" + assert harness.wakes == [], "no wake was acknowledged, so none may be reported" + assert harness.manager.shutdown_wake_failures == 2, ( + "both handoffs were abandoned -- the one the cancellation landed inside " + f"and the one never reached -- and {harness.manager.shutdown_wake_failures} " + "were counted" + ) + assert sorted(harness.metric) == ["tokens-1", "tokens-2"], ( + f"the metric must fire once per abandoned subscription: {harness.metric}" + ) + + +@pytest.mark.asyncio +async def test_a_probe_that_cannot_answer_is_not_reported_as_nothing_owed( + backend: MemoryStreamBackend, +) -> None: + """ "We could not tell" is not "nothing was owed". + + A Run whose status cannot be read may be holding a buffered record with + nowhere to announce it. Sending a wake anyway would race a Workflow Task that + might be open, so the sweep sends nothing -- but passing over the Run without + counting it tears the Run down and reports a clean shutdown, which is the same + silent loss by a different route. + """ + + class FailingProbeHarness(Harness): + async def _probe(self, run_id: str) -> str: + self.probes.append(run_id) + raise ConnectionError("service unavailable") + + harness = FailingProbeHarness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + for wait_id in (1, 2): + harness.manager.register( + run_id=RUN_ID, + wait_id=wait_id, + stream_key=StreamKey("ns", "wf", "first-run", f"tokens-{wait_id}"), + start_cursor=BEGINNING, + ) + + await asyncio.wait_for(harness.manager.shutdown(grace=timedelta(seconds=2)), 5) + + assert harness.wakes == [] + assert harness.manager.shutdown_wake_failures == 2, ( + "a Run this Worker could say nothing about was torn down reporting a " + "clean shutdown" + ) + assert sorted(harness.metric) == ["tokens-1", "tokens-2"] + assert harness.manager._runs == {}, "shutdown must still complete" + + +@pytest.mark.asyncio +async def test_a_hanging_probe_counts_the_runs_it_never_answered_for( + backend: MemoryStreamBackend, +) -> None: + """The grace period expiring inside the probe is the same loss as inside the wake.""" + + class HangingProbeHarness(Harness): + async def _probe(self, run_id: str) -> str: + self.probes.append(run_id) + await asyncio.sleep(60) + return RunStatus.NO_OPEN_WORKFLOW_TASK + + harness = HangingProbeHarness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + harness.manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=StreamKey("ns", "wf", "first-run", "tokens-1"), + start_cursor=BEGINNING, + ) + + await harness.manager.shutdown(grace=timedelta(milliseconds=200)) + + assert harness.wakes == [] + assert harness.manager.shutdown_wake_failures == 1 + assert harness.metric == ["tokens-1"] + + +@pytest.mark.asyncio +async def test_a_run_with_nothing_owed_is_not_counted_as_a_failure( + backend: MemoryStreamBackend, +) -> None: + """The counter has to stay quiet where it should, or it says nothing at all. + + A parked Run is woken by a producer's append through the ordinary path, and a + Run with an open Workflow Task is Core's to finish. Counting either as an + abandoned handoff would make the metric fire on every clean shutdown and + become unalertable. + """ + for status in (RunStatus.PARKED, RunStatus.WFT_OPEN): + harness = Harness(backend, status) + harness.manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=StreamKey("ns", "wf", "first-run", "tokens"), + start_cursor=BEGINNING, + ) + + await asyncio.wait_for(harness.manager.shutdown(grace=timedelta(seconds=2)), 5) + + assert harness.manager.shutdown_wake_failures == 0, ( + f"a Run reported as {status} owes no wake, so nothing was abandoned" + ) + assert harness.metric == [] + + +@pytest.mark.asyncio +async def test_a_wake_the_live_path_delivered_is_not_counted_as_abandoned( + backend: MemoryStreamBackend, +) -> None: + """The sweep's set is a snapshot, and the live path empties it underneath. + + Readiness that comes back `RunNotFound` owes a wake, sends it, and *then* drops + the subscription -- and `RunNotFound` is a likely answer during shutdown, while + watchers keep running for the whole grace window and the sweep awaits inside + itself for them to interleave with. A subscription that leaves the Run that way + has had its handoff made, so counting it reports a loss that did not happen on + the counter operators are told to alert on. + """ + other_run = "run-2" + harness = Harness(backend, RunStatus.PARKED) + for run_id in (RUN_ID, other_run): + harness.manager.register( + run_id=run_id, + wait_id=1, + stream_key=StreamKey("ns", "wf", run_id, "tokens"), + start_cursor=BEGINNING, + ) + + # While the sweep is busy with the first Run, the live path takes the second + # Run's subscription away -- which is what it does on `RunNotFound`, and only + # once its own owed wake has been acknowledged. The sweep then reaches that Run, + # finds no subscriptions, and moves on without saying anything about it. + # + # Both halves of that drop are reproduced, in the order the production path + # uses them: the flag first, then the pop. Popping alone takes the + # subscription out of the sweep's reach while leaving its watcher looping on + # `while not subscription._cancelled`, so the watcher outlives the test's + # event loop and reads from the backend after it closes -- an unraisable + # `RuntimeError: Event loop is closed` that this test's own shortcut, not the + # manager, is responsible for. + async def drop_the_other(run_id: str) -> str: + if run_id == RUN_ID: + dropped = harness.manager._runs.get(other_run, {}).pop(1, None) + if dropped is not None: + dropped._cancelled = True + return RunStatus.PARKED + + harness.manager._run_status = drop_the_other # type: ignore[assignment] + + await asyncio.wait_for(harness.manager.shutdown(grace=timedelta(seconds=2)), 5) + + assert harness.manager.shutdown_wake_failures == 0, ( + "a subscription whose wake the live path had already delivered was counted " + "as an abandoned handoff, on the counter operators are told to alert on" + ) + assert harness.metric == [] + + +@pytest.mark.asyncio +async def test_a_manager_with_no_probe_owes_nothing_and_reports_nothing( + backend: MemoryStreamBackend, +) -> None: + """No probe wired means no sweep, which means nothing to have failed at. + + The sweep is defined entirely in terms of what Core answers, so a manager with + no run-status probe has no handoff obligation at all. Counting its + subscriptions as abandoned wakes would make the metric fire wherever the + mechanism simply is not configured -- which is noise in exactly the series an + operator is expected to alert on. + """ + manager = StreamSubscriptionManager( + backend=backend, + notify_ready=lambda *_: _accepted(), + watch_block=timedelta(milliseconds=10), + ) + manager.register( + run_id=RUN_ID, + wait_id=1, + stream_key=StreamKey("ns", "wf", "first-run", "tokens"), + start_cursor=BEGINNING, + ) + + await asyncio.wait_for(manager.shutdown(grace=timedelta(seconds=2)), 5) + + assert manager.shutdown_wake_failures == 0 + assert manager._runs == {} + + +# --- teardown ordering -------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_sweep_runs_before_any_subscription_is_torn_down( + harness_factory, +) -> None: + """A subscription torn down first has nothing left to name in its wake. + + Teardown resets cursors to the committed boundary and cancels the watcher; + sweeping afterwards would send a wake for a wait whose buffered records had + already been discarded. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + key = harness.register() + await harness.backend.append(key, StreamRecord(RecordKind.DATA, b"x", "p", 0)) + await asyncio.sleep(0.1) + + await harness.manager.shutdown() + + assert harness.wakes == [(RUN_ID, 1)] + assert harness.manager._runs == {}, "teardown still happens, just afterwards" + + +@pytest.mark.asyncio +async def test_teardown_removes_every_run(harness_factory) -> None: + """The backstop half of the obligation: no Run outlives the manager. + + A Run whose watchers survived shutdown would hold a backend connection open + for the life of the process. + """ + harness = harness_factory(RunStatus.PARKED) + harness.register(wait_id=1, stream_name="a") + harness.register(wait_id=2, stream_name="b") + + await harness.manager.shutdown() + + assert harness.manager._runs == {} + + +@pytest.mark.asyncio +async def test_eviction_remains_the_normal_teardown_path(harness_factory) -> None: + """The sweep is a backstop, not a replacement. + + Per-Run teardown is driven by ``RemoveFromCache``, which is what guarantees a + finalization in flight is answered before the Run's state disappears. A + shutdown hook that tore Runs down itself would have no such ordering. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + harness.register() + + await harness.manager.evict_run(RUN_ID) + await harness.manager.shutdown() + + assert harness.probes == [], ( + "a Run already evicted has nothing left to sweep, and probing it would " + "ask Core about a Run this Worker has finished with" + ) + assert harness.wakes == [] + + +# --- the retry within the grace period ---------------------------------------- + + +class FlakyWakeHarness(Harness): + """Fails a fixed number of attempts, then succeeds.""" + + def __init__(self, backend: MemoryStreamBackend, failures: int) -> None: + super().__init__(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + self.remaining_failures = failures + self.attempts = 0 + + async def _wake(self, subscription) -> None: # type: ignore[no-untyped-def] + self.attempts += 1 + if self.remaining_failures > 0: + self.remaining_failures -= 1 + raise ConnectionError("service unavailable") + self.wakes.append((subscription.run_id, subscription.wait_id)) + + +@pytest.mark.asyncio +async def test_a_momentarily_failing_wake_is_retried_and_succeeds( + backend: MemoryStreamBackend, +) -> None: + """A Worker shutting down is often shutting down because something is unhealthy. + + That makes the first attempt the one most likely to land in the middle of + it, and giving up there would report a clean shutdown that lost a record. + """ + harness = FlakyWakeHarness(backend, failures=1) + harness.register() + + await harness.manager.shutdown() + + assert harness.attempts == 2, f"expected one retry, got {harness.attempts} attempts" + assert harness.wakes == [(RUN_ID, 1)] + assert harness.metric == [], "a wake that eventually succeeded is not a failure" + + +@pytest.mark.asyncio +async def test_the_retry_is_bounded_and_then_reported( + backend: MemoryStreamBackend, +) -> None: + """Retrying forever would trade a lost record for a Worker that never exits. + + The metric is what makes giving up visible, which is the only reason giving + up is acceptable. + """ + harness = FlakyWakeHarness(backend, failures=99) + harness.register() + + await harness.manager.shutdown() + + assert harness.attempts == SHUTDOWN_WAKE_ATTEMPTS + assert harness.metric == ["tokens"] + assert harness.wakes == [] + + +@pytest.mark.asyncio +async def test_the_retry_is_the_same_wake_not_a_second_one( + backend: MemoryStreamBackend, +) -> None: + """Derived from the wake's identity, so the server deduplicates it. + + This is what makes retrying safe at all: without a stable request ID the + retry would ask for a second Workflow Task, and an attempt that had in fact + arrived would be duplicated rather than resolved. + """ + harness = FlakyWakeHarness(backend, failures=1) + subscription = harness.register() + del subscription + + seen: list[str] = [] + + async def recording_wake(sub) -> None: # type: ignore[no-untyped-def] + seen.append(_swept_request_id(harness, sub)) + if len(seen) == 1: + raise ConnectionError("service unavailable") + + harness.manager._send_wake = recording_wake + + await harness.manager.shutdown() + + assert len(seen) == 2 + assert seen[0] == seen[1], ( + "the retry must derive the same request ID; a fresh one would ask for a " + "second Workflow Task rather than resolve the first attempt" + ) + + +@pytest.mark.asyncio +async def test_two_workers_sweeps_do_not_deduplicate_each_other( + backend: MemoryStreamBackend, +) -> None: + """The opposite half of the same requirement, and the reason it is per instance. + + Two Workers shutting down at different times are two separate asks, and they + share a ``Client`` -- so a sender identity taken from the client identity + would give both first unparked wakes the same request ID. The server would + deduplicate the second, and the Run the surviving Worker picked up would + never get its Workflow Task. + """ + harnesses = [Harness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) for _ in range(2)] + seen: list[str] = [] + + for harness in harnesses: + harness.register() + + async def recording_wake(sub, harness=harness) -> None: # type: ignore[no-untyped-def] + seen.append(_swept_request_id(harness, sub)) + + harness.manager._send_wake = recording_wake + + for harness in harnesses: + await harness.manager.shutdown() + + assert len(seen) == 2 + assert seen[0] != seen[1], ( + "both Workers' sweeps derived the same request ID, so the second " + "Worker's wake would be deduplicated away and its Run would stall" + ) + + +# --- the sweep driven through the Worker's real wake callback ----------------- + + +def _worker_wake_callback(manager): # type: ignore[no-untyped-def] + """The Worker's own sender, bound to this manager and nothing else. + + The retry and the metric are the manager's, but whether they can ever run is + the callback's: a callback that returns normally after a failed Signal makes + an unacknowledged wake indistinguishable from a delivered one, and the loop + below exits after its first attempt. Driving the real method is the only way + that shows. + """ + from temporalio.worker._workflow import _WorkflowWorker + + worker = object.__new__(_WorkflowWorker) + worker._client = object() # only ever handed to the patched sender + worker._external_stream_manager = manager + return worker._send_external_stream_wake + + +@pytest.fixture +def failing_signals(monkeypatch): # type: ignore[no-untyped-def] + """Makes the raw Signal call fail a fixed number of times, and counts it.""" + import temporalio.contrib.external_workflow_streams._wake as wake_module + + attempts: list[str] = [] + + def install(failures: int) -> list[str]: + remaining = [failures] + + async def fake_send( + client, # pyright: ignore[reportUnusedParameter] + wake_request, + *, + producer_session_id: str = "", # pyright: ignore[reportUnusedParameter] + ) -> str: + request_id = wake_request_id(wake_request) + attempts.append(request_id) + if remaining[0] > 0: + remaining[0] -= 1 + raise ConnectionError("service unavailable") + return request_id + + monkeypatch.setattr(wake_module, "send_wake_signal", fake_send) + return attempts + + return install + + +@pytest.mark.asyncio +async def test_a_signal_failure_reaches_the_sweeps_retry( + backend: MemoryStreamBackend, failing_signals +) -> None: + """The retry is only real if the callback tells the sweep it failed. + + A callback that logs and returns reports success, the three-attempt loop + ends after one call, and the record the Worker was holding is never + announced -- while shutdown reports itself clean. + """ + attempts = failing_signals(failures=2) + harness = Harness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + harness.register() + harness.manager._send_wake = _worker_wake_callback(harness.manager) + + await harness.manager.shutdown() + + assert len(attempts) == 3, ( + f"the sweep made {len(attempts)} Signal attempts; a failure the callback " + "swallows leaves the retry loop with nothing to retry" + ) + assert len(set(attempts)) == 1, ( + "the retries must be the same wake, not three separate asks" + ) + assert harness.metric == [], "a wake that eventually succeeded is not a failure" + assert harness.manager.shutdown_wake_failures == 0 + + +@pytest.mark.asyncio +async def test_a_wake_that_never_lands_is_retried_then_counted( + backend: MemoryStreamBackend, failing_signals +) -> None: + """And the metric fires, which is the only thing that makes giving up acceptable.""" + attempts = failing_signals(failures=99) + harness = Harness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + harness.register() + harness.manager._send_wake = _worker_wake_callback(harness.manager) + + await harness.manager.shutdown() + + assert len(attempts) == SHUTDOWN_WAKE_ATTEMPTS + assert harness.metric == ["tokens"], ( + "the wake was never acknowledged and shutdown reported nothing" + ) + assert harness.manager.shutdown_wake_failures == 1 + + +class _RecordedSignals(list): # type: ignore[type-arg] + """The wake requests each Signal would carry, and what a send costs. + + The request ID alone is not enough for these tests: what is under test is + *which park* the wake names, and an obsolete generation reaches the service + exactly as happily as a current one. + + ``delay`` is what makes a send a round trip rather than a function call. A + shutdown step that merely *schedules* a Signal passes every assertion a + zero-cost send can make, because the task it left behind runs on the way out + of the next await either way. + """ + + delay: float = 0.0 + + +@pytest.fixture +def recorded_signals(monkeypatch) -> _RecordedSignals: # type: ignore[no-untyped-def] + import temporalio.contrib.external_workflow_streams._wake as wake_module + + requests = _RecordedSignals() + + async def fake_send( + client, # pyright: ignore[reportUnusedParameter] + wake_request, + *, + producer_session_id: str = "", # pyright: ignore[reportUnusedParameter] + ) -> str: + if requests.delay: + await asyncio.sleep(requests.delay) + requests.append(wake_request) + return wake_request_id(wake_request) + + monkeypatch.setattr(wake_module, "send_wake_signal", fake_send) + return requests + + +def _removals_failing_until(harness: Harness, resume: Callable[[], bool]) -> None: + """Fails every park-intent removal until ``resume`` says the backend recovered.""" + original = harness.backend.remove_park_intent_if_matches + + async def flaky_remove( + key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + if not resume(): + raise ConnectionError("backend unavailable") + return await original( + key, + wait_id, + run_id=run_id, + park_generation=park_generation, + ) + + harness.backend.remove_park_intent_if_matches = ( # type: ignore[method-assign] + flaky_remove + ) + + +@pytest.mark.asyncio +async def test_the_sweep_does_not_wake_on_an_intent_it_has_decided_to_remove( + backend: MemoryStreamBackend, recorded_signals: _RecordedSignals +) -> None: + """A Signal the service accepts is not the same as a wake Core accepts. + + The sweep composed its wake from whatever ``current_park_generation`` + answered, and a stale intent is precisely a generation Core has already + discarded. So the send succeeded, the subscription was resolved as a handoff + made, and ``shutdown_wake_failures`` reported a clean exit for a Worker whose + only wake was ignored on arrival. The manager holds the ledger that says the + intent is stale, so it is the only thing that can tell the two apart -- and + the honest composition for an intent already decided against is the unparked + wake, which carries no generation to be discarded. + """ + harness = Harness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + key = harness.register() + harness.manager._send_wake = _worker_wake_callback(harness.manager) + # Never recovers, so the intent is still installed when the sweep runs and + # the only Signal of this shutdown is the sweep's own. + _removals_failing_until(harness, lambda: False) + await harness.manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + await harness.manager.resolve_park(RUN_ID) + assert await harness.backend.current_park_generation(key, 1) == 4, ( + "the failed removal must leave the stale intent installed" + ) + + await harness.manager.shutdown(grace=timedelta(seconds=1)) + + assert [request.park_generation for request in recorded_signals] == [0], ( + "the sweep named the generation behind an intent it had already decided " + "to remove, so Core discards the wake while the send reports success" + ) + assert harness.manager.shutdown_wake_failures == 0 + assert harness.metric == [] + + +@pytest.mark.asyncio +async def test_a_removal_the_final_pass_confirms_carries_its_own_handoff( + backend: MemoryStreamBackend, recorded_signals: _RecordedSignals +) -> None: + """The last removal of a Worker's life ends a suppression window too. + + An evicted Run has no subscription for the sweep to wake and none for the + cleanup to announce through, which is the whole reason the ledger is not + kept on the Subscription. When the backend recovers only after the retry + loops are cancelled -- the case :meth:`_final_owed_removal_pass` exists for + -- that pass is what ends the suppression, and it has to carry the handoff + itself: nothing runs after it, and a wake merely *scheduled* as the process + exits is a wake never sent. + + Which is what the send delay pins. The cleanup handoff is posted as a task, + because every drain reaches it holding the Run's park lock under a budget + meant for one backend call rather than for a Signal with retries behind it -- + so with a send that costs nothing, a shutdown that never waits for that task + still looks like one that did. + """ + harness = Harness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + key = harness.register() + harness.manager._send_wake = _worker_wake_callback(harness.manager) + recorded_signals.delay = 0.05 + # Recovers exactly when the autonomous retries have been cancelled, so the + # final pass is the attempt that succeeds rather than the loop. + _removals_failing_until( + harness, lambda: harness.manager._stopping_owed_removal_retries + ) + await harness.manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + await harness.manager.resolve_park(RUN_ID) + await harness.manager.evict_run(RUN_ID) + assert harness.manager.subscription(RUN_ID, 1) is None, ( + "eviction must drop the Subscription -- that is the premise" + ) + assert await harness.backend.current_park_generation(key, 1) == 4 + + await harness.manager.shutdown(grace=timedelta(seconds=1)) + + assert await harness.backend.current_park_generation(key, 1) is None, ( + "the final pass did not retire the intent, so this proves nothing" + ) + assert [request.park_generation for request in recorded_signals] == [0], ( + "shutdown reported itself clean while the record the retired intent had " + "silenced was announced to nobody" + ) + assert harness.manager.shutdown_wake_failures == 0 + + +@pytest.mark.asyncio +async def test_the_sweeps_wake_is_not_sent_twice_by_the_final_pass( + backend: MemoryStreamBackend, recorded_signals: _RecordedSignals +) -> None: + """One obligation, not two. + + The sweep already composed this wait's wake as unparked *because* the ledger + said its intent was stale, which makes it the same generation-0 handoff the + removal owes once it finally goes through. Sending it again asks the server + for a second, empty Workflow Task. + """ + harness = Harness(backend, RunStatus.NO_OPEN_WORKFLOW_TASK) + key = harness.register() + harness.manager._send_wake = _worker_wake_callback(harness.manager) + _removals_failing_until( + harness, lambda: harness.manager._stopping_owed_removal_retries + ) + await harness.manager.prepare_park(RUN_ID, 4, {1: BEGINNING}) + await harness.manager.resolve_park(RUN_ID) + + await harness.manager.shutdown(grace=timedelta(seconds=1)) + + assert await harness.backend.current_park_generation(key, 1) is None + assert [request.park_generation for request in recorded_signals] == [0], ( + "the sweep's wake and the cleanup's are one handoff; sending both costs " + "an empty Workflow Task" + ) + + +# --- the park intent's lifetime ----------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_resolved_park_leaves_no_intent_behind(harness_factory) -> None: + """A park intent exists only while that park is actually outstanding. + + An aborted park was never the only park that ends. A **confirmed** one ends + too -- when a wake Signal or a fresh quiescent snapshot clears Core's + ``park_generation`` -- and nothing about that is visible in the backend, so + only the manager can take the intent back out. + + What a left-behind intent costs is not tidiness. It *is* the answer + ``current_park_generation`` gives everyone who asks: a producer choosing what + its wake Signal names gets a generation Core has already discarded and its + wake is ignored as stale, and the shutdown sweep gets the same answer and + sends a parked wake whose request ID -- deliberately independent of the + sender -- is byte-identical to the wake that resolved the park in the first + place, so the server deduplicates it and no Workflow Task is ever created. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + key = harness.register() + + confirmed = not await harness.manager.prepare_park(RUN_ID, 1, {1: BEGINNING}) + assert confirmed, "nothing was appended, so this park must have confirmed" + assert await harness.backend.current_park_generation(key, 1) == 1 + + await harness.manager.resolve_park(RUN_ID) + + assert await harness.backend.park_intent(key, 1) is None, ( + "the intent of a park that is over is still installed" + ) + assert await harness.backend.current_park_generation(key, 1) is None, ( + "a resolved park still reports a generation, so the next wake -- a " + "producer's or the shutdown sweep's -- will name a park Core no longer " + "recognises instead of the unparked wake it owes" + ) + + +@pytest.mark.asyncio +async def test_a_resolve_with_no_park_installed_touches_nothing( + harness_factory, +) -> None: + """A resolve is the ordinary delivery path, not a rare one. + + Every record delivered live arrives on a resolve activation, so removing + intents there unconditionally would put a backend write on the hot path for + a park that never existed. The manager mirrors what it installed and removes + only that. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + key = harness.register() + removals: list[int] = [] + + original = harness.backend.remove_park_intent_if_matches + + async def recording_remove( + removed_key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + removals.append(wait_id) + return await original( + removed_key, + wait_id, + run_id=run_id, + park_generation=park_generation, + ) + + harness.backend.remove_park_intent_if_matches = ( # type: ignore[method-assign] + recording_remove + ) + + await harness.manager.resolve_park(RUN_ID) + assert removals == [], "a Run that never parked owes the backend nothing" + + await harness.manager.prepare_park(RUN_ID, 1, {1: BEGINNING}) + await harness.manager.resolve_park(RUN_ID) + await harness.manager.resolve_park(RUN_ID) + + assert removals == [1], ( + "the intent must come out exactly once: once for the park that was " + "installed, and never again for resolves with nothing outstanding" + ) + assert await harness.backend.park_intent(key, 1) is None + + +@pytest.mark.asyncio +async def test_a_backend_failure_during_removal_leaves_it_owed( + harness_factory, +) -> None: + """Cleanup does not get to fail a Workflow Task, and does not give up either. + + A resolve is a delivery activation; raising here would trade a stale intent + for a repeated Workflow Task on the healthy path. Swallowing the failure + *and* forgetting the intent would be the original defect back again, so the + removal stays owed and the next resolve retries it. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + key = harness.register() + await harness.manager.prepare_park(RUN_ID, 1, {1: BEGINNING}) + + original = harness.backend.remove_park_intent_if_matches + failures = [True] + + async def flaky_remove( + removed_key: StreamKey, + wait_id: int, + *, + run_id: str, + park_generation: int, + ) -> ParkIntentRemoval: + if failures: + failures.pop() + raise ConnectionError("backend unavailable") + return await original( + removed_key, + wait_id, + run_id=run_id, + park_generation=park_generation, + ) + + harness.backend.remove_park_intent_if_matches = ( # type: ignore[method-assign] + flaky_remove + ) + + await harness.manager.resolve_park(RUN_ID) + assert await harness.backend.current_park_generation(key, 1) == 1 + + await harness.manager.resolve_park(RUN_ID) + assert await harness.backend.current_park_generation(key, 1) is None, ( + "a removal that failed once was never retried, so the intent is stale " + "for the rest of the Run's life" + ) + + +# --- when the probe is asked --------------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_sweep_acts_on_what_the_probe_heard(harness_factory) -> None: + """The probe and the wakes belong at different moments, so they are separate. + + Core keeps the state the probe reports only until the Worker's shutdown is + initiated: an idle cached Run has no pending work, so ``shutdown_done`` is + satisfied by the first input after the shutdown token is cancelled and the + workflow-state lane ends there. Everything afterwards answers + ``RunNotFound``. + + That answer owes a wake, which is what makes the degradation invisible: a + sweep that asks too late still sends its wake and still looks correct, while + the two answers that mean *do not* send one stop occurring. This Run is + parked -- a producer's append reaches it through the ordinary path and it + needs nothing from the sweep -- and the sweep must still know that after + Core has forgotten. + """ + harness = harness_factory(RunStatus.PARKED) + harness.register() + + await harness.manager.probe_runs() + harness.status = RunStatus.RUN_NOT_FOUND + + await harness.manager.shutdown() + + assert harness.wakes == [], ( + "a parked Run was woken on the way out. Its state was read after Core " + "had dropped it, so the Parked branch could not be taken" + ) + + +@pytest.mark.asyncio +async def test_a_run_the_probe_never_reached_is_still_swept(harness_factory) -> None: + """Probing early must not become a way to miss a Run. + + A Workflow Task already in flight when the probe ran can still cache a Run + afterwards. Nothing is known about it, so it is asked -- and both answers + Core has left to give owe a wake. + """ + harness = harness_factory(RunStatus.NO_OPEN_WORKFLOW_TASK) + + await harness.manager.probe_runs() + harness.register() + + await harness.manager.shutdown() + + assert harness.wakes == [(RUN_ID, 1)] + assert harness.probes == [RUN_ID], ( + "a Run the probe phase never saw must still be asked about" + ) diff --git a/tests/contrib/external_workflow_streams/test_wake.py b/tests/contrib/external_workflow_streams/test_wake.py new file mode 100644 index 0000000000..5c3fdbe2bb --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_wake.py @@ -0,0 +1,2336 @@ +"""P14 — the producer wake-signal path. + +The wakeup used whenever no open Workflow Task can accept local readiness. Two of +its properties are the reason it does not reuse the public Signal API, and both +are asserted here rather than assumed: the envelope bypasses the user's +``DataConverter``, and the request ID is a pure function of the wake's identity. + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +# pyright: reportUnusedImport=false, reportMissingParameterType=false +from __future__ import annotations + +import asyncio +import dataclasses +import pathlib +import re +import uuid +from datetime import timedelta + +import pytest + +import temporalio.api.common.v1 +import temporalio.api.enums.v1 +import temporalio.api.notification.v1 +import temporalio.api.workflowservice.v1 +import temporalio.bridge +import temporalio.converter +import temporalio.service +from temporalio import workflow +from temporalio.bridge.proto.external_stream.external_stream_pb2 import WakeSignal +from temporalio.client import Client +from temporalio.common import Execution +from temporalio.contrib.external_workflow_streams._backend import ( + AppendConflictError, + ParkIntent, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._producer import ( + AppendNotAcknowledgedError, + ExternalStreamProducer, + WakeNotAcknowledgedError, + WorkflowChainKey, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._wake import ( + PROBE_CHANNEL, + UNPARKED_WAKE_GENERATION, + WAKE_SIGNAL_ENCODING, + WAKE_SIGNAL_ENVELOPE_VERSION, + WAKE_SIGNAL_MESSAGE_TYPE, + WAKE_SIGNAL_NAME, + ChannelAddress, + ChannelSupport, + WakeRequest, + build_signal_request, + channel_for, + channel_support, + send_wake, + send_wake_signal, + server_has_channels, + wake_request_for, + wake_request_id, +) +from temporalio.worker import Worker +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +CHAIN = WorkflowChainKey("ns", "wf-1", "first-run-1") + + +def request(**overrides) -> WakeRequest: # type: ignore[no-untyped-def] + fields = dict( + namespace="ns", + workflow_id="wf-1", + first_execution_run_id="first-run-1", + stream_name="tokens", + wait_id=1, + park_generation=7, + ) + fields.update(overrides) + return WakeRequest(**fields) # type: ignore[arg-type] + + +# --- the stable request ID ---------------------------------------------------- + + +def test_two_producers_retrying_one_wake_derive_the_identical_request_id() -> None: + """Which is what lets the server deduplicate them into a single wake. + + A generation is woken once. Producers racing to wake it must not each + produce a Workflow Task, and the public Signal path -- which draws a fresh + UUID per attempt -- would give exactly that. + """ + first_producer = request() + second_producer = request() + + assert wake_request_id(first_producer) == wake_request_id(second_producer) + + +def test_the_request_id_is_stable_across_processes() -> None: + """Pinned, because a derivation that drifted would silently double every wake. + + Hashing anything process-local -- ``hash()``, an object id, a random seed -- + would still pass a same-process equality test and fail in production. + """ + assert wake_request_id(request()) == "f5d6b411-3047-5b8e-a755-46cb74ad0ed8" + + +@pytest.mark.parametrize( + "field", + ["namespace", "workflow_id", "first_execution_run_id", "stream_name"], +) +def test_every_identity_field_changes_the_request_id(field: str) -> None: + """Otherwise two different wakes would deduplicate into one, losing one.""" + assert wake_request_id(request()) != wake_request_id(request(**{field: "other"})) + + +def test_the_wait_id_and_generation_change_the_request_id() -> None: + assert wake_request_id(request()) != wake_request_id(request(wait_id=2)) + assert wake_request_id(request()) != wake_request_id(request(park_generation=8)) + + +def test_fields_cannot_be_confused_with_each_other() -> None: + """A stream named with a separator must not collide with a different tuple. + + Delimiter-joining the identity would make ``("a", "b")`` and ``("a:b", "")`` + the same wake. + """ + left = request(workflow_id="a", stream_name="b") + right = request(workflow_id="a\x00b", stream_name="") + + assert wake_request_id(left) != wake_request_id(right) + + +# --- unparked wakes ----------------------------------------------------------- + + +def test_two_unparked_wakes_from_different_senders_differ() -> None: + """Generation 0 carries no attempt identity, so the sender must supply one. + + Without this, two Workers shutting down at different times would derive the + same request ID and the server would deduplicate the second wake away -- + turning a correct retry mechanism into silent loss. + """ + first = request(park_generation=0, sender_identity="worker-a", wake_counter=1) + second = request(park_generation=0, sender_identity="worker-b", wake_counter=1) + + assert wake_request_id(first) != wake_request_id(second) + + +def test_one_senders_successive_unparked_wakes_differ() -> None: + """They are two separate asks, not a retry of one.""" + first = request(park_generation=0, sender_identity="worker-a", wake_counter=1) + second = request(park_generation=0, sender_identity="worker-a", wake_counter=2) + + assert wake_request_id(first) != wake_request_id(second) + + +def test_retrying_one_unparked_wake_keeps_its_request_id() -> None: + """The counter is per attempt, not per send -- that is what makes it a retry.""" + attempt = request(park_generation=0, sender_identity="worker-a", wake_counter=3) + + assert wake_request_id(attempt) == wake_request_id( + request(park_generation=0, sender_identity="worker-a", wake_counter=3) + ) + + +def test_an_unparked_wake_without_a_sender_identity_is_refused() -> None: + """Refused at construction rather than producing a colliding request ID.""" + with pytest.raises(ValueError, match="sender identity"): + request(park_generation=UNPARKED_WAKE_GENERATION) + + +def test_a_parked_wake_needs_no_sender_identity() -> None: + """Its generation already identifies the attempt, for every sender alike.""" + assert wake_request_id(request(park_generation=7)) == wake_request_id( + request(park_generation=7, sender_identity="", wake_counter=0) + ) + + +# --- the Worker's sender identity --------------------------------------------- + + +class _StubBackend: + """Reports no confirmed park, so every wake here is an unparked one.""" + + async def current_park_generation(self, stream_key, wait_id): # type: ignore[no-untyped-def] + return None + + +class _StubSubscription: + def __init__(self, wake_counter: int = 1) -> None: + #: The manager keys its owed-removal ledger by Run, and the sender asks + #: it what generation this wake must name, so a target without one is + #: not a target the real sender can compose from. + self.run_id = "run-1" + self.stream_key = StreamKey("ns", "wf-1", "first-run-1", "tokens") + self.wait_id = 1 + #: The sender's sequence number for this wake, which is what the + #: unparked request ID is derived from. Held by the subscription but + #: drawn from the manager, so it does not restart when a Run comes back. + self.wake_counter = wake_counter + self.wakes_owed = wake_counter + self.backend = _StubBackend() + + +def _worker_sending_wakes(client_identity: str): # type: ignore[no-untyped-def] + """One Worker's real wake sender, wired to a manager and nothing else. + + Built around the actual ``_WorkflowWorker`` method rather than a + reimplementation of it, because the defect this guards was in *which* + identity that method passed, not in the derivation it passed it to. + """ + from temporalio.contrib.external_workflow_streams._manager import ( + StreamSubscriptionManager, + ) + from temporalio.worker._workflow import _WorkflowWorker + + async def notify_ready( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] + ) -> str: + raise AssertionError("readiness is not part of this path") + + worker = object.__new__(_WorkflowWorker) + worker._client = object() # only ever handed to the patched sender + worker._external_stream_manager = StreamSubscriptionManager( + backend=None, + notify_ready=notify_ready, + client_identity=client_identity, + ) + return worker + + +@pytest.fixture +def sent_request_ids(monkeypatch): # type: ignore[no-untyped-def] + """Records the request ID each wake would actually be sent under.""" + import temporalio.contrib.external_workflow_streams._wake as wake_module + + recorded: list[str] = [] + + async def fake_send( + client, # pyright: ignore[reportUnusedParameter] + wake_request, + *, + producer_session_id: str = "", # pyright: ignore[reportUnusedParameter] + ) -> str: + request_id = wake_request_id(wake_request) + recorded.append(request_id) + return request_id + + monkeypatch.setattr(wake_module, "send_wake_signal", fake_send) + return recorded + + +@pytest.mark.asyncio +async def test_two_workers_sharing_one_client_derive_different_request_ids( + sent_request_ids: list[str], +) -> None: + """The client identity is shared; the sender identity must not be. + + Two Workers in one process share a ``Client`` and so share its identity, and + each one's counter restarts at 1 -- so deriving from the client identity + gives both first unparked wakes the same request ID. The server + deduplicates the second, no Workflow Task is created, and the Run the + surviving Worker picked up stalls. + """ + first = _worker_sending_wakes("one-shared-client") + second = _worker_sending_wakes("one-shared-client") + + await first._send_external_stream_wake(_StubSubscription(wake_counter=1)) + await second._send_external_stream_wake(_StubSubscription(wake_counter=1)) + + assert sent_request_ids[0] != sent_request_ids[1], ( + "both Workers derived the same request ID, so the server would " + "deduplicate the second wake away and the Run would never be woken" + ) + + +@pytest.mark.asyncio +async def test_one_workers_retry_of_an_unparked_wake_keeps_its_request_id( + sent_request_ids: list[str], +) -> None: + """The identity is fixed for the Worker's lifetime, which is what makes it a retry. + + The shutdown sweep re-sends one owed wake within its grace period. If the + sender identity were redrawn per attempt the retry would ask for a second + Workflow Task instead of resolving the attempt that may in fact have + arrived. + """ + worker = _worker_sending_wakes("one-shared-client") + subscription = _StubSubscription(wake_counter=1) + + await worker._send_external_stream_wake(subscription) + await worker._send_external_stream_wake(subscription) + + assert len(sent_request_ids) == 2 + assert sent_request_ids[0] == sent_request_ids[1], ( + "the retry derived a fresh request ID, so it would wake the Workflow a " + "second time rather than deduplicate against the first attempt" + ) + + +@pytest.mark.asyncio +async def test_one_worker_retries_an_unacknowledged_wake_with_the_same_request_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failed send remains retriable under the same request identity.""" + import temporalio.contrib.external_workflow_streams._wake as wake_module + + recorded: list[str] = [] + + async def fail_once( + client, # pyright: ignore[reportUnusedParameter] + wake_request, + *, + producer_session_id: str = "", # pyright: ignore[reportUnusedParameter] + ) -> str: + request_id = wake_request_id(wake_request) + recorded.append(request_id) + if len(recorded) == 1: + raise ConnectionError("service unavailable") + return request_id + + monkeypatch.setattr(wake_module, "send_wake_signal", fail_once) + worker = _worker_sending_wakes("one-shared-client") + subscription = _StubSubscription(wake_counter=1) + + with pytest.raises(ConnectionError, match="service unavailable"): + await worker._send_external_stream_wake(subscription) + await worker._send_external_stream_wake(subscription) + + assert len(recorded) == 2 + assert recorded[0] == recorded[1] + + +@pytest.mark.asyncio +async def test_a_workers_sender_identity_still_names_its_client() -> None: + """So a request ID stays traceable to a client in server-side logs. + + The per-instance part is what makes two Workers distinct; the client + identity is what makes either of them identifiable afterwards. + """ + worker = _worker_sending_wakes("client-identity-here") + + assert worker._external_stream_manager.wake_sender_identity.startswith( + "client-identity-here" + ) + + +# --- the envelope ------------------------------------------------------------- + + +def test_the_signal_bypasses_the_user_data_converter() -> None: + """Core must read this Signal, and Core has no access to a user codec. + + Asserted structurally: the payload's encoding is the protocol's own, and its + bytes parse as the protobuf envelope. A ``DataConverter`` that encrypted + payloads would make it unreadable to the only reader that matters. + """ + signal = build_signal_request(request(), identity="producer-1") + + (payload,) = signal.input.payloads + assert payload.metadata["encoding"] == WAKE_SIGNAL_ENCODING + assert payload.metadata["messageType"].decode() == WAKE_SIGNAL_MESSAGE_TYPE + + envelope = WakeSignal() + envelope.ParseFromString(payload.data) + assert envelope.envelope_version == WAKE_SIGNAL_ENVELOPE_VERSION + assert envelope.stream_name == "tokens" + assert envelope.wait_id == 1 + assert envelope.park_generation == 7 + assert envelope.first_execution_run_id == "first-run-1" + + +def test_the_signal_name_is_the_reserved_one_and_not_the_other_features() -> None: + """``workflow_streams`` is a different feature that coexists with this one.""" + signal = build_signal_request(request(), identity="producer-1") + + assert signal.signal_name == WAKE_SIGNAL_NAME == "__temporal_external_stream_wake" + assert not signal.signal_name.startswith("__temporal_workflow_stream") + + +def test_the_signal_carries_no_run_id() -> None: + """So it always lands on the current Run of the chain. + + A wake sent while a Continue-As-New is in flight must reach the successor, + not fail against a Run that has already closed. Chain identity travels inside + the envelope instead, which is what lets Core reject a reused Workflow ID. + """ + signal = build_signal_request(request(), identity="producer-1") + + assert signal.workflow_execution.workflow_id == "wf-1" + assert signal.workflow_execution.run_id == "" + + +def test_the_envelope_carries_no_stream_payload() -> None: + """The record's bytes stay in the backend; the Signal is pure identity.""" + signal = build_signal_request( + request(), identity="producer-1", producer_session_id="session-1" + ) + + raw = signal.SerializeToString() + envelope = WakeSignal() + envelope.ParseFromString(signal.input.payloads[0].data) + assert set(envelope.DESCRIPTOR.fields_by_name) == { + "envelope_version", + "stream_name", + "wait_id", + "park_generation", + "first_execution_run_id", + "producer_session_id", + }, "a new envelope field must be reviewed for whether it can carry user data" + assert b"session-1" in raw # diagnostics only, and the only free-form string + + +# --- the producer send sequence ----------------------------------------------- + + +class RecordingClient: + """Captures the raw service call without a server. + + The wake path is a raw ``SignalWorkflowExecution``, so what needs asserting + is the request itself -- not that some client method was reached. + """ + + def __init__(self, fail: bool = False) -> None: + self.sent: list = [] + self.fail = fail + self.namespace = "ns" + self.data_converter = temporalio.converter.DataConverter.default + self.service_client = self + self.config = self + self.identity = "producer-identity" + self.workflow_service = self + + async def signal_workflow_execution(self, request) -> None: # type: ignore[no-untyped-def] + if self.fail: + raise ConnectionError("service unavailable") + self.sent.append(request) + + +def make_producer(backend, client) -> ExternalStreamProducer: # type: ignore[no-untyped-def] + return ExternalStreamProducer( + backend=backend, + workflow=CHAIN, + data_converter=temporalio.converter.DataConverter.default, + session_id="session-1", + client=client, # type: ignore[arg-type] + ) + + +async def park(backend: MemoryStreamBackend, key: StreamKey, wait_id: int, gen: int): + await backend.install_park_intent( + key, + ParkIntent( + wait_id=wait_id, cursor=BEGINNING, park_generation=gen, run_id="run-1" + ), + ) + + +@pytest.mark.asyncio +async def test_a_producer_that_finds_a_wakeable_generation_sends_exactly_one_signal() -> ( + None +): + backend = MemoryStreamBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + await topic.publish("a", wake=False) + sent = await topic.wake() + + assert len(sent) == 1 + assert len(client.sent) == 1 + envelope = WakeSignal() + envelope.ParseFromString(client.sent[0].input.payloads[0].data) + assert envelope.park_generation == 4 + assert envelope.wait_id == 1 + + +@pytest.mark.asyncio +async def test_a_producer_that_loses_the_claim_sends_the_same_wake_anyway() -> None: + """A lost claim says someone *intends* to send. It is not evidence one did. + + The Signal goes out either way, and it costs no second Workflow Task: a + parked wake's request ID is derived from the generation and ignores sender + identity, so both producers issue byte-identical requests that the server + deduplicates into one wake. Staying silent here is what strands a parked + Workflow when the claim holder crashed between claiming and signalling. + """ + backend = MemoryStreamBackend() + key = StreamKey("ns", "wf-1", "first-run-1", "tokens") + await park(backend, key, wait_id=1, gen=4) + await backend.claim_park_generation( + key, 1, 4, claimant="someone-else", lease=timedelta(seconds=30) + ) + + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await topic.publish("a", wake=False) + + assert len(await topic.wake()) == 1 + assert len(client.sent) == 1 + assert client.sent[0].request_id == wake_request_id(request(park_generation=4)), ( + "the claim holder's wake and this one must be the same request, or the " + "duplicate the claim exists to avoid becomes a second Workflow Task" + ) + + +@pytest.mark.asyncio +async def test_an_expired_claim_is_taken_over_rather_than_stranding_the_wake() -> None: + """A producer crashing between claim and Signal must not park the Workflow forever. + + Without lease expiry every other producer concludes the wake is already + handled, and the Workflow waits with data sitting in the stream. + """ + backend = MemoryStreamBackend() + key = StreamKey("ns", "wf-1", "first-run-1", "tokens") + await park(backend, key, wait_id=1, gen=4) + # The crashed producer's claim, already expired. + await backend.claim_park_generation( + key, 1, 4, claimant="crashed", lease=timedelta(milliseconds=1) + ) + await backend.expire_claims_for_test() + + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await topic.publish("a", wake=False) + + assert len(await topic.wake()) == 1, "an expired claim must be takeable" + + +@pytest.mark.asyncio +async def test_a_stream_with_no_park_intent_gets_an_unparked_wake() -> None: + """Cached-with-no-open-task and evicted are invisible from out here (ADR-023). + + Staying silent because no park was observed loses the record until something + else happens to wake the Workflow. + """ + backend = MemoryStreamBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + + await topic.publish("a", wake=False) + assert len(await topic.wake()) == 1 + + envelope = WakeSignal() + envelope.ParseFromString(client.sent[0].input.payloads[0].data) + assert envelope.park_generation == UNPARKED_WAKE_GENERATION + + +@pytest.mark.asyncio +async def test_two_subscriptions_to_one_stream_are_woken_independently() -> None: + """They are two waits with their own cursors, generations, and claims. + + Waking only one leaves the other parked on a record it can already see. + """ + backend = MemoryStreamBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + await park(backend, topic.stream_key, wait_id=2, gen=4) + + await topic.publish("a", wake=False) + await topic.wake() + + woken = [] + for signal in client.sent: + envelope = WakeSignal() + envelope.ParseFromString(signal.input.payloads[0].data) + woken.append(envelope.wait_id) + assert sorted(woken) == [1, 2] + + +@pytest.mark.asyncio +async def test_a_failed_wake_reports_unacknowledged_and_the_retry_completes_it() -> ( + None +): + """The record is durable, the wake is not, and only one of them may be retried. + + Reporting success here would make the "durable producer" row of the + wakeup-durability boundary false. + """ + backend = MemoryStreamBackend() + client = RecordingClient(fail=True) + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + offset = await topic.publish("a", wake=False) + + with pytest.raises(WakeNotAcknowledgedError) as caught: + await topic.wake() + + assert caught.value.pending, "the retry must know what is still owed" + + # The record stayed put: the append is not what failed. + assert await backend.read_range(topic.stream_key, offset, offset) + + client.fail = False + assert len(await topic.retry_wake(caught.value.pending)) == 1 + + +@pytest.mark.asyncio +async def test_a_retried_wake_reuses_the_original_request_id() -> None: + """Recomputing it would draw a fresh counter and defeat the deduplication.""" + backend = MemoryStreamBackend() + client = RecordingClient(fail=True) + topic = make_producer(backend, client).topic("tokens", type=str) + await topic.publish("a", wake=False) + + with pytest.raises(WakeNotAcknowledgedError) as caught: + await topic.wake() + expected = wake_request_id(caught.value.pending[0]) + + client.fail = False + assert await topic.retry_wake(caught.value.pending) == [expected] + + +class FlakyCoordinationBackend(MemoryStreamBackend): + """A backend whose append works and whose coordination calls do not. + + One named method fails once. Appending keeps working throughout, which is the + whole point: the record is durable before the failure happens, so what the + caller has to be told is which half of ``publish()`` failed. + """ + + def __init__(self, failing: str) -> None: + super().__init__() + self.failing = failing + self.failures = 0 + + def _maybe_fail(self, name: str) -> None: + if name == self.failing: + self.failing = "" + self.failures += 1 + raise ConnectionError(f"{name} is unavailable") + + async def parked_wait_ids(self, key): # type: ignore[no-untyped-def] + self._maybe_fail("parked_wait_ids") + return await super().parked_wait_ids(key) + + async def current_park_generation(self, key, wait_id): # type: ignore[no-untyped-def] + self._maybe_fail("current_park_generation") + return await super().current_park_generation(key, wait_id) + + async def claim_park_generation( + self, + key: StreamKey, + wait_id: int, + park_generation: int, + *, + claimant: str, + lease: timedelta, + ) -> bool: + self._maybe_fail("claim_park_generation") + return await super().claim_park_generation( + key, + wait_id, + park_generation, + claimant=claimant, + lease=lease, + ) + + +@pytest.mark.parametrize( + "failing", + ["parked_wait_ids", "current_park_generation", "claim_park_generation"], +) +@pytest.mark.asyncio +async def test_a_coordination_failure_after_the_append_is_still_unacknowledged( + failing: str, +) -> None: + """The wake is three steps, and all three are after a durable append. + + Only the Signal used to be inside the guarantee. The observe and claim steps + raised whatever the provider raised -- a bare ``ConnectionError`` -- which + passed straight through ``publish()``, because ``publish()`` catches only the + durable-but-unacknowledged error. The caller then had neither the offset nor + any statement about what had already landed, and its only obvious move -- + retry ``publish()`` -- appends the record a *second* time, under a new + sequence number and therefore a new idempotency key. + """ + backend = FlakyCoordinationBackend(failing) + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + with pytest.raises(WakeNotAcknowledgedError) as caught: + await topic.publish("a") + + assert backend.failures == 1, "the test did not exercise the failure it names" + assert caught.value.offset is not None, ( + "the caller has no other way to learn that the append it must not retry " + "did succeed" + ) + assert caught.value.restart, ( + "no Signal was composed, so `pending` is empty and only a fresh wake() " + "recovers -- a caller told to retry_wake([]) would get a no-op that " + "looks like success" + ) + assert not caught.value.pending + with pytest.raises(ValueError, match=r"call wake\(\) again"): + await topic.retry_wake(caught.value.pending) + + # Recovering the wake alone leaves exactly one record in the stream. + assert await topic.wake() + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [r.sequence for r in records if r.kind == RecordKind.DATA] == [0], ( + "the stream holds more than the one record that was published" + ) + + +@pytest.mark.asyncio +async def test_a_coordination_failure_after_a_fence_is_still_unacknowledged() -> None: + """A fence is the record most likely to find the Workflow parked. + + Same guarantee as ``publish()``, and the duplicate it prevents is worse: + retrying ``finish_writing()`` appends a second fence, which reads back as a + producer session that ended twice. + """ + backend = FlakyCoordinationBackend("parked_wait_ids") + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + with pytest.raises(WakeNotAcknowledgedError) as caught: + await topic.finish_writing() + + assert caught.value.offset is not None + assert caught.value.restart + + assert await topic.wake() + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert len([r for r in records if r.kind == RecordKind.WRITE_FENCE]) == 1, ( + "the stream holds more than the one fence that was written" + ) + + +class BlockingCoordinationBackend(MemoryStreamBackend): + """Appends, then blocks once in one named coordination call. + + One-shot, so the recovery the test performs afterwards can get through. The + append is never blocked: the record is durable before the block is reached, + which is the state the whole contract is about. + """ + + def __init__(self, blocking: str) -> None: + super().__init__() + self.blocking = blocking + self.reached = asyncio.Event() + + async def _maybe_block(self, name: str) -> None: + if name != self.blocking: + return + self.blocking = "" + self.reached.set() + await asyncio.Event().wait() + + async def parked_wait_ids(self, key): # type: ignore[no-untyped-def] + await self._maybe_block("parked_wait_ids") + return await super().parked_wait_ids(key) + + async def current_park_generation(self, key, wait_id): # type: ignore[no-untyped-def] + await self._maybe_block("current_park_generation") + return await super().current_park_generation(key, wait_id) + + async def claim_park_generation(self, key, wait_id, generation, **kwargs): # type: ignore[no-untyped-def,override] + await self._maybe_block("claim_park_generation") + return await super().claim_park_generation(key, wait_id, generation, **kwargs) + + +class BlockingClient(RecordingClient): + """A client whose first Signal send never completes.""" + + def __init__(self) -> None: + super().__init__() + self.reached = asyncio.Event() + self.blocking = True + + async def signal_workflow_execution(self, request) -> None: # type: ignore[no-untyped-def] + if self.blocking: + self.blocking = False + self.reached.set() + await asyncio.Event().wait() + await super().signal_workflow_execution(request) + + +async def _recover(topic, error: WakeNotAcknowledgedError) -> None: # type: ignore[no-untyped-def] + """The recovery the error names, and only that one.""" + assert bool(error.pending) != bool(error.restart), ( + "the error must name exactly one recovery: `pending` to re-send, or " + "`restart` to compose a fresh wake. Both or neither leaves the caller " + "guessing about a record that is already durable" + ) + if error.pending: + await topic.retry_wake(error.pending) + else: + await topic.wake() + + +@pytest.mark.parametrize( + "stage", + ["parked_wait_ids", "current_park_generation", "claim_park_generation", "signal"], +) +@pytest.mark.asyncio +async def test_cancellation_after_the_append_is_an_unacknowledged_wake( + stage: str, +) -> None: + """Cancellation is not a third outcome, and it must not read like the first. + + ``CancelledError`` is a ``BaseException``, so it passed through every handler + on this path -- the coordination block re-raised it deliberately, the Signal + loop caught only ``Exception``, and ``publish()`` catches only the + durable-but-unacknowledged error. What escaped was a bare ``CancelledError`` + with no offset, no ``pending`` and no ``restart``: indistinguishable from + cancellation *before* the append, and so unrecoverable in both directions. The + caller could neither wake the record it did not know about nor re-publish the + value safely -- retrying ``publish()`` draws a new sequence number and appends + a second record. + + Parameterized over all four post-append stages, because the state they leave + is the same and only the recovery differs. + """ + backend = BlockingCoordinationBackend("" if stage == "signal" else stage) + signalling = BlockingClient() if stage == "signal" else None + client: RecordingClient = signalling or RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + reached = signalling.reached if signalling is not None else backend.reached + + task = asyncio.ensure_future(topic.publish("a")) + await asyncio.wait_for(reached.wait(), 2) + task.cancel() + + with pytest.raises(WakeNotAcknowledgedError) as caught: + await task + + assert caught.value.cancelled, ( + "the cancellation is reported rather than swallowed: a caller that wants " + "to honour it after recovering the wake still has to know it was asked " + "to stop" + ) + assert caught.value.offset is not None, ( + "a cancellation carrying no offset cannot be told apart from one before " + "the append, so the caller cannot know whether the value is published" + ) + await _recover(topic, caught.value) + + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [r.sequence for r in records if r.kind == RecordKind.DATA] == [0], ( + "recovering the wake alone left more than the one record that was published" + ) + assert client.sent, "the recovery sent no Signal, so the record stays unannounced" + + +@pytest.mark.asyncio +async def test_cancellation_after_a_fence_is_an_unacknowledged_wake() -> None: + """A fence carries the identical contract, and a worse duplicate. + + Retrying ``finish_writing()`` appends a second fence, which reads back as a + producer session that ended twice. + """ + backend = BlockingCoordinationBackend("parked_wait_ids") + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + task = asyncio.ensure_future(topic.finish_writing()) + await asyncio.wait_for(backend.reached.wait(), 2) + task.cancel() + + with pytest.raises(WakeNotAcknowledgedError) as caught: + await task + + assert caught.value.cancelled and caught.value.offset is not None + await _recover(topic, caught.value) + + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert len([r for r in records if r.kind == RecordKind.WRITE_FENCE]) == 1, ( + "the stream holds more than the one fence that was written" + ) + + +class BlockingEncodeCodec(temporalio.converter.PayloadCodec): + """A codec whose first ``encode`` never completes. + + The one place a cancellation is *knowably* pre-append: the record has not + been built, let alone handed to the backend, so no provider anywhere can have + committed it. + """ + + def __init__(self) -> None: + self.reached = asyncio.Event() + + async def encode(self, payloads): # type: ignore[no-untyped-def] + self.reached.set() + await asyncio.Event().wait() + return list(payloads) + + async def decode(self, payloads): # type: ignore[no-untyped-def] + return list(payloads) + + +@pytest.mark.asyncio +async def test_cancellation_before_the_append_stays_a_cancellation() -> None: + """The other half of the distinction, without which the first is meaningless. + + Nothing was sent to the backend, so there is nothing to say about a durable + record and nothing to recover: reporting an unacknowledged wake here would + send the caller to wake a record that does not exist, and refusing a + cancellation that costs nothing to honour buys nothing. + + "Before the append" has to mean *before the backend was called*, not merely + before it answered. Cancellation while the call is in flight is a different + state, and it is the one below (ADR-038). + """ + backend = MemoryStreamBackend() + client = RecordingClient() + codec = BlockingEncodeCodec() + producer = ExternalStreamProducer( + backend=backend, + workflow=CHAIN, + data_converter=dataclasses.replace( + temporalio.converter.DataConverter.default, payload_codec=codec + ), + session_id="session-1", + client=client, # type: ignore[arg-type] + ) + topic = producer.topic("tokens", type=str) + + task = asyncio.ensure_future(topic.publish("a")) + await asyncio.wait_for(codec.reached.wait(), 2) + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + + assert not await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert not client.sent + assert not producer._unresolved, ( + "an append that was never made owes no recovery, and holding one would " + "block the stream for a record that cannot exist" + ) + + +# --- the append acknowledgement window ---------------------------------------- + + +class CommittingThenBlockingBackend(MemoryStreamBackend): + """A backend whose first ``append`` **commits** and then loses its answer. + + This is the shape a remote provider actually has: the Redis backend runs an + atomic script server-side and receives its result in a separate client-side + step, so a cancellation or a dropped connection between the two leaves a + durable record that the caller was never told about. A backend that blocks + *before* storing is indistinguishable from this one from the producer's side + -- which is the whole reason the window needs an outcome of its own. + """ + + def __init__(self, fail: BaseException | None = None) -> None: + super().__init__() + self.committed = asyncio.Event() + self.blocking = True + self._fail = fail + + async def append(self, key, record): # type: ignore[no-untyped-def] + placed = await super().append(key, record) + if self.blocking: + self.blocking = False + self.committed.set() + if self._fail is not None: + raise self._fail + await asyncio.Event().wait() + return placed + + +async def _cancel_after_commit(backend, call): # type: ignore[no-untyped-def] + """Runs ``call`` and cancels it once the backend has committed but not answered.""" + task = asyncio.ensure_future(call()) + await asyncio.wait_for(backend.committed.wait(), 2) + task.cancel() + return task + + +@pytest.mark.asyncio +async def test_cancellation_after_backend_commit_before_append_ack_is_recoverable() -> ( + None +): + """The record is durable and the caller was never told. Both must be true. + + An ``append()`` that does not return is not an append that did not happen: a + backend commits on its own side and only then answers. A bare + ``CancelledError`` here says nothing about the record, and the caller's + obvious move -- publishing the same value again -- draws a fresh sequence + number and puts it in the stream twice. + + So the window gets its own outcome, carrying the exact record, and the + recovery re-appends *that* record: byte-identical under the same + ``(session_id, sequence)``, which the backend contract makes a no-op + returning the original offset. + """ + backend = CommittingThenBlockingBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + + task = await _cancel_after_commit(backend, lambda: topic.publish("a", wake=False)) + + with pytest.raises(AppendNotAcknowledgedError) as caught: + await task + + error = caught.value + assert error.cancelled, ( + "a caller that wants to honour the cancellation after settling the " + "append still has to know it was asked to stop" + ) + assert error.record.sequence == 0 and error.record.kind == RecordKind.DATA + assert error.record.offset is None, ( + "the record is carried in the state it was sent in; one carrying an " + "offset would mean the append had been acknowledged after all" + ) + + committed = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [r.sequence for r in committed] == [0], ( + "the fixture must actually have committed, or this asserts nothing" + ) + + offset = await topic.resolve_append(error.record, wake=error.wake) + + assert offset == committed[0].offset, ( + "the recovery must recover the original offset, not mint a second " + "position for a record that already had one" + ) + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [(r.sequence, r.kind) for r in records] == [(0, RecordKind.DATA)], ( + "settling the append appended a second copy of a record that had already landed" + ) + + +@pytest.mark.asyncio +async def test_a_lost_append_response_is_recoverable_the_same_way() -> None: + """Cancellation is not what makes the window ambiguous -- the commit is. + + A provider exception raised after the server committed carries exactly as + little information as a cancellation does, so it must not reach the caller as + an ordinary failure either. + """ + backend = CommittingThenBlockingBackend(fail=ConnectionError("connection reset")) + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + + with pytest.raises(AppendNotAcknowledgedError) as caught: + await topic.publish("a", wake=False) + + assert not caught.value.cancelled + assert isinstance(caught.value.__cause__, ConnectionError) + + await topic.resolve_append(caught.value.record, wake=False) + + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [(r.sequence, r.kind) for r in records] == [(0, RecordKind.DATA)] + + +@pytest.mark.asyncio +async def test_an_unresolved_append_refuses_the_publish_that_would_duplicate_it() -> ( + None +): + """The failure the outcome exists to prevent, prevented rather than described. + + A caller that treats the unknown outcome as a failure retries `publish()`, + which draws sequence 1 and appends the value a second time. The stream will + not take it. + """ + backend = CommittingThenBlockingBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + + task = await _cancel_after_commit(backend, lambda: topic.publish("a", wake=False)) + with pytest.raises(AppendNotAcknowledgedError) as caught: + await task + + with pytest.raises(AppendNotAcknowledgedError) as refused: + await topic.publish("a", wake=False) + assert refused.value.record.idempotency_key == caught.value.record.idempotency_key + assert "resolve_append" in str(refused.value) + + with pytest.raises(AppendNotAcknowledgedError): + await topic.finish_writing(wake=False) + + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [(r.sequence, r.kind) for r in records] == [(0, RecordKind.DATA)], ( + "the refused publish still appended, which is the duplicate this whole " + "outcome exists to prevent" + ) + + await topic.resolve_append(caught.value.record, wake=False) + assert await topic.publish("b", wake=False), ( + "once the append is settled the stream has to be usable again, or the " + "refusal is a permanent wedge rather than a guard" + ) + + +@pytest.mark.asyncio +async def test_settling_an_unacknowledged_append_wakes_exactly_once() -> None: + """The recovery finishes the call, rather than half of it. + + The wake-enabled form owes both halves: the record's offset and one Signal. + Recovering only the append would leave a parked Workflow on a durable record, + which is the state the acknowledged-wake contract exists to refuse. + """ + backend = CommittingThenBlockingBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + task = await _cancel_after_commit(backend, lambda: topic.publish("a")) + with pytest.raises(AppendNotAcknowledgedError) as caught: + await task + + assert caught.value.wake, "the recovery has to know the call owed a wake" + assert not client.sent, "nothing can have been signalled before the append landed" + + await topic.resolve_append(caught.value.record) + + assert len(client.sent) == 1, ( + "the settled record was announced once, or not at all -- the recovery " + "defaults to what the interrupted call was doing rather than to a wake " + "policy of its own" + ) + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [(r.sequence, r.kind) for r in records] == [(0, RecordKind.DATA)] + + +@pytest.mark.asyncio +async def test_an_unacknowledged_fence_settles_to_exactly_one_fence() -> None: + """A duplicate fence reads back as a producer session that ended twice.""" + backend = CommittingThenBlockingBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + + task = await _cancel_after_commit(backend, lambda: topic.finish_writing(wake=False)) + with pytest.raises(AppendNotAcknowledgedError) as caught: + await task + + assert caught.value.record.kind == RecordKind.WRITE_FENCE + await topic.resolve_append(caught.value.record) + + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert len([r for r in records if r.kind == RecordKind.WRITE_FENCE]) == 1, ( + "the stream holds more than the one fence that was written" + ) + assert not client.sent, ( + "the fence was appended with wake=False and the recovery invented a " + "Signal for it; the recovery finishes the interrupted call, it does not " + "choose a different one" + ) + + +@pytest.mark.asyncio +async def test_a_refused_append_preserves_the_unresolved_operations_recovery() -> None: + """The refusal reports the *unsettled* operation, not the call it refused. + + An append outcome is more than its record: whether a wake was owed, under + which lease, and whether cancellation is still to be honoured once the state + is settled. Filling those from whichever later call happened to be refused + makes the error's own instructions wrong -- a caller following them settles a + record that owed a Signal with ``wake=False``, and the parked Workflow stays + parked on a durable record nobody announced. + """ + backend = CommittingThenBlockingBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + lease = timedelta(seconds=97) + + task = await _cancel_after_commit( + backend, lambda: topic.publish("a", wake=True, lease=lease) + ) + with pytest.raises(AppendNotAcknowledgedError) as caught: + await task + assert (caught.value.wake, caught.value.lease, caught.value.cancelled) == ( + True, + lease, + True, + ) + + with pytest.raises(AppendNotAcknowledgedError) as refused: + await topic.publish("b", wake=False, lease=timedelta(seconds=5)) + + assert refused.value.record == caught.value.record, ( + "the refusal must name the append that is actually unsettled" + ) + assert refused.value.wake is True, ( + "the refusal reported the refused call's wake policy beside the older " + "call's record, so recovering by its own fields drops a Signal" + ) + assert refused.value.lease == lease + assert refused.value.cancelled is True, ( + "the caller still has to know it was asked to stop; the refusal is not " + "a fresh, uncancelled attempt" + ) + assert refused.value.stream_key == topic.stream_key + + # Recovering by the refusal's own fields, which is what its message says. + await topic.resolve_append( + refused.value.record, wake=refused.value.wake, lease=refused.value.lease + ) + + assert len(client.sent) == 1, "the record is durable and nobody was told" + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [(r.sequence, r.kind) for r in records] == [(0, RecordKind.DATA)] + + +@pytest.mark.asyncio +async def test_a_reinterrupted_append_recovery_preserves_its_latest_wake() -> None: + """A second lost response must not restore the first attempt's wake policy. + + ``resolve_append`` may deliberately override the unresolved operation's wake + and lease. If that append also commits without returning, the error describes + the recovery attempt that was interrupted. Retaining the older operation + beside that newer error makes the next defaulted recovery silently use stale + instructions -- here it returns success without the Signal the error says is + owed. + """ + + class LoseThreeResponsesAfterCommit(MemoryStreamBackend): + def __init__(self) -> None: + super().__init__() + self.attempt = 0 + self.second_committed = asyncio.Event() + + async def append(self, key, record): # type: ignore[no-untyped-def] + placed = await super().append(key, record) + self.attempt += 1 + if self.attempt == 1: + raise ConnectionError("the append committed but its response was lost") + if self.attempt == 2: + self.second_committed.set() + await asyncio.Event().wait() + if self.attempt == 3: + raise ConnectionError("the recovery response was also lost") + return placed + + backend = LoseThreeResponsesAfterCommit() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + with pytest.raises(AppendNotAcknowledgedError) as first: + await topic.publish("a", wake=False) + + recovery_lease = timedelta(seconds=97) + recovery = asyncio.ensure_future( + topic.resolve_append(first.value.record, wake=True, lease=recovery_lease) + ) + await asyncio.wait_for(backend.second_committed.wait(), 2) + recovery.cancel() + with pytest.raises(AppendNotAcknowledgedError) as second: + await recovery + + assert second.value.record == first.value.record + assert second.value.wake is True + assert second.value.lease == recovery_lease + assert second.value.cancelled is True + assert not client.sent + + with pytest.raises(AppendNotAcknowledgedError) as third: + await topic.resolve_append(second.value.record) + + assert third.value.record == second.value.record + assert third.value.wake is True + assert third.value.lease == recovery_lease + assert third.value.cancelled is True, ( + "a later connection failure erased the cancellation the caller still " + "has to honour after settling the operation" + ) + + with pytest.raises(AppendNotAcknowledgedError) as refused: + await topic.publish("b", wake=False, lease=timedelta(seconds=5)) + + assert refused.value.record == third.value.record + assert refused.value.wake is True + assert refused.value.lease == recovery_lease + assert refused.value.cancelled is True + + offset = await topic.resolve_append(refused.value.record) + + assert len(client.sent) == 1, ( + "the second recovery defaulted to the original publish's wake=False " + "instead of the interrupted resolve_append's wake=True" + ) + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [(r.sequence, r.offset) for r in records] == [(0, offset)], ( + "repeated recovery appended more than the one record being settled" + ) + + +@pytest.mark.asyncio +async def test_an_unknown_append_can_only_be_resolved_on_its_originating_topic() -> ( + None +): + """Idempotency is scoped to the stream, so the recovery has to be too. + + A ``StreamRecord`` names its producer session and sequence but not its + stream. On any *other* topic that key has never been used, so settling there + does not deduplicate against the copy that may already exist -- it appends + the value a second time, onto a stream no consumer of it is watching, and + leaves the real stream still blocked. + """ + backend = CommittingThenBlockingBackend() + client = RecordingClient() + producer = make_producer(backend, client) + tokens = producer.topic("tokens", type=str) + events = producer.topic("events", type=str) + + task = await _cancel_after_commit(backend, lambda: tokens.publish("a", wake=False)) + with pytest.raises(AppendNotAcknowledgedError) as caught: + await task + assert caught.value.stream_key == tokens.stream_key, ( + "the error has to name where it must be settled, since the record cannot" + ) + + with pytest.raises(ValueError, match="not on"): + await events.resolve_append(caught.value.record) + assert not await backend.read_after(events.stream_key, BEGINNING, max_records=10), ( + "the wrong topic took the record, so one logical input is now on two streams" + ) + + # Nor do different bytes under the outstanding key count as that record. + tampered = dataclasses.replace(caught.value.record, payload=b'"z"') + with pytest.raises(ValueError, match="does not hold these bytes"): + await tokens.resolve_append(tampered) + assert producer._unresolved, ( + "a rejected recovery must not clear the state that is still unsettled" + ) + + offset = await tokens.resolve_append(caught.value.record) + records = await backend.read_after(tokens.stream_key, BEGINNING, max_records=10) + assert [(r.sequence, r.offset) for r in records] == [(0, offset)] + assert await tokens.publish("b", wake=False), "the settled stream stays usable" + + +@pytest.mark.asyncio +async def test_recovery_is_bound_to_the_producer_instance_that_made_the_append() -> ( + None +): + """A same-session replacement producer is not the one that owes the append. + + Its sequence and wake counters start at zero, so settling there leaves both + invalid: the next publish reuses a sequence number the recovered record + already holds, and the next unparked wake re-derives a request ID an earlier, + *different* wake already used -- which the server deduplicates away, leaving + a durable record unannounced. + + The recovery for a producer that is gone is the one the Activity retry + already performs: rebuild with the same session id and re-run the same calls + in the same order. That re-derives the same sequences, so the backend + deduplicates the appends, and it leaves the new session's counters correct. + """ + backend = CommittingThenBlockingBackend() + backend.blocking = False # the first publish has to land normally + client = RecordingClient() + first = make_producer(backend, client) + topic = first.topic("tokens", type=str) + + first_offset = await topic.publish("a") + assert len(client.sent) == 1 + first_request_id = client.sent[0].request_id + + backend.blocking = True + task = await _cancel_after_commit(backend, lambda: topic.publish("b")) + with pytest.raises(AppendNotAcknowledgedError) as caught: + await task + + replacement = make_producer(backend, client) + assert replacement.session_id == first.session_id + with pytest.raises(ValueError, match="no unsettled append"): + await replacement.topic("tokens", type=str).resolve_append(caught.value.record) + + # The recovery that refusal names, and what makes it the right one. + replacement_topic = replacement.topic("tokens", type=str) + assert await replacement_topic.publish("a") == first_offset, ( + "re-running the same call has to deduplicate onto the original record" + ) + assert client.sent[1].request_id == first_request_id, ( + "re-running one attempt is a retry of its wake, which the server " + "collapses rather than turning into a second Workflow Task" + ) + + await replacement_topic.publish("b") + assert client.sent[2].request_id != first_request_id, ( + "the second value's wake collided with the first value's, so the server " + "deduplicates it away and the record stays unannounced" + ) + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [r.sequence for r in records] == [0, 1], ( + "each value must appear exactly once, under the sequence its call drew" + ) + + +@pytest.mark.asyncio +async def test_a_conflicting_append_is_a_refusal_not_an_unknown_outcome() -> None: + """The one failure the contract lets the producer read as "did not land". + + ``AppendConflictError`` says the key was used with *different* bytes, so this + record definitely did not land and re-appending it would raise the identical + error. Folding it into the unknown outcome would send the caller to settle an + append that has no settlement, and would wedge the stream behind it. + """ + + class ConflictingBackend(MemoryStreamBackend): + async def append(self, key, record): # type: ignore[no-untyped-def] + raise AppendConflictError(record.idempotency_key) + + backend = ConflictingBackend() + producer = make_producer(backend, RecordingClient()) + topic = producer.topic("tokens", type=str) + + with pytest.raises(AppendConflictError): + await topic.publish("a", wake=False) + + assert not producer._unresolved + with pytest.raises(AppendConflictError): + await topic.publish("b", wake=False) + + +@pytest.mark.asyncio +async def test_settling_an_append_that_never_landed_appends_it_once() -> None: + """The other history the same call has to be right for. + + The producer cannot tell the two apart -- that is the premise -- so the + recovery has to leave one record whichever one happened. + """ + + class LosingBackend(MemoryStreamBackend): + """Loses its first append entirely, then behaves.""" + + def __init__(self) -> None: + super().__init__() + self.reached = asyncio.Event() + + async def append(self, key, record): # type: ignore[no-untyped-def] + if not self.reached.is_set(): + self.reached.set() + raise ConnectionError("connection reset before the write") + return await super().append(key, record) + + backend = LosingBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + + with pytest.raises(AppendNotAcknowledgedError) as caught: + await topic.publish("a", wake=False) + + assert not await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + + offset = await topic.resolve_append(caught.value.record, wake=False) + + records = await backend.read_after(topic.stream_key, BEGINNING, max_records=10) + assert [(r.sequence, r.offset) for r in records] == [(0, offset)], ( + "settling an append that had not landed has to land it, under the " + "sequence number the interrupted call drew rather than a fresh one" + ) + + +@pytest.mark.asyncio +async def test_settling_a_record_from_another_session_is_refused() -> None: + """Identity is the whole recovery; a foreign record is a different record.""" + backend = MemoryStreamBackend() + topic = make_producer(backend, RecordingClient()).topic("tokens", type=str) + foreign = StreamRecord( + kind=RecordKind.DATA, + payload=b'"a"', + producer_session_id="someone-else", + sequence=0, + ) + + with pytest.raises(ValueError, match="not by this one"): + await topic.resolve_append(foreign, wake=False) + with pytest.raises(ValueError, match="already carries offset"): + await topic.resolve_append( + StreamRecord( + kind=RecordKind.DATA, + payload=b'"a"', + producer_session_id="session-1", + sequence=0, + ).placed_at(Offset("1-1")), + wake=False, + ) + + +@pytest.mark.asyncio +async def test_waking_without_a_client_says_so() -> None: + backend = MemoryStreamBackend() + producer = ExternalStreamProducer( + backend=backend, + workflow=CHAIN, + data_converter=temporalio.converter.DataConverter.default, + session_id="session-1", + ) + + with pytest.raises(RuntimeError, match="requires a Temporal client"): + await producer.topic("tokens", type=str).wake() + + +# --- the two halves of the envelope live in different repos ------------------- + + +def test_the_python_constants_match_cores() -> None: + """Core is the reader; a drifted constant here is an unreadable Signal there. + + Nothing else would catch it: Python's own tests would still pass, Core's own + tests would still pass, and the wake would simply stop working -- appearing + as a Workflow that waits out its idle timeout for no visible reason. + """ + core = ( + pathlib.Path(temporalio.bridge.__file__).parent + / "sdk-core" + / "crates" + / "protos" + / "src" + / "protos" + / "mod.rs" + ) + if not core.exists(): + pytest.skip("the Core submodule is not checked out") + source = core.read_text() + + for name, value in ( + ("WAKE_SIGNAL_NAME", f'"{WAKE_SIGNAL_NAME}"'), + ("WAKE_SIGNAL_MESSAGE_TYPE", f'"{WAKE_SIGNAL_MESSAGE_TYPE}"'), + ("WAKE_SIGNAL_ENVELOPE_VERSION", str(WAKE_SIGNAL_ENVELOPE_VERSION)), + ("UNPARKED_WAKE_GENERATION", str(UNPARKED_WAKE_GENERATION)), + ): + assert re.search(rf"{name}:\s*\S+\s*=\s*{re.escape(value)}\s*;", source), ( + f"{name} is {value} in Python but Core declares something else" + ) + + +# --- P6b: publish()'s acknowledged-wake contract ------------------------------ + + +@pytest.mark.asyncio +async def test_publish_completes_only_once_its_wake_is_acknowledged() -> None: + """Returning means durable *and* signalled, which is the whole contract. + + An append that lands but is never signalled leaves the Workflow parked on + data already sitting in the stream -- and a publish() that returned success + there would be reporting a delivery that never happened. + """ + backend = MemoryStreamBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + offset = await topic.publish("a") + + assert offset is not None + assert len(client.sent) == 1, "publish must not return before it has signalled" + + +@pytest.mark.asyncio +async def test_publish_acknowledges_the_wake_even_when_another_producer_claimed_it() -> ( + None +): + """A claim held by a producer that crashed must not report a wake nobody sent. + + The lease permits takeover *after* it expires; it schedules nobody to take + over. If this publish is the last producer action -- and here it is the only + one -- a claim-based silence leaves the Workflow parked on a durable record + with a ``publish()`` that reported success. + """ + backend = MemoryStreamBackend() + key = StreamKey("ns", "wf-1", "first-run-1", "tokens") + await park(backend, key, wait_id=1, gen=4) + # A producer that claimed the generation and then died before signalling. + # Its lease is unexpired, so the claim is not takeable. + await backend.claim_park_generation( + key, 1, 4, claimant="crashed", lease=timedelta(seconds=30) + ) + + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + + offset = await topic.publish("a") + + assert offset is not None + assert len(client.sent) == 1, ( + "publish returned an acknowledged wake that nobody sent; the parked " + "Workflow is stranded on a durable record" + ) + envelope = WakeSignal() + envelope.ParseFromString(client.sent[0].input.payloads[0].data) + assert envelope.park_generation == 4 + assert client.sent[0].request_id == wake_request_id(request(park_generation=4)), ( + "the wake the crashed claimant owed and this one are the same request, " + "so the server collapses them rather than creating a second task" + ) + + +@pytest.mark.asyncio +async def test_a_publish_whose_wake_fails_reports_unacknowledged() -> None: + """Not success. This is the "durable producer" row of the boundary table.""" + backend = MemoryStreamBackend() + client = RecordingClient(fail=True) + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + with pytest.raises(WakeNotAcknowledgedError) as caught: + await topic.publish("a") + + assert caught.value.offset is not None, ( + "the caller has no other way to learn that the append it must not retry " + "did succeed" + ) + assert caught.value.pending + + +@pytest.mark.asyncio +async def test_retrying_the_wake_completes_an_unacknowledged_publish() -> None: + """And appends nothing: only the half that failed is retried.""" + backend = MemoryStreamBackend() + client = RecordingClient(fail=True) + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + with pytest.raises(WakeNotAcknowledgedError) as caught: + await topic.publish("a") + before = await backend.read_after( + topic.stream_key, BEGINNING, max_records=100, block=None + ) + + client.fail = False + assert len(await topic.retry_wake(caught.value.pending)) == 1 + + after = await backend.read_after( + topic.stream_key, BEGINNING, max_records=100, block=None + ) + assert [r.offset for r in after] == [r.offset for r in before], ( + "the retry must complete the wake without re-appending the record" + ) + + +@pytest.mark.asyncio +async def test_the_unacknowledged_state_is_explicit_not_hidden() -> None: + """``wake=False`` says "durable but un-signalled" in the call itself. + + The point is that a caller cannot end up in that state by accident -- it is + reachable only by asking for it, or by a wake that failed loudly. + """ + backend = MemoryStreamBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + await topic.publish("a", wake=False) + assert client.sent == [] + + await topic.wake() + assert len(client.sent) == 1 + + +@pytest.mark.asyncio +async def test_a_fence_is_signalled_too() -> None: + """It is the record most likely to find the Workflow parked. + + A fence is what a producer appends when it has nothing more to say, so an + unsignalled one strands the Workflow for its whole idle timeout at exactly + the moment it was waiting to be told. + """ + backend = MemoryStreamBackend() + client = RecordingClient() + topic = make_producer(backend, client).topic("tokens", type=str) + await park(backend, topic.stream_key, wait_id=1, gen=4) + + await topic.finish_writing() + + assert len(client.sent) == 1 + + +# --- deduplication, against a real server ------------------------------------- + + +@workflow.defn +class WaitForeverWorkflow: + """Exists to receive Signals. Never completes on its own.""" + + @workflow.run + async def run(self) -> None: + await asyncio.Future() + + +async def _signalled_events(handle) -> int: # type: ignore[no-untyped-def] + return len( + [ + e + async for e in handle.fetch_history_events() + if e.HasField("workflow_execution_signaled_event_attributes") + ] + ) + + +async def _settled_signal_count(handle, expected: int) -> int: # type: ignore[no-untyped-def] + """Waits for the count to reach ``expected`` and then stay there. + + Counting once races persistence in both directions: too early and a wake + that *was* recorded is missed, too eager and a duplicate that is about to + appear is not. Waiting for the count to reach the expected value and hold + across a further interval distinguishes "the server collapsed them" from + "the second one has not landed yet". + """ + for _ in range(40): + if await _signalled_events(handle) >= expected: + break + await asyncio.sleep(0.1) + await asyncio.sleep(1) + return await _signalled_events(handle) + + +async def _start_waiter(client: Client, task_queue: str): # type: ignore[no-untyped-def] + handle = await client.start_workflow( + WaitForeverWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + return handle, description.raw_description.workflow_execution_info.first_run_id + + +async def test_two_producers_retrying_one_wake_are_deduplicated_by_the_server( + client: Client, +) -> None: + """ID equality is only half the claim; this is the half that matters. + + Two producers racing to wake one generation must cost one Workflow Task, not + two. Asserting that they derive the same string proves nothing about what + the server does with it -- and the server is the component that has to + collapse them. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker(client, task_queue=task_queue, workflows=[WaitForeverWorkflow]): + handle, first_run = await _start_waiter(client, task_queue) + try: + wake = request( + namespace=client.namespace, + workflow_id=handle.id, + first_execution_run_id=first_run, + park_generation=4, + ) + + # Two producers, one generation, sent independently. + for _ in range(2): + await send_wake_signal(client, wake) + + assert await _settled_signal_count(handle, 1) == 1, ( + "the server recorded both wakes; a producer retrying after an " + "ambiguous failure would then cost a second Workflow Task" + ) + finally: + await handle.terminate() + + +async def test_two_workers_unparked_wakes_are_both_delivered( + client: Client, +) -> None: + """The opposite requirement, and why the counter exists. + + Two Workers shutting down at different times are two separate asks. If their + request IDs collided the server would deduplicate the second away -- turning + a correct retry mechanism into silent loss, which is the failure the sender + identity and counter are there to prevent. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker(client, task_queue=task_queue, workflows=[WaitForeverWorkflow]): + handle, first_run = await _start_waiter(client, task_queue) + try: + for identity in ("worker-a", "worker-b"): + await send_wake_signal( + client, + request( + namespace=client.namespace, + workflow_id=handle.id, + first_execution_run_id=first_run, + park_generation=0, + sender_identity=identity, + wake_counter=1, + ), + ) + + assert await _settled_signal_count(handle, 2) == 2, ( + "one of the two Workers' wakes was deduplicated away; both are " + "separate asks and both must reach the Workflow" + ) + finally: + await handle.terminate() + + +async def test_one_senders_retry_stays_a_single_wake(client: Client) -> None: + """The unparked case still deduplicates a genuine retry. + + The counter distinguishes *attempts*, not sends, so re-sending one attempt + must collapse exactly as a parked wake does -- otherwise the shutdown + sweep's retry would wake the Workflow twice. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker(client, task_queue=task_queue, workflows=[WaitForeverWorkflow]): + handle, first_run = await _start_waiter(client, task_queue) + try: + attempt = request( + namespace=client.namespace, + workflow_id=handle.id, + first_execution_run_id=first_run, + park_generation=0, + sender_identity="worker-a", + wake_counter=7, + ) + await send_wake_signal(client, attempt) + await send_wake_signal(client, attempt) + + assert await _settled_signal_count(handle, 1) == 1 + finally: + await handle.terminate() + + +# --- the wake transport ------------------------------------------------------- + + +def _rpc_error(status: temporalio.service.RPCStatusCode) -> temporalio.service.RPCError: + return temporalio.service.RPCError(status.name, status, b"") + + +UNIMPLEMENTED = temporalio.service.RPCStatusCode.UNIMPLEMENTED +NOT_FOUND = temporalio.service.RPCStatusCode.NOT_FOUND +UNAVAILABLE = temporalio.service.RPCStatusCode.UNAVAILABLE + + +class ChannelServiceClient(RecordingClient): + """A client whose service answers the channel calls, or refuses them.""" + + def __init__( + self, + refuse_channel: temporalio.service.RPCStatusCode | None = None, + listeners: int = 1, + describe: temporalio.service.RPCStatusCode | None = None, + linked: bool = False, + ) -> None: + super().__init__() + self.refuse_channel = refuse_channel + self.listeners = listeners + self.describe_status = describe + #: Whether a describe addressed to a workflow finds its linked channel. + self.linked = linked + self.notified: list = [] + self.described: list = [] + + async def notify_channel(self, request): # type: ignore[no-untyped-def] + self.notified.append(request) + if self.refuse_channel is not None: + raise _rpc_error(self.refuse_channel) + return temporalio.api.workflowservice.v1.NotifyChannelResponse( + listener_count=self.listeners + ) + + async def describe_channel(self, request): # type: ignore[no-untyped-def] + self.described.append(request) + if self.describe_status is not None: + raise _rpc_error(self.describe_status) + if self.linked and request.HasField("execution"): + return temporalio.api.workflowservice.v1.DescribeChannelResponse( + kind=temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_LINKED, + linked_to=temporalio.api.common.v1.Execution( + type=request.execution.type, + business_id=request.execution.business_id, + run_id="run", + ), + ) + return temporalio.api.workflowservice.v1.DescribeChannelResponse() + + +TOKENS_CHANNEL = channel_for(CHAIN.stream_key("tokens")).channel +WORKFLOW_TYPE = temporalio.api.enums.v1.ExecutionType.EXECUTION_TYPE_WORKFLOW + + +def channel_request(**overrides) -> WakeRequest: # type: ignore[no-untyped-def] + return request(channel=TOKENS_CHANNEL, **overrides) + + +@pytest.mark.asyncio +async def test_an_unknown_counter_falls_back_to_the_clock() -> None: + import time + + client = ChannelServiceClient() + before = time.time_ns() + await send_wake(client, channel_request(), transport="channel") # type: ignore[arg-type] + after = time.time_ns() + + assert before <= client.notified[0].notification.counter <= after + + +@pytest.mark.asyncio +async def test_auto_steps_down_to_the_signal_once_and_remembers_the_client() -> None: + client = ChannelServiceClient(refuse_channel=UNIMPLEMENTED) + + first = await send_wake(client, channel_request()) # type: ignore[arg-type] + second = await send_wake(client, channel_request()) # type: ignore[arg-type] + + assert len(client.notified) == 1, "the refusal was not remembered for the client" + assert [signal.request_id for signal in client.sent] == [first, second] + assert first == wake_request_id(channel_request()) + + +@pytest.mark.asyncio +async def test_the_signal_transport_never_tries_the_channel() -> None: + client = ChannelServiceClient() + + await send_wake(client, channel_request(), transport="signal") # type: ignore[arg-type] + + assert client.notified == [] + assert len(client.sent) == 1 + + +@pytest.mark.asyncio +async def test_an_ended_chain_raises_what_a_signal_to_it_raises() -> None: + """``NOT_FOUND`` is the chain being over, under both transports. + + Not a reason to fall back: a Signal addressed by Workflow ID could reach a + later chain that reused the ID, which is the wake the server just refused. + """ + client = ChannelServiceClient(refuse_channel=NOT_FOUND) + + with pytest.raises(temporalio.service.RPCError) as raised: + await send_wake(client, channel_request()) # type: ignore[arg-type] + + assert raised.value.status == NOT_FOUND + assert client.sent == [] + + +@pytest.mark.asyncio +async def test_a_producer_reports_an_ended_chain_as_the_cause_of_its_wake_failure() -> ( + None +): + backend = MemoryStreamBackend() + client = ChannelServiceClient(refuse_channel=NOT_FOUND) + topic = make_producer(backend, client).topic("tokens") + + with pytest.raises(WakeNotAcknowledgedError) as raised: + await topic.publish("hello") + + cause = raised.value.__cause__ + assert isinstance(cause, temporalio.service.RPCError) + assert cause.status == NOT_FOUND + + +class CountingBackend(MemoryStreamBackend): + """Orders offsets the way the Redis provider does, and asks for the channel.""" + + wake_transport = "channel" + + def wake_counter_for(self, offset: Offset) -> int: + ms, seq = offset.token.split("-") + return int(ms) * 2**20 + int(seq) + + +@pytest.mark.asyncio +async def test_a_producer_wakes_with_the_position_it_appended() -> None: + backend = CountingBackend() + client = ChannelServiceClient() + topic = make_producer(backend, client).topic("tokens") + + offset = await topic.publish("hello") + + [notify] = client.notified + assert notify.notification.position == offset.token.encode() + assert notify.notification.counter == backend.wake_counter_for(offset) + assert client.sent == [] + + +@pytest.mark.asyncio +async def test_a_bare_wake_after_a_batch_reports_where_the_batch_ended() -> None: + """Without it the bare wake falls back to the clock and folds behind the store.""" + backend = CountingBackend() + client = ChannelServiceClient() + topic = make_producer(backend, client).topic("tokens") + + await topic.publish("a", wake=False) + last = await topic.publish("b", wake=False) + await topic.wake() + + [notify] = client.notified + assert notify.notification.position == last.token.encode() + assert notify.notification.counter == backend.wake_counter_for(last) + + +@pytest.mark.asyncio +async def test_a_retried_wake_carries_its_original_position() -> None: + backend = CountingBackend() + client = ChannelServiceClient(refuse_channel=UNAVAILABLE) + topic = make_producer(backend, client).topic("tokens") + + with pytest.raises(WakeNotAcknowledgedError) as raised: + await topic.publish("hello") + client.refuse_channel = None + await topic.retry_wake(raised.value.pending) + + first, retried = client.notified + assert retried.notification.position == first.notification.position + assert retried.notification.counter == first.notification.counter + + +class _ReadSubscription(_StubSubscription): + def __init__(self, backend, cursor) -> None: # type: ignore[no-untyped-def] + super().__init__() + self.backend = backend + self.prefetch_cursor = cursor + + +@pytest.mark.asyncio +async def test_a_worker_wakes_with_the_position_it_read() -> None: + class _Backend(CountingBackend): + async def current_park_generation(self, stream_key, wait_id): # type: ignore[no-untyped-def] + return None + + backend = _Backend() + client = ChannelServiceClient() + worker = _worker_sending_wakes("client") + worker._client = client + read = Offset("1700000000000-5") + + await worker._send_external_stream_wake(_ReadSubscription(backend, AFTER(read))) + + [notify] = client.notified + assert notify.notification.position == read.token.encode() + assert notify.notification.counter == backend.wake_counter_for(read) + + +@pytest.mark.asyncio +async def test_a_wake_reports_a_position_appended_outside_the_producer() -> None: + backend = CountingBackend() + client = ChannelServiceClient() + topic = make_producer(backend, client).topic("tokens") + outside = Offset("1700000000000-9") + + await topic.wake(position=outside) + + [notify] = client.notified + assert notify.notification.position == outside.token.encode() + assert notify.notification.counter == backend.wake_counter_for(outside) + + +# --- the notification channel ------------------------------------------------- + + +def test_the_channel_is_the_stream_identity_without_the_namespace() -> None: + key = StreamKey("ns", "wf-1", "first-run-1", "tokens") + + assert channel_for(key).channel == "external-stream/wf-1/first-run-1/input/tokens" + # The server scopes a channel to the namespace of the call. + assert channel_for(dataclasses.replace(key, namespace="other")) == channel_for(key) + + +def test_the_channel_is_linked_to_the_streams_workflow() -> None: + """A stream key names a chain, so its channel is the owner's, by id alone.""" + address = channel_for(StreamKey("ns", "wf-1", "first-run-1", "tokens")) + + # The chain's id without a run, so the current run is the one reached. + assert address.execution == Execution.workflow("wf-1", run_id=None) + assert address.workflow_id == "wf-1" + # A channel without an owner is addressed by name alone. + independent = ChannelAddress(channel="orders", execution=None) + assert independent.execution is None and independent.workflow_id is None + # An owner that is not a workflow has no workflow id to give. + owned = ChannelAddress(channel="orders", execution=Execution.activity("act")) + assert owned.execution is not None and owned.workflow_id is None + + +def test_the_channel_tells_the_two_directions_apart() -> None: + from temporalio.contrib.external_workflow_streams._backend import StreamDirection + + key = StreamKey("ns", "wf-1", "first-run-1", "tokens") + output = dataclasses.replace(key, direction=StreamDirection.OUTPUT) + + assert channel_for(key) != channel_for(output) + + +def test_a_slash_in_a_name_cannot_borrow_another_streams_segments() -> None: + first = StreamKey("ns", "wf/1", "first-run-1", "tokens") + second = StreamKey("ns", "wf", "1/first-run-1", "tokens") + + assert channel_for(first) != channel_for(second) + assert channel_for(first).channel.split("/")[1] == "wf%2F1" + + +def test_the_request_composed_for_a_chain_names_its_channel() -> None: + composed = wake_request_for( + CHAIN, stream_name="tokens", wait_id=1, park_generation=7, sender_identity="" + ) + + assert composed.channel == TOKENS_CHANNEL + assert composed.channel_execution == Execution.workflow("wf-1") + assert composed.channel_address == channel_for(CHAIN.stream_key("tokens")) + + +def test_the_channel_is_not_part_of_the_request_id() -> None: + assert wake_request_id(channel_request()) == wake_request_id(request()) + + +@pytest.mark.asyncio +async def test_auto_notifies_the_channel_before_anything_else() -> None: + client = ChannelServiceClient() + + sent = await send_wake( + client, # type: ignore[arg-type] + channel_request(position=b"1700000000000-3", position_counter=42), + ) + + assert sent == wake_request_id(channel_request()) + assert client.sent == [] + [notify] = client.notified + assert notify.namespace == "ns" + assert notify.notification.channel == TOKENS_CHANNEL + assert notify.notification.position == b"1700000000000-3" + assert notify.notification.counter == 42 + assert notify.identity == "producer-identity" + # The Signal's deduplication carried over: a retry is the same request. + assert notify.request_id == wake_request_id(channel_request()) + + +@pytest.mark.asyncio +async def test_a_channel_with_no_listeners_is_not_a_reason_to_step_down() -> None: + client = ChannelServiceClient(listeners=0) + + await send_wake(client, channel_request()) # type: ignore[arg-type] + await send_wake(client, channel_request()) # type: ignore[arg-type] + + assert len(client.notified) == 2 + assert client.sent == [] + + +@pytest.mark.asyncio +async def test_the_step_down_is_remembered_per_client() -> None: + old = ChannelServiceClient(refuse_channel=UNIMPLEMENTED) + new = ChannelServiceClient() + + await send_wake(old, channel_request()) # type: ignore[arg-type] + await send_wake(new, channel_request()) # type: ignore[arg-type] + + assert len(new.notified) == 1 + assert new.sent == [] + + +@pytest.mark.asyncio +async def test_the_channel_transport_propagates_unimplemented() -> None: + client = ChannelServiceClient(refuse_channel=UNIMPLEMENTED) + + with pytest.raises(temporalio.service.RPCError) as raised: + await send_wake(client, channel_request(), transport="channel") # type: ignore[arg-type] + + assert raised.value.status == UNIMPLEMENTED + assert client.sent == [] + # Asked for explicitly, so nothing is learned from it either. + client.refuse_channel = None + await send_wake(client, channel_request()) # type: ignore[arg-type] + assert len(client.notified) == 2 + assert client.sent == [] + + +@pytest.mark.asyncio +async def test_the_channel_transport_needs_a_channel() -> None: + client = ChannelServiceClient() + + with pytest.raises(ValueError, match="channel"): + await send_wake(client, request(), transport="channel") # type: ignore[arg-type] + + # Under "auto" a request without a channel goes by Signal and learns nothing. + await send_wake(client, request()) # type: ignore[arg-type] + assert client.notified == [] + assert len(client.sent) == 1 + await send_wake(client, channel_request()) # type: ignore[arg-type] + assert len(client.notified) == 1 + + +@pytest.mark.asyncio +async def test_a_client_without_the_notify_call_steps_down_under_auto() -> None: + client = RecordingClient() + + await send_wake(client, channel_request()) # type: ignore[arg-type] + + assert len(client.sent) == 1 + with pytest.raises(RuntimeError, match="notify_channel"): + await send_wake( + RecordingClient(), # type: ignore[arg-type] + channel_request(), + transport="channel", + ) + + +@pytest.mark.asyncio +async def test_the_probe_says_yes_for_a_server_with_channels() -> None: + client = ChannelServiceClient() + + assert await server_has_channels(client) is True # type: ignore[arg-type] + + [described] = client.described + assert described.namespace == "ns" + assert described.channel == PROBE_CHANNEL + # An unknown channel is an answer from a server that knows the call. + unknown = ChannelServiceClient(describe=NOT_FOUND) + assert await server_has_channels(unknown) is True # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_the_probe_remembers_a_server_without_channels() -> None: + client = ChannelServiceClient(describe=UNIMPLEMENTED) + + assert await server_has_channels(client) is False # type: ignore[arg-type] + + await send_wake(client, channel_request()) # type: ignore[arg-type] + assert client.notified == [], "the probe's answer was not shared with the writers" + assert len(client.sent) == 1 + assert await server_has_channels(client) is False # type: ignore[arg-type] + assert len(client.described) == 1, "a remembered answer was asked for again" + + +@pytest.mark.asyncio +async def test_the_probe_leaves_an_unanswered_question_open() -> None: + client = ChannelServiceClient(describe=UNAVAILABLE) + + assert await server_has_channels(client) is None # type: ignore[arg-type] + + client.describe_status = None + assert await server_has_channels(client) is True # type: ignore[arg-type] + # A client whose service lacks the call has the answer in its shape. + assert await server_has_channels(RecordingClient()) is False # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_the_probe_finds_the_linked_kind_on_a_running_workflow() -> None: + """A running workflow's linked channel exists by construction, so describing + the probe channel as that workflow's is answered, kind and all.""" + client = ChannelServiceClient(linked=True) + + assert await channel_support(client, "wf-1") is ChannelSupport.LINKED # type: ignore[arg-type] + + [described] = client.described + assert described.namespace == "ns" + assert described.channel == PROBE_CHANNEL + assert described.execution.type == WORKFLOW_TYPE + assert described.execution.business_id == "wf-1" + assert described.execution.run_id == "" + + +@pytest.mark.asyncio +async def test_a_server_with_only_independent_channels_answers_not_found() -> None: + # The owner is ignored there, and the probe channel has never been notified. + client = ChannelServiceClient(describe=NOT_FOUND) + + assert await channel_support(client, "wf-1") is ChannelSupport.INDEPENDENT # type: ignore[arg-type] + # A describe answered without a kind is from a server that predates them. + assert ( + await channel_support(ChannelServiceClient(), "wf-1") # type: ignore[arg-type] + is ChannelSupport.INDEPENDENT + ) + + +@pytest.mark.asyncio +async def test_a_server_without_channels_is_remembered_by_the_three_way_probe() -> None: + client = ChannelServiceClient(describe=UNIMPLEMENTED) + + assert await channel_support(client, "wf-1") is ChannelSupport.NONE # type: ignore[arg-type] + + await send_wake(client, channel_request()) # type: ignore[arg-type] + assert client.notified == [], "the probe's answer was not shared with the writers" + assert len(client.sent) == 1 + assert await channel_support(client, "wf-1") is ChannelSupport.NONE # type: ignore[arg-type] + assert len(client.described) == 1, "a remembered answer was asked for again" + # A client whose service lacks the call has the answer in its shape. + assert await channel_support(RecordingClient(), "wf-1") is ChannelSupport.NONE # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_the_three_way_probe_leaves_an_unanswered_question_open() -> None: + client = ChannelServiceClient(describe=UNAVAILABLE, linked=True) + + assert await channel_support(client, "wf-1") is None # type: ignore[arg-type] + + client.describe_status = None + assert await channel_support(client, "wf-1") is ChannelSupport.LINKED # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_a_producer_notifies_the_channel_of_the_stream_it_appended_to() -> None: + backend = MemoryStreamBackend() + client = ChannelServiceClient() + topic = make_producer(backend, client).topic("tokens") + + await topic.publish("hello") + + [notify] = client.notified + assert notify.notification.channel == channel_for(topic.stream_key).channel + # The owner rides along, so a server with linked channels delivers to it. + assert notify.execution.type == WORKFLOW_TYPE + assert notify.execution.business_id == "wf-1" + assert notify.execution.run_id == "" + assert client.sent == [] + + +class ClockRuleBackend(CountingBackend): + """States the Redis rule for a wake without a position: now, maximal sequence.""" + + def wake_counter_now(self) -> int: + return 1_700_000_000_000 * 2**20 + 0xFFFFF + + +def test_a_wake_without_a_position_takes_the_backends_clock_rule() -> None: + from temporalio.contrib.external_workflow_streams._wake import wake_position + + assert wake_position(ClockRuleBackend(), None) == ( + b"", + 1_700_000_000_000 * 2**20 + 0xFFFFF, + ) + # The base rule is the clock in nanoseconds, which orders like the base counter. + position, counter = wake_position(MemoryStreamBackend(), None) + assert position == b"" + assert counter > 1_700_000_000 * 10**9 + # A positioned wake keeps the position's own order. + assert wake_position(ClockRuleBackend(), Offset("1700000000000-3")) == ( + b"1700000000000-3", + 1_700_000_000_000 * 2**20 + 3, + ) diff --git a/tests/contrib/external_workflow_streams/test_worker_crash.py b/tests/contrib/external_workflow_streams/test_worker_crash.py new file mode 100644 index 0000000000..5987318e64 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_worker_crash.py @@ -0,0 +1,622 @@ +"""A Worker that dies before its marker commits, and the one that takes over. + +"Marker recording commits the cursor. Reading or delivering a record does not." +The unit tests assert the *consequence* of that inside one manager -- a reset +cursor re-delivers -- but the property is about two Workers and a Workflow Task +that never completed, and nothing in one process can show that. + +So the consuming Worker here runs in a child process and is sent ``SIGKILL`` +while it holds a retained Workflow Task with records consumed on it. No +shutdown, no finalization, no marker. The server times the task out, a second +Worker picks up the retry, and the question is whether it resumes from the last +*committed* cursor -- re-reading everything consumed after it, and nothing +before it. +""" + +from __future__ import annotations + +import asyncio +import sys +import uuid +from datetime import timedelta +from pathlib import Path +from typing import Any + +import pytest + +from temporalio.client import Client, WorkflowUpdateRPCTimeoutOrCancelledError +from temporalio.contrib.external_workflow_streams import ( + ExternalOutputStreamClient, + WorkflowChainKey, +) +from temporalio.contrib.external_workflow_streams._backend import ( + StreamDirection, + StreamKey, +) +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputStageStatus, +) +from temporalio.contrib.external_workflow_streams._record import BEGINNING +from temporalio.contrib.external_workflow_streams._redis import RedisStreamBackend +from temporalio.worker import Worker +from tests.contrib.external_workflow_streams.conftest import KEY_NAMESPACE, redis_url +from tests.contrib.external_workflow_streams.crash_worker import ( + STREAM_NAME, + CrashConsumeWorkflow, + CrashPublishWorkflow, + CrashUpdatePublishWorkflow, + output_stage_log_key, + read_log_key, + recording_backend, +) +from tests.contrib.external_workflow_streams.test_worker_integration import publish + +TASK_TIMEOUT_SECONDS = 6 +"""Short, because the crash is only visible once the server gives up on the +retained task, and that wait is this test's floor.""" + +FEED_GAP_SECONDS = 0.3 +"""Well under the idle timeout, so the retained task is never released. + +An idle park would complete the task and *commit* the records fed here, which is +exactly the state this test needs not to reach. +""" + +UNCOMMITTED = ["alpha", "beta", "gamma"] +"""Consumed on the retained task the Worker dies holding. Never committed.""" + +LATER = ["delta"] +"""Published to the replacement Worker, so the Run demonstrably continues.""" + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +async def _reads(redis_client: Any, prefix: str, reader: str) -> list[str]: + raw = await redis_client.lrange(read_log_key(prefix, reader), 0, -1) + return [value.decode() if isinstance(value, bytes) else value for value in raw] + + +async def _history(handle: Any) -> list[Any]: + return [e async for e in handle.fetch_history_events()] + + +def _markers(events: list[Any]) -> list[Any]: + return [e for e in events if e.HasField("marker_recorded_event_attributes")] + + +async def _wait_until(predicate: Any, timeout: float, message: str) -> None: + """Polls rather than sleeping a fixed time, so a slow start is not a flake.""" + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if await predicate(): + return + await asyncio.sleep(0.2) + raise AssertionError(message) + + +async def _spawn_crash_worker( + client: Client, + *, + task_queue: str, + prefix: str, + mode: str, +) -> asyncio.subprocess.Process: + return await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "tests.contrib.external_workflow_streams.crash_worker", + "--address", + client.service_client.config.target_host, + "--namespace", + client.namespace, + "--task-queue", + task_queue, + "--redis-url", + redis_url(), + "--prefix", + prefix, + "--mode", + mode, + cwd=str(REPO_ROOT), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + + +def _output_marker_tokens(events: list[Any]) -> list[str]: + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + + tokens: list[str] = [] + for event in events: + if not event.HasField("marker_recorded_event_attributes"): + continue + attributes = event.marker_recorded_event_attributes + if attributes.marker_name != "core_external_stream": + continue + payloads = attributes.details["external_stream"].payloads + if len(payloads) != 1: + continue + marker = ExternalStreamMarkerData() + marker.ParseFromString(payloads[0].data) + if marker.HasField("output"): + tokens.append(marker.output.stage_token) + return tokens + + +async def _output_stage_was_logged(redis_client: Any, prefix: str) -> bool: + return bool(await redis_client.llen(output_stage_log_key(prefix))) + + +@pytest.mark.timeout(240) +async def test_a_crash_before_the_marker_makes_the_next_worker_re_read( + client: Client, redis_worker_id: str +) -> None: + """The durability boundary, with a real dead Worker on one side of it. + + Three things have to hold together, and each fails differently: + + - **Nothing was committed.** The records were consumed on a Workflow Task + that was retained -- so its annotation was still accumulating, unwritten -- + and that task died with its Worker. Asserted against History, because a + marker written before the kill would make the rest of this vacuous. + - **The same offsets are read again.** The replacement Worker's read log has + to contain them. Comparing read logs rather than observing that records + eventually arrived is the difference between proving a re-read and proving + that Redis still had the data. + - **Nothing is lost and nothing is doubled.** The Workflow returns its + records in order, so a record delivered twice or dropped is visible in the + result rather than only in a cursor. + """ + pytest.importorskip("redis.asyncio", reason="redis is not installed") + + prefix = f"{KEY_NAMESPACE}:{redis_worker_id}:{uuid.uuid4().hex}" + publisher = recording_backend(url=redis_url(), prefix=prefix, reader="publisher") + try: + await publisher._client.ping() + except Exception as err: + await publisher.aclose() + pytest.skip(f"Redis is not reachable at {redis_url()}: {err}") + + task_queue = f"tq-{uuid.uuid4()}" + successor = recording_backend(url=redis_url(), prefix=prefix, reader="successor") + child = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "tests.contrib.external_workflow_streams.crash_worker", + "--address", + client.service_client.config.target_host, + "--namespace", + client.namespace, + "--task-queue", + task_queue, + "--redis-url", + redis_url(), + "--prefix", + prefix, + cwd=str(REPO_ROOT), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + handle = None + try: + expected = len(UNCOMMITTED) + len(LATER) + handle = await client.start_workflow( + CrashConsumeWorkflow.run, + expected, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=timedelta(seconds=TASK_TIMEOUT_SECONDS), + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + STREAM_NAME, + ) + + # The child has to be the one holding the task, or the records would + # simply be waiting for the successor and nothing would be read twice. + async def child_took_the_task() -> bool: + return any( + e.HasField("workflow_task_started_event_attributes") + for e in await _history(handle) + ) + + await _wait_until( + child_took_the_task, 60, "the child Worker never picked the task up" + ) + + # One producer session, one continuous sequence: `(session_id, + # sequence)` is the append idempotency key, so restarting the numbering + # would re-use a key with different content, the backend would reject + # it, and the Workflow would appear to hang for reasons far from here. + session = f"crash-{uuid.uuid4()}" + for value in UNCOMMITTED: + await publish(publisher, key, [value], session=session) + await asyncio.sleep(FEED_GAP_SECONDS) + + async def child_read_them() -> bool: + read = await _reads(publisher._client, prefix, "crashed") + return len(read) >= len(UNCOMMITTED) + + await _wait_until( + child_read_them, + 30, + "the child Worker never read the records it is supposed to die " + "holding, so there is nothing for the successor to re-read", + ) + consumed = await publisher.read_after( + key, BEGINNING, max_records=100, block=None + ) + uncommitted_offsets = [str(r.offset) for r in consumed] + + # The kill. Nothing runs: no finalization, no marker, no wake. + child.kill() + await child.wait() + + assert not _markers(await _history(handle)), ( + "a marker was written before the crash, so the records the Worker " + "died holding were committed after all and the re-read below would " + "prove nothing -- the Workflow Task must still have been retained" + ) + + async with Worker( + client, + task_queue=task_queue, + workflows=[CrashConsumeWorkflow], + external_stream_backend=successor, + ): + await _wait_until( + lambda: _read_anything(publisher._client, prefix, "successor"), + 120, + "the replacement Worker never read the stream at all", + ) + await publish(publisher, key, LATER, session=session) + result = await asyncio.wait_for(handle.result(), 120) + + successor_reads = await _reads(publisher._client, prefix, "successor") + assert set(uncommitted_offsets) <= set(successor_reads), ( + "the replacement Worker did not re-read the offsets the dead one " + "had already taken -- an uncommitted consumption was treated as " + f"progress. re-read: {successor_reads}, owed: {uncommitted_offsets}" + ) + assert result == UNCOMMITTED + LATER, ( + "the Run lost or repeated a record across the crash. A marker " + "commits the cursor and only the first record's did, so everything " + f"after it was owed again and exactly once, got {result}" + ) + + assert any( + e.HasField("workflow_task_timed_out_event_attributes") + for e in await _history(handle) + ), ( + "no Workflow Task timed out, so the retained task was handed over " + "rather than dying with its Worker -- which is not the case under " + "test" + ) + finally: + if child.returncode is None: + child.kill() + await child.wait() + if handle is not None: + try: + await handle.terminate() + except Exception: + pass + keys = [k async for k in publisher._client.scan_iter(match=f"{prefix}*")] + if keys: + await publisher._client.delete(*keys) + await publisher.aclose() + await successor.aclose() + + +@pytest.mark.timeout(240) +async def test_real_crash_after_output_stage_aborts_old_token_and_exposes_one_retry( + client: Client, redis_worker_id: str +) -> None: + """SIGKILL after Redis staging leaves no phantom output before retry. + + The child provider blocks only after the pending stage is atomically + durable. Killing that process therefore prevents both the activation report + and graceful reconciliation. A successor retries the Workflow Task with a + fresh token; the reader proves the dead attempt absent from exact-run + History, aborts it, and exposes only the accepted attempt. + """ + pytest.importorskip("redis.asyncio", reason="redis is not installed") + + prefix = f"{KEY_NAMESPACE}:{redis_worker_id}:{uuid.uuid4().hex}" + backend = RedisStreamBackend(url=redis_url(), key_prefix=prefix) + try: + await backend._client.ping() + except Exception as err: + await backend.aclose() + pytest.skip(f"Redis is not reachable at {redis_url()}: {err}") + + task_queue = f"tq-{uuid.uuid4()}" + child = await _spawn_crash_worker( + client, task_queue=task_queue, prefix=prefix, mode="publish" + ) + handle = None + try: + handle = await client.start_workflow( + CrashPublishWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=timedelta(seconds=TASK_TIMEOUT_SECONDS), + ) + await _wait_until( + lambda: _output_stage_was_logged(backend._client, prefix), + 60, + "the child never durably staged Workflow output", + ) + + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + ) + key = chain.stream_key("events", direction=StreamDirection.OUTPUT) + staged_read = await backend.read_output_after( + key, BEGINNING, max_records=100, block=timedelta(0) + ) + assert staged_read.records == () + assert staged_read.pending is not None + abandoned_manifest = staged_read.pending.manifest + abandoned = await backend.output_stage(abandoned_manifest) + assert abandoned is not None + assert abandoned.status is OutputStageStatus.PENDING + assert abandoned.records, "the pending stage is not physically durable" + + before_kill = await _history(handle) + assert _output_marker_tokens(before_kill) == [] + assert not any( + event.HasField("workflow_task_completed_event_attributes") + for event in before_kill + ), "the child reported the producing Workflow Task before it was killed" + + child.kill() + await child.wait() + + async with Worker( + client, + task_queue=task_queue, + workflows=[CrashPublishWorkflow], + external_stream_backend=backend, + ): + assert await asyncio.wait_for(handle.result(), 120) == "done" + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + visible = [ + item.data + async for item in output_client.topic("events", type=str).subscribe() + ] + + assert visible == ["accepted-after-crash"] + abandoned = await backend.output_stage(abandoned_manifest) + assert abandoned is not None + assert abandoned.status is OutputStageStatus.ABORTED + + events = await _history(handle) + accepted_tokens = _output_marker_tokens(events) + assert len(accepted_tokens) == 1 + assert accepted_tokens[0] != abandoned_manifest.stage_token + assert any( + event.HasField("workflow_task_timed_out_event_attributes") + for event in events + ) + + statuses = await backend._client.hvals(backend._output_status_key(key)) + decoded_statuses = sorted( + value.decode() if isinstance(value, bytes) else value for value in statuses + ) + assert decoded_statuses == ["aborted", "committed"] + finally: + if child.returncode is None: + child.kill() + await child.wait() + if handle is not None: + try: + await handle.terminate() + except Exception: + pass + keys = [key async for key in backend._client.scan_iter(match=f"{prefix}*")] + if keys: + await backend._client.delete(*keys) + await backend.aclose() + + +@pytest.mark.timeout(240) +async def test_disconnected_update_leaves_live_staged_output_undecided_until_timeout( + client: Client, + redis_worker_id: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Cancelling the Update caller cannot decide its staged output. + + This uses a real speculative Update and a child Worker stopped between durable + Redis staging and its server report. A live output reader repeatedly checks + exact-run History after the caller disconnects and must leave the barrier + pending. Only the dead Worker's Workflow Task timeout is a deciding durable + boundary, at which point the reader may abort the abandoned stage. + + Retention loss itself is covered separately with deterministic fake History; + the local test server cannot force expiry of one exact Run on demand. + """ + pytest.importorskip("redis.asyncio", reason="redis is not installed") + + prefix = f"{KEY_NAMESPACE}:{redis_worker_id}:{uuid.uuid4().hex}" + backend = RedisStreamBackend(url=redis_url(), key_prefix=prefix) + try: + await backend._client.ping() + except Exception as err: + await backend.aclose() + pytest.skip(f"Redis is not reachable at {redis_url()}: {err}") + + task_queue = f"tq-{uuid.uuid4()}" + child = await _spawn_crash_worker( + client, task_queue=task_queue, prefix=prefix, mode="update" + ) + handle = None + update_task: asyncio.Task[str] | None = None + read_task: asyncio.Task[Any] | None = None + try: + handle = await client.start_workflow( + CrashUpdatePublishWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=timedelta(seconds=TASK_TIMEOUT_SECONDS), + ) + + async def initial_task_completed() -> bool: + return any( + event.HasField("workflow_task_completed_event_attributes") + for event in await _history(handle) + ) + + await _wait_until( + initial_task_completed, + 60, + "the child never completed the Workflow's initial waiting task", + ) + description = await handle.describe() + chain = WorkflowChainKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + ) + key = chain.stream_key("events", direction=StreamDirection.OUTPUT) + + update_task = asyncio.create_task( + handle.execute_update( + CrashUpdatePublishWorkflow.publish, + "unreported-update-output", + ) + ) + await _wait_until( + lambda: _output_stage_was_logged(backend._client, prefix), + 30, + "the speculative Update handler never durably staged its output", + ) + staged_read = await backend.read_output_after( + key, BEGINNING, max_records=100, block=timedelta(0) + ) + assert staged_read.records == () + assert staged_read.pending is not None + pending = staged_read.pending + manifest = pending.manifest + + output_client = await ExternalOutputStreamClient.connect( + backend=backend, + workflow=chain, + client=client, + ) + topic = output_client.topic("events", type=str) + original_history_decision = topic._history_decision + reconciliation_attempts = 0 + + async def count_live_history_decisions(manifest_to_check: Any) -> Any: + nonlocal reconciliation_attempts + reconciliation_attempts += 1 + return await original_history_decision(manifest_to_check) + + monkeypatch.setattr(topic, "_history_decision", count_live_history_decisions) + + async def reconcile_pending_stage() -> None: + while not await topic._resolve_pending(pending): + await asyncio.sleep(1) + + read_task = asyncio.create_task(reconcile_pending_stage()) + + # This is the caller disconnect. It cancels only its wait for the Update + # result; the server-side speculative Update and its Workflow Task survive. + update_task.cancel() + with pytest.raises(WorkflowUpdateRPCTimeoutOrCancelledError): + await update_task + + async def reader_retried_history() -> bool: + return reconciliation_attempts >= 3 + + await _wait_until( + reader_retried_history, + TASK_TIMEOUT_SECONDS - 1, + "the live reader did not retry ambiguous exact-run History", + ) + undecided = await backend.output_stage(manifest) + assert undecided is not None + assert undecided.status is OutputStageStatus.PENDING + assert not read_task.done() + + events_before_loss = await _history(handle) + relevant = [ + event + for event in events_before_loss + if event.event_id > manifest.history_floor_event_id + ] + # This server delivered the Update speculatively: the handler ran (the + # stage above is durable proof of that), but admission itself cannot + # enter History until the withheld Workflow Task report is accepted. + assert not any( + event.HasField("workflow_execution_update_admitted_event_attributes") + for event in relevant + ) + assert _output_marker_tokens(relevant) == [] + assert not any( + event.HasField(field) + for event in relevant + for field in ( + "workflow_task_completed_event_attributes", + "workflow_task_failed_event_attributes", + "workflow_task_timed_out_event_attributes", + ) + ) + + child.kill() + await child.wait() + + async def abandoned_stage_was_aborted() -> bool: + stage = await backend.output_stage(manifest) + return stage is not None and stage.status is OutputStageStatus.ABORTED + + await _wait_until( + abandoned_stage_was_aborted, + 30, + "the dead Update task timed out but its pending output was not aborted", + ) + await asyncio.wait_for(read_task, 5) + events_after_loss = await _history(handle) + assert any( + event.event_id > manifest.history_floor_event_id + and event.HasField("workflow_task_timed_out_event_attributes") + for event in events_after_loss + ) + assert _output_marker_tokens(events_after_loss) == [] + finally: + if update_task is not None and not update_task.done(): + update_task.cancel() + if read_task is not None and not read_task.done(): + read_task.cancel() + try: + await read_task + except asyncio.CancelledError: + pass + if child.returncode is None: + child.kill() + await child.wait() + if handle is not None: + try: + await handle.terminate() + except Exception: + pass + keys = [key async for key in backend._client.scan_iter(match=f"{prefix}*")] + if keys: + await backend._client.delete(*keys) + await backend.aclose() + + +async def _read_anything(redis_client: Any, prefix: str, reader: str) -> bool: + return bool(await _reads(redis_client, prefix, reader)) diff --git a/tests/contrib/external_workflow_streams/test_worker_handoff.py b/tests/contrib/external_workflow_streams/test_worker_handoff.py new file mode 100644 index 0000000000..24294fc8b1 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_worker_handoff.py @@ -0,0 +1,958 @@ +"""Handing a Run to another Worker: the shutdown sweep, and a failed finalization. + +Two transitions that only exist between Workers, so neither can be shown inside +one: + +- **Shutdown with no open Workflow Task.** Nothing local can create + server-visible work, so the manager sweeps: it asks Core what state the Run is + in through the read-only probe, sends the reserved wake Signal, and waits for + the server to acknowledge it before the watchers go away. The point of the + Signal is that *another* Worker picks the Run up. +- **A finalization that cannot be answered.** "A marker is never written without + its terminal": if the Run's state is gone when Core asks for the terminal, + Python fails the activation, Core writes no marker, and the retry replays from + the previous marker -- losing no record and moving no cursor. +""" + +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.client import Client +from temporalio.contrib.external_workflow_streams._annotation import decode_annotation +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._manager import ( + ReadinessResult, + StreamSubscriptionManager, +) +from temporalio.contrib.external_workflow_streams._record import ( + AFTER, + BEGINNING, + Cursor, +) +from temporalio.worker import Worker +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend +from tests.contrib.external_workflow_streams.test_worker_integration import publish + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams._api import external_stream + +WAKE_SIGNAL_NAME = "__temporal_external_stream_wake" + +MARKER_DETAILS_KEY = "external_stream" +"""Where Core puts the marker's ``ExternalStreamMarkerData``.""" + +RETENTION_IDLE_TIMEOUT = timedelta(seconds=2) +"""How long a fed Workflow Task stays retained here -- stated, not inherited. + +Case 30 needs a Workflow Task that is *open* at the moment Core asks for the +terminal, and everything the test does between the last record reaching the +Workflow and the shutdown call is bookkeeping read out of local state. The +one-second default makes that bookkeeping share a window with the idle timer, +which is not what this case is about. + +Not larger, because the same timeout also bounds how long the Workflow waits for +its *first* record: a record appended before the subscription's watcher is +running is picked up by the idle timer's park recheck rather than by the +watcher, so raising this raises the test's own floor. Bounded above by the +Workflow Task timeout the test starts the Workflow with in any case -- a task +retained past that does not park, it fails (`wft-lifecycle.md`). +""" + + +# These tests exercise cross-Worker handoff, not the sandbox. Keeping their +# fixture Workflows unsandboxed avoids re-importing this pytest +# assertion-rewritten module while each Worker validates its registrations. +@workflow.defn(sandboxed=False) +class TimerThenStreamWorkflow: + """Consumes one record, takes a timer, then consumes the rest. + + The timer is what puts the Run in the two states this module needs. Its + completion is server-bound, so retention is suppressed: the marker commits + there, and the Run is left cached with **no open Workflow Task** and a live + subscription. Once the timer fires, the Workflow blocks on the stream again + and the next task is retained. + + The marker and the retained task are therefore separated by the *whole* + timer: the marker is written when the timer is **started**, and the retained + task exists only once it has **fired** and its replacement task has reached + this Worker. A test that treats the marker as the signal to start feeding is + reading a clock, not a state -- see the caller. + """ + + @workflow.run + async def run(self, expected: int) -> list[str]: + tokens = external_stream.with_options( + idle_timeout=RETENTION_IDLE_TIMEOUT + ).topic("tokens", type=str) + seen: list[str] = [] + iterator = tokens.subscribe().__aiter__() + seen.append(await iterator.__anext__()) + await asyncio.sleep(0.5) + while len(seen) < expected: + seen.append(await iterator.__anext__()) + return seen + + +@workflow.defn(sandboxed=False) +class LeftWithNoOpenTaskWorkflow: + """Blocks on the stream first, then starts a long timer. The order is the point. + + The ``NoOpenWorkflowTask`` window needs two things at once -- a wait set Core + knows about, and no Workflow Task holding it -- and only this order produces + both: + + 1. **Block before doing anything else.** A completion that carries no command + sends ``WorkflowStreamQuiescent``, and that command is the *only* thing + that registers a wait set with Core. A Workflow that starts a timer on the + same completion it first blocks on never registers one at all: retention + is suppressed, so no quiescent snapshot is ever sent, Core's wait set stays + empty, and a wake Signal then marks nothing ready and creates a Workflow + Task that Core completes with no activation. Such a Run cannot be resumed + by anything -- which is what this case used to try to shut down. + 2. **Start the timer only after a record has woken it.** That command + suppresses retention for *that* completion, so the Workflow Task ends with + the wait set still registered and the Run sits in the window for as long as + the test needs. The timer is also the "unrelated Workflow event" the + shutdown sweep must not wait for. + """ + + @workflow.run + async def run(self, expected: int) -> list[str]: + tokens = external_stream.topic("tokens", type=str) + seen: list[str] = [] + iterator = tokens.subscribe().__aiter__() + seen.append(await iterator.__anext__()) + timer = asyncio.ensure_future(asyncio.sleep(600)) + while len(seen) < expected: + seen.append(await iterator.__anext__()) + timer.cancel() + return seen + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + backend = MemoryStreamBackend() + # These cases read the reserved Signal's envelope, so the transport is + # pinned to it rather than left to step down from a server that has more. + backend.wake_transport = "signal" + return backend + + +async def history(handle: Any) -> list[Any]: + return [e async for e in handle.fetch_history_events()] + + +def markers(events: list[Any]) -> list[Any]: + return [e for e in events if e.HasField("marker_recorded_event_attributes")] + + +def wake_signals(events: list[Any]) -> list[Any]: + return [ + e + for e in events + if e.HasField("workflow_execution_signaled_event_attributes") + and e.workflow_execution_signaled_event_attributes.signal_name + == WAKE_SIGNAL_NAME + ] + + +def committed_boundary(marker_event: Any, wait_id: int) -> Cursor: + """Where a marker says one wait's consumption had reached. + + "The cursor did not move" is a statement about the marker; anything derived + from what was published instead would be a statement about timing. + + The annotation's terminal, when it has one. A marker written on a + command-producing completion currently carries no terminal frame at all -- + ``WorkflowStreamProgress`` emits segments only, and ``add_terminal()`` runs + only on the park and finalization paths -- so the end of the last recorded + run is used instead. It names the same position the terminal would. + """ + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + + details = marker_event.marker_recorded_event_attributes.details + data = ExternalStreamMarkerData() + data.ParseFromString(details[MARKER_DETAILS_KEY].payloads[0].data) + annotation = decode_annotation(data.replay_annotation) + if annotation.terminal is not None: + return annotation.terminal[wait_id] + reached = annotation.header.streams[wait_id].start_cursor + for segment in annotation.segments: + for run in segment.runs: + if run.wait_id == wait_id: + reached = AFTER(run.last_offset) + return reached + + +def marker_bytes(marker_event: Any) -> bytes: + return ( + marker_event.marker_recorded_event_attributes.details[MARKER_DETAILS_KEY] + .payloads[0] + .data + ) + + +def wake_envelope(signal_event: Any) -> Any: + """The wake Signal's own envelope, read the way Core reads it. + + Deliberately not through the ``DataConverter``: the envelope is defined at + the protocol level precisely so Core -- which has no converter -- can read + it. + """ + from temporalio.bridge.proto.external_stream import WakeSignal + + payload = signal_event.workflow_execution_signaled_event_attributes.input.payloads[ + 0 + ] + envelope = WakeSignal() + envelope.ParseFromString(payload.data) + return envelope + + +async def stream_key_for(client: Client, handle: Any) -> StreamKey: + description = await handle.describe() + return StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + + +async def wait_until( + predicate: Any, timeout: float, message: str, interval: float = 0.2 +) -> None: + """Polls rather than sleeping a fixed time, so a slow start is not a flake. + + ``interval`` is worth lowering when the condition is read out of local + process state rather than fetched from the server, and when what happens + *after* it is satisfied is itself time-bounded: the poll gap is then part of + the window the caller has to act in, not merely latency. + """ + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if await predicate(): + return + await asyncio.sleep(interval) + raise AssertionError(message) + + +async def close_worker(worker: Worker, task: asyncio.Task[Any]) -> None: + """Request orderly shutdown and observe the Worker's terminal task state.""" + if not task.done(): + await asyncio.wait_for(worker.shutdown(), 60) + await asyncio.gather(task, return_exceptions=True) + + +def workflow_worker(worker: Worker) -> Any: + """The private workflow worker these white-box integration tests require.""" + assert worker._workflow_worker is not None + return worker._workflow_worker + + +# --- case 29: shutdown in the NoOpenWorkflowTask window ----------------------- + + +@pytest.mark.timeout(180) +async def test_shutdown_with_no_open_task_hands_the_run_to_another_worker( + client: Client, backend: MemoryStreamBackend +) -> None: + """The window the sweep exists for, with the second Worker that proves it. + + The Run is cached, has a wait set Core knows about, and holds no Workflow + Task. **Nothing else will ever create work for it**: the only other thing + that could produce a Workflow Task here is the Workflow's own 600-second + timer, and waiting for an unrelated event is not a plan. That is why the + sweep exists, and why it runs whether or not a record happens to be waiting + -- the obligation is to hand the Run over, not to announce an append. + + The window is *asserted*, not assumed: the read-only probe is called on the + live Worker before shutdown and must answer ``NoOpenWorkflowTask``. Getting + there needs both of the Workflow's steps, which is what + :class:`LeftWithNoOpenTaskWorkflow` documents. + + Four separate claims, none of which the others imply: + + - no marker is written, because nothing was accumulated to write; + - the Run's state is resolved with the read-only probe rather than a + readiness notification, which would assert a buffered record that the + sweep has no business claiming; + - a **new** wake Signal reaches the Workflow's own History, and it is an + *unparked* one -- ``park_generation = 0``. Counted against the Signals + already there rather than merely found, because an earlier wake in the + same History proves nothing about this one; + - a second Worker takes the resulting Workflow Task, reconstructs the + subscription from the marker, and delivers a record published after the + handover -- without the timer ever firing. + """ + task_queue = f"tq-{uuid.uuid4()}" + session = f"handoff-{uuid.uuid4()}" + worker_a = Worker( + client, + task_queue=task_queue, + workflows=[LeftWithNoOpenTaskWorkflow], + external_stream_backend=backend, + ) + worker_a_task = asyncio.create_task(worker_a.run()) + handle = None + try: + handle = await client.start_workflow( + LeftWithNoOpenTaskWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=timedelta(seconds=10), + ) + key = await stream_key_for(client, handle) + + # Published only once the Run's first Workflow Task has ended, which is + # what registers the wait set with Core. A record that arrives before + # that lands on a Run whose wait set is still empty. + await wait_until( + lambda: _has_markers(handle), + 60, + "the Run's first Workflow Task never ended, so its wait set was " + "never registered with Core and no wake could reach it", + ) + await publish(backend, key, ["alpha"], session=session) + await wait_until( + lambda: _timer_started(handle), + 60, + "the record never reached the Workflow, so the completion that " + "leaves the Run in the no-open-task window never happened", + ) + + manager = workflow_worker(worker_a)._external_stream_manager + assert manager is not None, "the manager should exist by now" + real_probe, real_ready = manager._run_status, manager._notify_ready + await wait_until( + lambda: _run_is_registered(manager), + 30, + "the manager holds no Run with subscriptions, so the sweep has " + "nothing to sweep", + ) + run_id = next(iter(manager._runs)) + await wait_until( + lambda: _in_the_window(real_probe, run_id), + 30, + "the Run never reached the no-open-Workflow-Task window, so this " + "case is not testing the transition it names", + ) + + before = markers(await history(handle)) + signals_before = wake_signals(await history(handle)) + + probes: list[str] = [] + readiness: list[str] = [] + + async def counting_probe(probed_run_id: str) -> Any: + probes.append(probed_run_id) + return await real_probe(probed_run_id) + + async def counting_ready( + ready_run_id: str, wait_id: int, generation: int + ) -> Any: + readiness.append(ready_run_id) + return await real_ready(ready_run_id, wait_id, generation) + + manager._run_status = counting_probe + manager._notify_ready = counting_ready + + await asyncio.wait_for(worker_a.shutdown(), 60) + + assert probes, ( + "the sweep never asked Core what state the Run was in; without the " + "read-only probe it cannot know whether a wake is owed at all" + ) + assert not readiness, ( + "the sweep used the readiness call as a probe. Readiness means 'a " + "record is buffered', so using it here asserts something the sweep " + "has no business claiming and manufactures a Workflow Task on the " + "way out" + ) + after = markers(await history(handle)) + assert len(after) == len(before), ( + "a marker was written on the way out of the no-open-task window; " + "nothing was accumulated there, so there was nothing to write" + ) + signals_after = wake_signals(await history(handle)) + assert len(signals_after) > len(signals_before), ( + "the sweep's wake Signal never reached the Workflow's own History, " + f"which still holds the same {len(signals_before)} it did before " + "shutdown. Nothing will ever create a Workflow Task for this Run" + ) + envelope = wake_envelope(signals_after[-1]) + assert envelope.park_generation == 0, ( + "the shutdown wake must be an unparked one -- there is no confirmed " + f"park to name -- got generation {envelope.park_generation}" + ) + + async with Worker( + client, + task_queue=task_queue, + workflows=[LeftWithNoOpenTaskWorkflow], + external_stream_backend=backend, + ): + # Published after the handover, so delivering it proves the second + # Worker rebuilt a live subscription rather than replaying one. + await publish(backend, key, ["beta"], session=session) + result = await asyncio.wait_for(handle.result(), 60) + + assert result == ["alpha", "beta"], ( + "the second Worker did not reconstruct the subscription from the " + f"marker: {result}" + ) + assert not any( + e.HasField("timer_fired_event_attributes") for e in await history(handle) + ), ( + "the Run was resumed by its own 600-second timer rather than by the " + "wake Signal, which is the one thing this case forbids" + ) + finally: + if handle is not None: + try: + await handle.terminate() + except Exception: + pass + await close_worker(worker_a, worker_a_task) + + +async def _has_markers(handle: Any) -> bool: + return bool(markers(await history(handle))) + + +async def _timer_started(handle: Any) -> bool: + return any( + e.HasField("timer_started_event_attributes") for e in await history(handle) + ) + + +async def _run_is_registered(manager: Any) -> bool: + return bool(manager._runs) + + +async def _in_the_window(probe: Any, run_id: str) -> bool: + """Whether Core says this Run has waits registered and no task open. + + Asked through the read-only probe, which is the same call the sweep makes + and the only thing that can tell the three states apart. + """ + status = await probe(run_id) + return getattr(status, "value", status) == "NoOpenWorkflowTask" + + +class _InTheWindow: + """A Run left cached with a live wait set and no open Workflow Task. + + The state both tests below need, and getting there is neither quick nor + obvious: the Workflow must block on the stream *first* so a + ``WorkflowStreamQuiescent`` registers the wait set with Core, park, be woken + by a record, and only then issue a server-bound command so the Workflow Task + ends with the wait set still registered. See + :class:`LeftWithNoOpenTaskWorkflow`. + """ + + def __init__(self, worker: Worker, task: asyncio.Task, handle: Any, key: StreamKey): + self.worker = worker + self.task = task + self.handle = handle + self.key = key + self.manager = workflow_worker(worker)._external_stream_manager + self.run_id = next(iter(self.manager._runs)) + + async def close(self) -> None: + try: + await self.handle.terminate() + except Exception: + pass + await close_worker(self.worker, self.task) + + +async def _leave_a_run_in_the_window( + client: Client, backend: MemoryStreamBackend, session: str +) -> _InTheWindow: + task_queue = f"tq-{uuid.uuid4()}" + worker = Worker( + client, + task_queue=task_queue, + workflows=[LeftWithNoOpenTaskWorkflow], + external_stream_backend=backend, + ) + worker_task = asyncio.create_task(worker.run()) + handle = await client.start_workflow( + LeftWithNoOpenTaskWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=timedelta(seconds=10), + ) + key = await stream_key_for(client, handle) + + # The first Workflow Task ends by parking, which is also what registers the + # wait set with Core. A record published before that lands on a Run whose + # wait set is still empty. + await wait_until( + lambda: _has_markers(handle), + 60, + "the Run's first Workflow Task never ended, so it never parked and " + "never registered a wait set with Core", + ) + await wait_until( + lambda: _park_is_installed(backend, key), + 30, + "no park intent was installed, so this Run never confirmed a park and " + "there is no resolved park to reason about", + ) + + await publish(backend, key, ["alpha"], session=session) + await wait_until( + lambda: _timer_started(handle), + 60, + "the record never reached the Workflow, so the completion that leaves " + "the Run in the no-open-task window never happened", + ) + + manager = workflow_worker(worker)._external_stream_manager + assert manager is not None, "the manager should exist by now" + await wait_until( + lambda: _run_is_registered(manager), + 30, + "the manager holds no Run with subscriptions", + ) + return _InTheWindow(worker, worker_task, handle, key) + + +async def _park_is_installed(backend: MemoryStreamBackend, key: StreamKey) -> bool: + return bool(await backend.parked_wait_ids(key)) + + +@pytest.mark.timeout(180) +async def test_a_wake_that_resolves_a_park_removes_its_intent( + client: Client, backend: MemoryStreamBackend +) -> None: + """The invariant, end to end: an intent outlives its park nowhere. + + The park here is confirmed rather than aborted, and it is resolved the way a + live one actually is -- a producer appends, the watcher sends the wake, and + Core clears its ``park_generation``. None of that is visible in the backend, + so if the manager does not take the intent back out on the resolve, nothing + ever will. + + Asserted through ``current_park_generation`` because that is the call the + damage comes through. Every wake path asks it: a producer, to decide what its + Signal names, and this Worker's own shutdown sweep, to decide whether it owes + a parked or an unparked wake. Both get a generation Core has already + discarded, and both wakes are then discarded in turn -- the producer's as + stale, the sweep's by the server, which sees the request ID of the wake that + ended the park. + """ + session = f"resolve-{uuid.uuid4()}" + window = await _leave_a_run_in_the_window(client, backend, session) + try: + assert await backend.parked_wait_ids(window.key) == [], ( + "the park that the wake Signal resolved still has its intent " + "installed in the backend" + ) + for wait_id in (0, 1, 2): + assert await backend.current_park_generation(window.key, wait_id) is None, ( + "a park that is over still reports a generation, so the next " + "wake for this wait will name it instead of being the unparked " + "wake it owes" + ) + finally: + await window.close() + + +@pytest.mark.timeout(180) +async def test_the_shutdown_probe_is_asked_while_core_still_holds_the_run( + client: Client, backend: MemoryStreamBackend +) -> None: + """The sweep's probe has to be asked when its answer still describes something. + + P20 gives the probe four answers and three behaviours: ``WftOpen`` waits for + C15b, ``Parked`` owes nothing, and ``NoOpenWorkflowTask``/``RunNotFound`` + owe the wake. Asked after every activation has been answered, Core has + already dropped the Run and only ``RunNotFound`` is reachable -- so the + sweep still sends its wake and still looks correct, while a parked Run gets + a wake it does not need and a Run holding a Workflow Task gets one that + races Core's own shutdown transition. + + This Run is in the ``NoOpenWorkflowTask`` window, which is asserted on the + live Worker before shutdown, so a sweep asking at the right moment must see + the same thing. The assertion is on what the *sweep* saw, not on what a + probe outside it saw, because the ordering is the only thing under test. + """ + session = f"probe-{uuid.uuid4()}" + window = await _leave_a_run_in_the_window(client, backend, session) + try: + real_probe = window.manager._run_status + await wait_until( + lambda: _in_the_window(real_probe, window.run_id), + 30, + "the Run never reached the no-open-Workflow-Task window, so this " + "case is not testing the ordering it names", + ) + + answers: list[str] = [] + + async def recording_probe(probed_run_id: str) -> Any: + status = await real_probe(probed_run_id) + answers.append(getattr(status, "value", status)) + return status + + window.manager._run_status = recording_probe + + await asyncio.wait_for(window.worker.shutdown(), 60) + + assert answers, "the sweep never probed at all" + assert "RunNotFound" not in answers, ( + "the sweep probed after Core had already dropped the Run, so its " + f"answers were {answers}. Every state P20 distinguishes collapses " + "into RunNotFound there: the wake still goes out, so nothing looks " + "broken, but the Parked and WftOpen branches are unreachable" + ) + assert answers == ["NoOpenWorkflowTask"], ( + "the Run was in the no-open-Workflow-Task window immediately before " + f"shutdown, and the sweep saw {answers}" + ) + finally: + await window.close() + + +# --- case 30: teardown racing finalization ------------------------------------ + + +def _status_value(status: Any) -> Any: + """The probe's answer as a plain string, matched structurally as Core is.""" + return getattr(status, "value", status) + + +def _delivery_boundary(workflow_worker: Any, run_id: str) -> Any: + """Where this Run's deliveries have reached, read from the Run's own state. + + ``blocked_snapshot()`` is the same thing the finalization terminal is built + from, which is what makes it the right thing to wait on: nothing else says + "a Workflow Task is open here and its annotation is not written yet". The + History cannot -- a retained task writes nothing until it ends, which is + exactly the state this case must catch it in. + """ + runtime = workflow_worker._external_stream_runtimes.get(run_id) + return None if runtime is None else runtime.blocked_snapshot() + + +async def _delivery_moved(workflow_worker: Any, run_id: str, previous: Any) -> bool: + """Whether the boundary has moved off ``previous``. + + Movement rather than a value: a cursor is ordered by the provider's own + comparator, so "is it past the record just published" is a question only the + backend can answer, while "did the record land" is a question the boundary + answers by itself. + """ + return _delivery_boundary(workflow_worker, run_id) != previous + + +@pytest.mark.timeout(180) +async def test_a_finalization_that_cannot_be_answered_writes_no_marker( + client: Client, backend: MemoryStreamBackend +) -> None: + """The Run's state vanishes underneath a finalization, and nothing is lost. + + Finalization is answered from the Run's own out-of-sandbox state, and its + only failure mode is that state being gone. The obligation then is not to do + something approximate: **a marker is never written without its terminal**, + because an abandoned Workflow Task commits no cursor and loses no record, + while a truncated annotation is durable and wrong. + + So the Run's entry is removed exactly once, while Core is asking for the + terminal, and the assertions are about what did *not* happen: no marker for + that task, no cursor movement in the marker that already existed, and no + record missing from what the Workflow finally received. The retry replays + from the previous marker and re-consumes everything after it. + """ + task_queue = f"tq-{uuid.uuid4()}" + session = f"finalize-{uuid.uuid4()}" + values = ["alpha", "beta", "gamma", "delta"] + worker_a = Worker( + client, + task_queue=task_queue, + workflows=[TimerThenStreamWorkflow], + external_stream_backend=backend, + ) + worker_a_task = asyncio.create_task(worker_a.run()) + shutdown_task = None + handle = None + try: + handle = await client.start_workflow( + TimerThenStreamWorkflow.run, + len(values), + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + task_timeout=timedelta(seconds=10), + ) + key = await stream_key_for(client, handle) + + # The first record is consumed on a task that also starts a timer, so + # its marker commits: that marker is the "previous" one the retry must + # replay from. + await publish(backend, key, values[:1], session=session) + await wait_until( + lambda: _has_markers(handle), + 60, + "no marker committed the first record, so there is no previous " + "marker for the retry to replay from", + ) + + # Fed from here, and each record is *followed* to the Workflow rather + # than merely published on a schedule. The precondition this case needs + # is a Workflow Task that is open with an annotation still accumulating + # in it, and the marker above says nothing about when that exists: it is + # written when the timer is started, while the retained task appears + # only once the timer has fired and its replacement task has reached + # this Worker. Feeding on a fixed schedule from the marker races the + # timer, and a shutdown that wins the race finds no open Workflow Task + # at all -- Core issues no ``FinalizeExternalStreams``, so the sweep + # runs instead and nothing this case is about ever happens. + # + # The condition waited on is the Run's own delivery boundary, which is + # the snapshot the terminal would be built from. Once it has moved past + # what the previous marker committed, the open task and its unwritten + # annotation are both facts. + workflow_worker_impl = workflow_worker(worker_a) + manager = workflow_worker_impl._external_stream_manager + assert manager is not None, "the manager should exist by now" + await wait_until( + lambda: _run_is_registered(manager), + 30, + "the manager holds no Run with subscriptions, so there is no Run " + "state for a finalization to be answered from", + ) + run_id = next(iter(manager._runs)) + assert _delivery_boundary(workflow_worker_impl, run_id) is not None, ( + "the manager registered a Run the Worker holds no stream runtime " + "for, so the sabotage below would remove nothing" + ) + for value in values[1:3]: + delivered = _delivery_boundary(workflow_worker_impl, run_id) + await publish(backend, key, [value], session=session) + await wait_until( + lambda: _delivery_moved(workflow_worker_impl, run_id, delivered), + 60, + f"{value!r} never reached the Workflow, so no Workflow Task is " + "open here with an unwritten annotation in it and the " + "finalization this case forces would have nothing to fail " + "against", + # The retained task the last delivery leaves behind is what the + # steps after this loop have to run inside, so the poll gap is + # spent out of that window. Short because the condition is a + # dictionary in this process, not a fetch from the server. + interval=0.02, + ) + + assert _status_value(await manager._run_status(run_id)) == "WftOpen", ( + "the records were delivered but the Workflow Task holding them is " + "already gone, so Core has nothing to ask for a terminal on. The " + "task is retained for RETENTION_IDLE_TIMEOUT, which every step " + "since the last delivery is expected to fit inside" + ) + + # Read here rather than at the first marker: every Workflow Task that + # closes between the two commits its own marker, and the claim this + # case makes is about the one task that is open right now. + before = markers(await history(handle)) + committed_before = committed_boundary(before[-1], wait_id=1) + marker_before = marker_bytes(before[-1]) + + # The Run's entry disappears underneath the finalization, exactly once. + original = workflow_worker_impl._handle_external_stream_jobs + sabotaged: list[str] = [] + + async def losing_the_run(act: Any, running: Any) -> Any: + if not sabotaged and any( + j.HasField("finalize_external_streams") for j in act.jobs + ): + sabotaged.append(act.run_id) + workflow_worker_impl._external_stream_runtimes.pop(act.run_id, None) + await manager.evict_run(act.run_id) + return await original(act, running) + + workflow_worker_impl._handle_external_stream_jobs = losing_the_run + + # Shutdown is what makes Core ask for the terminal while the task is + # still open. Not awaited: this Worker is deliberately being left unable + # to answer, and the test's subject is what the *server* is left holding. + shutdown_task = asyncio.create_task(worker_a.shutdown()) + + async def the_task_failed() -> bool: + return any( + e.HasField("workflow_task_failed_event_attributes") + for e in await history(handle) + ) + + await wait_until( + the_task_failed, + 90, + "the Workflow Task never failed, so the finalization was answered " + "from somewhere other than the Run state that was removed", + ) + assert sabotaged, "the finalization job never arrived, so nothing was raced" + + events = await history(handle) + failure = next( + e for e in events if e.HasField("workflow_task_failed_event_attributes") + ) + message = failure.workflow_task_failed_event_attributes.failure.message + assert "finalize_external_streams" in message, ( + "the Workflow Task failed for some other reason than the " + f"unanswerable finalization: {message}" + ) + after = markers(events) + assert len(after) == len(before), ( + "Core wrote a marker for a Workflow Task whose terminal was never " + "supplied -- a durable, truncated annotation is exactly what the " + "abandon-and-retry rule exists to prevent" + ) + assert committed_boundary(after[-1], wait_id=1) == committed_before, ( + "the committed cursor moved across a Workflow Task that was " + "abandoned; nothing consumed on it was ever committed" + ) + assert marker_bytes(after[-1]) == marker_before, ( + "the marker that was already in History was rewritten by the " + "abandoned Workflow Task" + ) + + async with Worker( + client, + task_queue=task_queue, + workflows=[TimerThenStreamWorkflow], + external_stream_backend=backend, + ): + await publish(backend, key, values[3:], session=session) + result = await asyncio.wait_for(handle.result(), 90) + + assert result == values, ( + "the retry did not replay from the previous marker: every record " + "consumed on the abandoned task was owed again, exactly once, and " + f"in order, got {result}" + ) + finally: + # Terminated first, then the Worker given a moment to finish shutting + # down: a Worker abandoned mid-shutdown leaves a Core run behind, and + # this suite's later tests are timing-sensitive enough to notice. + if handle is not None: + try: + await handle.terminate() + except Exception: + pass + if shutdown_task is not None: + await asyncio.wait([shutdown_task], timeout=10) + if not shutdown_task.done(): + shutdown_task.cancel() + await asyncio.gather(shutdown_task, return_exceptions=True) + if not worker_a_task.done(): + worker_a_task.cancel() + await asyncio.gather(worker_a_task, return_exceptions=True) + + +# --- the intent the Worker that installed it left behind ----------------------- + + +HANDOFF_RUN_ID = "run-handed-over" + + +def _manager_for(backend: MemoryStreamBackend) -> StreamSubscriptionManager: + """One Worker's manager, wired with only what the park handshake touches. + + No Core and no Signal path: what is under test here is state that lives in + the *backend* and outlives both. + """ + + async def accepted( + run_id: str, # pyright: ignore[reportUnusedParameter] + wait_id: int, # pyright: ignore[reportUnusedParameter] + generation: int, # pyright: ignore[reportUnusedParameter] + ) -> str: + return ReadinessResult.ACCEPTED + + return StreamSubscriptionManager( + backend=backend, + notify_ready=accepted, + watch_block=timedelta(milliseconds=10), + ) + + +async def _nothing_is_parked(backend: MemoryStreamBackend, key: StreamKey) -> bool: + return await backend.parked_wait_ids(key) == [] + + +async def test_a_park_intent_installed_by_a_previous_worker_is_removed( + backend: MemoryStreamBackend, +) -> None: + """The half of the intent invariant that only a hand-off can show. + + ``installed_park_generation`` is a *mirror* of backend state and lives on + the Worker that installed the park. The intent is durable and survives + eviction, a Workflow Task that moved to another Worker, and shutdown; the + mirror survives none of them. So removal keyed on the mirror alone cannot + reach exactly the intents that most need reaching -- the ones whose + installer is gone -- and the Run is then stranded in a way no later resolve + can repair: ``current_park_generation`` keeps answering a generation Core + discarded, every producer wake names it and Core discards it as stale, and + because a parked wake's request ID ignores sender identity the second such + wake is byte-identical to the first and the server deduplicates it away. + + The ordering is the real Worker's: ``ResolveExternalStreamWaits`` is + answered *before* user Workflow code runs, so the new Worker has no + subscription to resolve against at that point and the reconstructed one + arrives afterwards. + """ + key = StreamKey("ns", "wf", "first-run", "tokens") + + worker_a = _manager_for(backend) + worker_a.register(run_id=HANDOFF_RUN_ID, wait_id=1, stream_key=key) + confirmed = not await worker_a.prepare_park(HANDOFF_RUN_ID, 7, {1: BEGINNING}) + assert confirmed, "nothing was appended, so this park must have confirmed" + await worker_a.shutdown() + + assert await backend.parked_wait_ids(key) == [1], ( + "this case is only meaningful if the intent outlived the Worker that " + "installed it -- it is durable backend state, not Worker state" + ) + + worker_b = _manager_for(backend) + try: + await worker_b.resolve_park(HANDOFF_RUN_ID) + worker_b.register( + run_id=HANDOFF_RUN_ID, + wait_id=1, + stream_key=key, + ) + + await wait_until( + lambda: _nothing_is_parked(backend, key), + 5, + "the intent of a park no Worker is sitting in is still installed, " + "so the next producer wake names a dead generation instead of the " + "unparked wake it owes", + interval=0.01, + ) + assert await backend.current_park_generation(key, 1) is None, ( + "a park that ended with the Worker that confirmed it still reports " + "a generation" + ) + + # What the producer actually asks, in the order it asks it: the record + # lands, and then the wake is chosen from what the backend says is + # parked. Nothing is, so it is the unparked wake -- which Core always + # accepts as a recheck request -- rather than generation 7, which it + # discards as stale. + await publish(backend, key, ["beta"], session=f"handoff-{uuid.uuid4()}") + assert await backend.parked_wait_ids(key) == [] + finally: + await worker_b.shutdown() diff --git a/tests/contrib/external_workflow_streams/test_worker_integration.py b/tests/contrib/external_workflow_streams/test_worker_integration.py new file mode 100644 index 0000000000..a88c8cc898 --- /dev/null +++ b/tests/contrib/external_workflow_streams/test_worker_integration.py @@ -0,0 +1,1901 @@ +"""P10a/P10b/P11/P19 — the wiring, exercised through a real Worker. + +Everything below runs against a live server with a real Workflow Task loop. The +unit tests elsewhere prove each piece in isolation; these prove the pieces are +actually connected to each other, which is the only thing isolation cannot show. + +PYTEST_DONT_REWRITE: sandboxed fixture Workflows re-import this module, so pytest's +injected imports would make sandbox validation depend on pytest's import locks. +""" + +# pyright: reportMissingParameterType=false +from __future__ import annotations + +import asyncio +import threading +import time +import uuid +from collections.abc import Sequence +from datetime import timedelta +from pathlib import Path +from typing import Any + +import pytest + +import temporalio.api.common.v1 +import temporalio.api.enums.v1 +import temporalio.converter +from temporalio import workflow +from temporalio.client import Client, WorkflowHandle +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._errors import ( + METRIC_DECODE, + METRIC_INTEGRITY, + METRIC_STORAGE, +) +from temporalio.contrib.external_workflow_streams._record import ( + Offset, + RecordKind, + StreamRecord, +) +from temporalio.contrib.external_workflow_streams._wake import ( + ChannelSupport, + WakeRequest, + channel_for, + send_wake_signal, + wake_request_id, +) +from temporalio.runtime import MetricBuffer, Runtime, TelemetryConfig +from temporalio.service import RPCError, RPCStatusCode +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Replayer, Worker +from tests.contrib.external_workflow_streams.conftest import server_channel_support +from tests.contrib.external_workflow_streams.memory_backend import MemoryStreamBackend + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams._api import external_stream + + +@workflow.defn +class ConsumeTokensWorkflow: + """Consumes a fixed number of records, then returns them.""" + + def __init__(self) -> None: + self._seen: list[str] = [] + + @workflow.run + async def run(self, expected: int) -> list[str]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + + async for token in tokens.subscribe(): + self._seen.append(token) + if len(self._seen) >= expected: + break + return self._seen + + @workflow.query + def seen(self) -> list[str]: + return self._seen + + +@workflow.defn +class CountTokensWorkflow: + """Consumes records and returns only how many, never the values. + + Deliberately distinct from :py:class:`ConsumeTokensWorkflow`: a Workflow that + returns its records puts them in its own result, and History would then + contain them for a reason that has nothing to do with streams. + """ + + @workflow.run + async def run(self, expected: int) -> int: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + + seen = 0 + async for _ in tokens.subscribe(): + seen += 1 + if seen >= expected: + break + return seen + + +@workflow.defn +class SubscribeOnlyWorkflow: + """Subscribes and blocks forever, so the task is retained.""" + + @workflow.run + async def run(self) -> None: + tokens = external_stream.topic("tokens", type=str) + async for _ in tokens.subscribe(): + pass + + +@workflow.defn +class TimerSuppressedSubscriptionWorkflow: + """Subscribes and waits, with a long timer stopping the task being retained. + + Deliberately distinct from :py:class:`SubscribeOnlyWorkflow`: a Workflow + blocked on the stream *alone* leaves a retained Workflow Task, which is a + different shutdown transition. The timer makes every task complete, so the + Run sits cached with a live subscription and no open Workflow Task -- the + state that receives no eviction activation at shutdown at all. + """ + + @workflow.run + async def run(self) -> None: + tokens = external_stream.topic("tokens", type=str) + timer = asyncio.ensure_future(asyncio.sleep(600)) + try: + async for _ in tokens.subscribe(): + pass + finally: + timer.cancel() + + +@workflow.defn +class SubscribeInInitWorkflow: + """Subscribes from its ``@workflow.init`` constructor and consumes in ``run``. + + The user's object does not exist yet while the constructor runs, so per-Run + stream state cannot hang off it. + """ + + @workflow.init + def __init__(self, expected: int) -> None: + self._tokens = ( + external_stream.with_options(idle_timeout=timedelta(seconds=30)) + .topic("tokens", type=str) + .subscribe() + ) + + @workflow.run + async def run(self, expected: int) -> list[str]: + seen: list[str] = [] + async for token in self._tokens: + seen.append(token) + if len(seen) >= expected: + break + return seen + + +@pytest.fixture +def backend() -> MemoryStreamBackend: + return MemoryStreamBackend() + + +async def publish( + backend: MemoryStreamBackend, + key: StreamKey, + values: list[str], + *, + session: str = "producer", +) -> None: + """Appends records the way a producer would, encoded for the topic's type. + + The sequence continues across calls within one session, because + ``(session_id, sequence)`` is the idempotency key: restarting it would + re-use a key with different content, which the contract rejects outright. + """ + import temporalio.converter + from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec + + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + start = _sequences.get((id(backend), session), 0) + for i, value in enumerate(values, start=start): + await backend.append( + key, + StreamRecord(RecordKind.DATA, await codec.encode(value), session, i), + ) + _sequences[(id(backend), session)] = start + len(values) + + +#: Per-producer-session sequence, so successive publishes do not collide. +_sequences: dict[tuple[int, str], int] = {} + + +async def test_a_workflow_consumes_records_it_never_read_itself( + client: Client, backend: MemoryStreamBackend +) -> None: + """The vertical slice: subscribe, retain, notify, resolve, drain, deliver. + + Nothing in the Workflow ever touches the backend. The Worker's manager reads + it on its own loop, buffers, and tells Core; Core turns that into an + activation; and the Workflow's drain is a buffer pop that cannot block. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[ConsumeTokensWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + ConsumeTokensWorkflow.run, + 3, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + + # Let the Workflow reach its subscription before anything is published, + # so this exercises the wake path rather than a buffer that was already + # full when the first activation ran. + await asyncio.sleep(1) + await publish(backend, key, ["alpha", "beta", "gamma"]) + + assert await asyncio.wait_for(handle.result(), 30) == [ + "alpha", + "beta", + "gamma", + ] + + +async def test_a_constructor_can_subscribe( + client: Client, backend: MemoryStreamBackend +) -> None: + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[SubscribeInInitWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + SubscribeInInitWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await publish(backend, key, ["alpha", "beta"]) + + assert await asyncio.wait_for(handle.result(), 30) == ["alpha", "beta"] + + +async def test_no_stream_payload_reaches_history( + client: Client, backend: MemoryStreamBackend +) -> None: + """The entire point of the feature, asserted against real History. + + History cost scales with consumption batches, not with item count -- so a + record's bytes must appear nowhere in it, and the marker that *is* there + must be bounded metadata rather than the payloads. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[CountTokensWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + CountTokensWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await asyncio.sleep(1) + await publish(backend, key, ["secret-payload-one", "secret-payload-two"]) + assert await asyncio.wait_for(handle.result(), 30) == 2 + + events = [e async for e in handle.fetch_history_events()] + raw = b"".join(e.SerializeToString() for e in events) + + assert b"secret-payload-one" not in raw, ( + "a stream payload reached Temporal History, which is the one thing " + "this feature exists to prevent" + ) + assert b"secret-payload-two" not in raw + + # And History did not grow an event per record either: cost scales with + # consumption batches, not with item count. + markers = [e for e in events if e.HasField("marker_recorded_event_attributes")] + assert len(markers) <= 2, ( + f"expected marker cost to be per Workflow Task, got {len(markers)} " + "markers for 2 records" + ) + + +async def test_a_blocked_workflow_retains_its_task_rather_than_completing_it( + client: Client, backend: MemoryStreamBackend +) -> None: + """Retention is what lets the next record arrive on the *same* task. + + Without it every record would cost a Workflow Task round trip, which is the + cost model the feature exists to avoid. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[SubscribeOnlyWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + SubscribeOnlyWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + await asyncio.sleep(2) + + try: + history = [e async for e in handle.fetch_history_events()] + completed = [ + e + for e in history + if e.HasField("workflow_task_completed_event_attributes") + ] + # Exactly one task has completed -- the one that ran up to the + # subscription. The one holding the subscription open has not, and + # will not until a record arrives or the idle timeout fires. + assert len(completed) <= 1, ( + f"expected the subscription's task to be retained, got " + f"{len(completed)} completed tasks" + ) + finally: + await handle.terminate() + + +@workflow.defn +class ParkedQueryWorkflow: + """Blocks on an empty stream with a short idle timeout, so the task parks.""" + + def __init__(self) -> None: + self._seen: list[str] = [] + + @workflow.run + async def run(self) -> list[str]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=1)).topic( + "tokens", type=str + ) + async for token in tokens.subscribe(): + self._seen.append(token) + break + return self._seen + + @workflow.query + def seen(self) -> list[str]: + return self._seen + + +async def test_a_query_against_a_parked_workflow_is_answered( + client: Client, backend: MemoryStreamBackend +) -> None: + """A parked Run has no open Workflow Task, so its query travels alone. + + The server dispatches it on a task of its own, and Core allows nothing + beside the answer on that task. The wait set is still registered from the + parked task, so reporting it again with the answer made Core refuse the + completion, and the query never returned. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[ParkedQueryWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + ParkedQueryWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + # Past the idle timeout the retained task is reported and the Run + # waits for a wake with no task open. + await asyncio.sleep(3) + + assert await asyncio.wait_for(handle.query(ParkedQueryWorkflow.seen), 10) == [] + + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await publish(backend, key, ["first"]) + assert await asyncio.wait_for(handle.result(), 60) == ["first"] + + events = [e async for e in handle.fetch_history_events()] + assert not any( + e.HasField("workflow_task_failed_event_attributes") for e in events + ), "a Workflow Task failed while the query was outstanding" + + +async def test_a_workflow_without_a_configured_backend_says_so( + client: Client, +) -> None: + """The failure names the Worker option rather than an attribute error.""" + task_queue = f"tq-{uuid.uuid4()}" + async with Worker(client, task_queue=task_queue, workflows=[SubscribeOnlyWorkflow]): + handle = await client.start_workflow( + SubscribeOnlyWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + await asyncio.sleep(2) + try: + history = [e async for e in handle.fetch_history_events()] + failures = [ + e + for e in history + if e.HasField("workflow_task_failed_event_attributes") + ] + assert failures, "expected the Workflow Task to fail" + message = failures[0].workflow_task_failed_event_attributes.failure.message + assert "external_stream_backend" in message, ( + f"the failure should name the Worker option, got: {message}" + ) + finally: + await handle.terminate() + + +@workflow.defn +class TimerThenConsumeWorkflow: + """Consumes one record, sleeps, then consumes another. + + The sleep is the point: a completion carrying a timer is server-bound, so + retention is suppressed and the Workflow Task ends. The second record + therefore arrives with **no open Workflow Task**, which is the window only + the wake Signal covers. + """ + + @workflow.run + async def run(self) -> list[str]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + seen: list[str] = [] + subscription = tokens.subscribe() + iterator = subscription.__aiter__() + + seen.append(await iterator.__anext__()) + # A real timer: the completion that carries it cannot ask for retention. + await asyncio.sleep(2) + seen.append(await iterator.__anext__()) + return seen + + +async def test_an_append_with_no_open_task_wakes_the_workflow( + client: Client, backend: MemoryStreamBackend +) -> None: + """The wake Signal path, end to end, with nothing else able to explain it. + + The second record is published while the Workflow is sleeping, so no + Workflow Task is open to accept local readiness. Nothing else will create + one -- the sleep's own timer fires on its own schedule and the Workflow + would then find the record only by luck of timing. The watcher observing + `NoOpenWorkflowTask` and sending the Signal is what makes the delivery + reliable rather than incidental. + """ + # The Signal is the mechanism under test, so it is asked for outright: on + # a server with channels, "auto" would leave no Signal. + backend.wake_transport = "signal" + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[TimerThenConsumeWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + TimerThenConsumeWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await asyncio.sleep(1) + await publish(backend, key, ["first"]) + + # Let the Workflow consume it and settle into its sleep, so the second + # append lands squarely in the no-open-task window. + await asyncio.sleep(1) + await publish(backend, key, ["second"]) + + assert await asyncio.wait_for(handle.result(), 60) == ["first", "second"] + + events = [e async for e in handle.fetch_history_events()] + signals = [ + e + for e in events + if e.HasField("workflow_execution_signaled_event_attributes") + and e.workflow_execution_signaled_event_attributes.signal_name + == "__temporal_external_stream_wake" + ] + assert signals, ( + "the record was delivered without a wake Signal, so this passed by " + "timing rather than by the mechanism under test" + ) + + +async def test_every_marker_a_run_writes_is_a_complete_annotation( + client: Client, backend: MemoryStreamBackend +) -> None: + """Each marker carries its own header and ends with its own terminal. + + A Run that writes two markers is where this can go wrong, and + :py:class:`TimerThenConsumeWorkflow` writes exactly two: the completion that + carries its timer ends one Workflow Task, and the completion that returns + ends the next. Core writes and clears the accumulated annotation on both, so + an accumulator that carried its header across the first of them would begin + the second annotation at whatever frame came next -- decoded as a schema + version, since that is what leads an annotation. + + Decoding every marker independently is the assertion: the annotation is + opaque to Core, so nothing between here and History would notice one that + cannot be read back, and the failure would surface only on the replay that + needed it. + """ + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + from temporalio.contrib.external_workflow_streams._annotation import ( + decode_annotation, + ) + + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[TimerThenConsumeWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + TimerThenConsumeWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await asyncio.sleep(1) + await publish(backend, key, ["first"]) + await asyncio.sleep(1) + await publish(backend, key, ["second"]) + + assert await asyncio.wait_for(handle.result(), 60) == ["first", "second"] + + events = [e async for e in handle.fetch_history_events()] + markers = [ + e + for e in events + if e.HasField("marker_recorded_event_attributes") + and e.marker_recorded_event_attributes.marker_name == "core_external_stream" + ] + assert len(markers) >= 2, ( + "this Run should write one marker per Workflow Task it ended, and " + f"it ended at least two; got {len(markers)}" + ) + # Decoded first, all of them: a marker that lost its header fails here, + # and it is the *second* one that loses it, so a per-marker assertion + # interleaved with the decoding would stop on the first marker's own + # shortcoming and never reach the one under test. + annotations = [] + for index, event in enumerate(markers): + data = ExternalStreamMarkerData() + data.ParseFromString( + event.marker_recorded_event_attributes.details["external_stream"] + .payloads[0] + .data + ) + annotations.append(decode_annotation(data.replay_annotation)) + + for index, annotation in enumerate(annotations): + assert annotation.header.streams, ( + f"marker {index} records no stream in its header, so replay of " + "it has nothing to start from" + ) + assert annotation.terminal is not None, ( + f"marker {index} has no terminal, so nothing in it says where " + "the Workflow Task's deliveries stopped (ADR-008)" + ) + + +@workflow.defn +class TimerThenFirstRecordWorkflow: + """Starts a timer and blocks on the stream in the **same** activation. + + Deliberately distinct from :py:class:`TimerThenConsumeWorkflow`, which + consumes a record *before* its timer: that one is quiescent once before the + server-bound command ever appears, so its wait set is already registered + with Core and only has to survive. Here the very first block rides a + completion that carries a timer, so nothing has registered the wait yet -- + which is the only difference between the two, and the whole case. + """ + + @workflow.run + async def run(self) -> list[str]: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + iterator = tokens.subscribe().__aiter__() + # Long enough that it cannot be what resumes this Workflow. If a record + # is ever returned, the stream delivered it. + timer = asyncio.ensure_future(asyncio.sleep(600)) + try: + return [await iterator.__anext__()] + finally: + timer.cancel() + + +async def test_a_first_block_that_rides_a_server_bound_command_is_still_wakeable( + client: Client, backend: MemoryStreamBackend +) -> None: + """The wait set has to be registered even by a completion that is reported. + + A completion carrying a timer must not ask for retention -- the server has + to be told about the timer -- but the subscriptions it leaves behind are + still active, and the wake Signal that covers that window can only resume a + Run whose waits Core knows about. Without the registration every append + produces a wake Signal, every wake Signal produces a Workflow Task with no + jobs in it, and the Workflow is unresumable for the life of the timer. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[TimerThenFirstRecordWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + TimerThenFirstRecordWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + + # Published after the Workflow has settled, so the record arrives with + # no open Workflow Task and the wake Signal is the only way in. + await asyncio.sleep(1) + await publish(backend, key, ["first"]) + + try: + # Short on purpose: the timer is 600 seconds, so nothing but the + # stream can finish this Workflow and a longer wait would only make + # a known gap slower to report. + assert await asyncio.wait_for(handle.result(), 15) == ["first"] + finally: + # The Workflow reaching its own end is the success case here, so + # terminating unconditionally turns a pass into a NOT_FOUND error -- + # which is how the same mistake presented in the rollover tests. + try: + await handle.terminate() + except RPCError as err: + if err.status is not RPCStatusCode.NOT_FOUND: + raise + + +def _stream_markers(events: list) -> list: # type: ignore[type-arg] + return [ + e + for e in events + if e.HasField("marker_recorded_event_attributes") + and e.marker_recorded_event_attributes.marker_name == "core_external_stream" + ] + + +async def _wait_for_markers( + handle, count: int, message: str, timeout: float = 30 +) -> None: # type: ignore[no-untyped-def] + """Polls until History holds ``count`` stream markers. + + Polled rather than slept: a marker is written when the Workflow Task ends, + and waiting a fixed time instead would make every assertion after it a + statement about how fast this machine is. + """ + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if ( + len(_stream_markers([e async for e in handle.fetch_history_events()])) + >= count + ): + return + await asyncio.sleep(0.3) + raise AssertionError(message) + + +class ReplayFailureTrace: + """Correlates the manager events that distinguish issues 2 and 7.""" + + def __init__(self) -> None: + self._started = time.monotonic() + self.events: list[dict[str, object]] = [] + self.managers: list[object] = [] + + def record(self, event: str, **details: object) -> None: + entry: dict[str, object] = { + "at_seconds": round(time.monotonic() - self._started, 6), + "event": event, + } + entry.update(details) + self.events.append(entry) + + def record_explicit_wake(self, request: WakeRequest) -> None: + self.record( + "explicit_wake", + request_id=wake_request_id(request), + workflow_id=request.workflow_id, + run_id=request.first_execution_run_id, + stream_name=request.stream_name, + wait_id=request.wait_id, + park_generation=request.park_generation, + sender_identity=request.sender_identity, + wake_counter=request.wake_counter, + ) + + def record_explicit_wake_acknowledged(self, request: WakeRequest) -> None: + self.record("explicit_wake_acknowledged", request_id=wake_request_id(request)) + + def manager_wake_bursts_after_explicit_ack(self) -> list[int]: + """Counts wakes between each Core-accepted task completion.""" + acknowledged = next( + ( + index + for index, event in enumerate(self.events) + if event["event"] == "explicit_wake_acknowledged" + ), + None, + ) + if acknowledged is None: + raise AssertionError("the explicit wake was never acknowledged") + bursts: list[int] = [] + current: set[object] = set() + for event in self.events[acknowledged + 1 :]: + if event["event"] == "manager_wake_started": + current.add(event["request_id"]) + elif event["event"] == "workflow_task_completion_accepted": + bursts.append(len(current)) + current = set() + bursts.append(len(current)) + return bursts + + +@pytest.fixture +def replay_failure_trace(monkeypatch: pytest.MonkeyPatch) -> ReplayFailureTrace: + """Captures diagnostics without changing readiness or wake behavior.""" + import temporalio.contrib.external_workflow_streams._wake as wake_module + from temporalio.contrib.external_workflow_streams._manager import ( + StreamSubscriptionManager, + ) + + trace = ReplayFailureTrace() + original_init = StreamSubscriptionManager.__init__ + original_notify = StreamSubscriptionManager._notify_ready_with_retries + original_evict = StreamSubscriptionManager.evict_run + original_started = StreamSubscriptionManager.note_workflow_task_started + original_completed = StreamSubscriptionManager.note_workflow_task_completed + original_send_wake = wake_module.send_wake_signal + + def traced_init(self, *args, **kwargs): # type: ignore[no-untyped-def] + original_init(self, *args, **kwargs) + trace.managers.append(self) + trace.record("manager_created", sender_identity=self.wake_sender_identity) + + async def traced_notify(self, subscription): # type: ignore[no-untyped-def] + generation = subscription.current_wait_generation() + result = await original_notify(self, subscription) + trace.record( + "readiness", + run_id=subscription.run_id, + wait_id=subscription.wait_id, + generation=generation, + result=result, + buffered=subscription.buffered, + wakes_owed=subscription.wakes_owed, + ) + return result + + async def traced_evict(self, run_id: str) -> None: # type: ignore[no-untyped-def] + trace.record("evict_started", run_id=run_id) + await original_evict(self, run_id) + trace.record("evict_finished", run_id=run_id) + + def traced_completed(self, run_id: str, *, terminal: bool) -> None: # type: ignore[no-untyped-def] + trace.record( + "workflow_task_completion_accepted", + run_id=run_id, + terminal=terminal, + ) + original_completed(self, run_id, terminal=terminal) + + def traced_started(self, run_id: str) -> None: # type: ignore[no-untyped-def] + trace.record("workflow_task_started", run_id=run_id) + original_started(self, run_id) + + async def traced_send_wake( + client: Client, + request: WakeRequest, + *, + producer_session_id: str = "", + ) -> str: + request_id = wake_request_id(request) + trace.record( + "manager_wake_started", + request_id=request_id, + workflow_id=request.workflow_id, + run_id=request.first_execution_run_id, + stream_name=request.stream_name, + wait_id=request.wait_id, + park_generation=request.park_generation, + sender_identity=request.sender_identity, + wake_counter=request.wake_counter, + ) + try: + result = await original_send_wake( + client, request, producer_session_id=producer_session_id + ) + except BaseException as err: + trace.record( + "manager_wake_failed", + request_id=request_id, + error=repr(err), + ) + raise + trace.record("manager_wake_acknowledged", request_id=request_id) + return result + + monkeypatch.setattr(StreamSubscriptionManager, "__init__", traced_init) + monkeypatch.setattr( + StreamSubscriptionManager, "_notify_ready_with_retries", traced_notify + ) + monkeypatch.setattr(StreamSubscriptionManager, "evict_run", traced_evict) + monkeypatch.setattr( + StreamSubscriptionManager, + "note_workflow_task_started", + traced_started, + ) + monkeypatch.setattr( + StreamSubscriptionManager, + "note_workflow_task_completed", + traced_completed, + ) + monkeypatch.setattr(wake_module, "send_wake_signal", traced_send_wake) + return trace + + +async def _replay_failure_diagnosis( + handle, # type: ignore[no-untyped-def] + backend: MemoryStreamBackend, + key: StreamKey, + trace: ReplayFailureTrace, + artifact_dir: Path, + phase: str, +) -> str: + """Preserves the evidence a timeout cleanup used to destroy.""" + lines = [f"diagnostic phase: {phase}"] + try: + history = await handle.fetch_history() + artifact = artifact_dir / f"{handle.id}-{phase}-history.json" + artifact.write_text(history.to_json()) + lines.append(f"complete History: {artifact}") + workflow_task_timeline = [] + for event in history.events: + attributes = event.WhichOneof("attributes") or "?" + if attributes.startswith("workflow_task_"): + item: dict[str, object] = { + "event_id": event.event_id, + "attributes": attributes, + } + if event.HasField("workflow_task_failed_event_attributes"): + failed = event.workflow_task_failed_event_attributes + item["cause"] = failed.cause + item["message"] = failed.failure.message[:300] + workflow_task_timeline.append(item) + lines.append(f"Workflow Task timeline: {workflow_task_timeline}") + except Exception as err: # noqa: BLE001 -- diagnostics must not mask timeout + lines.append(f"History unavailable: {err!r}") + try: + description = await handle.describe() + lines.append(f"workflow status: {description.status}") + except Exception as err: # noqa: BLE001 + lines.append(f"describe unavailable: {err!r}") + try: + lines.append( + "stream state: " + f"records={backend.all_records(key)} " + f"parked_wait_ids={await backend.parked_wait_ids(key)}" + ) + except Exception as err: # noqa: BLE001 + lines.append(f"stream state unavailable: {err!r}") + for manager in trace.managers: + subscriptions = getattr(manager, "_runs", {}) + for run_id, waits in subscriptions.items(): + for wait_id, subscription in waits.items(): + lines.append( + f"subscription {run_id}/{wait_id}: " + f"buffered={subscription.buffered} " + f"committed={subscription.committed_cursor} " + f"delivery={subscription.delivery_cursor} " + f"prefetch={subscription.prefetch_cursor} " + f"generation={subscription.current_wait_generation()} " + f"wakes_owed={subscription.wakes_owed} " + f"cancelled={subscription._cancelled} " + f"watcher_done=" + f"{None if subscription._watcher is None else subscription._watcher.done()}" + ) + lines.append(f"correlated manager trace: {trace.events}") + return "\n".join(lines) + + +async def _await_replayed_result( + handle, # type: ignore[no-untyped-def] + expected: list[str], + backend: MemoryStreamBackend, + key: StreamKey, + trace: ReplayFailureTrace, + artifact_dir: Path, +) -> None: + """Waits for the result and retains both timeout and late-observation state.""" + try: + result = await asyncio.wait_for(handle.result(), 30) + except asyncio.TimeoutError: + at_timeout = await _replay_failure_diagnosis( + handle, backend, key, trace, artifact_dir, "at-timeout" + ) + try: + late_result = await asyncio.wait_for(handle.result(), 10) + except asyncio.TimeoutError: + after_observation = await _replay_failure_diagnosis( + handle, backend, key, trace, artifact_dir, "after-observation" + ) + raise AssertionError( + "the replayed Run did not complete within 30 seconds and remained " + "non-terminal for the 10-second observation window\n\n" + f"{at_timeout}\n\n{after_observation}" + ) from None + raise AssertionError( + "the replayed Run missed the 30-second bound but completed during " + f"the observation window with {late_result!r}\n\n{at_timeout}" + ) from None + assert result == expected + + +@workflow.defn +class ParkedAcrossEvictionWorkflow: + """Consumes records with nothing else to do, so every task ends in a park. + + No timer, no other command: each Workflow Task is retained until the idle + timeout parks it, which is what puts a marker in History and leaves the Run + parked rather than merely cached. Both are needed here -- the marker is what + makes the Run *replay* after eviction, and the park is what makes the wake + Signal the only thing that can bring a Workflow Task back. + """ + + @workflow.run + async def run(self, expected: int) -> list[str]: + tokens = external_stream.topic("tokens", type=str) + seen: list[str] = [] + async for token in tokens.subscribe(): + seen.append(token) + if len(seen) >= expected: + break + return seen + + +@workflow.defn +class FillerWorkflow: + """Occupies the one cache slot, which is what evicts the Run under test.""" + + @workflow.run + async def run(self) -> str: + return "done" + + +async def test_a_replayed_run_re_registers_its_wait_set( + client: Client, + backend: MemoryStreamBackend, + replay_failure_trace: ReplayFailureTrace, + tmp_path: Path, +) -> None: + """A Run that comes back through replay has to end replay registered. + + Core's wait set is per-Worker runtime state: it is not in History and it + does not survive eviction, so a replayed Run rebuilds it or has none. The + only thing that builds it is ``WorkflowStreamQuiescent``, and a completion + that reports no progress because it is replaying must still report the + snapshot -- what is already in History is the *annotation*, not the + registration. + + The Run is deliberately left **parked** before it is evicted, so nothing + else can explain the resume: there is no timer to fire, no watcher left + alive on this Worker, and no open Workflow Task. A wake Signal creates the + replacement task, and everything after that depends on the replayed Run + knowing what it is subscribed to. Without it every later readiness is + answered as though the Run had no subscriptions at all, each wake produces a + Workflow Task with no activation in it, and the record sits in the stream + while the Workflow waits. + """ + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[ParkedAcrossEvictionWorkflow, FillerWorkflow], + external_stream_backend=backend, + # One slot, so running anything else evicts the Run under test. Eviction + # is what discards the wait set, the watchers, and the buffers, leaving + # replay to rebuild all three. + max_cached_workflows=1, + max_concurrent_workflow_tasks=2, + ): + handle = await client.start_workflow( + ParkedAcrossEvictionWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + first_run_id = description.raw_description.workflow_execution_info.first_run_id + key = StreamKey(client.namespace, handle.id, first_run_id, "tokens") + + # Each wait is for a *park*, not for a duration: the record is published + # only once the Workflow has actually parked, so it is delivered by a + # wake rather than into a Workflow Task that happened to still be open. + await _wait_for_markers(handle, 1, "the Run never parked at all") + + # Consumed live, so the Run has a marker to replay from and a cursor + # that must be honoured -- a replay that re-delivered "alpha" would show + # up in the result rather than in a timeout. + await publish(backend, key, ["alpha"]) + await _wait_for_markers( + handle, + 2, + "the Run never parked again after consuming its first record, so it " + "was never in the state this case evicts from", + ) + + # Evicts the Run: its wait set, watchers, and buffers all go with it. + await client.execute_workflow( + FillerWorkflow.run, id=f"filler-{uuid.uuid4()}", task_queue=task_queue + ) + + # Appended with nothing on this Worker watching for it, then woken the + # way a durable producer wakes a parked Run. The Signal is the only + # thing that creates a Workflow Task here, and it carries no records -- + # so if the replayed Run does not know its own subscription, the + # Workflow Task it creates has nothing in it. + await publish(backend, key, ["beta"]) + request = WakeRequest( + namespace=client.namespace, + workflow_id=handle.id, + first_execution_run_id=first_run_id, + stream_name="tokens", + wait_id=1, + park_generation=0, + sender_identity=f"test-{uuid.uuid4()}", + wake_counter=1, + ) + replay_failure_trace.record_explicit_wake(request) + await send_wake_signal(client, request) + replay_failure_trace.record_explicit_wake_acknowledged(request) + + try: + await _await_replayed_result( + handle, + ["alpha", "beta"], + backend, + key, + replay_failure_trace, + tmp_path, + ) + bursts = replay_failure_trace.manager_wake_bursts_after_explicit_ack() + assert max(bursts, default=0) <= 1, ( + "readiness produced multiple wakes before Core accepted the " + f"task completion that one wake caused: {replay_failure_trace.events}" + ) + finally: + try: + await handle.terminate() + except Exception: + pass + + +@workflow.defn +class FloodedCountWorkflow: + """Consumes far more records than one activation is allowed to deliver.""" + + def __init__(self) -> None: + self._seen = 0 + + @workflow.run + async def run(self, expected: int) -> int: + tokens = external_stream.with_options(idle_timeout=timedelta(seconds=30)).topic( + "tokens", type=str + ) + async for _ in tokens.subscribe(): + self._seen += 1 + if self._seen >= expected: + break + return self._seen + + @workflow.query + def seen(self) -> int: + return self._seen + + +async def test_a_flood_larger_than_one_activations_budget_is_delivered_in_full( + client: Client, backend: MemoryStreamBackend +) -> None: + """The whole budget mechanism, end to end, on a real Workflow Task loop. + + Three things have to be connected for this to finish, and each of them hangs + the Workflow permanently on its own: + + - the per-activation budget, or the first ``activate()`` never returns and + the Workflow Task dies on the 2-second deadlock timeout -- on every retry, + because the records are still there; + - the reset at the start of each activation, or delivery stops for good once + the first budget is spent; + - the readiness re-arm, or the records the budget left buffered are never + announced again, because the watcher moved its prefetch cursor past them + when it buffered them. + + So a plain "did it finish" assertion is not a weak one here: nothing about + this Workflow completes if any part of the mechanism is missing. + """ + from temporalio.contrib.external_workflow_streams._api import ( + MAX_RECORDS_PER_ACTIVATION, + ) + + total = 3 * MAX_RECORDS_PER_ACTIVATION + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[FloodedCountWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + FloodedCountWorkflow.run, + total, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await asyncio.sleep(1) + # One session, one continuous sequence: `(session_id, sequence)` is the + # idempotency key, so restarting the numbering would re-use a key with + # different content and the backend would reject the append. + await publish(backend, key, [f"t{i}" for i in range(total)]) + + assert await asyncio.wait_for(handle.result(), 60) == total + + +async def test_a_clean_shutdown_sweeps_and_tears_down_the_manager( + client: Client, backend: MemoryStreamBackend +) -> None: + """P20's sweep has to run on the path users actually take. + + A normal ``Worker.shutdown()`` completes with no worker task raising, so + nothing is replaced by ``drain_poll_queue()``. Wiring the sweep there alone + means the whole mechanism is dead on the ordinary path: the manager never + enters its shutting-down state, every Run stays registered, watchers go on + polling, and the buffers and backend connections go out with the process -- + while the wake Signal that would hand the Run to another Worker is never + even considered. + + The Workflow here keeps a long timer running alongside its subscription, so + every Workflow Task completes and the Run sits *cached, subscribed, and + holding no Workflow Task* when shutdown begins. That is the state the sweep + exists for, and the one that gets no eviction activation of its own. + """ + task_queue = f"tq-{uuid.uuid4()}" + worker = Worker( + client, + task_queue=task_queue, + workflows=[TimerSuppressedSubscriptionWorkflow], + external_stream_backend=backend, + ) + worker_task = asyncio.create_task(worker.run()) + handle = None + try: + handle = await client.start_workflow( + TimerSuppressedSubscriptionWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + # Wait for the subscription to actually reach the manager rather than + # sleeping: with no Run registered there is nothing to sweep and the + # assertions below would hold for the wrong reason. + deadline = asyncio.get_running_loop().time() + 30 + while asyncio.get_running_loop().time() < deadline: + manager = worker._workflow_worker._external_stream_manager # type: ignore[union-attr] + if manager is not None and manager.runs_with_subscriptions(): + break + await asyncio.sleep(0.2) + manager = worker._workflow_worker._external_stream_manager # type: ignore[union-attr] + assert manager is not None and manager.runs_with_subscriptions(), ( + "the Workflow never registered a subscription, so there is nothing " + "for a shutdown sweep to find" + ) + + await asyncio.wait_for(worker.shutdown(), 60) + + assert manager._shutting_down, ( + "the manager was never told the Worker is shutting down, so no Run " + "was probed and no owed wake was sent" + ) + assert manager.runs_with_subscriptions() == [], ( + "subscriptions survived shutdown: their watchers, buffers, and " + "backend connections leak with the process" + ) + finally: + if handle is not None: + try: + await handle.terminate() + except Exception: + pass + if not worker_task.done(): + worker_task.cancel() + await asyncio.gather(worker_task, return_exceptions=True) + + +# --- P19 / ADR-011: decoding is Worker-side work, not Workflow-thread work --- + +#: What the probe codec saw each time it decoded a stream record: the thread it +#: ran on, and the module the running event loop's class comes from. Both are +#: recorded because either one alone is arguable -- a Workflow's deterministic +#: loop is identified by its class, and the executor thread by its name. +_decode_sites: list[tuple[str, str]] = [] + +#: Marks the one payload this probe reacts to, so ordinary activation payloads +#: -- arguments, results, headers -- are passed through untouched and cannot be +#: mistaken for a stream record's decode. +_STREAM_SENTINEL = b"external-stream-decode-probe" + +#: Longer than the Worker's 2-second deadlock timeout. A codec is user code and +#: may legitimately take this long: it can be fetching an external payload or +#: talking to a KMS. Awaited on the Worker's loop it costs nothing but latency; +#: awaited on the Workflow thread it is either a deadlocked Workflow Task or a +#: Workflow command synthesized out of a codec's internals. +_SLOW_DECODE = timedelta(seconds=2.5) + + +class ProbeCodec(temporalio.converter.PayloadCodec): + """A pass-through codec that reports where the stream record was decoded. + + Only the record carrying :py:data:`_STREAM_SENTINEL` is treated specially, + so the Worker's own ``decode_activation`` work -- which is *supposed* to run + on the Worker's loop -- adds no observations of its own. + """ + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return list(payloads) + + def __init__(self, delay: float | None = None) -> None: + self.delay = _SLOW_DECODE.total_seconds() if delay is None else delay + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + for payload in payloads: + if _STREAM_SENTINEL not in payload.data: + continue + loop = asyncio.get_running_loop() + _decode_sites.append( + (threading.current_thread().name, type(loop).__module__) + ) + # Real asynchronous work, of the length a codec is allowed to + # take: an external-payload fetch or a KMS round trip is I/O, and + # the Worker's loop is where the Worker awaits every other + # payload's codec. + await asyncio.sleep(self.delay) + return list(payloads) + + +async def test_a_slow_codec_decodes_off_the_workflow_thread( + client: Client, backend: MemoryStreamBackend +) -> None: + """A stream record's payload codec must not run inside ``activate()``. + + ``DataConverter.decode`` is three things: an external-payload retrieval, a + user ``PayloadCodec``, and a payload *converter*. The first two are + arbitrary asynchronous work -- a network fetch, a KMS round trip -- and the + Worker awaits them for every ordinary activation payload *before* handing + the activation to the Workflow executor, precisely so that no Workflow Task + can be failed by them. Only the third, a synchronous conversion, belongs on + the Workflow thread. + + A stream record is a payload like any other, so the same split has to hold + for it: by the time a record reaches ``activate()`` its codec has already + run on the Worker's loop, and the Workflow thread does nothing but convert + already-prepared bytes into a value. + + Running the codec on the Workflow thread instead is wrong three ways at + once, and the assertions below name them: the awaited work happens inside a + deterministic event loop that cannot perform I/O, it can exceed the + 2-second deadlock timeout and fail the Workflow Task for a perfectly + healthy codec, and anything the codec awaits is synthesized as Workflow + commands. + """ + _decode_sites.clear() + config = client.config() + config["data_converter"] = temporalio.converter.DataConverter( + payload_codec=ProbeCodec() + ) + codec_client = Client(**config) + + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + codec_client, + task_queue=task_queue, + workflows=[CountTokensWorkflow], + external_stream_backend=backend, + ): + handle = await codec_client.start_workflow( + CountTokensWorkflow.run, + 1, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + + # Published after the Workflow is already blocked on its subscription, + # so the record arrives through the live wake path rather than sitting + # in a buffer the first activation happens to find. + await asyncio.sleep(1) + from temporalio.contrib.external_workflow_streams._codec import ( + StreamPayloadCodec, + ) + + producer_codec = StreamPayloadCodec(codec_client.data_converter, str) + await backend.append( + key, + StreamRecord( + RecordKind.DATA, + await producer_codec.encode(_STREAM_SENTINEL.decode()), + "producer", + 0, + ), + ) + + # A Workflow Task failed by a deadlock is *retried*, and every retry + # re-decodes the same record and deadlocks again, so a Workflow that + # never returns is the visible form of that failure. + assert await asyncio.wait_for(handle.result(), 30) == 1 + + assert _decode_sites, ( + "the probe codec never saw the stream record, so this test proved " + "nothing about where decoding runs" + ) + for thread_name, loop_module in _decode_sites: + assert not thread_name.startswith("temporal_workflow_"), ( + f"the stream record's codec ran on the Workflow executor thread " + f"({thread_name}): arbitrary async work inside activate(), under " + "the 2-second deadlock timeout" + ) + assert loop_module != "temporalio.worker._workflow_instance", ( + "the stream record's codec was awaited on the Workflow's " + "deterministic event loop, which performs no I/O and turns every " + "await into a Workflow command" + ) + + # The codec's own awaits must not have become Workflow commands. A timer in + # History here is not a slow Workflow -- it is a Workflow whose History + # depends on what a codec did internally, and replay reproduces it only for + # as long as the codec behaves identically. + events = [e async for e in handle.fetch_history_events()] + assert not [e for e in events if e.HasField("timer_started_event_attributes")], ( + "the codec's await was turned into a Workflow timer command" + ) + + +async def test_a_replayed_record_is_prepared_off_the_workflow_thread( + env: WorkflowEnvironment, + replay_failure_trace: ReplayFailureTrace, + tmp_path: Path, +) -> None: + """Replay delivers down the same drain, so it prepares the same way. + + A replayed record never passes a watcher: its bytes are read and validated + when the replay job is prepared, before the Workflow thread runs at all. If + preparation happened only in the watcher, replay would hand raw producer + bytes to a Workflow whose converter has a codec -- and the choice would be + between running that codec inside ``activate()`` and yielding whatever the + undecoded bytes convert to. Preparing both paths in the same place is what + makes replay indistinguishable from live delivery, which is the property the + whole buffer design rests on. + """ + _decode_sites.clear() + backend = MemoryStreamBackend() + client = await env.connect_client( + data_converter=temporalio.converter.DataConverter(payload_codec=ProbeCodec(0)) + ) + task_queue = f"tq-{uuid.uuid4()}" + handle = None + async with Worker( + client, + task_queue=task_queue, + workflows=[ParkedAcrossEvictionWorkflow, FillerWorkflow], + external_stream_backend=backend, + max_cached_workflows=1, + max_concurrent_workflow_tasks=2, + ): + try: + handle = await client.start_workflow( + ParkedAcrossEvictionWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + first_run_id = ( + description.raw_description.workflow_execution_info.first_run_id + ) + key = StreamKey(client.namespace, handle.id, first_run_id, "tokens") + + await _wait_for_markers(handle, 1, "the Run never parked at all") + await publish( + backend, + key, + [_STREAM_SENTINEL.decode()], + session=f"producer-{uuid.uuid4()}", + ) + await _wait_for_markers( + handle, + 2, + "the Run never committed a marker, so nothing would be replayed", + ) + + # Evicts the Run, so the next Workflow Task replays the marker -- + # and re-delivers the recorded record through the replay path. + await client.execute_workflow( + FillerWorkflow.run, id=f"filler-{uuid.uuid4()}", task_queue=task_queue + ) + await publish(backend, key, ["beta"], session=f"producer-{uuid.uuid4()}") + request = WakeRequest( + namespace=client.namespace, + workflow_id=handle.id, + first_execution_run_id=first_run_id, + stream_name="tokens", + wait_id=1, + park_generation=0, + sender_identity=f"test-{uuid.uuid4()}", + wake_counter=1, + ) + replay_failure_trace.record_explicit_wake(request) + await send_wake_signal(client, request) + replay_failure_trace.record_explicit_wake_acknowledged(request) + + await _await_replayed_result( + handle, + [_STREAM_SENTINEL.decode(), "beta"], + backend, + key, + replay_failure_trace, + tmp_path, + ) + bursts = replay_failure_trace.manager_wake_bursts_after_explicit_ack() + assert max(bursts, default=0) <= 1, ( + "replayed readiness produced multiple wakes before Core accepted " + f"the task completion one wake caused: {replay_failure_trace.events}" + ) + finally: + if handle is not None: + try: + await handle.terminate() + except Exception: + pass + + assert _decode_sites, "the probe codec never saw the replayed record" + for thread_name, loop_module in _decode_sites: + assert not thread_name.startswith("temporal_workflow_"), ( + f"a replayed record's codec ran on the Workflow executor thread " + f"({thread_name})" + ) + assert loop_module != "temporalio.worker._workflow_instance", ( + "a replayed record's codec was awaited on the Workflow's " + "deterministic event loop" + ) + + +# --- P18: the failure taxonomy, connected end to end ------------------------- + + +class TaxonomyCodec(temporalio.converter.PayloadCodec): + """A pass-through codec that can be made to reject one known value. + + Stands in for the ordinary way row three happens: a consumer whose + converter or codec stops matching the producer's, with the stream itself + untouched. Scoped to the one sentinel value so the Workflow's own arguments + and result -- which travel through this same codec -- are unaffected. + """ + + def __init__(self) -> None: + self.fail = False + + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + return list(payloads) + + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> list[temporalio.api.common.v1.Payload]: + if self.fail and any(_TAXONOMY_SENTINEL in p.data for p in payloads): + raise RuntimeError("this codec cannot read the producer's payloads") + return list(payloads) + + +class UnreachableBackend(MemoryStreamBackend): + """A backend whose recorded-range read can be made to fail. + + Only ``read_range`` -- the replay read -- so the live path that wrote the + marker is unaffected and the failure lands where row one describes it: + reading back a range the marker already committed. + """ + + def __init__(self) -> None: + super().__init__() + self.fail_reads = False + + async def read_range( + self, key: StreamKey, first: Offset, last: Offset + ) -> list[StreamRecord]: + if self.fail_reads: + raise ConnectionError("the backend is unreachable") + return await super().read_range(key, first, last) + + +_TAXONOMY_SENTINEL = b"taxonomy-sentinel" + +#: The three counters, so each case can assert that the other two stayed silent. +_TAXONOMY_METRICS = ( + METRIC_STORAGE, + METRIC_INTEGRITY, + METRIC_DECODE, +) + + +def _failure_types(failure) -> list[str]: # type: ignore[no-untyped-def] + """Every application failure type in a failure's cause chain.""" + types = [] + while True: + types.append(failure.application_failure_info.type) + if not failure.HasField("cause"): + return types + failure = failure.cause + + +async def _await_failed_task(handle, message: str, timeout: float = 30): # type: ignore[no-untyped-def] + """The first ``WorkflowTaskFailed`` event, or an assertion naming why not.""" + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + for event in [e async for e in handle.fetch_history_events()]: + if event.HasField("workflow_task_failed_event_attributes"): + return event.workflow_task_failed_event_attributes + await asyncio.sleep(0.3) + raise AssertionError(message) + + +@pytest.mark.parametrize( + ["row", "error_type", "metric"], + [ + ("storage", "StreamStorageError", METRIC_STORAGE), + ("integrity", "StreamIntegrityError", METRIC_INTEGRITY), + ("decode", "StreamDecodeError", METRIC_DECODE), + ], +) +async def test_each_failure_row_is_reported_as_its_own( + env: WorkflowEnvironment, row: str, error_type: str, metric: str +) -> None: + """Rows one to three, each reaching an operator as itself. + + All three are Workflow Task failures, and the server retries a Workflow Task + failure whatever caused it -- so the retry says nothing about which one + happened. What distinguishes them is exactly two things, and this asserts + both: the completion carries the **external-storage failure cause**, which + separates all three from an ordinary Workflow bug, and **one counter** + increments while the other two stay silent, which is what lets an alert on + integrity loss mean integrity loss. + + They are reached the way an operator would meet them, through a marker that + was written live and then replayed against a backend that has since changed: + + - the recorded range cannot be read at all -- transient, clears itself; + - the recorded range reads back short -- the record was trimmed or expired, + and an operator has to repair the backend or terminate the Run; + - the range reads back exactly as recorded and the consumer's codec cannot + decode it -- the stream is undamaged and the configuration is wrong. + + The third is the one the taxonomy is most easily collapsed on, because it + is the only one whose failure surfaces from *inside* ``activate()``: the + record is prepared on the Worker's loop, its failure travels with it, and + the delivery that would have yielded its value raises it. + """ + backend = UnreachableBackend() + codec = TaxonomyCodec() + buffer = MetricBuffer(10000) + runtime = Runtime(telemetry=TelemetryConfig(metrics=buffer)) + client = await env.connect_client( + runtime=runtime, + data_converter=temporalio.converter.DataConverter(payload_codec=codec), + ) + + task_queue = f"tq-{uuid.uuid4()}" + handle = None + async with Worker( + client, + task_queue=task_queue, + workflows=[ParkedAcrossEvictionWorkflow, FillerWorkflow], + external_stream_backend=backend, + # One slot, so the filler Workflow evicts the Run under test and the + # next Workflow Task has to replay the marker. + max_cached_workflows=1, + max_concurrent_workflow_tasks=2, + ): + try: + handle = await client.start_workflow( + ParkedAcrossEvictionWorkflow.run, + 2, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + first_run_id = ( + description.raw_description.workflow_execution_info.first_run_id + ) + key = StreamKey(client.namespace, handle.id, first_run_id, "tokens") + + await _wait_for_markers(handle, 1, "the Run never parked at all") + # Consumed and committed live, so there is a recorded range to + # replay -- and the codec is asked for this value twice, once here + # while it still works and once on replay. + await publish( + backend, + key, + [_TAXONOMY_SENTINEL.decode()], + session=f"producer-{uuid.uuid4()}", + ) + await _wait_for_markers( + handle, + 2, + "the Run never committed a marker for the record it consumed, " + "so there is no recorded range for replay to read", + ) + + # Evicts the Run: buffers, watchers, and wait set all go with it, + # and the next Workflow Task replays the marker. + await client.execute_workflow( + FillerWorkflow.run, id=f"filler-{uuid.uuid4()}", task_queue=task_queue + ) + + if row == "storage": + backend.fail_reads = True + elif row == "integrity": + data = [r for r in backend.all_records(key) if not r.is_control] + assert data, "nothing was ever appended, so nothing can be lost" + assert data[0].offset is not None + await backend.delete_for_test(key, data[0].offset) + else: + codec.fail = True + + # The wake is what creates the Workflow Task that replays. + await send_wake_signal( + client, + WakeRequest( + namespace=client.namespace, + workflow_id=handle.id, + first_execution_run_id=first_run_id, + stream_name="tokens", + wait_id=1, + park_generation=0, + sender_identity=f"test-{uuid.uuid4()}", + wake_counter=1, + ), + ) + + failed = await _await_failed_task( + handle, + f"the {row} damage produced no Workflow Task failure at all, so " + "the Run either never replayed or silently delivered something", + ) + finally: + if handle is not None: + try: + await handle.terminate() + except Exception: + pass + + assert error_type in _failure_types(failed.failure), ( + f"the {row} failure reached the server as " + f"{_failure_types(failed.failure)} rather than as {error_type}, so an " + "operator cannot tell which of the three rows happened" + ) + assert ( + failed.cause + == temporalio.api.enums.v1.WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_EXTERNAL_STORAGE_FAILURE + ), ( + "the Workflow Task failed without the external-storage cause, so it is " + "indistinguishable from a bug in the Workflow's own code" + ) + + counted = { + update.metric.name + for update in buffer.retrieve_updates() + if update.metric.name in _TAXONOMY_METRICS + } + assert counted == {metric}, ( + f"the {row} failure incremented {sorted(counted)} rather than only " + f"{metric}; an alert on one row must not be diluted by another" + ) + + +@pytest.mark.needs_channel_server +async def test_an_outside_producer_wakes_the_reader_through_the_channel( + client: Client, backend: MemoryStreamBackend +) -> None: + """The channel path end to end: subscribe by command, notify, wake, replay. + + The same shape as the Signal case, with the backend asking for the channel + outright so a fallback fails the History checks below instead of passing + quietly. The Run subscribes to the stream's channel on the task that opens + the reader, the watcher notifies that channel when no task is open, and the + task the server schedules carries the notification. History then holds the + subscription event and no Signal, and a cold replay of it emits the + subscribe command where the live run did. + """ + support = await server_channel_support(client) + if support is ChannelSupport.NONE: + pytest.skip("the server does not implement notification channels") + if support is ChannelSupport.LINKED: + pytest.skip("a workflow-owned stream listens on its linked channel there") + handle, key = await _consume_two_through_the_channel(client, backend) + events = [e async for e in handle.fetch_history_events()] + assert _signalled(events) == [], ( + "a Signal woke the reader, so the channel was not the transport" + ) + assert _subscribed(events) == [channel_for(key).channel] + notified = _notified(events) + assert notified, "no Workflow Task was scheduled with a notification" + assert {n.channel for n in notified} == {channel_for(key).channel} + assert not any(n.HasField("linked_to") for n in notified) + + result = await Replayer( + workflows=[TimerThenConsumeWorkflow], external_stream_backend=backend + ).replay_workflow(await handle.fetch_history()) + assert result.replay_failure is None, f"replay failed: {result.replay_failure}" + + +@pytest.mark.needs_linked_server +async def test_an_outside_producer_wakes_the_reader_through_its_linked_channel( + client: Client, backend: MemoryStreamBackend +) -> None: + """The linked kind end to end: no command, notify the owner, wake, replay. + + The stream's channel lives in the reading workflow's own state, so the + run subscribes to nothing: the task that opens the reader carries no + command, the watcher notifies the channel by the owner's id, and the task + the server schedules carries a notification naming the owner. History then + holds neither a Signal nor a subscription event, and a cold replay takes + the same path from the recorded flags. + """ + if await server_channel_support(client) is not ChannelSupport.LINKED: + pytest.skip("the server does not serve channels linked to a workflow") + handle, key = await _consume_two_through_the_channel(client, backend) + events = [e async for e in handle.fetch_history_events()] + assert _signalled(events) == [], ( + "a Signal woke the reader, so the channel was not the transport" + ) + assert _subscribed(events) == [], "the owner is the listener by construction" + notified = _notified(events) + assert notified, "no Workflow Task was scheduled with a notification" + assert {n.channel for n in notified} == {channel_for(key).channel} + assert {(n.linked_to.type, n.linked_to.business_id) for n in notified} == { + (temporalio.api.enums.v1.ExecutionType.EXECUTION_TYPE_WORKFLOW, handle.id) + } + + result = await Replayer( + workflows=[TimerThenConsumeWorkflow], external_stream_backend=backend + ).replay_workflow(await handle.fetch_history()) + assert result.replay_failure is None, f"replay failed: {result.replay_failure}" + + +async def _consume_two_through_the_channel( + client: Client, backend: MemoryStreamBackend +) -> tuple[WorkflowHandle, StreamKey]: + """Runs the timer-then-consume shape with the channel asked for outright. + + The channel rather than ``auto``, so a step down to the Signal fails the + History checks of the caller instead of passing quietly. + """ + backend.wake_transport = "channel" + task_queue = f"tq-{uuid.uuid4()}" + async with Worker( + client, + task_queue=task_queue, + workflows=[TimerThenConsumeWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + TimerThenConsumeWorkflow.run, + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + description = await handle.describe() + key = StreamKey( + client.namespace, + handle.id, + description.raw_description.workflow_execution_info.first_run_id, + "tokens", + ) + await asyncio.sleep(1) + await publish(backend, key, ["first"]) + await asyncio.sleep(1) + await publish(backend, key, ["second"]) + + assert await asyncio.wait_for(handle.result(), 60) == ["first", "second"] + return handle, key + + +def _signalled(events: Sequence[Any]) -> list[Any]: + return [ + e for e in events if e.HasField("workflow_execution_signaled_event_attributes") + ] + + +def _subscribed(events: Sequence[Any]) -> list[str]: + return [ + e.workflow_notification_channel_subscribed_event_attributes.channel + for e in events + if e.HasField("workflow_notification_channel_subscribed_event_attributes") + ] + + +def _notified(events: Sequence[Any]) -> list[Any]: + return [ + notification + for e in events + if e.HasField("workflow_task_scheduled_event_attributes") + for notification in e.workflow_task_scheduled_event_attributes.notifications + ] diff --git a/tests/contrib/test_server_streams.py b/tests/contrib/test_server_streams.py new file mode 100644 index 0000000000..540f7f7d7b --- /dev/null +++ b/tests/contrib/test_server_streams.py @@ -0,0 +1,161 @@ +"""The Workflow Streams surface over a server-side stream. + +Both producers and the consumer, since the point of the surface is that an +application does not have to know which of them wrote a given item. Needs a +Temporal server built from the AI-198 branch: + + TEMPORAL_STREAM_TARGET=127.0.0.1:7333 uv run pytest tests/contrib/test_server_streams.py +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from dataclasses import dataclass +from datetime import timedelta + +import pytest + +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.contrib.server_streams import WorkflowStream, WorkflowStreamClient +from temporalio.worker import Worker + +TARGET = os.environ.get("TEMPORAL_STREAM_TARGET") + +pytestmark = pytest.mark.skipif( + not TARGET, + reason="set TEMPORAL_STREAM_TARGET to a server with the stream service", +) + +TOPIC = "turn_events" + + +@dataclass +class Event: + source: str + text: str + + +@activity.defn +async def emit_from_activity(count: int) -> None: + async with WorkflowStreamClient.from_within_activity() as client: + events = client.topic(TOPIC, type=Event) + for i in range(count): + events.publish(Event(source="activity", text=f"token {i}")) + + +@workflow.defn +class Emitting: + def __init__(self) -> None: + self._events = WorkflowStream().topic(TOPIC, type=Event) + + @workflow.run + async def run(self, count: int) -> None: + self._events.publish(Event(source="workflow", text="turn started")) + await workflow.execute_activity( + emit_from_activity, + count, + start_to_close_timeout=timedelta(seconds=30), + ) + self._events.publish(Event(source="workflow", text="turn ended")) + + +async def test_both_producers_reach_one_subscriber() -> None: + client = await Client.connect(TARGET or "") + task_queue = "ss-tq-" + uuid.uuid4().hex[:8] + wf_id = "ss-wf-" + uuid.uuid4().hex[:8] + tokens = 4 + + seen: list[Event] = [] + offsets: list[int] = [] + + async with Worker( + client, + task_queue=task_queue, + workflows=[Emitting], + activities=[emit_from_activity], + max_cached_workflows=0, + ): + handle = await client.start_workflow( + Emitting.run, tokens, id=wf_id, task_queue=task_queue + ) + + # Subscribed before the Workflow has published anything, which is what + # a consumer attaching to a session does. + stream = WorkflowStreamClient.create(client, wf_id) + + async def read() -> None: + async for item in stream.subscribe( + topics=[TOPIC], from_offset=0, result_type=Event + ): + seen.append(item.data) + offsets.append(item.offset) + if len(seen) == tokens + 2: + return + + reading = asyncio.ensure_future(read()) + await asyncio.wait_for(handle.result(), timeout=60) + await asyncio.wait_for(reading, timeout=60) + + # The Workflow's own publishes bracket the Activity's, and both are on one + # log in the order the server took them. + assert [e.source for e in seen] == ["workflow"] + ["activity"] * tokens + [ + "workflow" + ] + assert seen[0].text == "turn started" + assert seen[-1].text == "turn ended" + assert offsets == list(range(tokens + 2)) + + # A consumer that arrives after the fact reads the same thing, and is not + # left tailing: the Workflow has ended, so its stream is finished and the + # subscription ends on its own. + late = WorkflowStreamClient.create(client, wf_id) + assert await late.get_offset() == tokens + 2 + replayed = [ + item.data.text + async for item in late.subscribe( + topics=[TOPIC], from_offset=0, result_type=Event + ) + ] + assert replayed == [e.text for e in seen] + + +async def test_a_reader_resumes_from_an_offset_it_was_given() -> None: + client = await Client.connect(TARGET or "") + task_queue = "ss-resume-tq-" + uuid.uuid4().hex[:8] + wf_id = "ss-resume-wf-" + uuid.uuid4().hex[:8] + + async with Worker( + client, + task_queue=task_queue, + workflows=[Emitting], + activities=[emit_from_activity], + max_cached_workflows=0, + ): + await client.execute_workflow(Emitting.run, 3, id=wf_id, task_queue=task_queue) + + stream = WorkflowStreamClient.create(client, wf_id) + first = [ + item + async for item in stream.subscribe( + topics=[TOPIC], from_offset=0, result_type=Event + ) + ] + assert len(first) == 5 + + # Resuming past the second item skips exactly the two before it, so the + # offset a reader was handed is the position it means. + resumed = [ + item.data.text + async for item in stream.subscribe( + topics=[TOPIC], from_offset=first[1].offset + 1, result_type=Event + ) + ] + assert resumed == [item.data.text for item in first[2:]] + + # The raw page carries the stored records themselves. + raw = await stream.poll_raw(topics=[TOPIC], from_offset=0, wait=False) + assert [item.data.topic for item in raw.items] == [TOPIC] * 5 + assert raw.closed and raw.next_offset == raw.head_offset == 5 diff --git a/tests/contrib/test_server_streams_flush.py b/tests/contrib/test_server_streams_flush.py new file mode 100644 index 0000000000..23a72106e4 --- /dev/null +++ b/tests/contrib/test_server_streams_flush.py @@ -0,0 +1,86 @@ +"""What the buffered client does with a batch, without a server. + +The parity claim in the module docstring is that an application swaps the +import and keeps its code, so the two things the shipped client does on every +flush have to hold here too: the append carries a producer identity, and a +batch whose append failed is kept for the retry rather than lost with it. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio.api.stream.v1 import StreamRecord +from temporalio.contrib.server_streams import WorkflowStreamClient +from temporalio.converter import DataConverter + + +class _Handle: + """A stream handle that records its appends and can fail the first one.""" + + owner_run_id = "run" + + def __init__(self, *, fail_first: bool = False) -> None: + self.sent: list[tuple[list[StreamRecord], str, int]] = [] + self._fail_first = fail_first + + def pin(self, run_id: str) -> None: + del run_id + + async def append( + self, *records: StreamRecord, producer_id: str = "", sequence: int = 0 + ) -> Any: + self.sent.append((list(records), producer_id, sequence)) + if self._fail_first: + self._fail_first = False + raise ConnectionResetError("the server took it, the reply was lost") + return None + + +def _client(handle: _Handle) -> WorkflowStreamClient: + return WorkflowStreamClient( + handle, # type: ignore[arg-type] + DataConverter.default.payload_converter, + timedelta(seconds=60), + producer_id="act#1", + ) + + +def _bodies(sent: tuple[list[StreamRecord], str, int]) -> list[bytes]: + return [record.body.data for record in sent[0]] + + +async def test_an_append_carries_who_wrote_it_and_where_it_sits() -> None: + handle = _Handle() + client = _client(handle) + client.topic("t").publish({"n": 1}) + client.topic("t").publish({"n": 2}) + await client.flush() + client.topic("t").publish({"n": 3}) + await client.flush() + + # Without an identity the append is at-least-once, which is what the + # shipped client refuses to be. + assert [(who, seq) for _, who, seq in handle.sent] == [("act#1", 0), ("act#1", 2)] + + +async def test_a_batch_whose_append_failed_goes_out_again() -> None: + handle = _Handle(fail_first=True) + client = _client(handle) + client.topic("t").publish({"n": 1}) + with pytest.raises(ConnectionResetError): + await client.flush() + # Taken off the buffer before the await, it would have gone with the + # failure. It goes out again under the sequence it already had, so a copy + # the server did accept is deduplicated and one it never saw lands. + await client.flush() + assert len(handle.sent) == 2 + assert _bodies(handle.sent[0]) == _bodies(handle.sent[1]) + assert [seq for _, _, seq in handle.sent] == [0, 0] + + client.topic("t").publish({"n": 2}) + await client.flush() + assert handle.sent[2][2] == 1, "the next batch continues past the first" diff --git a/tests/nexus/test_temporal_operation.py b/tests/nexus/test_temporal_operation.py index 66bc2290b4..68627e25aa 100644 --- a/tests/nexus/test_temporal_operation.py +++ b/tests/nexus/test_temporal_operation.py @@ -632,6 +632,10 @@ async def run_update(i: int) -> str: async def check_updates_on_completed_workflows_fail(): """The handler workflow already finished at this point, further updaes should just fail""" + # The last update's result does not mean the target has closed yet. + # An update that lands before the closure stays pending instead of + # failing. + await target_handle.result() wf_handle = await client.start_workflow( UpdateWorkflowCaller.run, Input( diff --git a/tests/streams/__init__.py b/tests/streams/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/streams/conftest.py b/tests/streams/conftest.py new file mode 100644 index 0000000000..feb1264e10 --- /dev/null +++ b/tests/streams/conftest.py @@ -0,0 +1,206 @@ +import pytest + +_ENVIRONMENTS_WITHOUT_CHANNELS = ("local", "time-skipping", "envconfig") + +# The Core the bridge pins decides whether a workflow's subscribe command +# reaches the server. The protos-only pin refuses it; the delivery pin that +# the native layers move to handles it. +PINNED_CORE_HANDLES_CHANNEL_COMMAND = True + +# A channel linked to a workflow needs no command, but a notification reaches +# workflow code as the `NotificationsReceived` job Core builds from the +# scheduled event, and the protos-only pin ignores that job. So a linked case +# in which the workflow receives waits for the delivery pin as well; one that +# only talks to the server from the client runs on every layer. +PINNED_CORE_HANDLES_LINKED_CHANNEL = True + +# The unsubscribe is a command like the subscribe, refused by the protos-only +# pin and matched against its event by the delivery pin. +PINNED_CORE_HANDLES_UNSUBSCRIBE = True + +# A native stream lives on the server, and only the native layers carry a +# provider that opens one; the providers here hold streams in memory or in a +# workflow's History. A case that produces to a native stream waits for the +# layer with that provider. +PINNED_LAYER_HOSTS_NATIVE_STREAMS = True + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "reports_positions: the case needs append() to return where records landed", + ) + config.addinivalue_line( + "markers", + "detects_divergent_retries: the case needs append() to compare a repeat's " + "content with what the store holds", + ) + config.addinivalue_line( + "markers", + "truncates: the case needs a way to drop a topic's oldest records", + ) + config.addinivalue_line( + "markers", + "encodes_bodies: the case needs the outside path to run each body through " + "the client's data converter", + ) + config.addinivalue_line( + "markers", + "standalone_activities: the case needs the streams of an activity outside " + "any workflow", + ) + config.addinivalue_line( + "markers", + "hosts_standalone_streams: the case needs a stream with an id of its own and " + "no owner", + ) + config.addinivalue_line( + "markers", + "wakes_by_notification: the case needs an outside append to wake a parked " + "workflow reader through the server", + ) + config.addinivalue_line( + "markers", + "wakes_by_linked_notification: the case needs an outside append to wake a " + "parked workflow reader through the channel linked to its workflow", + ) + config.addinivalue_line( + "markers", + "needs_channel_server: the case needs a server that serves notification " + "channels, named with -E host:port", + ) + config.addinivalue_line( + "markers", + "needs_channel_core: the case needs the pinned Core to handle the " + "subscribe-notification-channel command", + ) + config.addinivalue_line( + "markers", + "needs_linked_server: the case needs a server that serves channels linked " + "to a workflow, named with -E host:port; the case skips itself on one " + "with only independent channels", + ) + config.addinivalue_line( + "markers", + "needs_linked_core: the case needs the pinned Core to hand a linked " + "channel's notifications to workflow code", + ) + config.addinivalue_line( + "markers", + "needs_describe_server: the case needs a server whose workflow description " + "lists the channel subscriptions, named with -E host:port", + ) + config.addinivalue_line( + "markers", + "needs_unsubscribe_server: the case needs a server that accepts the " + "unsubscribe-notification-channel command, named with -E host:port", + ) + config.addinivalue_line( + "markers", + "needs_unsubscribe_core: the case needs the pinned Core to handle the " + "unsubscribe-notification-channel command", + ) + config.addinivalue_line( + "markers", + "needs_stream_channel_server: the case needs a server on which a native " + "stream notifies the channel named by the stream, named with -E host:port", + ) + config.addinivalue_line( + "markers", + "needs_execution_server: the case needs a server that addresses a linked " + "channel by execution, a standalone activity's included, named with " + "-E host:port", + ) + config.addinivalue_line( + "markers", + "needs_native_provider: the case needs a provider that opens native streams " + "on the server, which only the native layers carry", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + # The dev server the suite starts for itself does not accept the + # subscribe command, so the live channel cases run only against a server + # the caller points at. + skips: list[tuple[str, pytest.MarkDecorator]] = [] + if config.getoption("--workflow-environment") in _ENVIRONMENTS_WITHOUT_CHANNELS: + skips.append( + ( + "needs_channel_server", + pytest.mark.skip( + reason="needs a server that serves notification channels; " + "name one with -E" + ), + ) + ) + if not PINNED_CORE_HANDLES_CHANNEL_COMMAND: + skips.append( + ( + "needs_channel_core", + pytest.mark.skip( + reason="the pinned Core refuses the subscribe command; py-05 " + "pins one that handles it" + ), + ) + ) + if config.getoption("--workflow-environment") in _ENVIRONMENTS_WITHOUT_CHANNELS: + skips.append( + ( + "needs_linked_server", + pytest.mark.skip( + reason="needs a server with channels linked to a workflow; " + "name one with -E" + ), + ) + ) + if not PINNED_CORE_HANDLES_LINKED_CHANNEL: + skips.append( + ( + "needs_linked_core", + pytest.mark.skip( + reason="the pinned Core ignores the notifications job; py-05 " + "pins one that delivers it" + ), + ) + ) + if config.getoption("--workflow-environment") in _ENVIRONMENTS_WITHOUT_CHANNELS: + for marker, what in ( + ("needs_describe_server", "lists channel subscriptions on describe"), + ("needs_unsubscribe_server", "accepts the unsubscribe command"), + ("needs_stream_channel_server", "notifies a stream's channel"), + ("needs_execution_server", "addresses a linked channel by execution"), + ): + skips.append( + ( + marker, + pytest.mark.skip( + reason=f"needs a server that {what}; name one with -E" + ), + ) + ) + if not PINNED_CORE_HANDLES_UNSUBSCRIBE: + skips.append( + ( + "needs_unsubscribe_core", + pytest.mark.skip( + reason="the pinned Core refuses the unsubscribe command; py-05 " + "pins one that handles it" + ), + ) + ) + if not PINNED_LAYER_HOSTS_NATIVE_STREAMS: + skips.append( + ( + "needs_native_provider", + pytest.mark.skip( + reason="this layer carries no provider for native streams; the " + "native layers and the union do" + ), + ) + ) + for item in items: + for marker, skip in skips: + if item.get_closest_marker(marker): + item.add_marker(skip) diff --git a/tests/streams/test_activity_streams.py b/tests/streams/test_activity_streams.py new file mode 100644 index 0000000000..194046ded3 --- /dev/null +++ b/tests/streams/test_activity_streams.py @@ -0,0 +1,333 @@ +"""Streams owned by activities, and which stream an activity's accessor reaches. + +``activity.stream_handle()`` resolves by a static rule, never by what exists: +an activity a workflow scheduled reaches its workflow's stream, a standalone +activity reaches its own, and ``scope="activity"`` gives an activity a +workflow scheduled its own streams. An activity's own streams are one per +activity execution, so a retry writes to the same stream and a reader sees +the attempt change as ``SUPERSEDED``. + +The memory provider always runs, and so does Workflow Streams, whose store is +the workflow's own History. A storage provider adds itself to ``SETUPS`` +behind its own ``STREAMS_LIVE`` gate: its setup receives the environment's +client and hands back the provider and a client with it registered, which the +workers and the reads in these cases share, and says whether it holds the +streams of a standalone activity, so the cases marked +``standalone_activities`` are skipped with a reason where it does not. +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from collections.abc import AsyncIterator, Callable +from dataclasses import dataclass +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.common import RetryPolicy +from temporalio.streams import ( + BEGINNING, + RecordKind, + StreamProvider, + topic, +) +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.streams.providers.native import NativeStreams +from temporalio.streams.providers.redis import RedisStreams +from temporalio.streams.providers.workflow_streams import WorkflowStreamsProvider +from temporalio.testing import WorkflowEnvironment +from tests.helpers import new_worker + +TOKENS = topic("tokens", dict) + + +@dataclass +class ActivitySetup: + """A provider under test, registered on the client the cases use.""" + + name: str + provider: StreamProvider + client: Client + standalone_activities: bool = True + """The provider holds the streams of an activity outside any workflow.""" + + +async def _memory_setup(client: Client) -> AsyncIterator[ActivitySetup]: + provider = MemoryStreams(poll_interval=timedelta(milliseconds=50)) + config = client.config() + config["plugins"] = [provider] + yield ActivitySetup("memory", provider, Client(**config)) + provider.reset() + + +async def _native_setup(client: Client) -> AsyncIterator[ActivitySetup]: + # The store is a server built from the stream-carrying branch, which the + # test environment's own server is not; TEMPORAL_ADDRESS names it. + address = os.environ.get("TEMPORAL_ADDRESS") + if address: + client = await Client.connect( + address, namespace=os.environ.get("TEMPORAL_NAMESPACE", "default") + ) + provider = NativeStreams() + config = client.config() + config["plugins"] = [provider] + yield ActivitySetup("native", provider, Client(**config)) + await provider.close() + + +async def _redis_setup(client: Client) -> AsyncIterator[ActivitySetup]: + # The store is a Redis the test environment does not start; the server + # is the environment's own unless TEMPORAL_ADDRESS names another. + address = os.environ.get("TEMPORAL_ADDRESS") + if address: + client = await Client.connect( + address, namespace=os.environ.get("TEMPORAL_NAMESPACE", "default") + ) + provider = RedisStreams( + url=os.environ.get("TEMPORAL_TEST_REDIS_URL") + or os.environ.get("AI198_REDIS_URL", "redis://127.0.0.1:6379"), + # A prefix per setup, because the store keeps what earlier runs wrote. + key_prefix=f"streams-activity-{uuid.uuid4().hex}", + poll_interval=timedelta(milliseconds=100), + ) + config = client.config() + config["plugins"] = [provider] + yield ActivitySetup("redis", provider, Client(**config)) + await provider.close() + + +async def _workflow_streams_setup(client: Client) -> AsyncIterator[ActivitySetup]: + provider = WorkflowStreamsProvider(poll_cooldown=timedelta(milliseconds=20)) + config = client.config() + config["plugins"] = [provider] + # An activity's streams live in its workflow's log, so a standalone + # activity has nowhere to put them. See the provider's module docstring. + yield ActivitySetup( + "workflow_streams", provider, Client(**config), standalone_activities=False + ) + + +SETUPS: dict[str, Callable[[Client], AsyncIterator[ActivitySetup]]] = { + "memory": _memory_setup, + "workflow_streams": _workflow_streams_setup, +} +if os.environ.get("STREAMS_LIVE") == "native": + SETUPS["native"] = _native_setup +if os.environ.get("STREAMS_LIVE") == "redis": + SETUPS["redis"] = _redis_setup + + +@pytest.fixture(params=sorted(SETUPS)) +async def setup( + request: pytest.FixtureRequest, client: Client, env: WorkflowEnvironment +) -> AsyncIterator[ActivitySetup]: + if env.supports_time_skipping: + pytest.skip("the time-skipping test server has no standalone activities") + async for found in SETUPS[request.param](client): + if ( + request.node.get_closest_marker("standalone_activities") + and not found.standalone_activities + ): + pytest.skip( + f"the {found.name} provider does not hold a standalone activity's streams" + ) + yield found + + +async def read_all(records: Any, timeout: float = 30.0) -> list: + """Read until the stream ends, which is when its owner does.""" + + async def _collect() -> list: + return [record async for record in records] + + return await asyncio.wait_for(_collect(), timeout) + + +def summary(records: list) -> list[tuple[Any, ...]]: + return [ + (r.kind, r.attempt, r.value["token"] if r.kind is RecordKind.DATA else None) + for r in records + ] + + +@activity.defn +async def write_by_default(label: str) -> str: + # No arguments: where this lands depends only on where the activity runs. + producer = activity.stream_handle().producer(topic=TOKENS) + await producer.append({"token": label}) + await producer.finish() + return activity.info().activity_id + + +@activity.defn +async def write_to_own_streams(label: str) -> str: + producer = activity.stream_handle(scope="activity").producer(topic=TOKENS) + await producer.append({"token": label}) + await producer.finish() + return activity.info().activity_id + + +@workflow.defn +class RunsOneActivity: + """Runs one activity by name and returns what it returned.""" + + @workflow.run + async def run(self, name: str, label: str) -> str: + return await workflow.execute_activity( + name, + label, + activity_id="streamer", + start_to_close_timeout=timedelta(seconds=30), + ) + + +async def test_workflow_activity_defaults_to_its_workflow(setup: ActivitySetup): + client = setup.client + workflow_id = f"streams-wfa-{uuid.uuid4().hex}" + async with new_worker( + client, RunsOneActivity, activities=[write_by_default] + ) as worker: + await client.execute_workflow( + RunsOneActivity.run, + args=["write_by_default", "to the workflow"], + id=workflow_id, + task_queue=worker.task_queue, + ) + records = await read_all( + client.get_stream_handle(workflow_id).read(topic=TOKENS) + ) + assert summary(records) == [ + (RecordKind.DATA, 1, "to the workflow"), + (RecordKind.FINISH, 1, None), + ] + assert all(r.producer_id == "streamer" for r in records) + own = client.get_stream_handle(workflow_id, activity_id="streamer") + assert await own.latest(topic=TOKENS) == BEGINNING + + +async def test_scope_activity_gives_a_workflow_activity_its_own_streams( + setup: ActivitySetup, +): + client = setup.client + workflow_id = f"streams-wfa-own-{uuid.uuid4().hex}" + async with new_worker( + client, RunsOneActivity, activities=[write_to_own_streams] + ) as worker: + await client.execute_workflow( + RunsOneActivity.run, + args=["write_to_own_streams", "to the activity"], + id=workflow_id, + task_queue=worker.task_queue, + ) + own = client.get_stream_handle(workflow_id, activity_id="streamer") + assert summary(await read_all(own.read(topic=TOKENS))) == [ + (RecordKind.DATA, 1, "to the activity"), + (RecordKind.FINISH, 1, None), + ] + # The workflow's topic of the same name is a different stream. + workflow_stream = client.get_stream_handle(workflow_id) + assert await workflow_stream.latest(topic=TOKENS) == BEGINNING + + +@pytest.mark.standalone_activities +async def test_standalone_activity_defaults_to_its_own_stream(setup: ActivitySetup): + client = setup.client + activity_id = f"streams-saa-{uuid.uuid4().hex}" + async with new_worker(client, activities=[write_by_default]) as worker: + handle = await client.start_activity( + write_by_default, + "standalone", + id=activity_id, + task_queue=worker.task_queue, + start_to_close_timeout=timedelta(seconds=30), + ) + assert await handle.result() == activity_id + stream = client.get_stream_handle(activity_id=activity_id) + # The read ends by itself: the activity reached a terminal status. + records = await read_all(stream.read(topic=TOKENS)) + assert summary(records) == [ + (RecordKind.DATA, 1, "standalone"), + (RecordKind.FINISH, 1, None), + ] + assert all(r.producer_id == activity_id for r in records) + + +@activity.defn +async def fail_once_after_writing() -> None: + attempt = activity.info().attempt + producer = activity.stream_handle().producer(topic=TOKENS) + await producer.append({"token": f"attempt {attempt}"}) + if attempt == 1: + raise RuntimeError("the first attempt fails after writing") + await producer.finish() + + +@pytest.mark.standalone_activities +async def test_a_retry_inherits_the_stream_and_supersedes(setup: ActivitySetup): + client = setup.client + activity_id = f"streams-saa-retry-{uuid.uuid4().hex}" + async with new_worker(client, activities=[fail_once_after_writing]) as worker: + handle = await client.start_activity( + fail_once_after_writing, + id=activity_id, + task_queue=worker.task_queue, + start_to_close_timeout=timedelta(seconds=30), + retry_policy=RetryPolicy( + initial_interval=timedelta(milliseconds=200), maximum_attempts=2 + ), + ) + await handle.result() + records = await read_all( + client.get_stream_handle(activity_id=activity_id).read(topic=TOKENS) + ) + # One stream across both attempts, with the takeover reported between. + assert summary(records) == [ + (RecordKind.DATA, 1, "attempt 1"), + (RecordKind.SUPERSEDED, 2, None), + (RecordKind.DATA, 2, "attempt 2"), + (RecordKind.FINISH, 2, None), + ] + superseded = records[1].supersession + assert superseded is not None + assert (superseded.previous_attempt, superseded.attempt) == (1, 2) + + +@activity.defn +async def ask_for_misaddressed_handles() -> list[str]: + errors: list[str] = [] + try: + activity.stream_handle(scope="workflow") + except RuntimeError as error: + errors.append(f"RuntimeError: {error}") + try: + activity.stream_handle("some-workflow", scope="activity") + except ValueError as error: + errors.append(f"ValueError: {error}") + return errors + + +async def test_a_standalone_activity_has_no_workflow_to_address(setup: ActivitySetup): + client = setup.client + async with new_worker(client, activities=[ask_for_misaddressed_handles]) as worker: + errors = await client.execute_activity( + ask_for_misaddressed_handles, + id=f"streams-saa-errors-{uuid.uuid4().hex}", + task_queue=worker.task_queue, + start_to_close_timeout=timedelta(seconds=30), + ) + assert len(errors) == 2 + assert errors[0].startswith("RuntimeError: this activity belongs to no workflow") + assert errors[1].startswith("ValueError: scope='activity'") + + +async def test_get_stream_handle_needs_an_owner(setup: ActivitySetup): + with pytest.raises( + ValueError, match="workflow_id, the activity_id or the stream_id" + ): + setup.client.get_stream_handle() diff --git a/tests/streams/test_channels.py b/tests/streams/test_channels.py new file mode 100644 index 0000000000..afcdc4a44a --- /dev/null +++ b/tests/streams/test_channels.py @@ -0,0 +1,1272 @@ +"""The notification channel surface: the command, the delivery and the client calls. + +Both kinds of channel are covered: the independent one a workflow subscribes +to by command, and the one linked to the workflow, which needs none. The +workflow instance is driven with activations directly, the way Core drives +it, because the dev server this chain tests against does not accept the +subscribe command. The live cases at the end need a server that does, or one +with the linked kind, one whose describe lists the subscriptions, or one that +accepts the unsubscribe, and skip otherwise. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import uuid +from datetime import datetime, timedelta, timezone +from typing import Any + +import pytest + +import temporalio.api.common.v1 +import temporalio.api.enums.v1 +import temporalio.api.notification.v1 +import temporalio.api.workflow.v1 +import temporalio.api.workflowservice.v1 +import temporalio.bridge.proto.workflow_activation +import temporalio.bridge.proto.workflow_completion +import temporalio.common +import temporalio.converter +from temporalio import workflow +from temporalio.api.enums.v1 import EventType +from temporalio.client import ( + Callback, + ChannelAddress, + ChannelKind, + ChannelSubscriptionInfo, + Client, + WorkflowExecutionDescription, + stream_channel, +) +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import DEFAULT_TOPIC, StreamRef +from temporalio.worker._workflow_instance import ( + UnsandboxedWorkflowRunner, + WorkflowInstance, + WorkflowInstanceDetails, +) +from tests.helpers import assert_eventually, new_worker + +WorkflowActivation = temporalio.bridge.proto.workflow_activation.WorkflowActivation +WorkflowActivationJob = ( + temporalio.bridge.proto.workflow_activation.WorkflowActivationJob +) +WorkflowActivationCompletion = ( + temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion +) +Notification = temporalio.api.notification.v1.Notification + + +@workflow.defn +class ReceiveOne: + """Subscribes to one channel and reports the first notification.""" + + @workflow.run + async def run(self, channel: str) -> dict[str, Any]: + notification = await workflow.subscribe_channel(channel).receive() + return { + "channel": notification.channel, + "counter": notification.counter, + "position": notification.position.decode(), + "topic": ( + workflow.payload_converter().from_payload( + notification.metadata["topic"], str + ) + if "topic" in notification.metadata + else None + ), + } + + +@workflow.defn +class CountToTwo: + """Subscribes twice to one channel and counts notifications up to counter two.""" + + @workflow.run + async def run(self, channel: str) -> int: + first = workflow.subscribe_channel(channel) + second = workflow.subscribe_channel(channel) + assert first is second + seen = 0 + async for notification in second: + seen += 1 + if notification.counter >= 2: + break + return seen + + +@workflow.defn +class EmptyChannel: + """Asks for a channel with no name.""" + + @workflow.run + async def run(self) -> None: + workflow.subscribe_channel("") + + +@workflow.defn +class EmptyLinkedChannel: + """Asks for a linked channel with no name.""" + + @workflow.run + async def run(self) -> None: + workflow.linked_channel("") + + +def _describe(notification: workflow.Notification) -> dict[str, Any]: + return { + "channel": notification.channel, + "counter": notification.counter, + "position": notification.position.decode(), + "topic": ( + workflow.payload_converter().from_payload( + notification.metadata["topic"], str + ) + if "topic" in notification.metadata + else None + ), + "owner": ( + notification.linked_to.business_id if notification.linked_to else None + ), + "owner_run": notification.linked_to.run_id if notification.linked_to else None, + } + + +@workflow.defn +class ReceiveLinked: + """Listens on its linked channel and reports the first notification.""" + + @workflow.run + async def run(self, channel: str) -> dict[str, Any]: + handle = workflow.linked_channel(channel) + assert handle is workflow.linked_channel(channel) + assert handle.linked + return _describe(await handle.receive()) + + +@workflow.defn +class BothKinds: + """Holds both kinds of handle on one name and keeps what each receives. + + Ends once the linked handle has seen counter two. + """ + + @workflow.run + async def run(self, channel: str) -> dict[str, list[int]]: + independent = workflow.subscribe_channel(channel) + linked = workflow.linked_channel(channel) + assert not independent.linked and linked.linked + seen: dict[str, list[int]] = {"independent": [], "linked": []} + + async def collect_independent() -> None: + async for notification in independent: + assert notification.linked_to is None + seen["independent"].append(notification.counter) + + collector = asyncio.create_task(collect_independent()) + async for notification in linked: + assert notification.linked_to is not None + seen["linked"].append(notification.counter) + if notification.counter >= 2: + break + collector.cancel() + return seen + + +@workflow.defn +class CountLinked: + """Counts the notifications on its linked channel up to counter two.""" + + @workflow.run + async def run(self, channel: str) -> int: + seen = 0 + async for notification in workflow.linked_channel(channel): + seen += 1 + if notification.counter >= 2: + break + return seen + + +async def _drain(handle: workflow.ChannelSubscription) -> dict[str, Any]: + """What a closed handle still gives: the queue, then the end, then the refusal.""" + drained = [notification.counter async for notification in handle] + try: + await handle.receive() + except RuntimeError as err: + refused: str | None = str(err) + else: + refused = None + return {"closed": handle.closed, "drained": drained, "refused": refused} + + +@workflow.defn +class ReceiveThenUnsubscribe: + """Takes the first notification, unsubscribes twice, then waits to be finished. + + The wait keeps the run open so a late notification can be aimed at it. + """ + + def __init__(self) -> None: + self._done = False + + @workflow.run + async def run(self, channel: str) -> dict[str, Any]: + handle = workflow.subscribe_channel(channel) + first = await handle.receive() + assert not handle.closed + handle.unsubscribe() + handle.unsubscribe() + drained = await _drain(handle) + await workflow.wait_condition(lambda: self._done) + return {"first": first.counter, **drained} + + @workflow.signal + def finish(self) -> None: + self._done = True + + +@workflow.defn +class UnsubscribeWithOneQueued: + """Unsubscribes with a notification still queued and reads it afterwards.""" + + @workflow.run + async def run(self, channel: str) -> dict[str, Any]: + handle = workflow.subscribe_channel(channel) + first = await handle.receive() + handle.unsubscribe() + return {"first": first.counter, **(await _drain(handle))} + + +@workflow.defn +class UnsubscribeLinked: + """Tries to unsubscribe from its linked channel.""" + + @workflow.run + async def run(self, channel: str) -> None: + workflow.linked_channel(channel).unsubscribe() + + +@workflow.defn +class Resubscribe: + """Subscribes, unsubscribes and subscribes again, then receives on the new handle.""" + + @workflow.run + async def run(self, channel: str) -> dict[str, Any]: + first = workflow.subscribe_channel(channel) + first.unsubscribe() + second = workflow.subscribe_channel(channel) + assert second is not first + assert first.closed and not second.closed + notification = await second.receive() + return {"counter": notification.counter, **(await _drain(first))} + + +def _instance(workflow_class: type) -> WorkflowInstance: + """Build an instance the way the worker does, without a worker. + + Needs a running event loop, since the constructor puts the runtime on it. + """ + defn = workflow._Definition.must_from_class(workflow_class) + now = datetime.now(timezone.utc) + info = workflow.Info( + attempt=1, + continued_run_id=None, + cron_schedule=None, + execution_timeout=None, + first_execution_run_id="run", + headers={}, + namespace="default", + original_execution_run_id="run", + parent=None, + root=None, + priority=temporalio.common.Priority.default, + raw_memo={}, + retry_policy=None, + run_id="run", + run_timeout=None, + search_attributes={}, + start_time=now, + task_queue="tq", + task_timeout=timedelta(seconds=10), + typed_search_attributes=temporalio.common.TypedSearchAttributes.empty, + workflow_id="wf", + workflow_start_time=now, + workflow_type=defn.name or "", + ) + converter = temporalio.converter.DataConverter.default + return UnsandboxedWorkflowRunner().create_instance( + WorkflowInstanceDetails( + payload_converter_factory=converter._new_internal_payload_converter, + failure_converter_class=converter.failure_converter_class, + interceptor_classes=[], + defn=defn, + info=info, + randomness_seed=0, + extern_functions={}, + disable_eager_activity_execution=False, + worker_level_failure_exception_types=[], + patch_activation_callback=None, + last_completion_result=temporalio.api.common.v1.Payloads(), + last_failure=None, + ) + ) + + +def _start(workflow_class: type, *args: Any) -> WorkflowActivation: + job = WorkflowActivationJob() + init = job.initialize_workflow + init.workflow_type = workflow._Definition.must_from_class(workflow_class).name or "" + init.workflow_id = "wf" + init.arguments.extend( + temporalio.converter.PayloadConverter.default.to_payloads(args) + ) + return WorkflowActivation(run_id="run", jobs=[job]) + + +def _notified(*notifications: Notification) -> WorkflowActivation: + job = WorkflowActivationJob() + job.notifications_received.notifications.extend(notifications) + return WorkflowActivation(run_id="run", jobs=[job]) + + +def _linked(channel: str, counter: int, position: bytes = b"") -> Notification: + """A notification the way a linked channel's owner receives it.""" + return Notification( + channel=channel, + counter=counter, + position=position, + linked_to=temporalio.common.Execution.workflow("wf", "run").to_proto(), + ) + + +def _signalled(name: str) -> WorkflowActivation: + job = WorkflowActivationJob() + job.signal_workflow.signal_name = name + return WorkflowActivation(run_id="run", jobs=[job]) + + +def _subscribed(completion: WorkflowActivationCompletion) -> list[str]: + assert completion.HasField("successful"), completion.failed.failure.message + return [ + command.subscribe_notification_channel.channel + for command in completion.successful.commands + if command.HasField("subscribe_notification_channel") + ] + + +def _channel_commands( + completion: WorkflowActivationCompletion, +) -> list[tuple[str, str]]: + """The channel commands of a completion in order, as (verb, channel) pairs.""" + assert completion.HasField("successful"), completion.failed.failure.message + commands: list[tuple[str, str]] = [] + for command in completion.successful.commands: + if command.HasField("subscribe_notification_channel"): + commands.append( + ("subscribe", command.subscribe_notification_channel.channel) + ) + elif command.HasField("unsubscribe_notification_channel"): + commands.append( + ("unsubscribe", command.unsubscribe_notification_channel.channel) + ) + return commands + + +def _completed(completion: WorkflowActivationCompletion) -> bool: + assert completion.HasField("successful"), completion.failed.failure.message + return any( + command.HasField("complete_workflow_execution") + for command in completion.successful.commands + ) + + +def _result(completion: WorkflowActivationCompletion) -> Any: + assert completion.HasField("successful"), completion.failed.failure.message + [done] = [ + command + for command in completion.successful.commands + if command.HasField("complete_workflow_execution") + ] + return temporalio.converter.PayloadConverter.default.from_payload( + done.complete_workflow_execution.result + ) + + +async def test_the_first_subscription_is_a_command_and_the_second_shares_it(): + instance = _instance(CountToTwo) + completion = instance.activate(_start(CountToTwo, "orders")) + assert _subscribed(completion) == ["orders"] + assert not _completed(completion) + + +async def test_a_notifications_received_job_wakes_the_receiver(): + instance = _instance(ReceiveOne) + assert _subscribed(instance.activate(_start(ReceiveOne, "orders"))) == ["orders"] + [topic] = temporalio.converter.PayloadConverter.default.to_payloads(["inputs"]) + completion = instance.activate( + _notified( + Notification( + channel="orders", position=b"7-0", counter=7, metadata={"topic": topic} + ) + ) + ) + assert _result(completion) == { + "channel": "orders", + "counter": 7, + "position": "7-0", + "topic": "inputs", + } + + +async def test_notifications_arrive_in_order_and_other_channels_are_dropped(): + instance = _instance(CountToTwo) + instance.activate(_start(CountToTwo, "orders")) + # A channel this run never subscribed to is not the workflow's concern. + completion = instance.activate(_notified(Notification(channel="other", counter=9))) + assert not _completed(completion) + completion = instance.activate( + _notified( + Notification(channel="orders", counter=1), + Notification(channel="orders", counter=2), + ) + ) + assert _result(completion) == 2 + + +async def test_an_empty_channel_name_is_refused(): + completion = _instance(EmptyChannel).activate(_start(EmptyChannel)) + assert completion.HasField("failed") + assert "channel must not be empty" in completion.failed.failure.message + + +async def test_an_empty_linked_channel_name_is_refused(): + completion = _instance(EmptyLinkedChannel).activate(_start(EmptyLinkedChannel)) + assert completion.HasField("failed") + assert "channel must not be empty" in completion.failed.failure.message + + +async def test_a_linked_channel_issues_no_command_and_gets_its_own_notifications(): + instance = _instance(ReceiveLinked) + completion = instance.activate(_start(ReceiveLinked, "orders")) + assert completion.HasField("successful"), completion.failed.failure.message + assert list(completion.successful.commands) == [] + # Without an owner the notification is the independent channel's, which + # this run never subscribed to. + completion = instance.activate(_notified(Notification(channel="orders", counter=1))) + assert not _completed(completion) + [topic] = temporalio.converter.PayloadConverter.default.to_payloads(["inputs"]) + notification = _linked("orders", 7, b"7-0") + notification.metadata["topic"].CopyFrom(topic) + assert _result(instance.activate(_notified(notification))) == { + "channel": "orders", + "counter": 7, + "position": "7-0", + "topic": "inputs", + "owner": "wf", + "owner_run": "run", + } + + +async def test_the_owner_on_a_notification_picks_the_handle_of_its_kind(): + instance = _instance(BothKinds) + completion = instance.activate(_start(BothKinds, "orders")) + # Only the independent handle costs a command. + assert _subscribed(completion) == ["orders"] + assert len(completion.successful.commands) == 1 + assert not _completed(instance.activate(_notified(_linked("orders", 1)))) + assert not _completed( + instance.activate(_notified(Notification(channel="orders", counter=5))) + ) + completion = instance.activate(_notified(_linked("orders", 2))) + assert _result(completion) == {"independent": [5], "linked": [1, 2]} + + +_CLOSED = "channel subscription closed" + + +async def test_an_unsubscribe_is_one_command_and_a_late_notification_is_dropped(): + instance = _instance(ReceiveThenUnsubscribe) + assert _channel_commands( + instance.activate(_start(ReceiveThenUnsubscribe, "orders")) + ) == [("subscribe", "orders")] + # The first notification is taken, then the two unsubscribe calls cost one + # command between them. + completion = instance.activate(_notified(Notification(channel="orders", counter=1))) + assert _channel_commands(completion) == [("unsubscribe", "orders")] + assert not _completed(completion) + # The server may still hand the run a notification it folded onto a task + # before the command landed. Nothing listens, so it changes nothing. + completion = instance.activate(_notified(Notification(channel="orders", counter=2))) + assert _channel_commands(completion) == [] + assert not _completed(completion) + assert _result(instance.activate(_signalled("finish"))) == { + "first": 1, + "closed": True, + "drained": [], + "refused": _CLOSED, + } + + +async def test_a_queued_notification_survives_the_unsubscribe_then_the_iteration_ends(): + instance = _instance(UnsubscribeWithOneQueued) + instance.activate(_start(UnsubscribeWithOneQueued, "orders")) + completion = instance.activate( + _notified( + Notification(channel="orders", counter=1), + Notification(channel="orders", counter=2), + ) + ) + assert _channel_commands(completion) == [("unsubscribe", "orders")] + assert _result(completion) == { + "first": 1, + "closed": True, + "drained": [2], + "refused": _CLOSED, + } + + +async def test_a_linked_handle_has_no_subscription_to_end(): + completion = _instance(UnsubscribeLinked).activate( + _start(UnsubscribeLinked, "orders") + ) + assert completion.HasField("failed") + assert "a linked channel has no subscription" in completion.failed.failure.message + + +async def test_a_subscription_after_an_unsubscribe_is_a_new_one(): + instance = _instance(Resubscribe) + completion = instance.activate(_start(Resubscribe, "orders")) + assert _channel_commands(completion) == [ + ("subscribe", "orders"), + ("unsubscribe", "orders"), + ("subscribe", "orders"), + ] + assert not _completed(completion) + # The notification reaches the open handle, and the closed one stays closed. + assert _result( + instance.activate(_notified(Notification(channel="orders", counter=3))) + ) == { + "counter": 3, + "closed": True, + "drained": [], + "refused": _CLOSED, + } + + +async def test_a_stream_names_the_channel_it_notifies(): + owner = temporalio.common.Execution.workflow("wf") + assert stream_channel(StreamRef.for_workflow("wf", topic="out")) == ChannelAddress( + "stream/out", owner + ) + # The address names the owner without a run, so it follows the chain. + assert stream_channel(StreamRef.for_workflow("wf", run_id="run")) == ChannelAddress( + "stream/" + DEFAULT_TOPIC, owner + ) + assert stream_channel( + StreamRef.for_activity("act", workflow_id="wf", topic="out") + ) == ChannelAddress("stream/act/out", owner) + # A standalone activity is an execution of its own, so its stream's + # channel is linked to it under the topic's name alone. + assert stream_channel(StreamRef.for_activity("act", topic="out")) == ChannelAddress( + "stream/out", temporalio.common.Execution.activity("act") + ) + # A standalone stream's topics share one stream on the server, so the + # topic is not part of the name. + assert stream_channel( + StreamRef.for_standalone("sid", topic="out") + ) == ChannelAddress("stream/sid", None) + # The workflow id stays readable for a caller that addresses by it, and + # only names a workflow. + assert stream_channel(StreamRef.for_workflow("wf")).workflow_id == "wf" + assert stream_channel(StreamRef.for_activity("act")).workflow_id is None + assert stream_channel(StreamRef.for_standalone("sid")).workflow_id is None + + +async def test_the_description_maps_every_channel_subscription_field(): + [topic] = temporalio.converter.PayloadConverter.default.to_payloads(["inputs"]) + pending = Notification(channel="orders", position=b"4-0", counter=4) + pending.metadata["topic"].CopyFrom(topic) + raw = temporalio.api.workflowservice.v1.DescribeWorkflowExecutionResponse( + workflow_execution_info=temporalio.api.workflow.v1.WorkflowExecutionInfo( + execution=temporalio.api.common.v1.WorkflowExecution( + workflow_id="wf", run_id="run" + ), + type=temporalio.api.common.v1.WorkflowType(name="ReceiveOne"), + status=temporalio.api.enums.v1.WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING, + task_queue="tq", + ), + channel_subscriptions=[ + temporalio.api.workflow.v1.ChannelSubscriptionInfo( + channel="orders", + kind=temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_INDEPENDENT, + subscribed_event_id=5, + last_counter=3, + pending_notification=pending, + scheduled_counter=4, + ), + temporalio.api.workflow.v1.ChannelSubscriptionInfo( + channel="orders", + kind=temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_LINKED, + last_counter=2, + listener_count=1, + retained_count=2, + accepted_count=7, + ), + ], + ) + description = await WorkflowExecutionDescription._from_raw_description( + raw, "default", temporalio.converter.DataConverter.default + ) + assert description.id == "wf" + assert description.channel_subscriptions == ( + ChannelSubscriptionInfo( + channel="orders", + kind=ChannelKind.INDEPENDENT, + subscribed_event_id=5, + last_counter=3, + pending_notification=workflow.Notification( + channel="orders", position=b"4-0", counter=4, metadata={"topic": topic} + ), + scheduled_counter=4, + listener_count=0, + retained_count=0, + accepted_count=0, + ), + ChannelSubscriptionInfo( + channel="orders", + kind=ChannelKind.LINKED, + subscribed_event_id=0, + last_counter=2, + pending_notification=None, + scheduled_counter=0, + listener_count=1, + retained_count=2, + accepted_count=7, + ), + ) + raw.ClearField("channel_subscriptions") + description = await WorkflowExecutionDescription._from_raw_description( + raw, "default", temporalio.converter.DataConverter.default + ) + assert description.channel_subscriptions == () + + +async def test_the_client_addresses_a_linked_channel_by_execution( + client: Client, monkeypatch: pytest.MonkeyPatch +): + """Every channel call carries the owner it was given, and only then.""" + requests: list[Any] = [] + owner = temporalio.common.Execution.workflow("wf", "run") + describe_response = temporalio.api.workflowservice.v1.DescribeChannelResponse( + kind=temporalio.api.notification.v1.ChannelKind.CHANNEL_KIND_LINKED, + linked_to=owner.to_proto(), + latest=_linked("orders", 3, b"3-0"), + ) + responses = { + "notify_channel": temporalio.api.workflowservice.v1.NotifyChannelResponse(), + "poll_channel": temporalio.api.workflowservice.v1.PollChannelResponse( + notifications=[_linked("orders", 3, b"3-0")] + ), + "describe_channel": describe_response, + "register_channel_listener": ( + temporalio.api.workflowservice.v1.RegisterChannelListenerResponse( + listener_id="listener" + ) + ), + "unregister_channel_listener": ( + temporalio.api.workflowservice.v1.UnregisterChannelListenerResponse() + ), + } + for name, response in responses.items(): + + async def call(req: Any, *, _response: Any = response, **_: Any) -> Any: + requests.append(req) + return _response + + monkeypatch.setattr(client.workflow_service, name, call) + + callback = Callback(url="http://localhost:1/never-called", headers={}) + await client.notify_channel("orders", counter=1, workflow_id="wf", run_id="run") + [polled] = await client.poll_channel("orders", workflow_id="wf", wait=False) + description = await client.describe_channel("orders", workflow_id="wf") + await client.register_channel_listener("orders", callback, workflow_id="wf") + await client.unregister_channel_listener("orders", "listener", workflow_id="wf") + # The workflow id is shorthand for a workflow execution, run id and all. + by_workflow_id = temporalio.common.Execution.workflow("wf").to_proto() + assert [req.execution for req in requests] == [ + owner.to_proto(), + by_workflow_id, + by_workflow_id, + by_workflow_id, + by_workflow_id, + ] + assert polled.linked_to == owner + assert description.kind == ChannelKind.LINKED + assert description.linked_to == owner + assert description.latest is not None and description.latest.linked_to == owner + + # An execution names any owner, a standalone activity included. + requests.clear() + activity = temporalio.common.Execution.activity("act", "run") + await client.notify_channel("orders", counter=1, execution=activity) + await client.poll_channel("orders", execution=activity, wait=False) + await client.describe_channel("orders", execution=activity) + await client.register_channel_listener("orders", callback, execution=activity) + await client.unregister_channel_listener("orders", "listener", execution=activity) + assert [req.execution for req in requests] == [activity.to_proto()] * 5 + assert activity.to_proto().type == ( + temporalio.api.enums.v1.ExecutionType.EXECUTION_TYPE_ACTIVITY + ) + + # Without an owner the calls address the independent channel. + requests.clear() + await client.notify_channel("orders", counter=1) + await client.poll_channel("orders", wait=False) + await client.describe_channel("orders") + await client.register_channel_listener("orders", callback) + await client.unregister_channel_listener("orders", "listener") + assert [req.HasField("execution") for req in requests] == [False] * 5 + + with pytest.raises(ValueError, match="run_id needs workflow_id"): + await client.notify_channel("orders", counter=1, run_id="run") + # The shorthand and the execution are two ways to say one thing. + with pytest.raises(ValueError, match="not both"): + await client.notify_channel( + "orders", counter=1, execution=activity, workflow_id="wf" + ) + with pytest.raises(ValueError, match="not both"): + await client.poll_channel("orders", execution=activity, run_id="run") + + +@pytest.mark.needs_channel_server +@pytest.mark.needs_channel_core +async def test_a_workflow_receives_a_client_notification(client: Client): + channel = f"orders-{uuid.uuid4()}" + worker = new_worker(client, ReceiveOne) + running = asyncio.create_task(worker.run()) + handle = await client.start_workflow( + ReceiveOne.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + try: + + async def subscribed() -> None: + # The worker's Core has to carry the subscribe command to the + # server. One that refuses it fails every completion, and the task + # times out instead; say so rather than wait on the result forever. + events = [event.event_type async for event in handle.fetch_history_events()] + if EventType.EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT in events: + pytest.fail( + "the worker could not complete the task that subscribes; the Core " + "the bridge pins must carry the subscribe command" + ) + assert ( + EventType.EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED in events + ) + + await assert_eventually(subscribed, timeout=timedelta(seconds=30)) + + async def listening() -> None: + # The channel exists once the subscribe lands, so a describe that + # races it is answered with not found. + try: + description = await client.describe_channel(channel) + except RPCError as err: + assert err.status != RPCStatusCode.NOT_FOUND, "channel not created yet" + raise + assert [listener.workflow_id for listener in description.listeners] == [ + handle.id + ] + + await assert_eventually(listening) + listeners = await client.notify_channel( + channel, position=b"1-0", counter=1, metadata={"topic": "inputs"} + ) + assert listeners == 1 + assert await asyncio.wait_for(handle.result(), 30) == { + "channel": channel, + "counter": 1, + "position": "1-0", + "topic": "inputs", + } + polled = await client.poll_channel(channel, wait=False) + assert [n.counter for n in polled] == [1] + description = await client.describe_channel(channel) + assert description.latest is not None and description.latest.counter == 1 + finally: + # A Core that refuses the subscribe command leaves the task in a + # timeout loop and the worker's shutdown waiting on it, so end the run + # first and give the shutdown a bound. + with contextlib.suppress(RPCError): + await handle.terminate() + try: + await asyncio.wait_for(worker.shutdown(), 15) + except asyncio.TimeoutError: + running.cancel() + with contextlib.suppress(asyncio.CancelledError): + await running + + +@pytest.mark.needs_channel_server +async def test_a_callback_listener_registers_and_unregisters(client: Client): + channel = f"orders-{uuid.uuid4()}" + callback = Callback(url="http://localhost:1/never-called", headers={}) + listener_id = await client.register_channel_listener(channel, callback) + description = await client.describe_channel(channel) + assert [listener.listener_id for listener in description.listeners] == [listener_id] + assert description.listeners[0].callback == callback + await client.unregister_channel_listener(channel, listener_id) + description = await client.describe_channel(channel) + assert description.listeners == [] + + +@pytest.mark.needs_channel_server +async def test_a_channel_retains_notifications_for_pollers(client: Client): + channel = f"orders-{uuid.uuid4()}" + # A channel nobody has touched does not exist. + with pytest.raises(RPCError) as untouched: + await client.describe_channel(f"untouched-{uuid.uuid4()}") + assert untouched.value.status == RPCStatusCode.NOT_FOUND + # Nobody listens yet: the notification is kept for pollers and the count + # says zero. + assert await client.notify_channel(channel, position=b"2-0", counter=2) == 0 + description = await client.describe_channel(channel) + assert description.listeners == [] + assert description.latest is not None + assert (description.latest.position, description.latest.counter) == (b"2-0", 2) + assert description.retained_count == 1 + polled = await client.poll_channel(channel, wait=False) + assert [(n.position, n.counter) for n in polled] == [(b"2-0", 2)] + # At or below the latest counter a notify changes nothing and is not kept. + assert await client.notify_channel(channel, position=b"1-0", counter=1) == 0 + assert await client.notify_channel(channel, position=b"2-0", counter=2) == 0 + description = await client.describe_channel(channel) + assert description.latest is not None and description.latest.counter == 2 + assert description.retained_count == 1 + # Above it the notification is kept, metadata and all, and a poll after + # the earlier counter sees only the new one. + assert ( + await client.notify_channel( + channel, position=b"3-0", counter=3, metadata={"topic": "inputs"} + ) + == 0 + ) + [newest] = await client.poll_channel(channel, after_counter=2, wait=False) + assert newest.counter == 3 + assert ( + client.data_converter.payload_converter.from_payload( + newest.metadata["topic"], str + ) + == "inputs" + ) + polled = await client.poll_channel(channel, wait=False) + assert [n.counter for n in polled] == [2, 3] + # A poll above the latest waits its bound out and comes back empty. + polled = await client.poll_channel( + channel, after_counter=3, wait=timedelta(seconds=1) + ) + assert polled == [] + + +async def _stop(handle: Any, worker: Any, running: asyncio.Task[None]) -> None: + """End the run and the worker, with a bound on the shutdown.""" + with contextlib.suppress(RPCError): + await handle.terminate() + try: + await asyncio.wait_for(worker.shutdown(), 15) + except asyncio.TimeoutError: + running.cancel() + with contextlib.suppress(asyncio.CancelledError): + await running + + +@pytest.mark.needs_linked_server +@pytest.mark.needs_linked_core +async def test_a_workflow_receives_a_notification_on_its_linked_channel( + client: Client, +): + channel = f"orders-{uuid.uuid4()}" + worker = new_worker(client, ReceiveLinked) + running = asyncio.create_task(worker.run()) + handle = await client.start_workflow( + ReceiveLinked.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + try: + # The channel exists with the run: no listener registers, nothing is + # retained yet, and the owner is named. + description = await client.describe_channel(channel, workflow_id=handle.id) + assert description.kind == ChannelKind.LINKED + assert description.listeners == [] + assert description.latest is None + assert description.retained_count == 0 + assert description.linked_to is not None + assert description.linked_to.business_id == handle.id + # The owner is the one listener. + listeners = await client.notify_channel( + channel, + position=b"1-0", + counter=1, + metadata={"topic": "inputs"}, + workflow_id=handle.id, + ) + assert listeners == 1 + assert await asyncio.wait_for(handle.result(), 30) == { + "channel": channel, + "counter": 1, + "position": "1-0", + "topic": "inputs", + "owner": handle.id, + "owner_run": handle.first_execution_run_id, + } + # The owner listens by construction, so History holds no subscribe + # event; the notification rode a scheduled event. + events = [event.event_type async for event in handle.fetch_history_events()] + assert ( + EventType.EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED not in events + ) + finally: + await _stop(handle, worker, running) + + +@pytest.mark.needs_linked_server +async def test_a_linked_channel_lives_and_dies_with_its_workflow(client: Client): + """The client side alone: no worker polls, so the run stays open until ended.""" + channel = f"orders-{uuid.uuid4()}" + handle = await client.start_workflow( + CountLinked.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=f"nobody-polls-{uuid.uuid4()}", + ) + run_id = handle.first_execution_run_id + assert run_id is not None + # A name nobody has notified exists all the same, with nothing in it. + description = await client.describe_channel(channel, workflow_id=handle.id) + assert description.kind == ChannelKind.LINKED + assert (description.listeners, description.latest) == ([], None) + assert description.retained_count == 0 + assert description.linked_to is not None + assert (description.linked_to.business_id, description.linked_to.run_id) == ( + handle.id, + run_id, + ) + # The independent channel of that name is a different thing and does not + # exist. + with pytest.raises(RPCError) as independent: + await client.describe_channel(channel) + assert independent.value.status == RPCStatusCode.NOT_FOUND + # A run id names that run; one that is not the chain's is not found, and + # neither is a workflow that never ran. + description = await client.describe_channel( + channel, workflow_id=handle.id, run_id=run_id + ) + assert description.kind == ChannelKind.LINKED + for wrong in ( + client.describe_channel( + channel, workflow_id=handle.id, run_id=str(uuid.uuid4()) + ), + client.notify_channel( + channel, counter=1, workflow_id=handle.id, run_id=str(uuid.uuid4()) + ), + client.notify_channel(channel, counter=1, workflow_id=f"never-{uuid.uuid4()}"), + ): + with pytest.raises(RPCError) as missing: + await wrong + assert missing.value.status == RPCStatusCode.NOT_FOUND + # The channel ends with the run. + await handle.terminate() + with pytest.raises(RPCError) as closed: + await client.notify_channel(channel, counter=1, workflow_id=handle.id) + assert closed.value.status == RPCStatusCode.NOT_FOUND + + +@pytest.mark.needs_linked_server +@pytest.mark.needs_execution_server +async def test_a_linked_channel_names_its_owner_as_an_execution(client: Client): + """The client side alone: the owner comes back typed, by either spelling.""" + channel = f"orders-{uuid.uuid4()}" + handle = await client.start_workflow( + CountLinked.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=f"nobody-polls-{uuid.uuid4()}", + ) + run_id = handle.first_execution_run_id + assert run_id is not None + by_id = temporalio.common.Execution.workflow(handle.id) + by_run = temporalio.common.Execution.workflow(handle.id, run_id) + try: + description = await client.describe_channel(channel, execution=by_id) + assert description.kind == ChannelKind.LINKED + assert description.linked_to is not None + assert description.linked_to == by_run + assert description.linked_to.type is temporalio.common.ExecutionType.WORKFLOW + # The execution and the workflow id shorthand reach one channel. + assert ( + await client.notify_channel( + channel, position=b"1-0", counter=1, execution=by_id + ) + == 1 + ) + [polled] = await client.poll_channel(channel, workflow_id=handle.id, wait=False) + assert (polled.counter, polled.linked_to) == (1, by_run) + [polled] = await client.poll_channel(channel, execution=by_run, wait=False) + assert polled.counter == 1 + # The same id as an activity names an execution that never ran. + with pytest.raises(RPCError) as missing: + await client.describe_channel( + channel, execution=temporalio.common.Execution.activity(handle.id) + ) + assert missing.value.status == RPCStatusCode.NOT_FOUND + finally: + with contextlib.suppress(RPCError): + await handle.terminate() + + +@pytest.mark.needs_linked_server +@pytest.mark.needs_linked_core +async def test_a_linked_channel_is_polled_by_workflow_id(client: Client): + channel = f"orders-{uuid.uuid4()}" + worker = new_worker(client, CountLinked) + running = asyncio.create_task(worker.run()) + handle = await client.start_workflow( + CountLinked.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + try: + assert ( + await client.notify_channel( + channel, position=b"1-0", counter=1, workflow_id=handle.id + ) + == 1 + ) + polled = await client.poll_channel(channel, workflow_id=handle.id, wait=False) + assert [(n.position, n.counter) for n in polled] == [(b"1-0", 1)] + assert polled[0].linked_to is not None + assert polled[0].linked_to.business_id == handle.id + # The run id reaches the same channel. + assert [ + n.counter + for n in await client.poll_channel( + channel, + workflow_id=handle.id, + run_id=handle.first_execution_run_id, + wait=False, + ) + ] == [1] + description = await client.describe_channel(channel, workflow_id=handle.id) + assert description.kind == ChannelKind.LINKED + assert description.latest is not None and description.latest.counter == 1 + assert description.retained_count == 1 + # A poll above the latest waits its bound out and comes back empty. + polled = await client.poll_channel( + channel, workflow_id=handle.id, after_counter=1, wait=timedelta(seconds=1) + ) + assert polled == [] + assert ( + await client.notify_channel( + channel, position=b"2-0", counter=2, workflow_id=handle.id + ) + == 1 + ) + assert await asyncio.wait_for(handle.result(), 30) == 2 + # The ring went with the run, so a poll after the close finds nothing + # to read. + with pytest.raises(RPCError) as closed: + await client.poll_channel( + channel, workflow_id=handle.id, after_counter=1, wait=False + ) + assert closed.value.status == RPCStatusCode.NOT_FOUND + finally: + await _stop(handle, worker, running) + + +_SUBSCRIBED = EventType.EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED +_UNSUBSCRIBED = EventType.EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED + + +async def _event_ids(handle: Any, event_type: Any) -> list[int]: + """The ids of the events of ``event_type`` in the run's History so far.""" + return [ + event.event_id + async for event in handle.fetch_history_events() + if event.event_type == event_type + ] + + +async def _one_event(handle: Any, event_type: Any) -> int: + """The id of the one event of ``event_type``, failing until it is there.""" + ids = await _event_ids(handle, event_type) + assert len(ids) == 1, ids + return ids[0] + + +@pytest.mark.needs_describe_server +@pytest.mark.needs_channel_core +async def test_a_description_lists_an_independent_subscription(client: Client): + channel = f"orders-{uuid.uuid4()}" + worker = new_worker(client, CountToTwo) + running = asyncio.create_task(worker.run()) + handle = await client.start_workflow( + CountToTwo.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + try: + event_id = await assert_eventually( + lambda: _one_event(handle, _SUBSCRIBED), timeout=timedelta(seconds=30) + ) + # Listed from the subscribe event on, with nothing accepted yet. The + # counts belong to the channel execution and stay zero here. + description = await handle.describe() + assert description.channel_subscriptions == ( + ChannelSubscriptionInfo( + channel=channel, + kind=ChannelKind.INDEPENDENT, + subscribed_event_id=event_id, + last_counter=0, + pending_notification=None, + scheduled_counter=0, + listener_count=0, + retained_count=0, + accepted_count=0, + ), + ) + assert await client.notify_channel(channel, position=b"1-0", counter=1) == 1 + + async def accepted() -> None: + # Once the task that carried it completes, the counter is the + # run's and nothing is pending or scheduled any more. + [info] = (await handle.describe()).channel_subscriptions + assert info.last_counter == 1 + assert info.pending_notification is None + assert info.scheduled_counter == 0 + + await assert_eventually(accepted) + assert await client.notify_channel(channel, position=b"2-0", counter=2) == 1 + assert await asyncio.wait_for(handle.result(), 30) == 2 + # A closed run keeps listing what it stood on. + [info] = (await handle.describe()).channel_subscriptions + assert (info.kind, info.last_counter) == (ChannelKind.INDEPENDENT, 2) + finally: + await _stop(handle, worker, running) + + +@pytest.mark.needs_describe_server +async def test_a_description_lists_a_linked_channel_once_it_holds_state( + client: Client, +): + """The client side alone: no worker polls, so the run stays open until ended.""" + channel = f"orders-{uuid.uuid4()}" + handle = await client.start_workflow( + CountLinked.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=f"nobody-polls-{uuid.uuid4()}", + ) + try: + # An untouched linked name exists by construction and holds nothing, + # so it is not listed. + assert (await handle.describe()).channel_subscriptions == () + assert ( + await client.notify_channel( + channel, position=b"1-0", counter=1, workflow_id=handle.id + ) + == 1 + ) + [info] = (await handle.describe()).channel_subscriptions + assert (info.channel, info.kind) == (channel, ChannelKind.LINKED) + # The owner's state took the notification in the write that accepted + # it, so the counter is the run's at once. Nobody polls, and the first + # task was scheduled without a counter when the run started, so the + # notification waits behind it as the pending entry. + assert (info.subscribed_event_id, info.last_counter) == (0, 1) + assert info.pending_notification is not None + assert info.pending_notification.counter == 1 + assert info.pending_notification.linked_to is not None + assert info.pending_notification.linked_to.business_id == handle.id + assert info.scheduled_counter == 0 + assert (info.listener_count, info.retained_count, info.accepted_count) == ( + 0, + 1, + 1, + ) + # A callback on the linked channel shows up in the owner's count. + callback = Callback(url="http://localhost:1/never-called", headers={}) + listener_id = await client.register_channel_listener( + channel, callback, workflow_id=handle.id + ) + [info] = (await handle.describe()).channel_subscriptions + assert info.listener_count == 1 + await client.unregister_channel_listener( + channel, listener_id, workflow_id=handle.id + ) + [info] = (await handle.describe()).channel_subscriptions + assert info.listener_count == 0 + # A closed run keeps listing what it stood on. + await handle.terminate() + [info] = (await handle.describe()).channel_subscriptions + assert (info.kind, info.last_counter) == (ChannelKind.LINKED, 1) + finally: + with contextlib.suppress(RPCError): + await handle.terminate() + + +@pytest.mark.needs_unsubscribe_server +@pytest.mark.needs_unsubscribe_core +async def test_a_workflow_unsubscribes_and_a_later_notify_wakes_nothing( + client: Client, +): + channel = f"orders-{uuid.uuid4()}" + worker = new_worker(client, ReceiveThenUnsubscribe) + running = asyncio.create_task(worker.run()) + handle = await client.start_workflow( + ReceiveThenUnsubscribe.run, + channel, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + try: + subscribed_id = await assert_eventually( + lambda: _one_event(handle, _SUBSCRIBED), timeout=timedelta(seconds=30) + ) + assert await client.notify_channel(channel, position=b"1-0", counter=1) == 1 + unsubscribed_id = await assert_eventually( + lambda: _one_event(handle, _UNSUBSCRIBED), timeout=timedelta(seconds=30) + ) + assert unsubscribed_id > subscribed_id + # The event names the subscription it ended. + [event] = [ + event + async for event in handle.fetch_history_events() + if event.event_id == unsubscribed_id + ] + attrs = event.workflow_notification_channel_unsubscribed_event_attributes + assert (attrs.channel, attrs.subscribed_event_id) == (channel, subscribed_id) + # Gone from both sides: the channel's listeners and the run's standing. + description = await client.describe_channel(channel) + assert [listener.workflow_id for listener in description.listeners] == [] + assert (await handle.describe()).channel_subscriptions == () + # Nothing listens any more, so a notify wakes nobody and is only + # retained for pollers. + assert await client.notify_channel(channel, position=b"2-0", counter=2) == 0 + await handle.signal(ReceiveThenUnsubscribe.finish) + assert await asyncio.wait_for(handle.result(), 30) == { + "first": 1, + "closed": True, + "drained": [], + "refused": _CLOSED, + } + assert await _event_ids(handle, _SUBSCRIBED) == [subscribed_id] + assert await _event_ids(handle, _UNSUBSCRIBED) == [unsubscribed_id] + finally: + await _stop(handle, worker, running) diff --git a/tests/streams/test_memory_provider.py b/tests/streams/test_memory_provider.py new file mode 100644 index 0000000000..3fd83a9fba --- /dev/null +++ b/tests/streams/test_memory_provider.py @@ -0,0 +1,69 @@ +"""What the reference provider does that the conformance suite cannot see. + +The conformance suite is the public surface, so it can say that closing a read +returns and that the topic still works afterwards, but not that the provider +let go of what the read parked on. That is this file: a few assertions against +``MemoryStreams`` internals, where holding on would leak quietly. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from temporalio.api.stream.v1 import StreamRecord +from temporalio.streams import CONTENT_HASH_KEY, StreamProducerError, topic +from temporalio.streams.providers.memory import MemoryStreams + +OUT = topic("out", dict) + + +async def test_closing_a_parked_read_drops_its_waiter(): + provider = MemoryStreams() + stream = provider.get_stream_handle(None, "wf-parked") # type: ignore[arg-type] + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + await producer.append({"n": 1}) + store = provider._topic("wf-parked", OUT.name) + + records = stream.read(topic=OUT) + await asyncio.wait_for(records.__anext__(), 5.0) + + pending = asyncio.ensure_future(records.__anext__()) + await asyncio.sleep(0.2) + assert store._waiters, "the read should be parked on the topic by now" + + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + # Nothing left behind: a reader that comes and goes must not grow this + # list for the life of the topic. + assert store._waiters == [] + await asyncio.wait_for(records.aclose(), 5.0) + + +async def test_a_divergent_retry_leaves_the_store_alone(): + provider = MemoryStreams() + stream = provider.get_stream_handle(None, "wf-divergent") # type: ignore[arg-type] + store = provider._topic("wf-divergent", OUT.name) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + await producer.append({"n": 1}) + assert len(store.records) == 1 + + retry = stream.producer(topic=OUT, producer_id="model", attempt=1) + with pytest.raises( + StreamProducerError, match="already used with different content" + ): + await retry.append({"n": 2}) + assert len(store.records) == 1 + + +async def test_a_stored_record_carries_the_plaintext_hash(): + provider = MemoryStreams() + stream = provider.get_stream_handle(None, "wf-hash") # type: ignore[arg-type] + await stream.producer(topic=OUT, producer_id="model", attempt=1).append({"n": 1}) + stored = StreamRecord.FromString(provider._topic("wf-hash", OUT.name).records[0]) + # What the store holds is the record after encode_body: the hash the + # server-side dedupe reads is on it, under the shared key. + assert stored.metadata[CONTENT_HASH_KEY].data.decode().isalnum() + assert len(stored.metadata[CONTENT_HASH_KEY].data) == 64 diff --git a/tests/streams/test_native_fingerprint.py b/tests/streams/test_native_fingerprint.py new file mode 100644 index 0000000000..fccc54aaa7 --- /dev/null +++ b/tests/streams/test_native_fingerprint.py @@ -0,0 +1,151 @@ +"""The native provider stamps every body with the hash of its plaintext. + +The server deduplicates a producer's repeat on that hash when it is present, +so a codec that encrypts with a fresh nonce per call cannot turn a retry into +a divergent write. These pin where the stamp is taken: over the body as the +payload converter produced it, before the codec on the outside path and before +the worker's payload pass on the workflow path. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Sequence +from typing import Any + +import pytest + +import temporalio.converter +from temporalio import workflow +from temporalio.api.common.v1 import Payload +from temporalio.api.stream.v1 import StreamRecord +from temporalio.bridge.proto.workflow_completion import WorkflowActivationCompletion +from temporalio.bridge.worker import encode_completion +from temporalio.client_stream import Appended +from temporalio.converter import ( + DataConverter, + PayloadCodec, + StorageDriverWorkflowInfo, +) +from temporalio.streams import CONTENT_HASH_KEY +from temporalio.streams.providers import native +from temporalio.streams.providers.native import NativeProducer + + +class _NonceCodec(PayloadCodec): + """Encodes to something different every time, like a nonce-based cipher.""" + + def __init__(self) -> None: + self.calls = 0 + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + self.calls += 1 + return [ + Payload( + metadata={"encoding": b"binary/nonce"}, + data=f"{self.calls}:".encode() + p.data, + ) + for p in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [Payload(data=p.data.split(b":", 1)[1]) for p in payloads] + + +class _Handle: + """Records what a producer appends and answers as the server would.""" + + def __init__(self) -> None: + self.owner_run_id = "run" + self.appended: list[StreamRecord] = [] + + async def append(self, *records: StreamRecord, **_: Any) -> Appended: + self.appended.extend(records) + return Appended(first_offset=0, next_offset=len(records), count=len(records)) + + +def _plaintext_hash(value: Any) -> bytes: + payload = temporalio.converter.default().payload_converter.to_payloads([value])[0] + return hashlib.sha256(payload.SerializeToString()).hexdigest().encode() + + +def _target(_run_id: str) -> StorageDriverWorkflowInfo: + return StorageDriverWorkflowInfo(namespace="ns", id="wf") + + +async def test_an_outside_append_is_stamped_before_the_codec() -> None: + handle: Any = _Handle() + converter = DataConverter(payload_codec=_NonceCodec()) + producer: NativeProducer[Any] = NativeProducer( + handle, None, converter, _target, "t", "p", 1 + ) + + await producer.append({"n": 1}) + await producer.append({"n": 1}) + + first, second = handle.appended + # The bodies went out differently encoded and the stamps agree anyway. + assert first.body.data != second.body.data + assert first.metadata[CONTENT_HASH_KEY].data == _plaintext_hash({"n": 1}) + assert second.metadata[CONTENT_HASH_KEY].data == _plaintext_hash({"n": 1}) + assert first.metadata[CONTENT_HASH_KEY].metadata["encoding"] == b"binary/plain" + + +async def test_a_finish_record_carries_no_stamp() -> None: + handle: Any = _Handle() + producer: NativeProducer[Any] = NativeProducer( + handle, None, DataConverter.default, _target, "t", "p", 1 + ) + await producer.finish() + (record,) = handle.appended + assert CONTENT_HASH_KEY not in record.metadata + + +def test_a_workflow_publish_is_stamped_on_the_workflow_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + staged: list[StreamRecord] = [] + + def stage(records: Sequence[StreamRecord], *, stream_name: str) -> None: + assert stream_name == "t" + staged.extend(records) + + monkeypatch.setattr(workflow, "_append_stream_records", stage) + converter = temporalio.converter.default().payload_converter + body = converter.to_payloads([{"n": 2}])[0] + record = StreamRecord(topic="t", body=body) + + native._NativeWriteSink("t").publish(record) + + (stamped,) = staged + assert stamped.metadata[CONTENT_HASH_KEY].data == _plaintext_hash({"n": 2}) + # Two publishes of the same value stamp the same, which is what replay + # reissues. + native._NativeWriteSink("t").publish(StreamRecord(topic="t", body=body)) + assert staged[1].metadata[CONTENT_HASH_KEY] == stamped.metadata[CONTENT_HASH_KEY] + + +async def test_the_workers_payload_pass_leaves_the_stamp_alone() -> None: + """The codec and the store rewrite the body and the other metadata, never the hash. + + The server reads the declared hash as sent and refuses a value that is not + hex, so a codec that touched it would fail the workflow's own append. + """ + converter = DataConverter(payload_codec=_NonceCodec()) + body = converter.payload_converter.to_payloads([{"n": 3}])[0] + record = native._fingerprint(StreamRecord(topic="t", body=body)) + record.metadata["note"].CopyFrom(Payload(data=b"plain")) + completion = WorkflowActivationCompletion() + command = completion.successful.commands.add() + command.append_stream_records.stream_name = "t" + command.append_stream_records.records.append(record) + + await encode_completion( + completion, converter, encode_headers=False, storage_concurrency_limit=1 + ) + + sent = completion.successful.commands[0].append_stream_records.records[0] + assert sent.body.metadata["encoding"] == b"binary/nonce" + assert sent.metadata["note"].metadata["encoding"] == b"binary/nonce" + assert sent.metadata[CONTENT_HASH_KEY].data == _plaintext_hash({"n": 3}) + assert sent.metadata[CONTENT_HASH_KEY].metadata["encoding"] == b"binary/plain" diff --git a/tests/streams/test_native_handles.py b/tests/streams/test_native_handles.py new file mode 100644 index 0000000000..318c30494b --- /dev/null +++ b/tests/streams/test_native_handles.py @@ -0,0 +1,102 @@ +"""What the native handles answer without a server. + +A ref names the owner as the handle addresses it, a close on an owned stream +is refused, a standalone handle refuses a cursor from another stream, and a +create refuses a policy the server cannot hold before any call is made. +""" + +from __future__ import annotations + +from datetime import timedelta +from types import SimpleNamespace +from typing import Any + +import pytest + +import temporalio.converter +from temporalio.streams import DEFAULT_TOPIC, StreamCursorError, StreamRef, topic +from temporalio.streams.providers.native import ( + NativeActivityStreamHandle, + NativeStandaloneStreamHandle, + NativeStreamHandle, + NativeStreams, + _cursor, +) + +OUT = topic("out", dict) +_CLIENT: Any = SimpleNamespace( + data_converter=temporalio.converter.default(), namespace="ns" +) + + +def test_a_workflow_handle_refs_its_owner_as_it_is_pinned() -> None: + following = NativeStreamHandle(_CLIENT, "wf", None) + assert following.ref() == StreamRef.for_workflow("wf") + assert following.ref(topic=OUT) == StreamRef.for_workflow("wf", topic="out") + pinned = NativeStreamHandle(_CLIENT, "wf", "run-1") + assert pinned.ref(topic="a") == StreamRef.for_workflow( + "wf", run_id="run-1", topic="a" + ) + + +def test_an_activity_handle_refs_the_activity_and_its_workflow() -> None: + scheduled = NativeActivityStreamHandle(_CLIENT, "act", "wf", "run-1") + assert scheduled.ref() == StreamRef.for_activity( + "act", workflow_id="wf", run_id="run-1" + ) + standalone = NativeActivityStreamHandle(_CLIENT, "act", None, None) + assert standalone.ref(topic=OUT) == StreamRef.for_activity("act", topic="out") + assert standalone.ref().workflow_id is None + + +def test_a_standalone_handle_refs_its_id() -> None: + handle = NativeStandaloneStreamHandle(_CLIENT, "s1") + assert handle.stream_id == "s1" + assert handle.ref() == StreamRef.for_standalone("s1") + assert handle.ref().topic == DEFAULT_TOPIC + assert handle.ref(topic=OUT) == StreamRef.for_standalone("s1", topic="out") + + +async def test_only_a_standalone_stream_can_be_closed() -> None: + with pytest.raises(ValueError, match="standalone"): + await NativeStreamHandle(_CLIENT, "wf", None).close() + with pytest.raises(ValueError, match="standalone"): + await NativeActivityStreamHandle(_CLIENT, "act", "wf", None).close() + + +def test_a_standalone_handle_refuses_another_streams_cursor() -> None: + handle = NativeStandaloneStreamHandle(_CLIENT, "s1") + with pytest.raises(StreamCursorError, match="another stream"): + handle.read(topic=OUT, after=_cursor("s2", 3)) + with pytest.raises(StreamCursorError): + handle.read(topic=OUT, after=_cursor("", 3)) + + +def test_a_ref_opens_the_handle_it_names_on_the_provider() -> None: + provider = NativeStreams() + opened = provider.get_stream_handle( + _CLIENT, StreamRef.for_standalone("s1", topic="out") + ) + assert opened.ref() == StreamRef.for_standalone("s1", topic="out") + assert opened.ref(topic="b") == StreamRef.for_standalone("s1", topic="b") + workflow = provider.get_stream_handle( + _CLIENT, StreamRef.for_workflow("wf", run_id="r") + ) + assert workflow.ref() == StreamRef.for_workflow("wf", run_id="r") + with pytest.raises(ValueError, match="run_id"): + provider.get_stream_handle(_CLIENT, StreamRef.for_workflow("wf"), run_id="r") + + +async def test_a_create_refuses_a_policy_the_server_cannot_hold() -> None: + provider = NativeStreams() + for bad in ( + dict(max_records=0), + dict(max_bytes=-1), + dict(retention=timedelta(0)), + ): + with pytest.raises(ValueError): + await provider.create_standalone_stream(_CLIENT, "s1", **bad) # type: ignore[arg-type] + with pytest.raises(ValueError): + await provider.create_standalone_stream(_CLIENT, "") + with pytest.raises(ValueError): + provider.get_standalone_stream_handle(_CLIENT, "") diff --git a/tests/streams/test_native_standalone_e2e.py b/tests/streams/test_native_standalone_e2e.py new file mode 100644 index 0000000000..2b9a43773a --- /dev/null +++ b/tests/streams/test_native_standalone_e2e.py @@ -0,0 +1,95 @@ +"""Standalone streams on the native provider against a live server. + +The conformance suite covers what every provider owes a standalone stream. +These pin what native adds on top: a read on an id nobody has created yet +parks on the server and delivers the first record once the stream exists, a +ref carries the stream to another client, and a create of an id that exists +is answered from the stream's own policy. Needs a server built from the +AI-198 branch: + + TEMPORAL_STREAM_TARGET=127.0.0.1:7433 uv run pytest tests/streams/test_native_standalone_e2e.py +""" + +from __future__ import annotations + +import asyncio +import os +import uuid + +import pytest + +from temporalio.client import Client +from temporalio.streams import StreamClosedError, StreamNotFoundError, StreamRef, topic +from temporalio.streams.providers.native import NativeStreams +from tests.streams.test_streams_conformance import take + +TARGET = os.environ.get("TEMPORAL_STREAM_TARGET") + +pytestmark = pytest.mark.skipif( + not TARGET, + reason="set TEMPORAL_STREAM_TARGET to a server with the stream service", +) + +OUT = topic("out", dict) + + +async def _connect(provider: NativeStreams) -> Client: + return await Client.connect(TARGET or "", plugins=[provider]) + + +async def test_a_read_on_a_stream_not_yet_created_waits_for_it() -> None: + provider = NativeStreams() + client = await _connect(provider) + stream_id = "late-" + uuid.uuid4().hex[:8] + try: + reader = client.get_stream_handle(stream_id=stream_id) + # Nothing to wait on for these: the stream does not exist. + with pytest.raises(StreamNotFoundError): + await reader.latest(topic=OUT) + with pytest.raises(StreamNotFoundError): + await reader.producer(topic=OUT, producer_id="early", attempt=1).append( + {"n": 0} + ) + + parked = asyncio.ensure_future(take(reader.read(topic=OUT), 1, timeout=30)) + await asyncio.sleep(0.5) + assert not parked.done(), "the read is parked, not failed" + + created = await client.create_stream(stream_id) + await created.producer(topic=OUT, producer_id="writer", attempt=1).append( + {"n": 1} + ) + assert [r.value for r in await parked] == [{"n": 1}] + finally: + await provider.close() + + +async def test_a_ref_carries_a_standalone_stream_to_another_client() -> None: + provider = NativeStreams() + client = await _connect(provider) + other_provider = NativeStreams() + other = await _connect(other_provider) + stream_id = "ref-" + uuid.uuid4().hex[:8] + try: + created = await client.create_stream(stream_id, max_records=10) + await created.producer(topic=OUT, producer_id="writer", attempt=1).append( + {"n": 1} + ) + ref = created.ref(topic=OUT) + assert ref == StreamRef.for_standalone(stream_id, topic="out") + + opened = other.get_stream_handle(ref) + records = await take(opened.read(), 1) + assert [r.value for r in records] == [{"n": 1}] + assert await opened.latest() == records[0].cursor + + await opened.close() + with pytest.raises(StreamClosedError): + await created.producer(topic=OUT, producer_id="writer", attempt=2).append( + {"n": 2} + ) + # The tail stays readable and the read ends on its own. + assert [r.value async for r in created.read(topic=OUT)] == [{"n": 1}] + finally: + await provider.close() + await other_provider.close() diff --git a/tests/streams/test_nexus_consumer.py b/tests/streams/test_nexus_consumer.py new file mode 100644 index 0000000000..d015f9d890 --- /dev/null +++ b/tests/streams/test_nexus_consumer.py @@ -0,0 +1,1151 @@ +"""The Nexus operation that consumes a stream through its notification channel. + +The unit cases drive :class:`StreamConsumerOperation` with a client stand-in +over the memory store and the deliveries a server would post, and catch what +it posts back as completions. The live cases need a server with notification +channels and a Nexus HTTP ingress, named with ``-E host:port`` and +``TEMPORAL_HTTP``; they put the memory store behind the Nexus front, where +the producer notifies the stream's channel by hand, and with +``STREAMS_LIVE=redis`` the Redis provider, whose producer notifies it itself. +The consumer's HTTP port is ``STREAM_CONSUMER_PORT``, 8813 by default. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import email.utils +import json +import os +import uuid +from collections.abc import AsyncIterator, Callable, Mapping +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, cast + +import nexusrpc +import nexusrpc.handler +import pytest +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer +from google.protobuf import json_format + +import temporalio.api.notification.v1 +import temporalio.converter +from temporalio import workflow +from temporalio.api.nexus.v1 import EndpointSpec, EndpointTarget +from temporalio.api.operatorservice.v1 import ( + CreateNexusEndpointRequest, + DeleteNexusEndpointRequest, +) +from temporalio.client import Callback, ChannelAddress, Client, stream_channel +from temporalio.common import Execution +from temporalio.contrib.external_workflow_streams._wake import ( + channel_for as external_channel_for, +) +from temporalio.exceptions import NexusOperationError +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import BEGINNING, StreamProvider, StreamRecord, StreamRef +from temporalio.streams._ref import open_ref +from temporalio.streams.providers import nexus +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.streams.providers.native import NativeStreams +from temporalio.streams.providers.nexus import ( + STREAM_CONSUMER_TOKEN_HEADER, + NexusStreams, + StreamConsumerOperation, + TemporalStreamsHandler, + stream_consumer_operation, +) +from temporalio.streams.providers.nexus_consumer_service import ( + CollectStream, + CollectStreamHandler, + NexusHttpService, + _NeverCancelled, + collect_values, +) +from temporalio.streams.providers.redis import RedisStreams, _chain +from temporalio.worker import Worker +from tests.helpers import assert_eventually +from tests.streams.test_nexus_provider import _own_endpoint + +VALUES = "values" +LISTENER_URL = "http://127.0.0.1:8813/deliveries" +CALLBACK_URL = "http://caller.invalid/namespaces/default/nexus/callback" +CALLBACK_HEADERS = {"temporal-callback-token": "caller-token"} + +_HTTP = os.environ.get("TEMPORAL_HTTP", "http://127.0.0.1:7243") +_PORT = int(os.environ.get("STREAM_CONSUMER_PORT", "8813")) +_REDIS_URL = os.environ.get("TEMPORAL_TEST_REDIS_URL") or os.environ.get( + "AI198_REDIS_URL", "redis://127.0.0.1:6379" +) + + +# --------------------------------------------------------------------------- +# Stand-ins. +# --------------------------------------------------------------------------- + + +def _as_client(stand_in: object) -> Client: + """Pass a stand-in where a client is typed; the consumer uses only what both have.""" + return cast(Client, stand_in) + + +class _ClientStandIn: + """What the consumer asks of a client: a provider, a converter, the channel calls.""" + + def __init__(self, store: MemoryStreams) -> None: + self._store = store + self.data_converter = temporalio.converter.DataConverter.default + self.registered: list[tuple[str, Callback, Execution | None]] = [] + self.unregistered: list[tuple[str, str, Execution | None]] = [] + + def get_stream_handle(self, ref: StreamRef) -> Any: + # The memory store opens a handle without a client. + return open_ref(self._store, _as_client(None), ref) + + async def register_channel_listener( + self, channel: str, callback: Callback, *, execution: Execution | None = None + ) -> str: + self.registered.append((channel, callback, execution)) + return f"listener-{len(self.registered)}" + + async def unregister_channel_listener( + self, channel: str, listener_id: str, *, execution: Execution | None = None + ) -> None: + self.unregistered.append((channel, listener_id, execution)) + + +@dataclass +class _Posted: + url: str + headers: dict[str, str] + body: bytes + + @property + def state(self) -> str: + return self.headers["Nexus-Operation-State"] + + +@pytest.fixture +def posted(monkeypatch: pytest.MonkeyPatch) -> list[_Posted]: + """Catch the completions the consumer posts to the caller's callback.""" + caught: list[_Posted] = [] + + def record(url: str, body: bytes, headers: Mapping[str, str], timeout: Any) -> None: + del timeout + caught.append(_Posted(url, dict(headers), body)) + + monkeypatch.setattr(nexus, "_post_completion", record) + return caught + + +def _start_context( + callback_url: str | None = CALLBACK_URL, +) -> nexusrpc.handler.StartOperationContext: + return nexusrpc.handler.StartOperationContext( + service="CollectStream", + operation="collect", + headers={}, + request_id=uuid.uuid4().hex, + callback_url=callback_url, + callback_headers=dict(CALLBACK_HEADERS), + task_cancellation=_NeverCancelled(), + ) + + +def _cancel_context() -> nexusrpc.handler.CancelOperationContext: + return nexusrpc.handler.CancelOperationContext( + service="CollectStream", + operation="collect", + headers={}, + task_cancellation=_NeverCancelled(), + ) + + +def _notification( + channel: str, counter: int, *, position: bytes = b"", closed: bool = False +) -> bytes: + """The body the server posts: the notification as protobuf JSON.""" + proto = temporalio.api.notification.v1.Notification( + channel=channel, counter=counter, position=position + ) + if closed: + proto.metadata["closed"].CopyFrom( + temporalio.converter.DataConverter.default.payload_converter.to_payload( + True + ) + ) + return json_format.MessageToJson(proto).encode() + + +@dataclass +class _Rig: + store: MemoryStreams + client: _ClientStandIn + operation: StreamConsumerOperation[list[Any]] + ref: StreamRef + channel: str + producer: Any + + async def start(self) -> str: + result = await self.operation.start(_start_context(), self.ref) + assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) + await self.settled(result.token) + return result.token + + async def settled(self, token: str) -> None: + """Wait for the read the start kicked off.""" + opening = self.operation._states[token].opening + assert opening is not None + await asyncio.wait_for(opening, 10) + + def headers(self, token: str) -> dict[str, str]: + return {STREAM_CONSUMER_TOKEN_HEADER: token} + + async def deliver(self, token: str, counter: int, *, closed: bool = False) -> Any: + return await self.operation.deliver( + self.headers(token), _notification(self.channel, counter, closed=closed) + ) + + +async def _rig( + consume: Callable[[StreamRecord[Any], list[Any]], list[Any]] = collect_values, +) -> _Rig: + store = MemoryStreams() + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await store.create_standalone_stream(None, stream_id) + ref = handle.ref(topic=VALUES) + client = _ClientStandIn(store) + operation = stream_consumer_operation( + consume, + initial=list, + listener_url=LISTENER_URL, + client=_as_client(client), + ) + return _Rig( + store, + client, + operation, + ref, + stream_channel(ref).channel, + handle.producer(topic=VALUES, producer_id="writer", attempt=1), + ) + + +# --------------------------------------------------------------------------- +# The helper's state machine. +# --------------------------------------------------------------------------- + + +def test_the_default_rule_is_the_servers_naming_of_a_streams_channel(): + # The consumer registers where the server's stream component notifies: + # a standalone stream's channel is independent, an owned stream's is + # linked to the owning workflow. + assert stream_channel(StreamRef.for_standalone("s-1", topic="values")) == ( + ChannelAddress("stream/s-1", None) + ) + assert stream_channel(StreamRef.for_workflow("wf", topic="t")) == ChannelAddress( + "stream/t", Execution.workflow("wf") + ) + assert stream_channel( + StreamRef.for_activity("act", workflow_id="wf", topic="t") + ) == ChannelAddress("stream/act/t", Execution.workflow("wf")) + # A standalone activity is an execution of its own, so its stream's + # channel is linked to it. + assert stream_channel(StreamRef.for_activity("act", topic="t")) == ( + ChannelAddress("stream/t", Execution.activity("act")) + ) + + +async def test_a_rule_may_answer_with_a_pair(posted: list[_Posted]): + # A store's rule that returns (channel, workflow_id) fits as well as an + # address does. + store = MemoryStreams() + handle = await store.create_standalone_stream(None, "paired") + client = _ClientStandIn(store) + operation = stream_consumer_operation( + collect_values, + initial=list, + listener_url=LISTENER_URL, + client=_as_client(client), + channel_for=lambda ref: (f"custom/{ref.stream_id}", None), + ) + result = await operation.start(_start_context(), handle.ref(topic=VALUES)) + assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) + state = operation.state(result.token) + assert state is not None and (state.channel, state.owner) == ("custom/paired", None) + await operation.close() + assert client.unregistered == [("custom/paired", "listener-1", None)] + assert posted == [] + + +async def test_an_owned_stream_registers_on_the_owners_linked_channel( + posted: list[_Posted], +): + store = MemoryStreams() + ref = store.get_stream_handle(None, "wf-1").ref(topic=VALUES) + client = _ClientStandIn(store) + operation = stream_consumer_operation( + collect_values, + initial=list, + listener_url=LISTENER_URL, + client=_as_client(client), + ) + result = await operation.start(_start_context(), ref) + assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) + state = operation.state(result.token) + owner = Execution.workflow("wf-1") + assert state is not None and (state.channel, state.owner) == ( + "stream/values", + owner, + ) + assert [(channel, owner) for channel, _, owner in client.registered] == [ + ("stream/values", owner) + ] + await operation.close() + assert client.unregistered == [("stream/values", "listener-1", owner)] + assert posted == [] + + +async def test_start_registers_the_listener_and_reads_what_was_there( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a", "b") + token = await rig.start() + + assert rig.operation.tokens == [token] + assert rig.client.registered == [ + ( + rig.channel, + Callback(url=LISTENER_URL, headers={STREAM_CONSUMER_TOKEN_HEADER: token}), + None, + ) + ] + state = rig.operation.state(token) + assert state is not None + assert state.value == ["a", "b"] + assert (state.reads, state.deliveries, state.records) == (1, 0, 2) + assert state.cursor != BEGINNING + assert state.listener_id == "listener-1" + assert posted == [] + + +async def test_a_delivery_reads_from_the_cursor_once_and_in_order( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a") + token = await rig.start() + + # Records after the registration arrive on the next delivery, each once. + await rig.producer.append("b", "c") + delivery = await rig.deliver(token, 1) + assert (delivery.known, delivery.read, delivery.records, delivery.completed) == ( + True, + True, + 2, + False, + ) + state = rig.operation.state(token) + assert state is not None and state.value == ["a", "b", "c"] + + # A delivery that brings nothing new reads nothing. + delivery = await rig.deliver(token, 1) + assert (delivery.read, delivery.records) == (False, 0) + state = rig.operation.state(token) + assert state is not None and (state.reads, state.deliveries) == (2, 2) + + # A burst the channel folded into one delivery costs one read. + await rig.producer.append("d", "e", "f") + delivery = await rig.operation.deliver( + {STREAM_CONSUMER_TOKEN_HEADER.lower(): token}, _notification(rig.channel, 4) + ) + assert (delivery.read, delivery.records) == (True, 3) + state = rig.operation.state(token) + assert state is not None + assert state.value == ["a", "b", "c", "d", "e", "f"] + assert (state.reads, state.deliveries, state.records) == (3, 3, 6) + assert posted == [] + + +async def test_the_close_completes_through_the_callers_callback( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a") + token = await rig.start() + await rig.producer.append("b") + + delivery = await rig.deliver(token, 2, closed=True) + assert (delivery.read, delivery.records, delivery.completed) == (True, 1, True) + [completion] = posted + assert completion.url == CALLBACK_URL + assert completion.state == "succeeded" + assert completion.headers["temporal-callback-token"] == "caller-token" + assert completion.headers["Nexus-Operation-Token"] == token + assert completion.headers["Content-Type"] == "application/json" + assert email.utils.parsedate_to_datetime( + completion.headers["Nexus-Operation-Start-Time"] + ) + assert json.loads(completion.body) == ["a", "b"] + assert rig.client.unregistered == [(rig.channel, "listener-1", None)] + assert rig.operation.state(token) is None + assert rig.operation.tokens == [] + + # Once complete, the token is unknown and a late delivery is ignored. + delivery = await rig.deliver(token, 3) + assert (delivery.known, delivery.read) == (False, False) + assert len(posted) == 1 + + +async def test_a_finish_record_closes_the_operation(posted: list[_Posted]): + rig = await _rig() + token = await rig.start() + await rig.producer.append("only") + await rig.producer.finish() + + delivery = await rig.deliver(token, 1) + assert delivery.completed + [completion] = posted + assert completion.state == "succeeded" + assert json.loads(completion.body) == ["only"] + assert rig.client.unregistered == [(rig.channel, "listener-1", None)] + + +async def test_a_stream_finished_before_the_start_completes_at_once( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a") + await rig.producer.finish() + token = await rig.start() + assert rig.operation.state(token) is None + [completion] = posted + assert completion.state == "succeeded" + assert json.loads(completion.body) == ["a"] + + +async def test_cancel_unregisters_and_reports_canceled(posted: list[_Posted]): + rig = await _rig() + token = await rig.start() + + await rig.operation.cancel(_cancel_context(), token) + assert rig.client.unregistered == [(rig.channel, "listener-1", None)] + [completion] = posted + assert completion.state == "canceled" + assert completion.headers["Nexus-Operation-Token"] == token + assert completion.headers["Content-Type"] == "application/json" + assert "canceled" in json.loads(completion.body)["message"] + assert rig.operation.state(token) is None + + with pytest.raises(nexusrpc.HandlerError) as unknown: + await rig.operation.cancel(_cancel_context(), token) + assert unknown.value.type is nexusrpc.HandlerErrorType.NOT_FOUND + + +async def test_a_delivery_for_an_operation_not_held_here_is_ignored( + posted: list[_Posted], +): + rig = await _rig() + await rig.producer.append("a") + delivery = await rig.operation.deliver({}, _notification(rig.channel, 1)) + assert (delivery.token, delivery.known) == (None, False) + delivery = await rig.deliver("nobody", 1) + assert (delivery.token, delivery.known, delivery.read) == ("nobody", False, False) + assert posted == [] + + +async def test_a_consume_failure_fails_the_operation(posted: list[_Posted]): + def fussy(record: StreamRecord[Any], values: list[Any]) -> list[Any]: + if record.value == "bad": + raise ValueError("cannot take bad") + return collect_values(record, values) + + rig = await _rig(fussy) + token = await rig.start() + await rig.producer.append("good", "bad", "later") + delivery = await rig.deliver(token, 1) + assert (delivery.records, delivery.completed) == (1, False) + [completion] = posted + assert completion.state == "failed" + assert json.loads(completion.body)["message"] == "ValueError: cannot take bad" + assert rig.client.unregistered == [(rig.channel, "listener-1", None)] + assert rig.operation.state(token) is None + + +async def test_a_read_failure_leaves_the_operation_for_the_retry( + posted: list[_Posted], monkeypatch: pytest.MonkeyPatch +): + rig = await _rig() + token = await rig.start() + await rig.producer.append("a") + handle = rig.client.get_stream_handle + + def broken(ref: StreamRef) -> Any: + del ref + raise RuntimeError("the store is away") + + monkeypatch.setattr(rig.client, "get_stream_handle", broken) + with pytest.raises(RuntimeError, match="away"): + await rig.deliver(token, 1) + monkeypatch.setattr(rig.client, "get_stream_handle", handle) + delivery = await rig.deliver(token, 1) + assert (delivery.read, delivery.records) == (True, 1) + assert posted == [] + + +async def test_close_lets_go_of_every_listener_without_completing( + posted: list[_Posted], +): + rig = await _rig() + first = await rig.start() + second = await rig.start() + assert rig.operation.tokens == [first, second] + await rig.operation.close() + assert rig.operation.tokens == [] + assert rig.client.unregistered == [ + (rig.channel, "listener-1", None), + (rig.channel, "listener-2", None), + ] + assert posted == [] + + +async def test_a_null_result_travels_without_a_content_type(posted: list[_Posted]): + def nothing(record: StreamRecord[Any], value: None) -> None: + del record + return value + + store = MemoryStreams() + handle = await store.create_standalone_stream(None, "silent") + ref = handle.ref(topic=VALUES) + client = _ClientStandIn(store) + operation = stream_consumer_operation( + nothing, + initial=lambda: None, + listener_url=LISTENER_URL, + client=_as_client(client), + ) + result = await operation.start(_start_context(), ref) + assert isinstance(result, nexusrpc.handler.StartOperationResultAsync) + await handle.producer(topic=VALUES, producer_id="w", attempt=1).finish() + await operation.deliver( + {STREAM_CONSUMER_TOKEN_HEADER: result.token}, + _notification(stream_channel(ref).channel, 1), + ) + [completion] = posted + assert completion.state == "succeeded" + assert "Content-Type" not in completion.headers + assert completion.body == b"" + + +async def test_the_listener_url_must_be_given(): + with pytest.raises(ValueError, match="listener_url"): + stream_consumer_operation(collect_values, initial=list, listener_url="") + + +# --------------------------------------------------------------------------- +# The standalone service speaks the Nexus HTTP protocol. +# --------------------------------------------------------------------------- + + +async def test_the_service_speaks_the_nexus_http_protocol(posted: list[_Posted]): + rig = await _rig() + service = NexusHttpService( + nexusrpc.handler.Handler([CollectStreamHandler(rig.operation)]), [rig.operation] + ) + converter = temporalio.converter.DataConverter.default.payload_converter + ref_body = converter.to_payload(rig.ref).data + async with TestClient(TestServer(service.application())) as http: + # A start carries the completion callback in the query and its + # headers under the callback prefix; an asynchronous start is a 201. + answer = await http.post( + "/CollectStream/collect", + params={"callback": CALLBACK_URL}, + data=ref_body, + headers={ + "Content-Type": "application/json", + "Nexus-Request-Id": "req-1", + "Nexus-Callback-Temporal-Callback-Token": "caller-token", + "Request-Timeout": "9.5s", + }, + ) + assert answer.status == 201 + token = (await answer.json())["token"] + assert rig.operation.tokens == [token] + held = rig.operation._states[token] + assert held.callback_url == CALLBACK_URL + assert held.callback_headers == {"temporal-callback-token": "caller-token"} + await rig.settled(token) + + # The delivery route reaches the operation the token names. + await rig.producer.append("a") + answer = await http.post( + service.deliveries_path, + data=_notification(rig.channel, 1), + headers={ + "Content-Type": "application/json", + "Temporal-Notification-Channel": rig.channel, + STREAM_CONSUMER_TOKEN_HEADER: token, + }, + ) + assert answer.status == 200 + state = rig.operation.state(token) + assert state is not None and state.value == ["a"] + + # A cancel names the token in its header and is accepted; a second one + # finds nothing and says so. + answer = await http.post( + "/CollectStream/collect/cancel", headers={"Nexus-Operation-Token": token} + ) + assert answer.status == 202 + assert [p.state for p in posted] == ["canceled"] + answer = await http.post( + "/CollectStream/collect/cancel", headers={"Nexus-Operation-Token": token} + ) + assert answer.status == 404 + assert "token" in (await answer.json())["message"] + + # What is not a stream reference is the caller's fault. + answer = await http.post( + "/CollectStream/collect", + data=b'{"kind": "nowhere"}', + headers={"Content-Type": "application/json"}, + ) + assert answer.status == 400 + answer = await http.post("/Other/op", data=b"{}") + assert answer.status == 404 + + +# --------------------------------------------------------------------------- +# Live: an external stream whose producer notifies the stream's channel. +# --------------------------------------------------------------------------- + + +@dataclass +class ConsumeInput: + """Which endpoint and service to call, which stream to hand it, and whether to cancel.""" + + endpoint: str + service: str + ref: StreamRef + cancel: bool = False + + +@workflow.defn +class ConsumeCaller: + """Starts the consumer operation with a ref and awaits its result.""" + + def __init__(self) -> None: + self._go = False + + @workflow.signal + def go(self) -> None: + """Let a cancelling run cancel now.""" + self._go = True + + @workflow.run + async def run(self, input: ConsumeInput) -> Any: + """Return the operation's result, or ``"canceled"`` after a cancel.""" + client = workflow.create_nexus_client( + service=input.service, endpoint=input.endpoint + ) + handle = await client.start_operation("collect", input.ref, output_type=list) + if not input.cancel: + return await handle + await workflow.wait_condition(lambda: self._go) + handle.cancel() + try: + await handle + except NexusOperationError: + return "canceled" + return "completed" + + +@contextlib.asynccontextmanager +async def _own_external_endpoint( + client: Client, name: str, url: str +) -> AsyncIterator[str]: + """Register a Nexus endpoint whose target is ``url`` for one test's life.""" + created = await client.operator_service.create_nexus_endpoint( + CreateNexusEndpointRequest( + spec=EndpointSpec( + name=name, + target=EndpointTarget(external=EndpointTarget.External(url=url)), + ) + ) + ) + try: + yield name + finally: + await client.operator_service.delete_nexus_endpoint( + DeleteNexusEndpointRequest( + id=created.endpoint.id, version=created.endpoint.version + ) + ) + + +@dataclass +class _Live: + store: StreamProvider + fronted: Client + consumer: StreamConsumerOperation[list[Any]] + endpoint: str + service: str + + +@contextlib.asynccontextmanager +async def _consumer_service( + client: Client, + store: StreamProvider | None = None, + *, + channel_for: nexus.ChannelRule = stream_channel, +) -> AsyncIterator[_Live]: + """Stand up the front over ``store`` and the standalone consumer behind an endpoint.""" + store = store or MemoryStreams() + uid = uuid.uuid4().hex + stream_handler = TemporalStreamsHandler(store, client) + handler_tq = f"streams-{uid}" + async with ( + _own_endpoint(client, f"streams-{uid}", handler_tq) as front_endpoint, + Worker(client, task_queue=handler_tq, nexus_service_handlers=[stream_handler]), + ): + front = NexusStreams( + endpoint=front_endpoint, http_address=_HTTP, read_wait=timedelta(seconds=2) + ) + config = client.config() + config["plugins"] = [front] + fronted = Client(**config) + consumer = stream_consumer_operation( + collect_values, + initial=list, + listener_url=f"http://127.0.0.1:{_PORT}/deliveries", + client=fronted, + channel_for=channel_for, + ) + service = NexusHttpService( + nexusrpc.handler.Handler([CollectStreamHandler(consumer)]), + [consumer], + data_converter=fronted.data_converter, + ) + runner = web.AppRunner(service.application()) + await runner.setup() + await web.TCPSite(runner, "127.0.0.1", _PORT).start() + try: + async with _own_external_endpoint( + client, f"consumers-{uid}", f"http://127.0.0.1:{_PORT}" + ) as endpoint: + yield _Live(store, fronted, consumer, endpoint, CollectStream.__name__) + finally: + await consumer.close() + await runner.cleanup() + await stream_handler.close() + + +async def _listener_token( + client: Client, channel: str, *, workflow_id: str | None = None +) -> str: + """The operation token the one callback listener on ``channel`` carries.""" + + async def registered() -> str: + # The channel comes into being with the registration, so a describe + # that races it is answered with not found. + try: + description = await client.describe_channel( + channel, workflow_id=workflow_id + ) + except RPCError as err: + assert err.status != RPCStatusCode.NOT_FOUND, "channel not created yet" + raise + # A channel linked to a workflow lists its owner as a listener as well. + callbacks = [one for one in description.listeners if one.callback is not None] + assert len(callbacks) == 1, description.listeners + [listener] = callbacks + assert listener.callback is not None + # The server reports the registration's headers in lower case. + headers = { + key.lower(): value for key, value in listener.callback.headers.items() + } + return headers[STREAM_CONSUMER_TOKEN_HEADER.lower()] + + return await assert_eventually(registered, timeout=timedelta(seconds=30)) + + +async def _consumed( + consumer: StreamConsumerOperation[list[Any]], token: str, count: int +) -> None: + async def reached() -> None: + state = consumer.state(token) + assert state is not None and state.records == count, state + + await assert_eventually(reached, timeout=timedelta(seconds=30)) + + +@pytest.mark.needs_channel_server +async def test_a_standalone_service_consumes_an_external_stream_through_its_channel( + client: Client, +): + store = MemoryStreams() + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await store.create_standalone_stream(None, stream_id) + ref = handle.ref(topic=VALUES) + channel = stream_channel(ref).channel + producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) + + async with _consumer_service(client, store) as live: + # Written before anyone listens: the channel keeps the notification, + # and the registration is handed it once. + await producer.append("a", "b") + assert await client.notify_channel(channel, position=b"2", counter=2) == 0 + + async with Worker( + client, task_queue=f"callers-{stream_id}", workflows=[ConsumeCaller] + ) as worker: + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(live.endpoint, live.service, ref), + id=f"caller-{stream_id}", + task_queue=worker.task_queue, + ) + token = await _listener_token(client, channel) + await _consumed(live.consumer, token, 2) + + # Written after the registration: each notification is delivered + # and read from the cursor; the retained one brought nothing new. + await producer.append("c") + assert await client.notify_channel(channel, position=b"3", counter=3) == 1 + await _consumed(live.consumer, token, 3) + await producer.append("d", "e", "f") + for counter in (4, 5, 6): + await client.notify_channel( + channel, position=b"%d" % counter, counter=counter + ) + await _consumed(live.consumer, token, 6) + state = live.consumer.state(token) + assert state is not None + assert state.value == ["a", "b", "c", "d", "e", "f"] + # Every read moved the cursor, none repeated a record, and a burst + # of three cost at most three reads. + assert state.reads <= 1 + 1 + 3 + assert state.deliveries >= 2 + + # The close completes the operation with what was consumed. + await producer.append("g") + await producer.finish() + await client.notify_channel( + channel, position=b"7", counter=7, metadata={"closed": True} + ) + assert await asyncio.wait_for(run.result(), 60) == [ + "a", + "b", + "c", + "d", + "e", + "f", + "g", + ] + assert live.consumer.state(token) is None + assert (await client.describe_channel(channel)).listeners == [] + + +@pytest.mark.needs_channel_server +async def test_cancel_unregisters_the_listener(client: Client): + store = MemoryStreams() + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await store.create_standalone_stream(None, stream_id) + ref = handle.ref(topic=VALUES) + channel = stream_channel(ref).channel + + async with ( + _consumer_service(client, store) as live, + Worker( + client, task_queue=f"callers-{stream_id}", workflows=[ConsumeCaller] + ) as worker, + ): + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(live.endpoint, live.service, ref, cancel=True), + id=f"caller-{stream_id}", + task_queue=worker.task_queue, + ) + token = await _listener_token(client, channel) + await run.signal(ConsumeCaller.go) + assert await asyncio.wait_for(run.result(), 60) == "canceled" + assert live.consumer.state(token) is None + assert (await client.describe_channel(channel)).listeners == [] + + +@workflow.defn +class StreamOwner: + """Owns the stream an outside producer writes to, and reads nothing itself.""" + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: False) + + +@pytest.mark.needs_channel_server +@pytest.mark.skipif( + os.environ.get("STREAMS_LIVE") != "redis", + reason="needs a Redis; run with STREAMS_LIVE=redis", +) +async def test_a_redis_stream_is_consumed_through_the_channel_its_producer_notifies( + client: Client, +): + """The Redis provider notifies the stream's channel on every append, so no + one notifies by hand: the channel is the owner's, named by the chain.""" + provider = RedisStreams( + url=_REDIS_URL, key_prefix=f"nexus-consumer-{uuid.uuid4().hex}" + ) + config = client.config() + config["plugins"] = [provider] + redis_client = Client(**config) + owner_id = f"owner-{uuid.uuid4().hex}" + try: + async with Worker( + client, + task_queue=f"owners-{owner_id}", + workflows=[StreamOwner, ConsumeCaller], + ) as worker: + owner = await client.start_workflow( + StreamOwner.run, id=owner_id, task_queue=worker.task_queue + ) + try: + chain = await _chain(redis_client, owner_id) + address = external_channel_for(chain.stream_key(VALUES)) + ref = StreamRef.for_workflow(owner_id, topic=VALUES) + producer = redis_client.get_stream_handle(owner_id).producer( + topic=VALUES, producer_id="writer", attempt=1 + ) + async with _consumer_service( + redis_client, + provider, + channel_for=lambda ref: external_channel_for( + chain.stream_key(ref.topic) + ), + ) as live: + # Written before the consumer exists: the start reads them. + await producer.append("a", "b") + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(live.endpoint, live.service, ref), + id=f"caller-{owner_id}", + task_queue=worker.task_queue, + ) + token = await _listener_token( + client, address.channel, workflow_id=address.workflow_id + ) + await _consumed(live.consumer, token, 2) + + # Each append notifies the channel, and the consumer reads + # from its cursor on the delivery. + await producer.append("c") + await _consumed(live.consumer, token, 3) + await producer.append("d", "e", "f") + await _consumed(live.consumer, token, 6) + state = live.consumer.state(token) + assert state is not None + assert state.value == ["a", "b", "c", "d", "e", "f"] + assert state.deliveries >= 2 + + # The producer's FINISH is a record like any other, and the + # consumer completes on it. + await producer.finish() + assert await asyncio.wait_for(run.result(), 60) == [ + "a", + "b", + "c", + "d", + "e", + "f", + ] + assert live.consumer.state(token) is None + description = await client.describe_channel( + address.channel, workflow_id=address.workflow_id + ) + assert [one for one in description.listeners if one.callback] == [] + finally: + await owner.terminate() + finally: + await provider.close() + + +# --------------------------------------------------------------------------- +# Live: the handler hosted in a Temporal worker, deliveries through the +# frontend's Nexus route of a companion operation. +# --------------------------------------------------------------------------- + + +@nexusrpc.service +class CollectInWorker: + """The consumer next to the synchronous operation the channel posts to.""" + + collect: nexusrpc.Operation[StreamRef, list[Any]] + deliver: nexusrpc.Operation[dict[str, Any], None] + + +@nexusrpc.handler.service_handler(service=CollectInWorker) +class CollectInWorkerHandler: + """Hosts the consumer in a worker; ``deliver`` is the listener's URL target.""" + + def __init__(self, consumer: StreamConsumerOperation[list[Any]]) -> None: + self._consumer = consumer + + @nexusrpc.handler.operation_handler + def collect(self) -> nexusrpc.handler.OperationHandler[StreamRef, list[Any]]: + return self._consumer + + @nexusrpc.handler.sync_operation + async def deliver( + self, ctx: nexusrpc.handler.StartOperationContext, input: dict[str, Any] + ) -> None: + # The server's delivery arrives as a start of this operation: the + # notification is the input and the listener's headers are the + # request's. + await self._consumer.deliver(ctx.headers, input) + + +@pytest.mark.needs_channel_server +async def test_a_worker_hosted_consumer_is_reached_through_the_frontend( + client: Client, +): + """The frontend takes the channel's post as a start of the companion operation. + + What the worker-hosted shape still lacks is the completion: the start + context a worker hands the handler names ``temporal://system`` as the + callback, which only a workflow or activity run the server completes + can reach, not an HTTP post. So this case checks the deliveries and + stops before the close; the completion through a run is a follow-up. + """ + store = MemoryStreams() + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await store.create_standalone_stream(None, stream_id) + ref = handle.ref(topic=VALUES) + channel = stream_channel(ref).channel + producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) + uid = uuid.uuid4().hex + stream_handler = TemporalStreamsHandler(store, client) + + async with ( + _own_endpoint(client, f"streams-{uid}", f"streams-{uid}") as front_endpoint, + Worker( + client, task_queue=f"streams-{uid}", nexus_service_handlers=[stream_handler] + ), + ): + front = NexusStreams( + endpoint=front_endpoint, http_address=_HTTP, read_wait=timedelta(seconds=2) + ) + config = client.config() + config["plugins"] = [front] + fronted = Client(**config) + # The listener's URL is the frontend's route to the companion + # operation, so the endpoint has to exist before the consumer does. + created = await client.operator_service.create_nexus_endpoint( + CreateNexusEndpointRequest( + spec=EndpointSpec( + name=f"hosted-{uid}", + target=EndpointTarget( + worker=EndpointTarget.Worker( + namespace=client.namespace, task_queue=f"hosted-{uid}" + ) + ), + ) + ) + ) + consumer = stream_consumer_operation( + collect_values, + initial=list, + listener_url=( + f"{_HTTP}/nexus/endpoints/{created.endpoint.id}/services/" + f"{CollectInWorker.__name__}/deliver" + ), + ) + try: + async with ( + Worker( + fronted, + task_queue=f"hosted-{uid}", + nexus_service_handlers=[CollectInWorkerHandler(consumer)], + ), + Worker( + client, task_queue=f"callers-{uid}", workflows=[ConsumeCaller] + ) as callers, + ): + await producer.append("a") + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(f"hosted-{uid}", CollectInWorker.__name__, ref), + id=f"caller-{uid}", + task_queue=callers.task_queue, + ) + token = await _listener_token(client, channel) + await _consumed(consumer, token, 1) + await producer.append("b", "c") + assert ( + await client.notify_channel(channel, position=b"3", counter=3) == 1 + ) + await _consumed(consumer, token, 3) + state = consumer.state(token) + assert state is not None + assert state.value == ["a", "b", "c"] + assert state.deliveries >= 1 + held = consumer._states[token] + assert held.callback_url is not None + assert held.callback_url.startswith("temporal://"), held.callback_url + await run.terminate() + finally: + await consumer.close() + await stream_handler.close() + await client.operator_service.delete_nexus_endpoint( + DeleteNexusEndpointRequest( + id=created.endpoint.id, version=created.endpoint.version + ) + ) + + +# --------------------------------------------------------------------------- +# Live, waiting for a server: a native standalone stream notifies its own +# channel, so nobody notifies by hand. +# --------------------------------------------------------------------------- + + +@pytest.mark.needs_stream_channel_server +@pytest.mark.needs_native_provider +async def test_a_native_standalone_stream_is_consumed_through_its_channel( + client: Client, +): + provider = NativeStreams() + client = await Client.connect( + client.service_client.config.target_host, + namespace=client.namespace, + plugins=[provider], + ) + stream_id = f"consumed-{uuid.uuid4().hex}" + handle = await client.create_stream(stream_id) + ref = handle.ref(topic=VALUES) + producer = handle.producer(topic=VALUES, producer_id="writer", attempt=1) + + async with _consumer_service(client, provider) as live: + await producer.append("a", "b") + async with Worker( + client, task_queue=f"callers-{stream_id}", workflows=[ConsumeCaller] + ) as worker: + run = await client.start_workflow( + ConsumeCaller.run, + ConsumeInput(live.endpoint, live.service, ref), + id=f"caller-{stream_id}", + task_queue=worker.task_queue, + ) + token = await _listener_token(client, stream_channel(ref).channel) + await _consumed(live.consumer, token, 2) + # The server notifies the stream's channel on every append and on + # the close, so the records reach the consumer without a notify. + await producer.append("c") + await _consumed(live.consumer, token, 3) + await producer.finish() + await handle.close() + assert await asyncio.wait_for(run.result(), 60) == ["a", "b", "c"] + assert ( + await client.describe_channel(stream_channel(ref).channel) + ).listeners == [] diff --git a/tests/streams/test_nexus_provider.py b/tests/streams/test_nexus_provider.py new file mode 100644 index 0000000000..efec762c89 --- /dev/null +++ b/tests/streams/test_nexus_provider.py @@ -0,0 +1,1234 @@ +"""Conformance for the Nexus front. + +The caller talks only to the stream endpoint; the handler fronts a storage +provider's own handles, so these tests are the provider-hiding demonstration: +nothing on the caller side names or could name the store. + +The live tests run over the server's Nexus HTTP ingress and are gated behind +``STREAMS_LIVE=nexus`` because they need a dev server with an HTTP port. +Environment: ``TEMPORAL_ADDRESS`` (default ``localhost:7233``) and +``TEMPORAL_HTTP`` (default ``http://127.0.0.1:7243``). Each live test +registers its own Nexus endpoint through the operator service and deletes it +at the end. + +Everything else stands the endpoint up in this process, because what it +checks is the bytes the caller puts on the wire and the handler's own rules. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import dataclasses +import gc +import http.client +import json +import os +import uuid +from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, cast +from unittest import mock + +import nexusrpc +import nexusrpc.handler +import pytest + +import temporalio.converter +from temporalio import nexus as temporal_nexus +from temporalio import streams, workflow +from temporalio.api.common.v1 import Payload +from temporalio.api.nexus.v1 import EndpointSpec, EndpointTarget +from temporalio.api.operatorservice.v1 import ( + CreateNexusEndpointRequest, + DeleteNexusEndpointRequest, +) +from temporalio.client import Client +from temporalio.common import WorkflowIDConflictPolicy +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamCursorError, + StreamProducerError, + StreamRef, + StreamUnsupportedError, +) +from temporalio.streams._ref import open_ref +from temporalio.streams._wire import WireRecord +from temporalio.streams.providers import nexus +from temporalio.streams.providers._nexus_generated import AppendInput, ReadInput +from temporalio.streams.providers._nexus_generated import StreamRef as WireStreamRef +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.streams.providers.nexus import ( + NexusStreamHandle, + NexusStreams, + TemporalStreamsHandler, +) +from temporalio.streams.providers.workflow_streams import WorkflowStreamsProvider +from temporalio.worker import Worker +from tests.helpers import new_worker +from tests.streams.test_workflow_streams_provider import EchoLoop, take +from tests.streams.test_workflow_streams_provider import _feed as feed_echo_loop + +live_only = pytest.mark.skipif( + os.environ.get("STREAMS_LIVE") != "nexus", + reason="needs a live server and nexus endpoint; run with STREAMS_LIVE=nexus", +) + +APPEND_OPERATION = nexus._APPEND_OPERATION # pyright: ignore[reportPrivateUsage] +READ_OPERATION = nexus._READ_OPERATION # pyright: ignore[reportPrivateUsage] +INPUTS = "inputs" +DECISIONS = "decisions" + + +async def _live_client() -> Client: + return await Client.connect(os.environ.get("TEMPORAL_ADDRESS", "localhost:7233")) + + +def _live_front(endpoint: str) -> NexusStreams: + return NexusStreams( + endpoint=endpoint, + http_address=os.environ.get("TEMPORAL_HTTP", "http://127.0.0.1:7243"), + read_wait=timedelta(seconds=5), + ) + + +@contextlib.asynccontextmanager +async def _own_endpoint( + client: Client, name: str, task_queue: str +) -> AsyncIterator[str]: + """Register a Nexus endpoint routed to ``task_queue`` for one test's life.""" + created = await client.operator_service.create_nexus_endpoint( + CreateNexusEndpointRequest( + spec=EndpointSpec( + name=name, + target=EndpointTarget( + worker=EndpointTarget.Worker( + namespace=client.namespace, task_queue=task_queue + ) + ), + ) + ) + ) + try: + yield name + finally: + await client.operator_service.delete_nexus_endpoint( + DeleteNexusEndpointRequest( + id=created.endpoint.id, version=created.endpoint.version + ) + ) + + +@live_only +async def test_interface_loop_through_the_nexus_front(): + # The workflow worker and the handler worker both use the storage + # provider; only the caller goes through the front, and it names the + # endpoint the way an operator does, by name. + store = WorkflowStreamsProvider() + client = await _live_client() + workflow_id = f"streams-nexus-live-{uuid.uuid4().hex}" + handler = TemporalStreamsHandler(store, client) + handler_tq = f"handlers-{workflow_id}" + + async with ( + _own_endpoint(client, workflow_id, handler_tq) as endpoint, + Worker(client, task_queue=handler_tq, nexus_service_handlers=[handler]), + ): + front = _live_front(endpoint) + async with Worker( + client, + task_queue=f"tq-{workflow_id}", + workflows=[EchoLoop], + plugins=[store], + ): + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + + stream = front.get_stream_handle(client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + await producer.append({"n": 3}) + await producer.finish() + + records = await take(stream.read(topic=DECISIONS, result_type=dict), 4, 60) + assert [r.kind for r in records] == [ + RecordKind.DATA, + RecordKind.DATA, + RecordKind.DATA, + RecordKind.FINISH, + ] + assert [r.value["echo"] for r in records[:3]] == [1, 2, 3] + + # An opaque cursor from behind the front resumes a fresh reader + # just past the record it names. + checkpoint = records[0].cursor + again = await take( + stream.read(topic=DECISIONS, result_type=dict, after=checkpoint), 2, 60 + ) + assert [r.value["echo"] for r in again[:2]] == [2, 3] + + await handle.signal(EchoLoop.release) + assert await handle.result() == 3 + await handler.close() + + +class ScrambleCodec(temporalio.converter.PayloadCodec): + """Flips every byte and labels the result with an encoding only it can read.""" + + KEY = 0x5A + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + """Wrap each payload as opaque bytes nothing downstream can read.""" + return [ + Payload( + metadata={"encoding": b"binary/scrambled"}, + data=self._flip(payload.SerializeToString()), + ) + for payload in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + """Recover the payloads :meth:`encode` wrapped.""" + return [Payload.FromString(self._flip(payload.data)) for payload in payloads] + + @classmethod + def _flip(cls, data: bytes) -> bytes: + return bytes(byte ^ cls.KEY for byte in data) + + +class _NeverCancelled(nexusrpc.handler.OperationTaskCancellation): + def is_cancelled(self) -> bool: + return False + + def cancellation_reason(self) -> str | None: + return None + + def wait_until_cancelled_sync(self, timeout: float | None = None) -> bool: + del timeout + return False + + async def wait_until_cancelled(self) -> None: + await asyncio.Event().wait() + + +def _context( + operation: str, deadline: datetime | None = None +) -> nexusrpc.handler.StartOperationContext: + return nexusrpc.handler.StartOperationContext( + service=nexus._SERVICE_NAME, # pyright: ignore[reportPrivateUsage] + operation=operation, + headers={}, + task_cancellation=_NeverCancelled(), + request_deadline=deadline, + request_id=uuid.uuid4().hex, + ) + + +async def _dispatch( + handler: TemporalStreamsHandler, operation: str, request: Any +) -> Any: + if operation == APPEND_OPERATION: + return await handler.append(_context(operation), request) + return await handler.read(_context(operation), request) + + +def _in_process_endpoint( + handler: TemporalStreamsHandler, + posted: list[bytes], + answered: list[bytes], + *, + fail_after_applying: list[str] | None = None, +) -> Any: + """Serve the caller's posts from ``handler``, recording both directions. + + ``fail_after_applying`` names operations whose first call is applied and + then reported as a transport failure, the ambiguous case a retry has to + survive. + """ + contract = ( + temporalio.converter.DataConverter.default._get_internal_payload_converter() + ) + loop = asyncio.get_running_loop() + failing = set(fail_after_applying or []) + + def post( + url: str, body: bytes, headers: Mapping[str, str], timeout: timedelta + ) -> bytes: + del headers, timeout + posted.append(body) + operation = url.rsplit("/", 1)[1] + request_type = AppendInput if operation == APPEND_OPERATION else ReadInput + request: Any = contract.from_payloads( + [Payload(metadata={"encoding": b"json/plain"}, data=body)], [request_type] + )[0] + try: + answer = asyncio.run_coroutine_threadsafe( + _dispatch(handler, operation, request), loop + ).result() + except nexusrpc.HandlerError as error: + # The ingress answers a handler error with a failure body and the + # status its type maps to. + status = 404 if error.type is nexusrpc.HandlerErrorType.NOT_FOUND else 400 + raise nexus._EndpointFailure( # pyright: ignore[reportPrivateUsage] + json.dumps({"message": str(error)}), status + ) from error + raw = contract.to_payloads([answer])[0].data + answered.append(raw) + if operation in failing: + failing.discard(operation) + raise nexus._EndpointFailure( # pyright: ignore[reportPrivateUsage] + "connection reset after the handler answered", None + ) + return raw + + return post + + +def _front(codec: temporalio.converter.PayloadCodec | None) -> NexusStreams: + return NexusStreams( + endpoint="in-process", + # This endpoint has no other traffic to wait for, so park briefly + # rather than for the contract's default. + read_wait=timedelta(milliseconds=200), + data_converter=dataclasses.replace( + temporalio.converter.DataConverter.default, payload_codec=codec + ), + ) + + +async def _loop_through_an_endpoint( + monkeypatch: pytest.MonkeyPatch, + codec: temporalio.converter.PayloadCodec | None, +) -> tuple[list[Any], list[bytes], list[bytes]]: + store = MemoryStreams() + posted: list[bytes] = [] + answered: list[bytes] = [] + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, posted, answered)) + stream = _front(codec).get_stream_handle(None, "wf-codec") + + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"secret": "tuna"}) + await producer.finish() + + records = await take(stream.read(topic=INPUTS, result_type=dict), 2, timeout=30) + await handler.close() + return records, posted, answered + + +def _appended_bytes(posted: list[bytes]) -> bytes: + out = b"" + for body in posted: + for encoded in json.loads(body).get("payloads", []): + out += base64.b64decode(encoded) + return out + + +def _answered_bodies(answered: list[bytes]) -> bytes: + out = b"" + for body in answered: + for record in json.loads(body).get("records", []): + wire = WireRecord.FromString(base64.b64decode(record["record"])) + out += wire.body.SerializeToString() + return out + + +async def test_the_caller_encodes_records_with_the_configured_codec( + monkeypatch: pytest.MonkeyPatch, +): + records, posted, answered = await _loop_through_an_endpoint( + monkeypatch, ScrambleCodec() + ) + + # The value survives the hop although the handler cannot read the + # encoding, so the body passed through it untouched and the codec ran on + # the caller alone. + assert [record.kind for record in records] == [RecordKind.DATA, RecordKind.FINISH] + assert records[0].value == {"secret": "tuna"} + # Neither direction carried the plaintext. + assert b"tuna" not in _appended_bytes(posted) + assert b"tuna" not in _answered_bodies(answered) + + +async def test_without_a_codec_the_same_records_go_out_in_the_clear( + monkeypatch: pytest.MonkeyPatch, +): + # The contrast is the point: the bytes above differ because of the codec, + # not because the transport obscures them anyway. + records, posted, answered = await _loop_through_an_endpoint(monkeypatch, None) + + assert records[0].value == {"secret": "tuna"} + assert b"tuna" in _appended_bytes(posted) + assert b"tuna" in _answered_bodies(answered) + + +def _ref(workflow_id: str, topic: str = INPUTS) -> WireStreamRef: + return WireStreamRef(kind="workflow", workflow_id=workflow_id, topic=topic) + + +def _append( + workflow_id: str, + batch_index: int, + *values: Any, + sequence: int | None = None, + finish: bool = False, +) -> AppendInput: + converter = temporalio.converter.DataConverter.default.payload_converter + return AppendInput( + stream=_ref(workflow_id), + producer_id="model", + attempt=1, + # Batches of one, numbered in step with the batch index unless a + # case wants them out of step. + sequence=batch_index - 1 if sequence is None else sequence, + batch_index=batch_index, + payloads=[ + converter.to_payloads([value])[0].SerializeToString() for value in values + ], + finish=finish, + ) + + +async def _stored(store: MemoryStreams, workflow_id: str) -> list[Any]: + stream = store.get_stream_handle(None, workflow_id) + end = await stream.latest(topic=INPUTS) + if end == BEGINNING: + return [] + out: list[Any] = [] + async for record in stream.read(topic=INPUTS, result_type=dict): + if record.kind is RecordKind.DATA: + out.append(record.value) + if record.cursor == end: + break + return out + + +async def test_a_handler_without_state_for_a_producer_refuses_to_continue_it(): + # A second handler instance stands in for a restarted or load-balanced + # handler worker: it shares the store but not the dedupe state. + store = MemoryStreams() + first = TemporalStreamsHandler(store, None) + second = TemporalStreamsHandler(store, None) + await _dispatch(first, APPEND_OPERATION, _append("wf", 1, {"n": 1})) + with pytest.raises(nexusrpc.HandlerError) as failed: + await _dispatch(second, APPEND_OPERATION, _append("wf", 2, {"n": 2})) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + assert failed.value.retryable_override is False + # Named so the caller can raise the same class. + assert str(failed.value).startswith("StreamProducerError: ") + # The store holds the first batch and nothing was silently lost or doubled. + assert await _stored(store, "wf") == [{"n": 1}] + + +async def test_the_handler_answers_repeats_and_rejects_gaps(): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})) + second = await _dispatch(handler, APPEND_OPERATION, _append("wf", 2, {"n": 2})) + # A repeat of the last batch is written once and answers with where the + # original landed, so a retrying caller checkpoints the same position. + repeat = await _dispatch(handler, APPEND_OPERATION, _append("wf", 2, {"n": 2})) + assert repeat.cursor == second.cursor + with pytest.raises(nexusrpc.HandlerError) as skipped: + await _dispatch(handler, APPEND_OPERATION, _append("wf", 4, {"n": 4})) + assert skipped.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + with pytest.raises(nexusrpc.HandlerError) as behind: + await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})) + assert str(behind.value).startswith("StreamProducerError: ") + with pytest.raises(nexusrpc.HandlerError) as renumbered: + await _dispatch( + handler, APPEND_OPERATION, _append("wf", 3, {"n": 3}, sequence=7) + ) + assert "sequence 7 does not continue at 2" in str(renumbered.value) + assert await _stored(store, "wf") == [{"n": 1}, {"n": 2}] + + +async def test_a_malformed_payload_is_the_callers_fault(): + handler = TemporalStreamsHandler(MemoryStreams(), None) + request = _append("wf", 1) + request.payloads = [b"\xff\xfe not a payload"] + with pytest.raises(nexusrpc.HandlerError) as failed: + await _dispatch(handler, APPEND_OPERATION, request) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + + +async def test_the_caller_retries_an_ambiguous_append_under_the_same_index( + monkeypatch: pytest.MonkeyPatch, +): + posted: list[bytes] = [] + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr( + nexus, + "_post", + _in_process_endpoint( + handler, posted, [], fail_after_applying=[APPEND_OPERATION] + ), + ) + stream = _front(None).get_stream_handle(None, "wf") + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + # A transport failure is an RPCError, never the transport's own type. + with pytest.raises(RPCError) as failed: + await producer.append({"n": 1}) + assert failed.value.status == RPCStatusCode.UNAVAILABLE + landed = await producer.append({"n": 1}) + await producer.append({"n": 2}) + await producer.finish() + indexes = [json.loads(body)["batch_index"] for body in posted] + sequences = [json.loads(body)["sequence"] for body in posted] + # The retry re-sent index 1, which the handler answered as a repeat with + # the original's position, so the store holds each record once. + assert indexes == [1, 1, 2, 3] + assert sequences == [0, 0, 1, 2] + assert landed == await _memory_position(store, "wf", 0) + assert await _stored(store, "wf") == [{"n": 1}, {"n": 2}] + + +async def _memory_position( + store: MemoryStreams, workflow_id: str, index: int +) -> Cursor: + records = await take( + store.get_stream_handle(None, workflow_id).read(topic=INPUTS, result_type=dict), + index + 1, + ) + return records[index].cursor + + +async def test_a_failed_append_goes_out_before_the_next_one( + monkeypatch: pytest.MonkeyPatch, +): + posted: list[bytes] = [] + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr( + nexus, + "_post", + _in_process_endpoint( + handler, posted, [], fail_after_applying=[APPEND_OPERATION] + ), + ) + stream = _front(None).get_stream_handle(None, "wf") + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + with pytest.raises(RPCError): + await producer.append({"n": 1}) + # The caller moves on without retrying; the pending batch is replayed + # first under its own index and the new one takes the next. + await producer.append({"n": 2}) + indexes = [json.loads(body)["batch_index"] for body in posted] + assert indexes == [1, 1, 2] + assert await _stored(store, "wf") == [{"n": 1}, {"n": 2}] + + +async def test_a_stream_condition_crosses_the_endpoint_under_its_own_class( + monkeypatch: pytest.MonkeyPatch, +): + handler = TemporalStreamsHandler(MemoryStreams(), None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + stream = _front(None).get_stream_handle(None, "wf") + # The token is opaque on this side, so the store behind the endpoint is + # what refuses it, and the refusal arrives on the first read as the class + # the store raised. + with pytest.raises(StreamCursorError): + await take(stream.read(topic=INPUTS, after=Cursor("elsewhere:1")), 1) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}) + # A second handler has no state for the attempt; the caller learns that + # as a producer conflict. + fresh = TemporalStreamsHandler(MemoryStreams(), None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(fresh, [], [])) + with pytest.raises(StreamProducerError): + await producer.append({"n": 2}) + await handler.close() + + +def test_the_front_has_no_workflow_half(): + with pytest.raises(StreamUnsupportedError): + _front(None).workflow_provider() + + +async def test_the_front_registers_on_a_client(client: Client): + # The same accessor as any provider, so code outside a workflow does not + # change when the store moves behind an endpoint. + config = client.config() + config["plugins"] = [_front(None)] + registered = Client(**config) + assert isinstance(registered.get_stream_handle("wf"), NexusStreamHandle) + + +async def test_consecutive_reads_share_one_parked_subscription(): + handler = TemporalStreamsHandler(MemoryStreams(), None) + await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1}, {"n": 2})) + first = await _dispatch( + handler, + READ_OPERATION, + ReadInput(stream=_ref("wf"), max_records=1, wait_ms=200), + ) + assert len(first.records) == 1 + parked = handler._subscriptions # pyright: ignore[reportPrivateUsage] + pump = next(iter(parked.values())).pump + # A call that resumes where the last one stopped drains the same + # subscription instead of parking a second read on the store. + second = await _dispatch( + handler, + READ_OPERATION, + ReadInput( + stream=_ref("wf"), + after_token=first.next_token, + max_records=1, + wait_ms=200, + ), + ) + assert len(second.records) == 1 + assert second.next_token != first.next_token + assert len(parked) == 1 and next(iter(parked.values())).pump is pump + # A call from somewhere else replaces it. + await _dispatch( + handler, + READ_OPERATION, + ReadInput(stream=_ref("wf"), wait_ms=200), + ) + assert len(parked) == 1 and next(iter(parked.values())).pump is not pump + await handler.close() + assert not parked + + +async def test_the_read_wait_is_cut_to_the_request_deadline(): + handler = TemporalStreamsHandler(MemoryStreams(), None) + started = asyncio.get_running_loop().time() + answer = await handler.read( + _context(READ_OPERATION, datetime.now(timezone.utc) + timedelta(seconds=1)), + ReadInput(stream=_ref("wf"), wait_ms=60000), + ) + assert answer.records == [] and answer.next_token == "" and not answer.done + assert asyncio.get_running_loop().time() - started < 5 + await handler.close() + + +async def test_the_read_ends_when_the_store_ends_it( + client: Client, monkeypatch: pytest.MonkeyPatch +): + # Behind the endpoint sits the Workflow Streams store, whose read ends + # once the run has closed and its tail is served. The front carries that + # end to the caller, whose loop leaves by itself. + store = WorkflowStreamsProvider(poll_cooldown=timedelta(milliseconds=20)) + handler = TemporalStreamsHandler(store, client) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + workflow_id = f"streams-nexus-{uuid.uuid4().hex}" + async with new_worker(client, EchoLoop, plugins=[store]) as worker: + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + await feed_echo_loop(store, client, workflow_id) + await handle.signal(EchoLoop.release) + assert await handle.result() == 3 + + stream = _front(None).get_stream_handle(None, workflow_id) + + async def read_everything() -> list[Any]: + return [ + (r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + + records = await asyncio.wait_for(read_everything(), 30) + await handler.close() + assert records == [ + (RecordKind.DATA, {"echo": 1}), + (RecordKind.DATA, {"echo": 2}), + (RecordKind.DATA, {"echo": 3}), + (RecordKind.FINISH, None), + ] + + +class _SlowStore(MemoryStreams): + """The memory store with a real await inside ``append``. + + The handler's repeat check and its commit have an await between them. + Without a gate there, two in-flight copies of one batch both finish the + check before either commits and both reach the store. + """ + + def __init__(self) -> None: + super().__init__() + self.gate = asyncio.Event() + self.appends = 0 + + def get_stream_handle( + self, client: Any, workflow_id: str, *, run_id: str | None = None + ) -> Any: + handle = super().get_stream_handle(client, workflow_id, run_id=run_id) + outer = self + make_producer = handle.producer + + def producer(**kwargs: Any) -> Any: + delegate = make_producer(**kwargs) + inner_append = delegate.append + + async def append(*values: Any) -> Any: + outer.appends += 1 + await outer.gate.wait() + return await inner_append(*values) + + delegate.append = append # type: ignore[method-assign] + return delegate + + handle.producer = producer # type: ignore[method-assign] + return handle + + +async def test_two_copies_of_one_batch_reach_the_store_once(): + store = _SlowStore() + handler = TemporalStreamsHandler(store, None) + both = asyncio.gather( + _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})), + _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})), + ) + await asyncio.sleep(0.1) + store.gate.set() + first, second = await asyncio.wait_for(both, 10) + # One of them wrote and the other was answered from the state it left. + # Counted at the store because a store that dedupes by itself would hide + # a second commit the handler should never have made. + assert store.appends == 1 + assert first.cursor == second.cursor + assert await _stored(store, "wf") == [{"n": 1}] + + +async def test_a_finished_batch_can_be_retried(): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + await _dispatch(handler, APPEND_OPERATION, _append("wf", 1, {"n": 1})) + finished = await _dispatch( + handler, APPEND_OPERATION, _append("wf", 2, {"n": 2}, finish=True) + ) + # The caller never saw the answer. The byte-identical retry has to be + # answerable, because a finish the caller cannot repeat is a batch it can + # neither complete nor abandon. + again = await _dispatch( + handler, APPEND_OPERATION, _append("wf", 2, {"n": 2}, finish=True) + ) + assert again.cursor == finished.cursor + assert await _stored(store, "wf") == [{"n": 1}, {"n": 2}] + # And the attempt is over: a batch after the finish is refused rather + # than landing behind the marker. + with pytest.raises(nexusrpc.HandlerError) as after: + await _dispatch(handler, APPEND_OPERATION, _append("wf", 3, {"n": 3})) + assert "already finished" in str(after.value) + + +class _FailingReadStore(MemoryStreams): + """A store whose read fails once, the way a transient store failure does.""" + + def __init__(self) -> None: + super().__init__() + self.reads = 0 + + def get_stream_handle( + self, client: Any, workflow_id: str, *, run_id: str | None = None + ) -> Any: + handle = super().get_stream_handle(client, workflow_id, run_id=run_id) + outer = self + + def read(**kwargs: Any) -> Any: + outer.reads += 1 + if outer.reads == 1: + + async def failing() -> AsyncIterator[Any]: + for record in cast("list[Any]", []): + yield record + raise RuntimeError("the store went away") + + return failing() + return MemoryStreams.get_stream_handle( + outer, client, workflow_id, run_id=run_id + ).read(**kwargs) + + handle.read = read # type: ignore[method-assign] + return handle + + +async def test_a_failed_read_is_not_reported_as_the_end_of_the_stream(): + store = _FailingReadStore() + handler = TemporalStreamsHandler(store, None) + await ( + store.get_stream_handle(None, "wf") + .producer(topic=INPUTS, producer_id="model", attempt=1) + .append({"n": 1}) + ) + + request = ReadInput(stream=_ref("wf"), wait_ms=200, max_records=10) + with pytest.raises(RuntimeError, match="the store went away"): + await _dispatch(handler, READ_OPERATION, request) + # The retry from the same token has to re-subscribe rather than be + # answered done=True off the subscription the failed pump left behind. + answer = await _dispatch(handler, READ_OPERATION, request) + assert [WireRecord.FromString(r.record).sequence for r in answer.records] == [1] + assert answer.done is False + await handler.close() + + +async def test_the_handler_does_not_grow_a_lock_per_address(): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + for index in range(25): + workflow_id = f"wf-{index}" + await _dispatch( + handler, APPEND_OPERATION, _append(workflow_id, 1, {"n": index}) + ) + await _dispatch( + handler, + READ_OPERATION, + ReadInput(stream=_ref(workflow_id), wait_ms=0, max_records=1), + ) + gc.collect() + # Held weakly, so an address nobody is reading or appending to leaves + # nothing behind. A strong map would hold one entry per address forever. + assert len(handler._read_locks) == 0 # pyright: ignore[reportPrivateUsage] + assert len(handler._append_locks) == 0 # pyright: ignore[reportPrivateUsage] + await handler.close() + + +def test_the_read_bounds_are_checked_where_they_are_set(): + # The contract accepts a wait up to a minute and a batch up to a + # thousand; past that the endpoint answers with a payload validation + # error, which is neither a stream condition nor an RPC failure. + with pytest.raises(ValueError, match="read_wait"): + NexusStreams(endpoint="e", read_wait=timedelta(minutes=2)) + with pytest.raises(ValueError, match="read_wait"): + NexusStreams(endpoint="e", read_wait=timedelta(seconds=-1)) + with pytest.raises(ValueError, match="max_records"): + NexusStreams(endpoint="e", max_records=0) + with pytest.raises(ValueError, match="max_records"): + NexusStreams(endpoint="e", max_records=1001) + NexusStreams(endpoint="e", read_wait=timedelta(seconds=60), max_records=1000) + + +def test_a_socket_failure_never_reaches_the_caller_as_a_timeout(): + # urllib wraps only the request in URLError, so a read timeout comes out + # of getresponse() as builtins.TimeoutError, which on 3.11 and later is + # asyncio.TimeoutError and would be taken for the caller's own deadline. + def _raise(*args: Any, **kwargs: Any) -> Any: + del args, kwargs + raise TimeoutError("the socket read timed out") + + with mock.patch("urllib.request.urlopen", _raise): + with pytest.raises(nexus._EndpointFailure): # pyright: ignore[reportPrivateUsage] + nexus._post( # pyright: ignore[reportPrivateUsage] + "http://127.0.0.1:1/x", b"{}", {}, timedelta(seconds=1) + ) + + def _disconnect(*args: Any, **kwargs: Any) -> Any: + del args, kwargs + raise http.client.RemoteDisconnected("the endpoint hung up") + + with mock.patch("urllib.request.urlopen", _disconnect): + with pytest.raises(nexus._EndpointFailure): # pyright: ignore[reportPrivateUsage] + nexus._post( # pyright: ignore[reportPrivateUsage] + "http://127.0.0.1:1/x", b"{}", {}, timedelta(seconds=1) + ) + + # A url urllib cannot even build a request from does not escape either. + with pytest.raises(nexus._EndpointFailure): # pyright: ignore[reportPrivateUsage] + nexus._post("not a url", b"{}", {}, timedelta(seconds=1)) # pyright: ignore[reportPrivateUsage] + + +async def test_the_front_carries_every_read_start(monkeypatch: pytest.MonkeyPatch): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + stream = _front(None).get_stream_handle(None, "wf-starts") + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}) + + # last= rides the contract to the store behind the endpoint. + newest = await take( + stream.read(topic=INPUTS, result_type=dict, last=2), 2, timeout=30 + ) + assert [record.value for record in newest] == [{"n": 2}, {"n": 3}] + + # END is the endpoint's newest position when the read starts. + at_end = stream.read(topic=INPUTS, result_type=dict, after=END) + first = asyncio.ensure_future(at_end.__anext__()) + try: + for _ in range(100): + await producer.append({"n": "new"}) + done, _ = await asyncio.wait({first}, timeout=0.1) + if done: + break + record = await asyncio.wait_for(first, 30) + finally: + await at_end.aclose() + assert record.value == {"n": "new"} + + # A token resumes a read and last_n starts one, so the two are refused. + with pytest.raises(nexusrpc.HandlerError) as both: + await _dispatch( + handler, + READ_OPERATION, + ReadInput( + stream=_ref("wf-starts"), + after_token=newest[0].cursor.token, + last_n=1, + ), + ) + assert both.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + await handler.close() + + +async def test_an_activity_owner_crosses_the_endpoint(monkeypatch: pytest.MonkeyPatch): + # The reference carries the owner, so an activity's own stream is reached + # through the same two operations and lands on the store's activity + # accessor, apart from the workflow's topics of the same name. + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + front = _front(None) + own = front.get_activity_stream_handle(None, "act", workflow_id="wf") + await own.producer(topic=INPUTS, producer_id="model", attempt=1).append({"n": 1}) + records = await take(own.read(topic=INPUTS, result_type=dict), 1, timeout=30) + assert records[0].value == {"n": 1} + assert await _stored(store, "wf") == [] + stored = store.get_activity_stream_handle(None, "act", workflow_id="wf") + assert [r.value async for r in _take_data(stored, 1)] == [{"n": 1}] + await handler.close() + + +async def _take_data(stream: Any, count: int) -> AsyncIterator[Any]: + async for record in stream.read(topic=INPUTS, result_type=dict): + if record.kind is RecordKind.DATA: + yield record + count -= 1 + if count == 0: + return + + +async def test_an_owner_the_store_cannot_host_is_refused_under_its_own_class( + client: Client, monkeypatch: pytest.MonkeyPatch +): + # Whether an owner is supported is the store's answer, not the front's: + # the Workflow Streams store hosts only the streams of a workflow's + # activities, so a standalone activity's are refused, and its refusal + # reaches the caller as the class it raised. + handler = TemporalStreamsHandler(WorkflowStreamsProvider(), client) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + own = _front(None).get_activity_stream_handle(None, "act") + with pytest.raises(StreamUnsupportedError, match="activity"): + await own.producer(topic=INPUTS, producer_id="model", attempt=1).append( + {"n": 1} + ) + with pytest.raises(StreamUnsupportedError, match="activity"): + await take(own.read(topic=INPUTS), 1) + # No provider addresses a standalone stream yet, so that owner is refused + # for every store. + standalone = WireStreamRef(kind="standalone", stream_id="s-1", topic=INPUTS) + with pytest.raises(nexusrpc.HandlerError) as refused: + await _dispatch(handler, READ_OPERATION, ReadInput(stream=standalone)) + assert str(refused.value).startswith("StreamUnsupportedError: ") + await handler.close() + + +async def test_a_reference_missing_its_owner_id_is_the_callers_fault(): + handler = TemporalStreamsHandler(MemoryStreams(), None) + for ref in ( + WireStreamRef(kind="workflow", topic=INPUTS), + WireStreamRef(kind="activity", workflow_id="wf", topic=INPUTS), + WireStreamRef(kind="standalone", topic=INPUTS), + WireStreamRef(kind="workflow", workflow_id="wf", topic=""), + ): + with pytest.raises(nexusrpc.HandlerError) as failed: + await _dispatch(handler, READ_OPERATION, ReadInput(stream=ref)) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + with pytest.raises(nexusrpc.HandlerError) as failed: + request = _append("wf", 1, {"n": 1}) + request.stream = ref + await _dispatch(handler, APPEND_OPERATION, request) + assert failed.value.type is nexusrpc.HandlerErrorType.BAD_REQUEST + + +async def test_a_handle_names_its_stream_as_a_ref_that_opens_on_the_front( + monkeypatch: pytest.MonkeyPatch, +): + store = MemoryStreams() + handler = TemporalStreamsHandler(store, None) + monkeypatch.setattr(nexus, "_post", _in_process_endpoint(handler, [], [])) + front = _front(None) + handle = front.get_stream_handle(None, "wf", run_id="r1") + ref = handle.ref(topic=INPUTS) + assert ref == StreamRef.for_workflow("wf", run_id="r1", topic=INPUTS) + # Plain data: the default converter carries it the way an operation + # result or a workflow argument is carried. + converter = temporalio.converter.DataConverter.default.payload_converter + carried = converter.from_payloads([converter.to_payloads([ref])[0]], [StreamRef]) + assert carried == [ref] + # Opened on the front, the ref reaches the same stream and its topic is + # the handle's default; the client accessor goes through open_ref. The + # in-process endpoint is reached by id, so no client is needed to resolve it. + no_client: Any = None + opened = open_ref(front, no_client, ref) + await opened.producer(producer_id="model", attempt=1).append({"n": 1}) + records = await take(handle.read(topic=INPUTS, result_type=dict), 1, timeout=30) + assert records[0].value == {"n": 1} + assert opened.ref() == ref + # Every owner kind names itself. + assert front.get_activity_stream_handle( + None, "act", workflow_id="wf" + ).ref() == StreamRef.for_activity("act", workflow_id="wf") + assert front.get_standalone_stream_handle(None, "s-1").ref( + topic="t" + ) == StreamRef.for_standalone("s-1", topic="t") + await handler.close() + + +def test_an_sdk_stream_ref_crosses_the_wire_model_with_its_nulls(): + # The SDK's default converter writes null for every member a ref does not + # use. The wire model declares those members nullable, so a ref an + # operation returned as plain data parses into the same wire reference + # the front builds, and the wire form of that reference reads back as + # the SDK type with nothing lost. + converter = temporalio.converter.DataConverter.default.payload_converter + contract = ( + temporalio.converter.DataConverter.default._get_internal_payload_converter() + ) + for ref in ( + StreamRef.for_workflow("wf", topic=INPUTS), + StreamRef.for_workflow("wf", run_id="r1", topic=INPUTS), + StreamRef.for_activity("act", workflow_id="wf", topic=INPUTS), + StreamRef.for_standalone("s-1", topic=INPUTS), + ): + as_sdk = converter.to_payloads([ref])[0] + assert b"null" in as_sdk.data, as_sdk.data + wire = contract.from_payloads([as_sdk], [WireStreamRef])[0] + assert wire == WireStreamRef( + kind=ref.kind, + topic=ref.topic, + workflow_id=ref.workflow_id, + run_id=ref.run_id, + activity_id=ref.activity_id, + stream_id=ref.stream_id, + ) + back = converter.from_payloads([contract.to_payloads([wire])[0]], [StreamRef])[ + 0 + ] + assert back == ref + + +async def test_the_front_refuses_what_the_contract_cannot_carry(): + # No create and no seal operation: an owned stream cannot be closed by + # anyone, and a standalone stream is created and sealed on the store. + front = _front(None) + with pytest.raises(ValueError, match="only a standalone stream"): + await front.get_stream_handle(None, "wf").close() + with pytest.raises(StreamUnsupportedError, match="seals"): + await front.get_standalone_stream_handle(None, "s-1").close() + with pytest.raises(StreamUnsupportedError, match="creates"): + await front.create_standalone_stream(None, "s-1") + with pytest.raises(ValueError, match="stream_id"): + front.get_standalone_stream_handle(None, "") + + +# --------------------------------------------------------------------------- +# A stream as an operation result and as an operation input (June s8 b). +# --------------------------------------------------------------------------- + + +@dataclass +class ScoreUpdate: + """One score change.""" + + home: int + away: int + + +@dataclass +class GameRequest: + """Which game to start, and where its workflow runs.""" + + game_id: str + task_queue: str + + +@dataclass +class ScorePost: + """A score for the stream given, and whether it is the last one.""" + + stream: StreamRef + score: ScoreUpdate + last: bool = False + + +@dataclass +class CallerInput: + """The endpoint to call, the game to ask for, and the scores to post.""" + + endpoint: str + game_id: str + task_queue: str + scores: list[ScoreUpdate] + + +SCORES = streams.topic("scores", ScoreUpdate) +COMMANDS = streams.topic("commands", ScoreUpdate) + + +@workflow.defn +class ScoreboardGame: + """Reads posted scores on ``commands`` and publishes each on ``scores``.""" + + @workflow.run + async def run(self) -> int: + """Relay every posted score until the poster finishes, then finish.""" + posted = 0 + scores = workflow.stream_writer(SCORES) + async for record in workflow.stream_reader(COMMANDS): + if record.kind is RecordKind.FINISH: + break + if record.kind is RecordKind.DATA: + assert record.value is not None + scores.publish(record.value) + posted += 1 + scores.finish() + return posted + + +@nexusrpc.service +class ScoreboardService: + """Starts a game and hands back its stream; takes a stream to post to.""" + + start_game: nexusrpc.Operation[GameRequest, StreamRef] + post_score: nexusrpc.Operation[ScorePost, None] + + +@nexusrpc.handler.service_handler(service=ScoreboardService) +class ScoreboardHandler: + """Serves the two operations as a client of the stream front.""" + + @nexusrpc.handler.sync_operation + async def start_game( + self, _ctx: nexusrpc.handler.StartOperationContext, input: GameRequest + ) -> StreamRef: + """Start the game workflow and return a ref to its ``scores`` topic.""" + client = temporal_nexus.client() + # Reusing a running game makes a retried start hand back the same ref. + await client.start_workflow( + ScoreboardGame.run, + id=input.game_id, + task_queue=input.task_queue, + id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING, + ) + return client.get_stream_handle(input.game_id).ref(topic=SCORES) + + @nexusrpc.handler.sync_operation + async def post_score( + self, ctx: nexusrpc.handler.StartOperationContext, input: ScorePost + ) -> None: + """Append the score to the stream the caller named.""" + # The request id is the producer, so a retried post writes once. + producer = ( + temporal_nexus.client() + .get_stream_handle(input.stream) + .producer(producer_id=f"post-{ctx.request_id}", attempt=1) + ) + await producer.append(input.score) + if input.last: + await producer.finish() + + +@workflow.defn +class ScoreboardCaller: + """Calls the operations and passes the ref on; it never reads the stream.""" + + @workflow.run + async def run(self, input: CallerInput) -> StreamRef: + """Start a game, post the scores to it, and return where to read them.""" + scoreboard = workflow.create_nexus_client( + service=ScoreboardService, endpoint=input.endpoint + ) + ref = await scoreboard.execute_operation( + ScoreboardService.start_game, + GameRequest(input.game_id, input.task_queue), + ) + for index, score in enumerate(input.scores): + await scoreboard.execute_operation( + ScoreboardService.post_score, + ScorePost( + ref.with_topic(COMMANDS), score, last=index == len(input.scores) - 1 + ), + ) + return ref + + +@live_only +async def test_an_operation_returns_a_stream_ref_and_another_takes_one(): + # June s8 (b) without the emulation: the operation's result type is the + # SDK's StreamRef, the calling workflow passes it on as data, the client + # opens it on the front it carries and long-polls it, and a second + # operation appends to a stream it was given as input. The handlers are + # clients of the front like any process; only the game workflow's worker + # holds the store. + store = WorkflowStreamsProvider() + plain = await _live_client() + game_id = f"streams-nexus-ref-{uuid.uuid4().hex}" + handler_tq = f"handlers-{game_id}" + stream_handler = TemporalStreamsHandler(store, plain) + + async with ( + _own_endpoint(plain, game_id, handler_tq) as endpoint, + Worker( + plain, + task_queue=handler_tq, + nexus_service_handlers=[stream_handler], + ), + ): + front = _live_front(endpoint) + config = plain.config() + config["plugins"] = [front] + fronted = Client(**config) + async with ( + Worker( + fronted, + task_queue=f"scoreboard-{game_id}", + nexus_service_handlers=[ScoreboardHandler()], + ), + Worker( + plain, + task_queue=f"tq-{game_id}", + workflows=[ScoreboardGame, ScoreboardCaller], + plugins=[store], + ), + ): + # The scoreboard service sits behind its own endpoint. + async with _own_endpoint( + plain, f"scoreboard-{game_id}", f"scoreboard-{game_id}" + ) as scoreboard_endpoint: + posted = [ScoreUpdate(1, 0), ScoreUpdate(1, 1), ScoreUpdate(2, 1)] + ref = await plain.execute_workflow( + ScoreboardCaller.run, + CallerInput(scoreboard_endpoint, game_id, f"tq-{game_id}", posted), + id=f"{game_id}-caller", + task_queue=f"tq-{game_id}", + ) + assert ref == StreamRef.for_workflow(game_id, topic=SCORES) + + # The ref is plain data the client opens on its own provider, + # here the front, and reads with the endpoint's long poll. It + # names the topic, not the record type, so the reader says + # what to decode as, the way a runtime topic name does. + records = await take( + fronted.get_stream_handle(ref).read(result_type=ScoreUpdate), 4, 60 + ) + assert [record.kind for record in records] == [ + RecordKind.DATA, + RecordKind.DATA, + RecordKind.DATA, + RecordKind.FINISH, + ] + assert [record.value for record in records[:3]] == posted + assert await plain.get_workflow_handle(game_id).result() == 3 + await stream_handler.close() diff --git a/tests/streams/test_redis_activity_streams.py b/tests/streams/test_redis_activity_streams.py new file mode 100644 index 0000000000..9208fb54ff --- /dev/null +++ b/tests/streams/test_redis_activity_streams.py @@ -0,0 +1,288 @@ +"""Live checks for what the Redis provider decides about an activity's streams. + +``test_activity_streams`` runs the shared cases on this provider behind +``STREAMS_LIVE=redis``. This module covers what only this store does: how a +read learns that a workflow's activity is terminal without the server saying +so, that an activity that never wrote is read until its workflow closes, +that retention trims an activity's stream like any other, that nothing +gates an append once the owner is terminal, and that an activity's streams +belong to the run its execution is in, so an id started again in a new run +starts new ones. All need a dev server (``TEMPORAL_ADDRESS`` or the test +environment's own) and a Redis (``TEMPORAL_TEST_REDIS_URL`` or +``AI198_REDIS_URL``). +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from collections.abc import AsyncIterator +from datetime import timedelta + +import pytest + +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.streams import BEGINNING, RecordKind, StreamCursorError +from temporalio.streams.providers.redis import RedisStreams +from temporalio.testing import WorkflowEnvironment +from tests.helpers import new_worker +from tests.streams.test_activity_streams import ( + TOKENS, + RunsOneActivity, + read_all, + summary, + write_by_default, + write_to_own_streams, +) +from tests.streams.test_streams_conformance import StreamHost, take + +pytestmark = pytest.mark.skipif( + os.environ.get("STREAMS_LIVE") != "redis", + reason="needs a live server and redis; run with STREAMS_LIVE=redis", +) + + +def redis_url() -> str: + return os.environ.get("TEMPORAL_TEST_REDIS_URL") or os.environ.get( + "AI198_REDIS_URL", "redis://127.0.0.1:6379" + ) + + +@pytest.fixture +async def live(client: Client, env: WorkflowEnvironment) -> AsyncIterator[Client]: + """A client with a fresh provider registered, on a server with standalone activities.""" + if env.supports_time_skipping: + pytest.skip("the time-skipping test server has no standalone activities") + address = os.environ.get("TEMPORAL_ADDRESS") + if address: + client = await Client.connect( + address, namespace=os.environ.get("TEMPORAL_NAMESPACE", "default") + ) + provider = RedisStreams( + url=redis_url(), + # A prefix per case, because the store keeps what earlier cases wrote. + key_prefix=f"streams-redis-activity-{uuid.uuid4().hex}", + poll_interval=timedelta(milliseconds=100), + ) + config = client.config() + config["plugins"] = [provider] + try: + yield Client(**config) + finally: + await provider.close() + + +@activity.defn +async def write_nothing(label: str) -> str: + return label + + +@workflow.defn +class RunsOneActivityThenWaits: + """Runs one activity by name, then stays open until released.""" + + def __init__(self) -> None: + self._released = False + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self, name: str, label: str) -> str: + result = await workflow.execute_activity( + name, + label, + activity_id="streamer", + start_to_close_timeout=timedelta(seconds=30), + ) + await workflow.wait_condition(lambda: self._released) + return result + + +async def test_a_read_opened_after_the_activity_finished_ends_by_itself( + live: Client, +): + # The server does not describe a workflow's activity, so the provider ends + # the read once the activity is no longer pending and its stream exists. + workflow_id = f"streams-redis-wfa-{uuid.uuid4().hex}" + async with new_worker( + live, RunsOneActivityThenWaits, activities=[write_to_own_streams] + ) as worker: + handle = await live.start_workflow( + RunsOneActivityThenWaits.run, + args=["write_to_own_streams", "to the activity"], + id=workflow_id, + task_queue=worker.task_queue, + ) + own = live.get_stream_handle(workflow_id, activity_id="streamer") + # The first read follows the activity to its FINISH while it runs. + await take(own.read(topic=TOKENS), 2, 30) + # A read opened afterwards ends on its own, with the workflow still open. + records = await read_all(own.read(topic=TOKENS), timeout=10) + assert summary(records) == [ + (RecordKind.DATA, 1, "to the activity"), + (RecordKind.FINISH, 1, None), + ] + assert (await handle.describe()).status is not None + await handle.signal(RunsOneActivityThenWaits.release) + await handle.result() + + +async def test_an_activity_that_never_wrote_is_read_until_its_workflow_closes( + live: Client, +): + workflow_id = f"streams-redis-wfa-silent-{uuid.uuid4().hex}" + async with new_worker( + live, RunsOneActivityThenWaits, activities=[write_nothing] + ) as worker: + handle = await live.start_workflow( + RunsOneActivityThenWaits.run, + args=["write_nothing", "quiet"], + id=workflow_id, + task_queue=worker.task_queue, + ) + assert await handle.query("__temporal_workflow_metadata") is not None + own = live.get_stream_handle(workflow_id, activity_id="streamer") + reading = asyncio.ensure_future(read_all(own.read(topic=TOKENS), timeout=30)) + # No stream exists for an activity that wrote nothing, so the read has + # nothing to say the activity is over and waits for the workflow. + await asyncio.sleep(1.0) + assert not reading.done() + await handle.signal(RunsOneActivityThenWaits.release) + await handle.result() + assert await reading == [] + + +async def test_retention_trims_an_activity_stream_too(client: Client): + address = os.environ.get("TEMPORAL_ADDRESS") + if address: + client = await Client.connect(address) + provider = RedisStreams( + url=redis_url(), + key_prefix=f"streams-redis-activity-{uuid.uuid4().hex}", + poll_interval=timedelta(milliseconds=100), + max_len=2, + ) + config = client.config() + config["plugins"] = [provider] + live = Client(**config) + workflow_id = f"streams-redis-wfa-trim-{uuid.uuid4().hex}" + try: + async with new_worker(live, StreamHost) as worker: + host = await live.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = live.get_stream_handle(workflow_id, activity_id="tool") + producer = stream.producer(topic=TOKENS, producer_id="tool", attempt=1) + first = await producer.append({"token": "one"}) + assert first is not None + await producer.append({"token": "two"}, {"token": "three"}) + # The window holds two entries, so the first record is gone. + records = await take(stream.read(topic=TOKENS), 2, 10) + assert [r.value["token"] for r in records] == ["two", "three"] + with pytest.raises(StreamCursorError, match="retention has trimmed"): + await take(stream.read(topic=TOKENS, after=first), 1, 10) + await host.terminate() + finally: + await provider.close() + + +async def test_an_append_after_the_owner_is_terminal_still_lands(live: Client): + # The store has no gate the server would have: a late attempt writes, and + # a reader that already ended is not told. The handle names no run, so it + # describes the finished activity and lands on that run's stream. + activity_id = f"streams-redis-saa-late-{uuid.uuid4().hex}" + async with new_worker(live, activities=[write_by_default]) as worker: + handle = await live.start_activity( + write_by_default, + "standalone", + id=activity_id, + task_queue=worker.task_queue, + start_to_close_timeout=timedelta(seconds=30), + ) + assert await handle.result() == activity_id + stream = live.get_stream_handle(activity_id=activity_id) + before = await stream.latest(topic=TOKENS) + assert before != BEGINNING + late = stream.producer(topic=TOKENS, producer_id=activity_id, attempt=2) + after = await late.append({"token": "late"}) + assert after != before + records = await read_all(stream.read(topic=TOKENS), timeout=10) + assert summary(records) == [ + (RecordKind.DATA, 1, "standalone"), + (RecordKind.FINISH, 1, None), + (RecordKind.SUPERSEDED, 2, None), + (RecordKind.DATA, 2, "late"), + ] + # Pinned to the run, the same stream reads the same way. + pinned = live.get_stream_handle(activity_id=activity_id, run_id=handle.run_id) + assert summary(await read_all(pinned.read(topic=TOKENS), timeout=10)) == summary( + records + ) + + +async def test_a_standalone_activity_id_started_again_starts_a_new_stream( + live: Client, +): + # The stream belongs to the activity execution, which is its run, not to + # the id: the second execution under the same id writes to a new stream, + # a handle without a run reads the current one, and a run pins the other. + activity_id = f"streams-redis-saa-again-{uuid.uuid4().hex}" + async with new_worker(live, activities=[write_by_default]) as worker: + runs = [] + for label in ("first", "second"): + handle = await live.start_activity( + write_by_default, + label, + id=activity_id, + task_queue=worker.task_queue, + start_to_close_timeout=timedelta(seconds=30), + ) + assert await handle.result() == activity_id + runs.append(handle.run_id) + assert runs[0] != runs[1] + current = live.get_stream_handle(activity_id=activity_id) + assert summary(await read_all(current.read(topic=TOKENS), timeout=10)) == [ + (RecordKind.DATA, 1, "second"), + (RecordKind.FINISH, 1, None), + ] + earlier = live.get_stream_handle(activity_id=activity_id, run_id=runs[0]) + assert summary(await read_all(earlier.read(topic=TOKENS), timeout=10)) == [ + (RecordKind.DATA, 1, "first"), + (RecordKind.FINISH, 1, None), + ] + + +async def test_a_workflow_activity_in_a_new_run_starts_a_new_stream(live: Client): + # The same workflow id run again schedules the same activity id; keyed by + # the workflow's run, the two executions keep their streams apart. + workflow_id = f"streams-redis-wfa-again-{uuid.uuid4().hex}" + async with new_worker( + live, RunsOneActivity, activities=[write_to_own_streams] + ) as worker: + runs = [] + for label in ("first", "second"): + handle = await live.start_workflow( + RunsOneActivity.run, + args=["write_to_own_streams", label], + id=workflow_id, + task_queue=worker.task_queue, + ) + await handle.result() + runs.append(handle.result_run_id) + assert runs[0] != runs[1] + current = live.get_stream_handle(workflow_id, activity_id="streamer") + assert summary(await read_all(current.read(topic=TOKENS), timeout=10)) == [ + (RecordKind.DATA, 1, "second"), + (RecordKind.FINISH, 1, None), + ] + earlier = live.get_stream_handle( + workflow_id, activity_id="streamer", run_id=runs[0] + ) + assert summary(await read_all(earlier.read(topic=TOKENS), timeout=10)) == [ + (RecordKind.DATA, 1, "first"), + (RecordKind.FINISH, 1, None), + ] diff --git a/tests/streams/test_redis_provider.py b/tests/streams/test_redis_provider.py new file mode 100644 index 0000000000..0674d66b73 --- /dev/null +++ b/tests/streams/test_redis_provider.py @@ -0,0 +1,384 @@ +"""What the Redis provider decides without a store: cursors, the sync publish, the wake.""" + +from __future__ import annotations + +import asyncio +import time +from datetime import timedelta +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +from temporalio.client import WorkflowExecutionStatus +from temporalio.contrib.external_workflow_streams import ( + StreamDirection, + WakeNotAcknowledgedError, +) +from temporalio.contrib.external_workflow_streams import ( + StreamError as TransportStreamError, +) +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._record import Offset +from temporalio.contrib.external_workflow_streams._redis import RedisStreamBackend +from temporalio.converter import DataConverter +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import ( + BEGINNING, + Cursor, + StreamCursorError, + StreamError, +) +from temporalio.streams.providers import redis as redis_provider +from temporalio.streams.providers.redis import ( + DEFAULT_RETENTION, + RedisProducer, + RedisStreams, + _ActivityOwner, + _drive, + _position, + _StandaloneOwner, + _wake_counter, +) + + +class _ChainClient: + """A client whose describes of the chain answer from a script of statuses. + + The last entry repeats, so a chain that stays running keeps saying so. + """ + + def __init__(self, *answers: WorkflowExecutionStatus | Exception) -> None: + self.data_converter = DataConverter.default + self.namespace = "default" + self._answers = list(answers) + self.describes = 0 + + def get_workflow_handle(self, _workflow_id: str, **_: Any) -> Any: + return self + + async def describe(self) -> Any: + self.describes += 1 + answer = self._answers.pop(0) if len(self._answers) > 1 else self._answers[0] + if isinstance(answer, Exception): + raise answer + return SimpleNamespace(status=answer) + + +class _RefusingInput: + """The transport's input topic, refusing the first ``refusals`` wakes.""" + + def __init__(self, refusals: int, error: Exception | None = None) -> None: + self._refusals = refusals + self._error = error + self.calls = 0 + self.positions: list[Any] = [] + + async def wake(self, *, position: Any = None) -> list[str]: + self.calls += 1 + self.positions.append(position) + if self.calls <= self._refusals: + raise self._error or WakeNotAcknowledgedError( + "workflow operation can not be applied because workflow is closing", + pending=[], + ) + return ["sent"] + + +def _waking_producer(client: _ChainClient, wakes: _RefusingInput) -> RedisProducer[Any]: + producer: RedisProducer[Any] = RedisProducer( + RedisStreams(), cast(Any, client), "wf", "inputs", "console", 1 + ) + producer._input = wakes + return producer + + +@pytest.fixture +def quick_wake_retries(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + redis_provider, "_WAKE_RETRY_BACKOFF", timedelta(milliseconds=10) + ) + monkeypatch.setattr( + redis_provider, "_WAKE_RETRY_WINDOW", timedelta(milliseconds=300) + ) + + +@pytest.mark.usefixtures("quick_wake_retries") +async def test_a_wake_refused_by_a_closing_run_is_sent_again_to_its_successor(): + # The chain describes as running: the run that refused the wake handed over + # to a successor, which is where the records already are. + client = _ChainClient(WorkflowExecutionStatus.RUNNING) + wakes = _RefusingInput(refusals=1) + await _waking_producer(client, wakes)._wake(None) + assert (wakes.calls, client.describes) == (2, 1) + + +@pytest.mark.usefixtures("quick_wake_retries") +async def test_a_refused_wake_waits_for_the_successor_to_become_current(): + # In the instant between two runs the chain still describes as the + # predecessor that continued as new, and the next Signal is refused too. + client = _ChainClient( + WorkflowExecutionStatus.CONTINUED_AS_NEW, WorkflowExecutionStatus.RUNNING + ) + wakes = _RefusingInput(refusals=2) + await _waking_producer(client, wakes)._wake(None) + assert (wakes.calls, client.describes) == (3, 2) + + +@pytest.mark.usefixtures("quick_wake_retries") +@pytest.mark.parametrize( + "ending", + [ + WorkflowExecutionStatus.COMPLETED, + WorkflowExecutionStatus.FAILED, + WorkflowExecutionStatus.CANCELED, + WorkflowExecutionStatus.TERMINATED, + WorkflowExecutionStatus.TIMED_OUT, + RPCError("gone", RPCStatusCode.NOT_FOUND, b""), + ], + ids=lambda ending: getattr(ending, "name", "not found"), +) +async def test_a_wake_refused_by_a_finished_chain_is_the_ordinary_ending( + ending: WorkflowExecutionStatus | Exception, +): + client = _ChainClient(ending) + wakes = _RefusingInput(refusals=99) + await _waking_producer(client, wakes)._wake(None) + assert wakes.calls == 1 + + +@pytest.mark.usefixtures("quick_wake_retries") +async def test_a_wake_refused_as_closing_for_the_whole_window_is_dropped(): + # The run is inside a Workflow Task that tried to close it while the wake + # sat buffered. The record is in the log, so if the run stays open its + # next park rechecks the log and finds it; the producer is not failed. + client = _ChainClient(WorkflowExecutionStatus.RUNNING) + wakes = _RefusingInput(refusals=99) + await _waking_producer(client, wakes)._wake(None) + assert wakes.calls > 1 + + +@pytest.mark.usefixtures("quick_wake_retries") +async def test_a_wake_refused_for_another_reason_for_the_whole_window_is_raised(): + client = _ChainClient(WorkflowExecutionStatus.RUNNING) + wakes = _RefusingInput( + refusals=99, error=WakeNotAcknowledgedError("signal rate limited", pending=[]) + ) + with pytest.raises(WakeNotAcknowledgedError, match="rate limited"): + await _waking_producer(client, wakes)._wake(None) + assert wakes.calls > 1 + + +@pytest.mark.usefixtures("quick_wake_retries") +async def test_a_wake_refused_as_not_found_ends_without_a_describe(): + # The server answers NOT_FOUND for a chain that has ended, whether the + # wake went as a Signal or to the owner's linked channel, so nothing is + # left to describe. + client = _ChainClient(WorkflowExecutionStatus.RUNNING) + refusal = WakeNotAcknowledgedError("gone", pending=[]) + refusal.__cause__ = RPCError("gone", RPCStatusCode.NOT_FOUND, b"") + wakes = _RefusingInput(refusals=99, error=refusal) + await _waking_producer(client, wakes)._wake(None) + assert (wakes.calls, client.describes) == (1, 0) + + +async def test_a_wake_reports_the_entry_it_follows(): + client = _ChainClient(WorkflowExecutionStatus.RUNNING) + wakes = _RefusingInput(refusals=0) + placed = Offset("1700000000000-4") + await _waking_producer(client, wakes)._wake(placed) + assert wakes.positions == [placed] + + +def test_a_wake_counter_follows_the_entry_id_order(): + ids = ["1700000000000-0", "1700000000000-1", "1700000000001-0", "1800000000000-7"] + counters = [_wake_counter(Offset(entry)) for entry in ids] + assert counters == sorted(counters) + assert len(set(counters)) == len(counters) + assert _wake_counter(Offset("1700000000000-3")) == 1700000000000 * 2**20 + 3 + + +def test_a_wake_counter_caps_the_sequence_inside_its_millisecond(): + capped = _wake_counter(Offset(f"5-{2**20 + 9}")) + assert capped == 5 * 2**20 + 2**20 - 1 + assert capped < _wake_counter(Offset("6-0")) + + +def test_a_wake_counter_fits_the_servers_signed_64_bit_field(): + year_2200_ms = 7258118400000 + assert _wake_counter(Offset(f"{year_2200_ms}-{2**20}")) < 2**63 + + +def test_a_positionless_wake_counter_follows_the_entry_id_rule(): + backend = RedisStreams(client=_NoRedis())._require_backend() + before = _wake_counter(Offset(f"{int(time.time() * 1000)}-{2**20 - 1}")) + now = backend.wake_counter_now() + after = _wake_counter(Offset(f"{int(time.time() * 1000) + 1}-0")) + assert before <= now < after + # Any entry appended before the call orders below it. + assert now > _wake_counter(Offset(f"{int(time.time() * 1000) - 1}-7")) + + +def test_the_wake_transport_reaches_the_backend_both_sides_share(): + streams = RedisStreams(client=_NoRedis(), wake_transport="signal") + backend = streams._require_backend() + assert backend.wake_transport == "signal" + assert backend.wake_counter_for(Offset("2-1")) == _wake_counter(Offset("2-1")) + assert RedisStreams(client=_NoRedis())._require_backend().wake_transport == "auto" + channel = RedisStreams(client=_NoRedis(), wake_transport="channel") + assert channel._require_backend().wake_transport == "channel" + + +def test_an_unknown_wake_transport_is_refused_at_construction(): + with pytest.raises(ValueError, match="wake transport"): + RedisStreams(wake_transport=cast(Any, "carrier-pigeon")) + + +async def test_a_wake_the_store_could_not_send_is_a_storage_error(): + client = _ChainClient(WorkflowExecutionStatus.RUNNING) + wakes = _RefusingInput(refusals=1, error=TransportStreamError("no connection")) + with pytest.raises(StreamError, match="could not be sent"): + await _waking_producer(client, wakes)._wake(None) + assert (wakes.calls, client.describes) == (1, 0) + + +class _NoRedis: + """Enough of a Redis client to construct a backend and render its keys.""" + + def register_script(self, _script: str) -> None: + return None + + +def test_activity_keys_encode_their_ids_and_never_meet_chain_keys(): + # The run is part of the key: a workflow's activity is keyed by the + # workflow's run and a standalone one by its own, so an id started again + # in a new run starts a new stream. + assert ( + _ActivityOwner("ns", "wf", "act", "run").key("p", "t") + == "p:ns:activity/wf/run/act:t" + ) + assert ( + _ActivityOwner("ns", None, "act", "run").key("p", "t") + == "p:ns:activity//run/act:t" + ) + # An id holding a separator is encoded, so it cannot move a boundary. + assert ( + _ActivityOwner("n:s", "w/f", "a:c", "r/1").key("p", "t/u") + == "p:n%3As:activity/w%2Ff/r%2F1/a%3Ac:t%2Fu" + ) + # A key needs the run; a handle opened without one resolves it first. + with pytest.raises(RuntimeError, match="not resolved"): + _ActivityOwner("ns", "wf", "act", None).key("p", "t") + # A chain key percent-encodes every id, so none of its components holds a + # "/" however the ids are chosen, and the owner component here always does. + backend = RedisStreamBackend(client=_NoRedis(), key_prefix="p") + forged = StreamKey( + namespace="ns", + workflow_id="activity/wf/act", + first_execution_run_id="t", + stream_name="t", + direction=StreamDirection.OUTPUT, + ) + assert "/" not in backend.stream_key(forged) + assert backend.stream_key(forged) != _ActivityOwner("ns", "wf", "act", "t").key( + "p", "t" + ) + + +def test_an_activity_owner_names_itself_for_messages(): + assert str(_ActivityOwner("ns", None, "act", None)) == "activity 'act'" + assert ( + str(_ActivityOwner("ns", "wf", "act", "run")) + == "activity 'act' of workflow 'wf'" + ) + + +def test_cursors_name_entries_of_the_topics_one_log(): + assert _position(BEGINNING) is None + position = _position(Cursor("redis:1700000000000-3")) + assert position is not None and position.token == "1700000000000-3" + # A workflow reader and an outside reader name the same log, so either + # side's cursor seeds the other. The form the two-key layout minted for + # workflow readers named an entry of the input key, which is the log now. + legacy = _position(Cursor("redis:in:1700000000000-3")) + assert legacy is not None and legacy.token == "1700000000000-3" + with pytest.raises(StreamCursorError): + _position(Cursor("memory:3")) + with pytest.raises(StreamCursorError): + _position(Cursor("redis:not-an-id")) + with pytest.raises(StreamCursorError): + _position(Cursor("redis:in:not-an-id")) + + +def test_a_publish_that_completes_at_once_is_driven_to_the_end(): + done = [] + + async def publish() -> None: + done.append(True) + + _drive(publish()) + assert done == [True] + + +def test_a_publish_that_would_wait_fails_loudly(): + async def publish() -> None: + await asyncio.get_running_loop().create_future() + + async def run() -> None: + with pytest.raises(StreamError, match="batch is full"): + _drive(publish()) + + asyncio.run(run()) + + +def test_retention_options_are_checked_at_construction(): + with pytest.raises(ValueError, match="retention"): + RedisStreams(retention=timedelta(0)) + with pytest.raises(ValueError, match="max_len"): + RedisStreams(max_len=0) + RedisStreams(retention=timedelta(hours=1), max_len=10) + + +async def test_a_client_the_caller_opened_gets_the_providers_layout_and_stays_open(): + # The layout and the trims are the provider's whichever connection it + # runs on, and closing the provider does not close a caller's client. + client = _NoRedis() + provider = RedisStreams(client=client, max_len=10) + backend = provider._require_backend() + assert backend._client is client + assert backend.describe_window() == f"retention={DEFAULT_RETENTION}, max_len=10" + await provider.close() + assert provider._require_backend() is not backend + assert provider._require_backend()._client is client + + +async def test_the_default_window_is_an_age_and_can_be_turned_off(): + # Nothing is trimmed on a topic nobody appends to, so the default has to + # be a window that every append applies; a count cap would refuse a task + # whose batch does not fit under it, so that one stays off. + provider = RedisStreams() + try: + backend = provider._require_backend() + assert backend._retention == DEFAULT_RETENTION == timedelta(days=7) + assert backend._max_len is None + assert backend.describe_window() == f"retention={DEFAULT_RETENTION}" + finally: + await provider.close() + unbounded = RedisStreams(retention=None) + try: + assert unbounded._require_backend().describe_window() == "no retention" + finally: + await unbounded.close() + + +def test_standalone_keys_have_their_own_owner_component(): + owner = _StandaloneOwner("ns", "shared") + assert owner.meta("p") == "p:ns:standalone/shared" + assert owner.key("p", "t") == "p:ns:standalone/shared:t" + # A topic log always has one more component than the hash, and every id and + # topic is encoded, so no name can land on the hash or on another topic. + assert owner.key("p", "meta") != owner.meta("p") + tricky = _StandaloneOwner("n:s", "a/b:c") + assert tricky.meta("p") == "p:n%3As:standalone/a%2Fb%3Ac" + assert tricky.key("p", "t/u") == "p:n%3As:standalone/a%2Fb%3Ac:t%2Fu" + assert str(owner) == "standalone stream 'shared'" diff --git a/tests/streams/test_redis_replay.py b/tests/streams/test_redis_replay.py new file mode 100644 index 0000000000..72922c64bc --- /dev/null +++ b/tests/streams/test_redis_replay.py @@ -0,0 +1,1396 @@ +"""Live checks for the client-side (Redis) provider inside a workflow. + +The conformance suite covers the outside surface when ``STREAMS_LIVE=redis``. +This module runs the interface loop inside a workflow over the staged commit, +lets a read end with the workflow, shares a topic between an outside producer +and the workflow, queries a completed run, which replays it, trims by +retention and shows what a replay and a read past the trim do, and seeds a +workflow reader from a cursor. All need a dev server (``TEMPORAL_ADDRESS``) +and a Redis (``TEMPORAL_TEST_REDIS_URL`` or ``AI198_REDIS_URL``). The worker +keeps a warm cache because the transport holds the task open between records. +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from collections.abc import AsyncIterator +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.client import Client +from temporalio.common import Execution, ExecutionType +from temporalio.contrib.external_workflow_streams import ( + RecordKind as TransportRecordKind, +) +from temporalio.contrib.external_workflow_streams import StreamDirection +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._output_backend import ( + OutputStageManifest, + OutputStageStatus, + StagedOutputRecord, +) +from temporalio.converter import WorkflowSerializationContext +from temporalio.streams import ( + BEGINNING, + END, + Cursor, + RecordKind, + StreamCursorError, + StreamProducerError, +) +from temporalio.streams._wire import to_wire +from temporalio.streams.providers import redis as redis_provider +from temporalio.streams.providers.redis import RedisStreams, _chain, _TopicLogBackend +from temporalio.worker import Replayer, Worker +from tests.streams.test_streams_conformance import StreamHost, take +from tests.streams.test_streams_workflow import ( + DECISIONS, + INPUTS, + ContractLoop, + OneLine, +) + +pytestmark = pytest.mark.skipif( + os.environ.get("STREAMS_LIVE") != "redis", + reason="needs a live server and redis; run with STREAMS_LIVE=redis", +) + + +def redis_url() -> str: + return os.environ.get("TEMPORAL_TEST_REDIS_URL") or os.environ.get( + "AI198_REDIS_URL", "redis://127.0.0.1:6379" + ) + + +@pytest.fixture +async def provider() -> AsyncIterator[RedisStreams]: + # A prefix per case, because the store keeps what earlier cases wrote. + streams = RedisStreams( + url=redis_url(), key_prefix=f"streams-redis-{uuid.uuid4().hex}" + ) + try: + yield streams + finally: + await streams.close() + + +@pytest.fixture +async def live_client(client: Client) -> Client: + # The test environment's own server, unless TEMPORAL_ADDRESS names another. + address = os.environ.get("TEMPORAL_ADDRESS") + return await Client.connect(address) if address else client + + +async def test_interface_loop_over_redis(live_client: Client, provider: RedisStreams): + workflow_id = f"streams-redis-live-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[ContractLoop], + plugins=[provider], + max_cached_workflows=100, + ): + handle = await live_client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + await producer.append({"n": 3}) + await producer.finish() + + records = await take(stream.read(topic=DECISIONS), 4, 60) + assert [r.kind for r in records] == [ + RecordKind.DATA, + RecordKind.DATA, + RecordKind.DATA, + RecordKind.FINISH, + ] + assert [r.value["decided"] for r in records[:3]] == [1, 2, 3] + assert all(r.producer_id == "" and r.topic == DECISIONS.name for r in records) + assert await handle.result() == [ + {"kind": "decision", "n": 1, "attempt": 1}, + {"kind": "decision", "n": 2, "attempt": 1}, + {"kind": "decision", "n": 3, "attempt": 1}, + {"kind": "finish", "producer": "model"}, + ] + + # The read ends by itself once the workflow is closed and every + # promoted record has been handed over. + async def read_everything() -> list[Any]: + return [r.value async for r in stream.read(topic=DECISIONS)] + + assert await asyncio.wait_for(read_everything(), 60) == [ + {"decided": 1}, + {"decided": 2}, + {"decided": 3}, + None, + ] + # The producer's own records are readable from outside as well, on + # the topic it wrote. + inputs = await take(stream.read(topic=INPUTS), 4, 60) + assert [r.value for r in inputs[:3]] == [{"n": 1}, {"n": 2}, {"n": 3}] + assert inputs[3].kind is RecordKind.FINISH + assert all(r.producer_id == "model" and r.attempt == 1 for r in inputs) + + +async def test_an_outside_producer_and_the_workflow_share_a_topic( + live_client: Client, provider: RedisStreams +): + workflow_id = f"streams-redis-shared-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[OneLine], + plugins=[provider], + ): + handle = await live_client.start_workflow( + OneLine.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = provider.get_stream_handle(live_client, workflow_id) + await stream.producer(topic=DECISIONS, producer_id="tool", attempt=1).append( + {"from": "producer"} + ) + await handle.result() + + async def read_everything() -> list[Any]: + return [r async for r in stream.read(topic=DECISIONS)] + + records = await asyncio.wait_for(read_everything(), 60) + # Both writers land on one topic, each under its own identity. The order + # between them is whatever the store took first. + assert sorted( + ((r.producer_id, r.kind, r.value) for r in records), key=str + ) == sorted( + [ + ("tool", RecordKind.DATA, {"from": "producer"}), + ("", RecordKind.DATA, {"from": "workflow"}), + ("", RecordKind.FINISH, None), + ], + key=str, + ) + + +@workflow.defn +class SignalWokenPublish: + """Publishes in the task a signal wakes, then completes in that same task.""" + + def __init__(self) -> None: + self._closed = False + + @workflow.signal + def close(self) -> None: + self._closed = True + + @workflow.run + async def run(self) -> int: + out = workflow.stream_writer("out") + out.publish({"n": 0}) + await workflow.wait_condition(lambda: self._closed) + for n in range(1, 4): + out.publish({"n": n}) + return 4 + + @workflow.query + def probe(self) -> int: + return 1 + + +async def test_query_after_completion_replays_the_final_task( + live_client: Client, provider: RedisStreams +): + # A query against a completed run replays it. The final task's publishes + # were woken by a signal, which the activation applies before the replay + # marker, so the marker's expectations have to be installed before that + # task's code runs or the replay records nothing against a manifest of + # three. + workflow_id = f"streams-redis-replay-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[SignalWokenPublish], + plugins=[provider], + ): + handle = await live_client.start_workflow( + SignalWokenPublish.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = provider.get_stream_handle(live_client, workflow_id) + await take(stream.read(topic="out", result_type=dict), 1, timeout=60) + await handle.signal(SignalWokenPublish.close) + assert await handle.result() == 4 + assert await handle.query(SignalWokenPublish.probe) == 1 + values = [r.value async for r in stream.read(topic="out", result_type=dict)] + assert values == [{"n": 0}, {"n": 1}, {"n": 2}, {"n": 3}] + + +async def _log_length( + streams: RedisStreams, client: Client, workflow_id: str, topic: str +) -> int: + """How many entries the topic's log holds. + + Both of the transport's keys for the topic render onto the log, which is + what makes it one; asked through both so a test would notice if they came + apart. + """ + import redis.asyncio + + backend = streams._require_backend() + chain = await _chain(client, workflow_id) + store = redis.asyncio.from_url(redis_url()) + try: + by_input = backend.stream_key( + chain.stream_key(topic, direction=StreamDirection.INPUT) + ) + by_output = backend.stream_key( + chain.stream_key(topic, direction=StreamDirection.OUTPUT) + ) + assert by_input == by_output + return await store.xlen(by_input) + finally: + await store.aclose() + + +async def _trim_everything( + streams: RedisStreams, client: Client, workflow_id: str, topic: str +) -> None: + """What retention elsewhere, or an operator, does to a topic's output key.""" + import redis.asyncio + + backend = streams._require_backend() + chain = await _chain(client, workflow_id) + store = redis.asyncio.from_url(redis_url()) + try: + await store.xtrim( + backend.stream_key( + chain.stream_key(topic, direction=StreamDirection.OUTPUT) + ), + maxlen=0, + approximate=False, + ) + finally: + await store.aclose() + + +async def test_a_replay_past_the_retention_window_fails_loudly(live_client: Client): + # Six entries per key: the loop's four input records stay while it runs, + # and six more appends afterwards push them out. + streams = RedisStreams( + url=redis_url(), key_prefix=f"streams-redis-{uuid.uuid4().hex}", max_len=6 + ) + workflow_id = f"streams-redis-retention-{uuid.uuid4().hex}" + try: + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[ContractLoop], + plugins=[streams], + max_cached_workflows=100, + ): + handle = await live_client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = streams.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}) + await producer.finish() + assert len(await handle.result()) == 4 + before = await take(stream.read(topic=INPUTS), 4, 60) + history = await handle.fetch_history() + assert await _log_length(streams, live_client, workflow_id, INPUTS.name) == 4 + + # Inside the window the recorded ranges read back and the replay passes. + await Replayer(workflows=[ContractLoop], plugins=[streams]).replay_workflow( + history + ) + + late = stream.producer(topic=INPUTS, producer_id="late", attempt=1) + cursors = [await late.append({"n": n}) for n in range(10, 16)] + assert await _log_length(streams, live_client, workflow_id, INPUTS.name) == 6 + + # The recorded input ranges are gone, and the replay says so rather + # than delivering fewer records. + with pytest.raises(Exception) as failure: + await Replayer(workflows=[ContractLoop], plugins=[streams]).replay_workflow( + history + ) + # The task fails under the transport's integrity row: its type, the + # external storage cause, and a message that names the window. + message = str(failure.value) + assert "StreamIntegrityError" in message and "ExternalStorageFailure" in message + assert "past the redis provider's retention (" in message + assert "max_len=6)" in message + + # An outside cursor below the trim is refused, not resumed from the + # first retained record. + with pytest.raises(StreamCursorError, match="retention has trimmed"): + await take(stream.read(topic=INPUTS, after=before[0].cursor), 1, 10) + + # Inside the window a read still works and lands where append said. + records = [r async for r in stream.read(topic=INPUTS, after=cursors[1])] + assert [r.value for r in records] == [{"n": n} for n in range(12, 16)] + assert [r.cursor for r in records] == cursors[2:] + finally: + await streams.close() + + +async def test_retention_by_age_trims_older_entries(live_client: Client): + streams = RedisStreams( + url=redis_url(), + key_prefix=f"streams-redis-{uuid.uuid4().hex}", + retention=timedelta(milliseconds=300), + ) + workflow_id = f"streams-redis-age-{uuid.uuid4().hex}" + try: + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[StreamHost], + plugins=[streams], + ): + handle = await live_client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = streams.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + first = await producer.append({"n": 1}) + await producer.append({"n": 2}) + await asyncio.sleep(0.6) + # The append that crosses the window is what trims the two before it. + third = await producer.append({"n": 3}) + assert ( + await _log_length(streams, live_client, workflow_id, INPUTS.name) == 1 + ) + assert await stream.latest(topic=INPUTS) == third + with pytest.raises(StreamCursorError, match="retention has trimmed"): + await take(stream.read(topic=INPUTS, after=first), 1, 10) + + # A fully trimmed topic answers the way an empty one does. + await _trim_everything(streams, live_client, workflow_id, INPUTS.name) + assert await stream.latest(topic=INPUTS) == BEGINNING + await handle.signal(StreamHost.release) + await handle.result() + assert [r async for r in stream.read(topic=INPUTS)] == [] + finally: + await streams.close() + + +@workflow.defn +class ResumeAfter: + """Reads ``inputs``; the first run hands its first record's cursor to the next.""" + + def __init__(self) -> None: + self._seen: list[int] = [] + + @workflow.query + def seen(self) -> list[int]: + return self._seen + + @workflow.run + async def run(self, after: str | None) -> list[int]: + reader = workflow.stream_reader( + INPUTS, after=Cursor(after) if after else BEGINNING + ) + first: str | None = None + async for record in reader: + if record.kind is RecordKind.FINISH: + break + assert record.value is not None + self._seen.append(record.value["n"]) + first = first or record.cursor.token + if after is None and len(self._seen) == 2: + workflow.continue_as_new(first) + return self._seen + + +async def test_a_workflow_reader_started_from_a_cursor_skips_the_earlier_records( + live_client: Client, provider: RedisStreams +): + # The first run reads two records and continues as new with the cursor of + # the first. Without the cursor the successor would resume after the + # second, which is where the chain left off; with it, it reads the second + # again and then the rest. + workflow_id = f"streams-redis-resume-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[ResumeAfter], + plugins=[provider], + max_cached_workflows=100, + ): + handle = await live_client.start_workflow( + ResumeAfter.run, None, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}) + await producer.finish() + assert await handle.result() == [2, 3] + # The completed run is evicted, so the query replays it cold with the + # seeded position and the recorded ranges. + assert await handle.query(ResumeAfter.seen) == [2, 3] + + # Both runs replay offline against the store, the seeded one included. + replayer = Replayer(workflows=[ResumeAfter], plugins=[provider]) + await replayer.replay_workflow( + await live_client.get_workflow_handle( + workflow_id, run_id=handle.first_execution_run_id + ).fetch_history() + ) + await replayer.replay_workflow(await handle.fetch_history()) + + +@workflow.defn +class BatchConsumer: + """Applies one batch of ``inputs`` per run, handing over at the sender's FINISH.""" + + @workflow.run + async def run(self, applied: list[str]) -> list[str]: + stop = False + async for record in workflow.stream_reader(INPUTS): + if record.kind is RecordKind.FINISH: + break + if record.kind is not RecordKind.DATA: + continue + assert record.value is not None + if record.value["op"] == "stop": + stop = True + continue + applied.append(record.value["op"]) + if stop: + return applied + workflow.continue_as_new(applied) + + +async def _current_open_run( + client: Client, workflow_id: str, previous: str | None +) -> str: + """Wait until the chain's newest run is a new one and still open.""" + while True: + description = await client.get_workflow_handle(workflow_id).describe() + assert description.run_id is not None + if description.run_id != previous and description.close_time is None: + return description.run_id + await asyncio.sleep(0.1) + + +async def test_a_wake_refused_by_a_run_continuing_as_new_reaches_its_successor( + live_client: Client, provider: RedisStreams +): + # The consumer reads the sender's FINISH straight from the store while it + # holds its task open and continues as new on it, so the wake Signal for + # that record resolves to a run that is closing and the server refuses it. + # The records are keyed by the chain and already where the successor reads + # them; the wake has to follow, and finish() must not fail the sender. + workflow_id = f"streams-redis-batches-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[BatchConsumer], + plugins=[provider], + max_cached_workflows=100, + ): + handle = await live_client.start_workflow( + BatchConsumer.run, [], id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = provider.get_stream_handle(live_client, workflow_id) + run_id: str | None = None + for number, batch in enumerate([["a", "b"], ["c"], ["d", "stop"]], start=1): + run_id = await asyncio.wait_for( + _current_open_run(live_client, workflow_id, run_id), 30 + ) + # A sender per batch: the stream spans the chain, so one identity + # numbering its records from the start again would collide with + # the batch before. + sender = stream.producer( + topic=INPUTS, producer_id=f"console-{number}", attempt=1 + ) + for op in batch: + await sender.append({"op": op}) + await sender.finish() + assert await asyncio.wait_for(handle.result(), 30) == ["a", "b", "c", "d"] + + +async def _stage_one( + streams: RedisStreams, + client: Client, + workflow_id: str, + run_id: str, + topic: str, + values: list[Any], +) -> tuple[Any, Any]: + """Stage ``values`` on ``topic``'s output key without committing them. + + Staged the way the worker stages a task's publishes, so the records read back as + records rather than as bytes the reader has to skip. + """ + backend = streams._require_backend() + chain = await _chain(client, workflow_id) + key = chain.stream_key(topic, direction=StreamDirection.OUTPUT) + codec: StreamPayloadCodec[bytes] = StreamPayloadCodec( + client.data_converter.with_context( + WorkflowSerializationContext( + namespace=client.namespace, workflow_id=workflow_id + ) + ), + bytes, + ) + records = [] + for index, value in enumerate(values): + wire = to_wire( + client.data_converter.payload_converter, + topic=topic, + kind=RecordKind.DATA, + value=value, + ) + records.append( + StagedOutputRecord( + kind=TransportRecordKind.DATA, + payload=await codec.encode(wire.SerializeToString()), + publish_index=index, + ) + ) + manifest = OutputStageManifest( + stream_key=key, + provider_id=backend.provider_id, + provider_format_version=backend.provider_format_version, + stage_token=uuid.uuid4().hex, + run_id=run_id, + history_floor_event_id=3, + sub_batch_id=0, + fingerprint_version=1, + fingerprint=b"\x01" * 32, + record_count=len(records), + logical_byte_count=sum(len(r.payload) for r in records), + ) + await backend.stage_output(manifest, records) + return backend, manifest + + +async def test_a_pending_stage_is_settled_by_the_reader_rather_than_wedging_it( + live_client: Client, provider: RedisStreams +): + # The commit protocol's own failure modes, which nothing else here reaches: a + # stage whose task never landed in History is a barrier until the reader + # reconciles it, the reconcile aborts it, and the records it held are never + # handed over while everything behind it flows. + workflow_id = f"streams-redis-stage-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[StreamHost], + plugins=[provider], + ) as worker: + handle = await live_client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}) + + backend, manifest = await _stage_one( + provider, + live_client, + workflow_id, + handle.first_execution_run_id or "", + INPUTS.name, + [{"n": 10}, {"n": 11}], + ) + assert ( + await backend.output_stage(manifest) + ).status is OutputStageStatus.PENDING + + # Behind the stage so the read has to get past it to reach this. + await producer.append({"n": 2}) + + assert [r.value for r in await take(stream.read(topic=INPUTS), 2, 30)] == [ + {"n": 1}, + {"n": 2}, + ] + # The task never reached History, so the reader settled the stage as + # aborted and its records are gone rather than pending forever. + assert ( + await backend.output_stage(manifest) + ).status is OutputStageStatus.ABORTED + + await handle.signal(StreamHost.release) + await handle.result() + + +async def test_an_aborted_stage_yields_nothing_and_stops_being_a_barrier( + live_client: Client, provider: RedisStreams +): + workflow_id = f"streams-redis-abort-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[StreamHost], + plugins=[provider], + ) as worker: + handle = await live_client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}) + backend, manifest = await _stage_one( + provider, + live_client, + workflow_id, + handle.first_execution_run_id or "", + INPUTS.name, + [{"n": 10}], + ) + await backend.abort_output(manifest) + await producer.append({"n": 2}) + + await handle.signal(StreamHost.release) + await handle.result() + assert [r.value async for r in stream.read(topic=INPUTS)] == [ + {"n": 1}, + {"n": 2}, + ] + + +async def test_a_committed_stage_is_released_in_the_order_it_was_staged( + live_client: Client, provider: RedisStreams +): + workflow_id = f"streams-redis-commit-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[StreamHost], + plugins=[provider], + ) as worker: + handle = await live_client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(live_client, workflow_id) + backend, manifest = await _stage_one( + provider, + live_client, + workflow_id, + handle.first_execution_run_id or "", + INPUTS.name, + [{"n": 10}, {"n": 11}], + ) + await backend.commit_output(manifest) + + assert await stream.latest(topic=INPUTS) != BEGINNING + await handle.signal(StreamHost.release) + await handle.result() + assert [r.value async for r in stream.read(topic=INPUTS)] == [ + {"n": 10}, + {"n": 11}, + ] + + +async def test_a_batch_the_window_cannot_hold_is_refused_at_the_stage( + live_client: Client, +): + # max_len at or below a task's batch trims the stage before its commit, and + # the commit then fails on a missing record for as long as the task retries. + streams = RedisStreams( + url=redis_url(), key_prefix=f"streams-redis-{uuid.uuid4().hex}", max_len=2 + ) + workflow_id = f"streams-redis-window-{uuid.uuid4().hex}" + try: + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[StreamHost], + plugins=[streams], + ) as worker: + handle = await live_client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + run_id = handle.first_execution_run_id or "" + with pytest.raises(ValueError, match="has to exceed the largest batch"): + await _stage_one( + streams, + live_client, + workflow_id, + run_id, + INPUTS.name, + [{"n": 1}, {"n": 2}], + ) + # One below the window still stages. + await _stage_one( + streams, live_client, workflow_id, run_id, INPUTS.name, [{"n": 1}] + ) + await handle.signal(StreamHost.release) + await handle.result() + finally: + await streams.close() + + +async def test_a_producer_record_lands_once_however_often_it_is_sent( + live_client: Client, provider: RedisStreams +): + # One log per topic: an outside record is written once, where the + # workflow's subscription and outside readers both find it. + workflow_id = f"streams-redis-once-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[StreamHost], + plugins=[provider], + ) as worker: + handle = await live_client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + producer = provider.get_stream_handle(live_client, workflow_id).producer( + topic=INPUTS, producer_id="model", attempt=1 + ) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}) + assert await _log_length(provider, live_client, workflow_id, INPUTS.name) == 3 + + # A producer that comes back with a fresh sequence re-appends the same + # identities, which the script reuses rather than doubling. + again = provider.get_stream_handle(live_client, workflow_id).producer( + topic=INPUTS, producer_id="model", attempt=1 + ) + await again.append({"n": 1}, {"n": 2}, {"n": 3}) + assert await _log_length(provider, live_client, workflow_id, INPUTS.name) == 3 + + await handle.signal(StreamHost.release) + await handle.result() + + +async def test_a_repeat_under_one_identity_with_other_bytes_writes_nothing( + live_client: Client, provider: RedisStreams +): + workflow_id = f"streams-redis-conflict-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[StreamHost], + plugins=[provider], + ) as worker: + handle = await live_client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(live_client, workflow_id) + await stream.producer(topic=INPUTS, producer_id="model", attempt=1).append( + {"n": 1} + ) + other = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + with pytest.raises(StreamProducerError, match="sequence 1"): + await other.append({"n": 99}) + # The refusal wrote nothing. + assert await _log_length(provider, live_client, workflow_id, INPUTS.name) == 1 + await handle.signal(StreamHost.release) + await handle.result() + + +async def test_an_identity_claimed_with_other_bytes_refuses_the_first_append( + live_client: Client, provider: RedisStreams +): + # The idempotency hash is read before the log is touched, so a claim that + # disagrees with the record refuses it without writing. + workflow_id = f"streams-redis-claimed-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[StreamHost], + plugins=[provider], + ) as worker: + handle = await live_client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + backend = provider._require_backend() + chain = await _chain(live_client, workflow_id) + key = chain.stream_key(INPUTS.name, direction=StreamDirection.OUTPUT) + await backend._client.hset( + backend._idempotency_key(key), "model#1/1", "1-0|" + "0" * 64 + ) + + producer = provider.get_stream_handle(live_client, workflow_id).producer( + topic=INPUTS, producer_id="model", attempt=1 + ) + with pytest.raises(StreamProducerError, match="sequence 1"): + await producer.append({"n": 1}) + + assert await _log_length(provider, live_client, workflow_id, INPUTS.name) == 0 + await handle.signal(StreamHost.release) + await handle.result() + + +@workflow.defn +class NudgedLoop: + """The contract loop with a signal that does nothing but complete a task.""" + + def __init__(self) -> None: + self._nudges = 0 + + @workflow.signal + def nudge(self) -> None: + self._nudges += 1 + + @workflow.run + async def run(self) -> list[int]: + decisions = workflow.stream_writer(DECISIONS) + seen: list[int] = [] + async for record in workflow.stream_reader(INPUTS): + if record.kind is RecordKind.FINISH: + break + if record.kind is not RecordKind.DATA: + continue + assert record.value is not None + decisions.publish({"decided": record.value["n"]}) + seen.append(record.value["n"]) + decisions.finish() + return seen + + +async def _completed_tasks(client: Client, workflow_id: str, run_id: str) -> list[int]: + return [ + event.event_id + async for event in client.get_workflow_handle( + workflow_id, run_id=run_id + ).fetch_history_events() + if event.HasField("workflow_task_completed_event_attributes") + ] + + +_MARKER_DETAILS_KEY = "external_stream" + + +async def _consuming_tasks(client: Client, workflow_id: str, run_id: str) -> list[int]: + """The completed Workflow Tasks whose marker delivered a record, in order. + + Read from the markers rather than counted by position: which task opens + the reader, and whether that task can stay open for the first record, + depends on the server, so a task index names a different task from one + server to the next while the marker says what each task consumed. + """ + from temporalio.bridge.proto.external_data import ExternalStreamMarkerData + from temporalio.contrib.external_workflow_streams._annotation import ( + decode_annotation, + ) + + handle = client.get_workflow_handle(workflow_id, run_id=run_id) + completed: int | None = None + consuming: list[int] = [] + async for event in handle.fetch_history_events(): + if event.HasField("workflow_task_completed_event_attributes"): + completed = event.event_id + continue + if not event.HasField("marker_recorded_event_attributes"): + continue + details = event.marker_recorded_event_attributes.details + if _MARKER_DETAILS_KEY not in details or completed is None: + continue + data = ExternalStreamMarkerData() + data.ParseFromString(details[_MARKER_DETAILS_KEY].payloads[0].data) + annotation = decode_annotation(data.replay_annotation) + if any(segment.runs for segment in annotation.segments): + consuming.append(completed) + return consuming + + +async def _reset_at( + client: Client, workflow_id: str, run_id: str, *, finish_event_id: int +) -> str: + """Reset ``run_id`` at the Workflow Task completed by ``finish_event_id``. + + The server keeps History up to that task's completion and runs the task + again, so the tasks before it are inherited and the task itself is not. + Returns the new run id. + """ + from temporalio.api.common.v1 import WorkflowExecution + from temporalio.api.workflowservice.v1 import ResetWorkflowExecutionRequest + + response = await client.workflow_service.reset_workflow_execution( + ResetWorkflowExecutionRequest( + namespace=client.namespace, + workflow_execution=WorkflowExecution( + workflow_id=workflow_id, run_id=run_id + ), + reason="streams: reset mid-stream", + workflow_task_finish_event_id=finish_event_id, + request_id=uuid.uuid4().hex, + ) + ) + return response.run_id + + +async def _consume_two_then_nudge( + live_client: Client, provider: RedisStreams, workflow_id: str, task_queue: str +) -> tuple[str, Any, Any]: + """Two records, each consumed by a task of its own, then a nudged task; the base run.""" + handle = await live_client.start_workflow( + NudgedLoop.run, id=workflow_id, task_queue=task_queue + ) + base_run = handle.result_run_id + assert base_run is not None + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}) + await take(stream.read(topic=DECISIONS), 1, 60) + await producer.append({"n": 2}) + await take(stream.read(topic=DECISIONS), 2, 60) + before = len(await _completed_tasks(live_client, workflow_id, base_run)) + await handle.signal(NudgedLoop.nudge) + for _ in range(300): + if len(await _completed_tasks(live_client, workflow_id, base_run)) > before: + break + await asyncio.sleep(0.1) + return base_run, stream, producer + + +async def test_a_reset_run_replays_the_inherited_ranges_and_continues( + live_client: Client, provider: RedisStreams +): + # Reset at the completion of the task the nudge woke, which consumed and + # published nothing: both consuming tasks are inherited. Their ranges are + # re-read from the log and replayed against the inherited markers, so + # their decisions are not published again, and reading continues from the + # last inherited boundary. + workflow_id = f"streams-redis-reset-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[NudgedLoop], + plugins=[provider], + max_cached_workflows=100, + ): + base_run, stream, producer = await _consume_two_then_nudge( + live_client, provider, workflow_id, f"tq-{workflow_id}" + ) + last_task = (await _completed_tasks(live_client, workflow_id, base_run))[-1] + reset_run = await _reset_at( + live_client, workflow_id, base_run, finish_event_id=last_task + ) + assert reset_run != base_run + + await producer.append({"n": 3}) + await producer.finish() + continued = live_client.get_workflow_handle(workflow_id, run_id=reset_run) + assert await asyncio.wait_for(continued.result(), 90) == [1, 2, 3] + decisions = [r async for r in stream.read(topic=DECISIONS)] + assert [(r.kind, r.value) for r in decisions] == [ + (RecordKind.DATA, {"decided": 1}), + (RecordKind.DATA, {"decided": 2}), + (RecordKind.DATA, {"decided": 3}), + (RecordKind.FINISH, None), + ] + + # Offline, the reset run's History reads the inherited ranges from the log. + await Replayer(workflows=[NudgedLoop], plugins=[provider]).replay_workflow( + await continued.fetch_history() + ) + + +async def test_a_reset_point_task_is_run_again_from_the_log( + live_client: Client, provider: RedisStreams +): + # Reset at the completion of the task that consumed the second record: the + # tasks before it are inherited, the one that consumed the first record + # among them, and this one is run again. The record it consumed is still + # in the log, so the reset run reads it again and publishes again, and an + # outside reader sees that decision from both runs. + workflow_id = f"streams-redis-reset-rerun-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[NudgedLoop], + plugins=[provider], + max_cached_workflows=100, + ): + base_run, stream, producer = await _consume_two_then_nudge( + live_client, provider, workflow_id, f"tq-{workflow_id}" + ) + second = (await _consuming_tasks(live_client, workflow_id, base_run))[1] + reset_run = await _reset_at( + live_client, workflow_id, base_run, finish_event_id=second + ) + + await producer.append({"n": 3}) + await producer.finish() + continued = live_client.get_workflow_handle(workflow_id, run_id=reset_run) + assert await asyncio.wait_for(continued.result(), 90) == [1, 2, 3] + decisions = [r.value async for r in stream.read(topic=DECISIONS)] + assert decisions == [ + {"decided": 1}, + {"decided": 2}, + {"decided": 2}, + {"decided": 3}, + None, + ] + + await Replayer(workflows=[NudgedLoop], plugins=[provider]).replay_workflow( + await continued.fetch_history() + ) + + +@workflow.defn +class EchoOnOneTopic: + """Reads ``inputs`` and answers each value on the same topic. + + Its own records land in the log it reads, so what it returns says whether + it read them back. + """ + + @workflow.run + async def run(self) -> list[Any]: + seen: list[Any] = [] + out = workflow.stream_writer(INPUTS) + async for record in workflow.stream_reader(INPUTS): + if record.kind is RecordKind.FINISH: + break + if record.kind is not RecordKind.DATA: + continue + seen.append(record.value) + out.publish({"echo": record.value}) + out.finish() + return seen + + @workflow.query + def probe(self) -> int: + return 1 + + +async def test_a_workflow_does_not_read_its_own_records_from_the_shared_log( + live_client: Client, provider: RedisStreams +): + workflow_id = f"streams-redis-echo-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[EchoOnOneTopic], + plugins=[provider], + ) as worker: + handle = await live_client.start_workflow( + EchoOnOneTopic.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + # The echoes are promoted into the same log the producer wrote to. + first_four = await take(stream.read(topic=INPUTS), 4, 60) + assert sorted( + ((r.producer_id, r.value) for r in first_four), key=str + ) == sorted( + [ + ("model", {"n": 1}), + ("model", {"n": 2}), + ("", {"echo": {"n": 1}}), + ("", {"echo": {"n": 2}}), + ], + key=str, + ) + await producer.finish() + # The workflow saw the producer's records and none of its echoes. + assert await asyncio.wait_for(handle.result(), 60) == [{"n": 1}, {"n": 2}] + assert await _log_length(provider, live_client, workflow_id, INPUTS.name) == 6 + + # A cold query replays the run against the log with its own entries + # in every recorded range; the read filters them the way the live one did. + assert await handle.query(EchoOnOneTopic.probe) == 1 + everything = [r async for r in stream.read(topic=INPUTS)] + assert [(r.producer_id, r.kind) for r in everything if r.producer_id == ""] == [ + ("", RecordKind.DATA), + ("", RecordKind.DATA), + ("", RecordKind.FINISH), + ] + + await Replayer(workflows=[EchoOnOneTopic], plugins=[provider]).replay_workflow( + await handle.fetch_history() + ) + + +@workflow.defn +class NewestTwoOnGo: + """Opens ``inputs`` at its newest two records once told to, and returns them.""" + + def __init__(self) -> None: + self._go = False + + @workflow.signal + def go(self) -> None: + self._go = True + + @workflow.run + async def run(self) -> list[Any]: + await workflow.wait_condition(lambda: self._go) + reader = workflow.stream_reader(INPUTS, last=2) + values: list[Any] = [] + async for value in reader.values(): + values.append(value["n"]) + if len(values) == 2: + reader.close() + return values + + +@workflow.defn +class FromNowOnGo: + """Opens ``inputs`` at its end once told to, and returns the first value.""" + + def __init__(self) -> None: + self._go = False + + @workflow.signal + def go(self) -> None: + self._go = True + + @workflow.run + async def run(self) -> Any: + await workflow.wait_condition(lambda: self._go) + reader = workflow.stream_reader(INPUTS, after=END) + async for value in reader.values(): + reader.close() + return value["n"] + return None + + +async def test_a_workflow_reader_starts_at_the_last_n_records_and_replays_there( + live_client: Client, provider: RedisStreams +): + # The worker positions the subscription against the log after the task that + # opened it and records the entry with it; replay takes the entry from + # History, so what lands in the log later does not move the start. The + # producer needs the run to exist, so the reader opens on a signal. + workflow_id = f"streams-redis-last-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[NewestTwoOnGo], + plugins=[provider], + ): + handle = await live_client.start_workflow( + NewestTwoOnGo.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}) + await handle.signal(NewestTwoOnGo.go) + assert await asyncio.wait_for(handle.result(), 60) == [3, 4] + history = await handle.fetch_history() + await producer.append({"n": 5}, {"n": 6}) + # Offline, with two more records in the log than the run ever saw. + await Replayer(workflows=[NewestTwoOnGo], plugins=[provider]).replay_workflow( + history + ) + + +async def test_a_workflow_reader_at_end_skips_what_was_there_and_replays( + live_client: Client, provider: RedisStreams +): + workflow_id = f"streams-redis-end-{uuid.uuid4().hex}" + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[FromNowOnGo], + plugins=[provider], + ): + handle = await live_client.start_workflow( + FromNowOnGo.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + await producer.append({"n": "old"}) + await handle.signal(FromNowOnGo.go) + result = asyncio.ensure_future(handle.result()) + # The subscription is positioned when the worker gets to it, which the + # test does not observe, so appends keep coming until the run takes one. + for _ in range(150): + await producer.append({"n": "new"}) + done, _ = await asyncio.wait({result}, timeout=0.2) + if done: + break + assert await asyncio.wait_for(result, 60) == "new" + history = await handle.fetch_history() + await Replayer(workflows=[FromNowOnGo], plugins=[provider]).replay_workflow(history) + + +async def test_a_batch_staged_for_a_rejected_task_does_not_stay_in_the_log( + live_client: Client, monkeypatch: pytest.MonkeyPatch +): + # The first stage takes longer than the Workflow Task timeout, so the server + # times the task out and the worker's completion is rejected; the run is + # evicted and run again, and the second attempt stages the same batch under + # a new token. The first stage can never be named by a marker. It is + # settled against History when the run is evicted and its entries leave the + # log, so the log holds the batch once and nothing counts the dead one. + class SlowFirstStage(_TopicLogBackend): + delayed = False + + async def stage_output(self, manifest: Any, records: Any) -> Any: + if not SlowFirstStage.delayed: + SlowFirstStage.delayed = True + await asyncio.sleep(3) + return await super().stage_output(manifest, records) + + monkeypatch.setattr(redis_provider, "_TopicLogBackend", SlowFirstStage) + provider = RedisStreams( + url=redis_url(), key_prefix=f"streams-redis-{uuid.uuid4().hex}" + ) + workflow_id = f"streams-redis-rejected-{uuid.uuid4().hex}" + try: + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[OneLine], + plugins=[provider], + ): + handle = await live_client.start_workflow( + OneLine.run, + id=workflow_id, + task_queue=f"tq-{workflow_id}", + task_timeout=timedelta(seconds=1), + ) + await asyncio.wait_for(handle.result(), 60) + assert SlowFirstStage.delayed + # Counted before any reader could settle a barrier: the worker did. + assert ( + await _log_length(provider, live_client, workflow_id, DECISIONS.name) + == 2 + ) + stream = provider.get_stream_handle(live_client, workflow_id) + records = [r async for r in stream.read(topic=DECISIONS)] + assert [(r.kind, r.value) for r in records] == [ + (RecordKind.DATA, {"from": "workflow"}), + (RecordKind.FINISH, None), + ] + finally: + await provider.close() + + +async def _channel_support(client: Client) -> Any: + """What the server offers the readers: no channels, independent ones or linked ones.""" + from tests.contrib.external_workflow_streams.conftest import server_channel_support + + return await server_channel_support(client) + + +@pytest.mark.needs_channel_server +async def test_an_outside_producer_wakes_the_reader_through_the_channel( + live_client: Client, +): + from temporalio.contrib.external_workflow_streams._wake import ChannelSupport + + support = await _channel_support(live_client) + if support is ChannelSupport.NONE: + pytest.skip("the server does not implement notification channels") + if support is ChannelSupport.LINKED: + pytest.skip("a workflow-owned stream listens on its linked channel there") + events = await _consume_three_through_the_channel(live_client) + subscribed = _subscribed(events) + assert len(subscribed) == 1, "the run subscribes once per channel" + [index] = [ + i + for i, e in enumerate(events) + if e.HasField("workflow_notification_channel_subscribed_event_attributes") + ] + # Core issues the subscription after the marker of the completion that + # ends the task, so the task that opened the reader stayed retained. + assert events[index - 1].HasField("marker_recorded_event_attributes"), ( + "the subscription did not wait for the completion that ends the task" + ) + notified = _notified(events) + assert {n.channel for n in notified} == set(subscribed) + assert not any(n.HasField("linked_to") for n in notified) + + +@pytest.mark.needs_linked_server +async def test_an_outside_producer_wakes_the_reader_through_its_linked_channel( + live_client: Client, +): + """The stream's channel is the reading workflow's own, so nothing subscribes.""" + from temporalio.contrib.external_workflow_streams._wake import ChannelSupport + + if await _channel_support(live_client) is not ChannelSupport.LINKED: + pytest.skip("the server does not serve channels linked to a workflow") + events = await _consume_three_through_the_channel(live_client) + assert _subscribed(events) == [], "the owner is the listener by construction" + notified = _notified(events) + assert len({n.channel for n in notified}) == 1 + [workflow_id] = { + e.workflow_execution_started_event_attributes.workflow_id + for e in events + if e.HasField("workflow_execution_started_event_attributes") + } or {""} + owners = {Execution.from_proto(n.linked_to) for n in notified} + assert {(owner.type, owner.business_id) for owner in owners} == { + (ExecutionType.WORKFLOW, workflow_id) + } + + +async def _consume_three_through_the_channel(live_client: Client) -> list[Any]: + """Runs the contract loop with three spaced appends and returns its History. + + The channel outright rather than ``"auto"``, so a step down to the Signal + fails the checks here instead of passing quietly. Common to both channel + kinds: no Signal, a notification per wake, and the entry id as the + position with the counter derived from it, so producers and workers order + the same wakes alike. + """ + from temporalio.contrib.external_workflow_streams._record import Offset + + provider = RedisStreams( + url=redis_url(), + key_prefix=f"streams-redis-{uuid.uuid4().hex}", + wake_transport="channel", + ) + workflow_id = f"streams-redis-channel-{uuid.uuid4().hex}" + try: + async with Worker( + live_client, + task_queue=f"tq-{workflow_id}", + workflows=[ContractLoop], + plugins=[provider], + max_cached_workflows=100, + ): + handle = await live_client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=f"tq-{workflow_id}" + ) + stream = provider.get_stream_handle(live_client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + # Spaced past the reader's idle timeout, so the reader parks between + # appends and only a notification from outside can move it. + for n in (1, 2, 3): + await producer.append({"n": n}) + await asyncio.sleep(2) + await producer.finish() + + result = await asyncio.wait_for(handle.result(), 60) + assert [entry.get("n") for entry in result[:3]] == [1, 2, 3] + assert result[3] == {"kind": "finish", "producer": "model"} + + events = [e async for e in handle.fetch_history_events()] + finally: + await provider.close() + signalled = [ + e for e in events if e.HasField("workflow_execution_signaled_event_attributes") + ] + assert signalled == [], "a Signal woke the reader" + notified = _notified(events) + assert notified, "no Workflow Task was scheduled with a notification" + positioned = [note for note in notified if note.position] + assert positioned, "no notification carried the store's position" + for note in positioned: + offset = Offset(note.position.decode()) + assert note.counter == redis_provider._wake_counter(offset) + return events + + +def _subscribed(events: list[Any]) -> list[str]: + return [ + e.workflow_notification_channel_subscribed_event_attributes.channel + for e in events + if e.HasField("workflow_notification_channel_subscribed_event_attributes") + ] + + +def _notified(events: list[Any]) -> list[Any]: + return [ + n + for e in events + if e.HasField("workflow_task_scheduled_event_attributes") + for n in e.workflow_task_scheduled_event_attributes.notifications + ] diff --git a/tests/streams/test_stream_accessors.py b/tests/streams/test_stream_accessors.py new file mode 100644 index 0000000000..4fd3f9b4e6 --- /dev/null +++ b/tests/streams/test_stream_accessors.py @@ -0,0 +1,324 @@ +"""The accessors each context asks for its stream, with one registration. + +The provider is registered on the client alone. A worker built from that +client inherits it, an activity on that worker reaches its own workflow's +stream through ``activity.stream_handle()``, and any code holding the client +reaches a stream through ``client.get_stream_handle()``. Without a +registration, both say so with the same error. +""" + +from __future__ import annotations + +import asyncio +import uuid +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.streams import ( + BEGINNING, + DEFAULT_TOPIC, + END, + RecordKind, + StreamClosedError, + StreamRef, + StreamUnsupportedError, + topic, +) +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.testing import ActivityEnvironment, WorkflowEnvironment +from tests.helpers import new_worker + +INPUTS = topic("inputs", dict) +DECISIONS = topic("decisions", dict) + + +@pytest.fixture +def provider(env: WorkflowEnvironment): # pyright: ignore[reportUnusedFunction] + if env.supports_time_skipping: + pytest.skip( + "the memory provider polls on a timer, which time skipping turns into a spin" + ) + streams = MemoryStreams() + yield streams + streams.reset() + + +def _with_provider(client: Client, provider: MemoryStreams) -> Client: + # The same connection, with the provider registered the way an + # application registers it: once, on the client. + config = client.config() + config["plugins"] = [provider] + return Client(**config) + + +@activity.defn +async def emit(count: int) -> str: + # No workflow id and no run id: the handle is this activity's own + # workflow, pinned to its run, and the producer's identity is the + # activity's. + model = activity.stream_handle().producer(topic=INPUTS) + for n in range(count): + await model.append({"n": n}) + await model.finish() + return activity.info().activity_id + + +@workflow.defn +class Echo: + """Runs the emitting activity and echoes what arrives on ``inputs``.""" + + @workflow.run + async def run(self, count: int) -> list[Any]: + inputs = workflow.stream_reader(INPUTS) + decisions = workflow.stream_writer(DECISIONS) + emitting = workflow.start_activity( + emit, count, start_to_close_timeout=timedelta(seconds=30) + ) + seen: list[Any] = [] + async for record in inputs: + if record.kind is RecordKind.FINISH: + seen.append(("finish", record.producer_id)) + break + assert record.value is not None + seen.append(record.value["n"]) + decisions.publish({"echo": record.value["n"]}) + decisions.finish() + producer_id = await emitting + return [*seen, ("activity", producer_id)] + + +async def test_one_registration_on_the_client_serves_every_context( + client: Client, provider: MemoryStreams +): + registered = _with_provider(client, provider) + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + # No plugins on the worker: it inherits the client's provider. + async with new_worker(registered, Echo, activities=[emit]) as worker: + handle = await registered.start_workflow( + Echo.run, 2, id=workflow_id, task_queue=worker.task_queue + ) + result = await handle.result() + # The activity wrote as itself onto its own workflow's topic. + assert result[:2] == [0, 1] + assert result[2] == ["finish", result[3][1]] + + stream = registered.get_stream_handle(workflow_id) + + async def read_everything() -> list[Any]: + return [(r.kind, r.value) async for r in stream.read(topic=DECISIONS)] + + assert await asyncio.wait_for(read_everything(), 30) == [ + (RecordKind.DATA, {"echo": 0}), + (RecordKind.DATA, {"echo": 1}), + (RecordKind.FINISH, None), + ] + + +@activity.defn +async def emit_on_the_default_topic() -> None: + await activity.stream_handle().producer().append({"from": "activity"}) + + +@workflow.defn +class Answers: + """Waits for the activity's record on the default topic and answers there.""" + + @workflow.run + async def run(self) -> None: + reader = workflow.stream_reader(result_type=dict) + await workflow.execute_activity( + emit_on_the_default_topic, start_to_close_timeout=timedelta(seconds=30) + ) + async for value in reader.values(): + workflow.stream_writer().publish({"answer": value["from"]}) + reader.close() + + +async def test_every_accessor_defaults_to_the_same_topic( + client: Client, provider: MemoryStreams +): + registered = _with_provider(client, provider) + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker( + registered, Answers, activities=[emit_on_the_default_topic] + ) as worker: + stream = registered.get_stream_handle(workflow_id) + assert await stream.latest() == BEGINNING + await registered.execute_workflow( + Answers.run, id=workflow_id, task_queue=worker.task_queue + ) + + async def read_everything() -> list[Any]: + return [(r.topic, r.value) async for r in stream.read()] + + assert await asyncio.wait_for(read_everything(), 30) == [ + (DEFAULT_TOPIC, {"from": "activity"}), + (DEFAULT_TOPIC, {"answer": "activity"}), + ] + + +async def test_get_stream_handle_needs_a_registered_provider(client: Client): + with pytest.raises(StreamUnsupportedError, match="plugins="): + client.get_stream_handle("wf") + + +async def test_stream_handle_needs_a_provider_on_the_worker(): + async def ask() -> None: + activity.stream_handle() + + with pytest.raises(StreamUnsupportedError, match="plugins="): + await ActivityEnvironment().run(ask) + + +@activity.defn +def ask_from_a_sync_activity() -> str: + try: + activity.stream_handle() + except RuntimeError as error: + return str(error) + return "opened" + + +@workflow.defn +class RunsASyncActivity: + """Runs the `def` activity that reaches for a handle.""" + + @workflow.run + async def run(self) -> str: + return await workflow.execute_activity( + ask_from_a_sync_activity, start_to_close_timeout=timedelta(seconds=30) + ) + + +async def test_a_sync_activity_is_told_it_cannot_have_a_handle( + client: Client, provider: MemoryStreams +): + # The worker has a provider. What a `def` activity is missing is the + # client, so that is what the error has to say, rather than sending the + # reader to register a provider that is already there. + registered = _with_provider(client, provider) + with ThreadPoolExecutor(max_workers=1) as executor: + async with new_worker( + registered, + RunsASyncActivity, + activities=[ask_from_a_sync_activity], + activity_executor=executor, + ) as worker: + result = await registered.execute_workflow( + RunsASyncActivity.run, + id=f"streams-wf-{uuid.uuid4().hex}", + task_queue=worker.task_queue, + ) + assert "only available in `async def` activities" in result + assert "plugins=" not in result + + +async def test_an_activity_owned_stream_takes_every_read_start( + client: Client, provider: MemoryStreams +): + registered = _with_provider(client, provider) + stream = registered.get_stream_handle(activity_id=f"act-{uuid.uuid4().hex}") + producer = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}) + + async def first(records: Any) -> Any: + async for record in records: + await records.aclose() + return record.value + return None + + assert await first(stream.read(topic=INPUTS, after=BEGINNING)) == {"n": 1} + assert await first(stream.read(topic=INPUTS, last=1)) == {"n": 3} + at_end = stream.read(topic=INPUTS, after=END) + await producer.append({"n": 4}) + assert await asyncio.wait_for(first(at_end), 10) == {"n": 4} + + +NOTES = topic("notes", dict) + + +@activity.defn +async def append_to_ref(ref: StreamRef) -> str: + # The ref arrived as an argument and names the stream in full, so it + # takes no scope; the handle it opens writes to the ref's topic. + try: + activity.stream_handle(ref, scope="activity") + except ValueError as error: + refused = str(error) + else: + refused = "accepted" + await activity.stream_handle(ref).producer().append({"via": "activity"}) + return refused + + +@workflow.defn +class PublishToRef: + """Hands a stream ref to an activity and returns it, both as plain data.""" + + @workflow.run + async def run(self, ref: StreamRef) -> tuple[StreamRef, str]: + refused = await workflow.execute_activity( + append_to_ref, ref, start_to_close_timeout=timedelta(seconds=30) + ) + return ref, refused + + +async def test_a_ref_travels_as_data_and_opens_from_every_context( + client: Client, provider: MemoryStreams +): + registered = _with_provider(client, provider) + shared = await registered.create_stream(f"shared-{uuid.uuid4().hex}") + ref = shared.ref(topic=NOTES) + async with new_worker( + registered, PublishToRef, activities=[append_to_ref] + ) as worker: + returned, refused = await registered.execute_workflow( + PublishToRef.run, + ref, + id=f"streams-wf-{uuid.uuid4().hex}", + task_queue=worker.task_queue, + ) + # The workflow argument and result went through the data converter. + assert returned == ref + assert "takes no run_id or scope" in refused + # The activity wrote to the referenced topic, and the client reads it + # from the ref without naming the topic. Sealing the stream is what ends + # a full read of it. + await shared.close() + records = [ + (r.topic, r.value) async for r in registered.get_stream_handle(ref).read(last=1) + ] + assert records == [("notes", {"via": "activity"})] + with pytest.raises(ValueError, match="takes no run_id"): + registered.get_stream_handle(ref, run_id="r") + + +async def test_the_client_creates_reaches_and_closes_a_standalone_stream( + client: Client, provider: MemoryStreams +): + registered = _with_provider(client, provider) + stream_id = f"shared-{uuid.uuid4().hex}" + created = await registered.create_stream(stream_id, max_records=3) + producer = created.producer(topic=NOTES, producer_id="writer", attempt=1) + await producer.append({"n": 1}) + + reached = registered.get_stream_handle(stream_id=stream_id) + assert await reached.latest(topic=NOTES) != BEGINNING + await reached.close() + with pytest.raises(StreamClosedError): + await producer.append({"n": 2}) + assert [r.value async for r in reached.read(topic=NOTES)] == [{"n": 1}] + + with pytest.raises(ValueError, match="no workflow_id"): + registered.get_stream_handle(stream_id=stream_id, workflow_id="wf") + with pytest.raises(ValueError, match="empty"): + await registered.create_stream("") + with pytest.raises(StreamUnsupportedError, match="plugins="): + await client.create_stream(stream_id) + with pytest.raises(StreamUnsupportedError, match="plugins="): + client.get_stream_handle(stream_id=stream_id) diff --git a/tests/streams/test_stream_channel_e2e.py b/tests/streams/test_stream_channel_e2e.py new file mode 100644 index 0000000000..6a632556a4 --- /dev/null +++ b/tests/streams/test_stream_channel_e2e.py @@ -0,0 +1,254 @@ +"""A native stream notifies the channel named by the stream, on a live server. + +Every append and the close of a native stream notify the channel +:func:`temporalio.client.stream_channel` derives from the stream's ref, so a +client follows a stream the way it follows an external one: by polling the +channel or registering a callback on it. The workflow's own consumption of a +stream is untouched by this and is covered elsewhere. Needs a server on which +streams drive channels, named with ``-E host:port``; skipped otherwise. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import uuid +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import activity, workflow +from temporalio.client import ( + Callback, + ChannelAddress, + ChannelKind, + Client, + stream_channel, +) +from temporalio.common import Execution, ExecutionType +from temporalio.service import RPCError +from temporalio.streams import StreamRef, topic +from temporalio.streams.providers.native import NativeStreams +from tests.helpers import assert_eventually, new_worker + +OUT = topic("out", dict) + +pytestmark = pytest.mark.needs_stream_channel_server + + +async def _native(client: Client, provider: NativeStreams) -> Client: + """A client on the same server as ``client`` with the native provider on it.""" + return await Client.connect( + client.service_client.config.target_host, + namespace=client.namespace, + plugins=[provider], + ) + + +def _closed(client: Client, notification: Any) -> bool: + if "closed" not in notification.metadata: + return False + return client.data_converter.payload_converter.from_payload( + notification.metadata["closed"], bool + ) + + +async def test_a_standalone_stream_notifies_the_channel_named_by_its_id( + client: Client, +): + provider = NativeStreams() + native = await _native(client, provider) + stream_id = f"chan-{uuid.uuid4().hex[:8]}" + try: + created = await native.create_stream(stream_id) + address = stream_channel(created.ref(topic=OUT)) + # A standalone stream's topics share one stream, so the topic is not in + # the name and the channel is an independent one. + assert address == ChannelAddress(f"stream/{stream_id}", None) + producer = created.producer(topic=OUT, producer_id="writer", attempt=1) + + async def changes(count: int) -> list[Any]: + polled = await native.poll_channel(address.channel, wait=False) + assert [n.counter for n in polled] == list(range(1, count + 1)) + return polled + + # A standalone stream reaches its channel through a task of its own + # that hands over the latest change, so the notifications trail the + # appends and a burst arrives as its newest change. Each append waits + # for its notification so that every change is seen. + await producer.append({"n": 1}) + await assert_eventually(lambda: changes(1)) + await producer.append({"n": 2}) + polled = await assert_eventually(lambda: changes(2)) + for notification in polled: + assert notification.channel == address.channel + assert notification.linked_to is None + # The position is the head after the change, as a native cursor + # names one: the stream id in place of a run, then the offset. + assert notification.position.decode().startswith(f"{stream_id}:") + assert not _closed(native, notification) + await created.close() + [third] = await native.poll_channel( + address.channel, after_counter=2, wait=timedelta(seconds=10) + ) + assert third.counter == 3 + assert _closed(native, third) + description = await native.describe_channel(address.channel) + assert description.kind == ChannelKind.INDEPENDENT + assert description.latest is not None and description.latest.counter == 3 + assert description.retained_count == 3 + # A callback registers on the derived name like on any channel. + callback = Callback(url="http://localhost:1/never-called", headers={}) + listener_id = await native.register_channel_listener(address.channel, callback) + description = await native.describe_channel(address.channel) + assert [listener.callback for listener in description.listeners] == [callback] + await native.unregister_channel_listener(address.channel, listener_id) + finally: + await provider.close() + + +@workflow.defn +class WriteOnNudge: + """Writes one record at the start and one more per nudge, until ``rounds``.""" + + def __init__(self) -> None: + self._nudges = 0 + self._written = 0 + + @workflow.run + async def run(self, rounds: int) -> int: + writer = workflow.stream_writer(OUT) + writer.publish({"n": 0}) + while self._written < rounds: + await workflow.wait_condition(lambda: self._nudges > self._written) + self._written += 1 + writer.publish({"n": self._written}) + return self._written + + @workflow.signal + def nudge(self) -> None: + self._nudges += 1 + + +async def test_a_workflow_stream_notifies_the_channel_linked_to_its_owner( + client: Client, +): + provider = NativeStreams() + native = await _native(client, provider) + worker = new_worker(native, WriteOnNudge) + running = asyncio.create_task(worker.run()) + # More rounds than nudges: the linked ring dies with the run, so the run + # stays open until the test has read it and is ended by hand. + handle = await native.start_workflow( + WriteOnNudge.run, 10, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + ) + try: + address = stream_channel(StreamRef.for_workflow(handle.id, topic=OUT)) + assert address == ChannelAddress("stream/out", Execution.workflow(handle.id)) + assert address.workflow_id == handle.id + assert address == stream_channel( + native.get_stream_handle(handle.id).ref(topic=OUT) + ) + + async def changes(count: int) -> list[Any]: + polled = await native.poll_channel( + address.channel, workflow_id=address.workflow_id, wait=False + ) + assert [n.counter for n in polled] == list(range(1, count + 1)) + return polled + + # The first task's publish is one append, so one notification. + await assert_eventually(lambda: changes(1), timeout=timedelta(seconds=30)) + await handle.signal(WriteOnNudge.nudge) + await assert_eventually(lambda: changes(2), timeout=timedelta(seconds=30)) + await handle.signal(WriteOnNudge.nudge) + polled = await assert_eventually( + lambda: changes(3), timeout=timedelta(seconds=30) + ) + run_id = handle.first_execution_run_id + for notification in polled: + assert notification.linked_to is not None + assert notification.linked_to.business_id == handle.id + assert notification.position.decode().startswith(f"{run_id}:") + description = await native.describe_channel( + address.channel, workflow_id=address.workflow_id + ) + assert description.kind == ChannelKind.LINKED + assert description.latest is not None and description.latest.counter == 3 + finally: + with contextlib.suppress(RPCError): + await handle.terminate() + try: + await asyncio.wait_for(worker.shutdown(), 15) + except asyncio.TimeoutError: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + await provider.close() + + +_released: dict[str, bool] = {} +"""Activity ids the test has let go, read by the activity in the same process.""" + + +@activity.defn +async def write_then_wait() -> str: + """Writes one record to the activity's own stream and lingers until released.""" + activity_id = activity.info().activity_id + producer = activity.stream_handle().producer(topic=OUT) + await producer.append({"n": 1}) + while not _released.get(activity_id): + activity.heartbeat() + await asyncio.sleep(0.2) + return activity_id + + +@pytest.mark.needs_execution_server +async def test_a_standalone_activity_stream_notifies_the_channel_linked_to_it( + client: Client, +): + provider = NativeStreams() + native = await _native(client, provider) + activity_id = f"act-{uuid.uuid4().hex[:8]}" + async with new_worker(native, activities=[write_then_wait]) as worker: + handle = await native.start_activity( + write_then_wait, + id=activity_id, + task_queue=worker.task_queue, + start_to_close_timeout=timedelta(seconds=60), + ) + try: + # A standalone activity is an execution of its own, so its stream + # notifies a channel linked to it, under the topic's name alone. + address = stream_channel(StreamRef.for_activity(activity_id, topic=OUT)) + assert address == ChannelAddress( + "stream/out", Execution.activity(activity_id) + ) + assert address.workflow_id is None + + async def changes(count: int) -> list[Any]: + polled = await native.poll_channel( + address.channel, execution=address.execution, wait=False + ) + assert [n.counter for n in polled] == list(range(1, count + 1)) + return polled + + [first] = await assert_eventually( + lambda: changes(1), timeout=timedelta(seconds=30) + ) + assert first.linked_to is not None + assert (first.linked_to.type, first.linked_to.business_id) == ( + ExecutionType.ACTIVITY, + activity_id, + ) + description = await native.describe_channel( + address.channel, execution=address.execution + ) + assert description.kind == ChannelKind.LINKED + assert description.linked_to is not None + assert description.linked_to.business_id == activity_id + finally: + _released[activity_id] = True + with contextlib.suppress(Exception): + await asyncio.wait_for(handle.result(), 30) + await provider.close() diff --git a/tests/streams/test_stream_hooks.py b/tests/streams/test_stream_hooks.py new file mode 100644 index 0000000000..7908dfc576 --- /dev/null +++ b/tests/streams/test_stream_hooks.py @@ -0,0 +1,169 @@ +"""The lifecycle interceptor's rules about when the finish hook runs. + +Driven directly, with a stand-in runtime on the loop, because the two cases +that matter here are the ones a workflow test cannot stage on purpose: a +coroutine collected after its worker went away, and a run evicted from the +cache. Both used to reach the finish hook, and at collection time the hook +acted on whichever workflow was running on the thread. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, cast + +import pytest + +from temporalio import workflow +from temporalio.worker._interceptor import ( + ExecuteWorkflowInput, + WorkflowInboundInterceptor, +) +from temporalio.worker._workflow import _StreamHooksInterceptor +from temporalio.worker._workflow_instance import _WorkflowInstanceImpl + + +class _Provider: + def __init__(self) -> None: + self.calls: list[str] = [] + + def on_workflow_start(self) -> None: + self.calls.append("start") + + async def on_workflow_finish(self) -> None: + self.calls.append("finish") + + +class _Streams: + def __init__(self, provider: _Provider) -> None: + self.provider = provider + + +class _FakeRuntime: + """Only what the interceptor reads, and only through the runtime interface. + + It deliberately carries no ``_deleting`` attribute. An interceptor that + reads the eviction state by attribute name instead of by method would see + a runtime that is never evicting here, and the eviction case below would + catch it. + """ + + def __init__(self, provider: _Provider, *, evicting: bool = False) -> None: + self._streams = _Streams(provider) + self._evicting = evicting + + def workflow_streams(self) -> _Streams: + return self._streams + + def workflow_is_evicting(self) -> bool: + return self._evicting + + def call_soon(self, callback: Any) -> None: + # The real runtime is the workflow's event loop; here the test's loop + # stands in for it. + asyncio.get_running_loop().call_soon(callback) + + +class _Body(WorkflowInboundInterceptor): + """The workflow function's stand-in: returns, raises or parks forever.""" + + def __init__(self, outcome: Any) -> None: # type: ignore[reportMissingSuperCall] + self._outcome = outcome + + def init(self, outbound: Any) -> None: + del outbound + + async def execute_workflow(self, input: ExecuteWorkflowInput) -> Any: + del input + if self._outcome is _PARK: + await asyncio.Event().wait() + if isinstance(self._outcome, BaseException): + raise self._outcome + return self._outcome + + +_PARK = object() + + +async def _unused_run_fn() -> None: + pass + + +_INPUT = ExecuteWorkflowInput(type=object, run_fn=_unused_run_fn, args=(), headers={}) + + +async def _hooked(body: _Body) -> _StreamHooksInterceptor: + """The interceptor over ``body``, initialised the way the instance does it. + + ``init`` queues the start hook for the loop's first turn, so one turn + runs it, as the instance's loop does before any task takes a step. + """ + interceptor = _StreamHooksInterceptor(body) + interceptor.init(cast(Any, None)) + await asyncio.sleep(0) + return interceptor + + +@pytest.fixture +async def provider() -> Any: + fake = _Provider() + loop = asyncio.get_running_loop() + workflow._Runtime.set_on_loop(loop, _FakeRuntime(fake)) # type: ignore[arg-type] + yield fake + workflow._Runtime.set_on_loop(loop, None) + + +def _evicting(fake: _Provider) -> None: + loop = asyncio.get_running_loop() + workflow._Runtime.set_on_loop(loop, _FakeRuntime(fake, evicting=True)) # type: ignore[arg-type] + + +async def test_the_finish_hook_runs_on_return(provider: _Provider): + assert await (await _hooked(_Body("done"))).execute_workflow(_INPUT) == "done" + assert provider.calls == ["start", "finish"] + + +async def test_the_finish_hook_runs_when_the_function_raises(provider: _Provider): + with pytest.raises(RuntimeError, match="boom"): + await (await _hooked(_Body(RuntimeError("boom")))).execute_workflow(_INPUT) + assert provider.calls == ["start", "finish"] + + +async def test_the_finish_hook_runs_on_continue_as_new(provider: _Provider): + error = workflow.ContinueAsNewError.__new__(workflow.ContinueAsNewError) + with pytest.raises(workflow.ContinueAsNewError): + await (await _hooked(_Body(error))).execute_workflow(_INPUT) + assert provider.calls == ["start", "finish"] + + +async def test_the_finish_hook_runs_on_a_workflow_cancellation(provider: _Provider): + # A cancelled primary task is the run ending, so the provider lets go. + with pytest.raises(asyncio.CancelledError): + await (await _hooked(_Body(asyncio.CancelledError()))).execute_workflow(_INPUT) + assert provider.calls == ["start", "finish"] + + +async def test_the_finish_hook_does_not_run_when_the_coroutine_is_collected( + provider: _Provider, +): + coroutine = (await _hooked(_Body(_PARK))).execute_workflow(_INPUT) + # Run up to the park, the way a worker that shut down without evicting + # leaves the primary task, then close it as garbage collection would. + coroutine.send(None) + coroutine.close() + assert provider.calls == ["start"] + + +async def test_the_finish_hook_does_not_run_during_eviction(provider: _Provider): + _evicting(provider) + with pytest.raises(asyncio.CancelledError): + await (await _hooked(_Body(asyncio.CancelledError()))).execute_workflow(_INPUT) + assert provider.calls == ["start"] + + +def test_the_runtime_answers_the_eviction_question_itself(): + # The interceptor asks the runtime rather than reading a private field, + # so the coupling is declared on the base class and the real instance has + # to answer it or fail to construct. + assert "workflow_is_evicting" in workflow._Runtime.__abstractmethods__ + assert "workflow_is_evicting" not in _WorkflowInstanceImpl.__abstractmethods__ diff --git a/tests/streams/test_stream_reader.py b/tests/streams/test_stream_reader.py new file mode 100644 index 0000000000..d3918fe405 --- /dev/null +++ b/tests/streams/test_stream_reader.py @@ -0,0 +1,178 @@ +"""The reader's rules that a workflow test cannot pin down. + +``stream_reader`` promises one subscription per topic per run, that a second +loop on one reader shares its buffer rather than racing the source, and that +closing it and opening the topic again is a new subscription. The memory +provider's source hands back everything it has without ever suspending, so a +workflow test cannot tell a reader that serialises its fetches from one that +does not. These drive the reader directly, with a source that suspends where +a real provider would, and a stand-in runtime on the loop. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.converter import DataConverter +from temporalio.streams import BEGINNING, Cursor, RecordKind, topic +from temporalio.streams._wire import WireRecord, to_wire +from temporalio.workflow._streams import _WorkflowStreams + +INPUTS = topic("inputs", dict) + +_CONVERTER = DataConverter.default.payload_converter + + +def _record(n: int) -> WireRecord: + return to_wire( + _CONVERTER, + topic=INPUTS.name, + kind=RecordKind.DATA, + value={"n": n}, + producer_id="model", + attempt=1, + sequence=n, + ) + + +class _SlowSource: + """Hands over one record per batch, suspending first the way a real one does.""" + + def __init__(self, count: int) -> None: + self.opened = True + self.next_calls = 0 + self.in_flight = 0 + self.overlapped = False + self._offset = 0 + self._count = count + + async def next_batch(self) -> list[tuple[Cursor, WireRecord]]: + self.next_calls += 1 + self.in_flight += 1 + if self.in_flight > 1: + self.overlapped = True + try: + # Where a provider waits for the worker to deliver. Two fetches + # that both get here have both left the reader's buffer behind. + await asyncio.sleep(0) + await asyncio.sleep(0) + if self._offset >= self._count: + raise StopAsyncIteration + self._offset += 1 + return [(Cursor(f"fake:{self._offset - 1}"), _record(self._offset - 1))] + finally: + self.in_flight -= 1 + + def close(self) -> None: + self.opened = False + + +class _Half: + """A workflow half that hands out one source per topic and counts opens.""" + + def __init__(self) -> None: + self.sources: list[_SlowSource] = [] + + def open_reader(self, topic: str, *, after: Cursor) -> Any: + del topic, after + source = _SlowSource(4) + self.sources.append(source) + return source + + def open_writer(self, topic: str) -> Any: + del topic + raise NotImplementedError + + def on_workflow_start(self) -> None: + pass + + async def on_workflow_finish(self) -> None: + pass + + +class _FakeRuntime: + """Only what the reader reads: the stream state and the payload converter.""" + + def __init__(self, half: _Half) -> None: + self._streams = _WorkflowStreams(half) # type: ignore[arg-type] + + def workflow_streams(self) -> _WorkflowStreams: + return self._streams + + def workflow_payload_converter(self) -> Any: + return _CONVERTER + + +@pytest.fixture +async def half() -> Any: + fake = _Half() + loop = asyncio.get_running_loop() + workflow._Runtime.set_on_loop(loop, _FakeRuntime(fake)) # type: ignore[arg-type] + yield fake + workflow._Runtime.set_on_loop(loop, None) + + +async def test_two_loops_on_one_reader_split_the_records(half: _Half): + reader = workflow.stream_reader(INPUTS) + seen: list[int] = [] + + async def pull(count: int) -> None: + for _ in range(count): + record = await reader.__anext__() + assert record.value is not None + seen.append(record.value["n"]) + + await asyncio.wait_for(asyncio.gather(pull(2), pull(2)), 5.0) + # Every record once and none lost, whichever loop got there first, and + # the source was never asked for two batches at the same time. + assert sorted(seen) == [0, 1, 2, 3] + assert half.sources[0].overlapped is False + + +@pytest.mark.usefixtures("half") +async def test_a_cancelled_read_does_not_lose_the_record_it_waited_for(): + reader = workflow.stream_reader(INPUTS) + pending = asyncio.ensure_future(reader.__anext__()) + await asyncio.sleep(0) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + + # The next read picks up where the cancelled one was, rather than finding + # the reader wedged behind a lock the cancelled fetch still holds. + record = await asyncio.wait_for(reader.__anext__(), 5.0) + assert record.value == {"n": 0} + + +async def test_closing_a_reader_lets_the_topic_be_opened_again(half: _Half): + reader = workflow.stream_reader(INPUTS) + assert workflow.stream_reader(INPUTS) is reader + reader.close() + assert half.sources[0].opened is False + + # A new subscription, not the closed one handed back. It is also a new + # command on a real provider, which is why the docstring says to gate it. + again = workflow.stream_reader(INPUTS) + assert again is not reader + assert len(half.sources) == 2 + assert (await asyncio.wait_for(again.__anext__(), 5.0)).value == {"n": 0} + + +@pytest.mark.usefixtures("half") +async def test_a_closed_reader_stops_iterating(): + reader = workflow.stream_reader(INPUTS) + assert (await asyncio.wait_for(reader.__anext__(), 5.0)).value == {"n": 0} + reader.close() + with pytest.raises(StopAsyncIteration): + await asyncio.wait_for(reader.__anext__(), 5.0) + + +@pytest.mark.usefixtures("half") +async def test_a_reader_ends_when_the_source_ends(): + reader = workflow.stream_reader(INPUTS, after=BEGINNING) + values = [record.value async for record in reader] + assert values == [{"n": 0}, {"n": 1}, {"n": 2}, {"n": 3}] diff --git a/tests/streams/test_streams_conformance.py b/tests/streams/test_streams_conformance.py new file mode 100644 index 0000000000..0d68b167fd --- /dev/null +++ b/tests/streams/test_streams_conformance.py @@ -0,0 +1,1533 @@ +"""Conformance tests for the stream contract's outside surface. + +Written against the public surface, parametrised over the providers this +tree can stand up. The memory provider always runs, with no server and no +store. A storage provider adds itself to ``SETUPS``, behind its own +``STREAMS_LIVE`` gate when it needs a store the test environment does not +start: its setup receives the environment's client and hands back a provider +instance, the client the cases should use, a ``host`` that starts the +workflow owning a stream when the store lives inside a running workflow, and +which capabilities it lacks, so the cases marked ``reports_positions`` are +skipped with a reason on a provider whose ``append()`` learns positions at +read time. + +``NexusStreams`` is deliberately absent from ``SETUPS``. It is a front with no +workflow half (its ``workflow_provider()`` raises), so there is no store for a +host workflow to own; ``test_nexus_provider`` covers it through an endpoint +that fronts a storage provider. + +What this file pins down is what a provider owes: producer identity, retry +deduplication, positions, supersession, topic addressing, cursor resumption, +cursor ownership, releasing a read the caller stopped early, naming a stream +as a ``StreamRef``, and running bodies through the client's data converter so +external storage applies and a retry through a nondeterministic codec still +matches its original. Every case here goes through the public surface, so a +new provider answers this file and nothing else. The shared pieces no provider +implements are unit-tested in ``test_streams_internals``; the workflow-side +handles and the two rules about Workflow Tasks live in +``test_streams_workflow``. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import os +import uuid +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.api.common.v1 import Payload +from temporalio.client import Client, WorkflowHandle +from temporalio.common import Execution, ExecutionType, RawValue +from temporalio.contrib.external_workflow_streams._wake import ChannelSupport +from temporalio.converter import ( + DataConverter, + ExternalStorage, + PayloadCodec, + StorageDriver, + StorageDriverClaim, + StorageDriverRetrieveContext, + StorageDriverStoreContext, +) +from temporalio.service import RPCError +from temporalio.streams import ( + BEGINNING, + DEFAULT_TOPIC, + END, + Cursor, + RecordKind, + StreamClosedError, + StreamCursorError, + StreamError, + StreamHandle, + StreamNotFoundError, + StreamProducerError, + StreamProvider, + StreamRef, + StreamUnsupportedError, + Supersession, + topic, +) +from temporalio.streams._ref import open_ref +from temporalio.streams.providers import workflow_streams +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.streams.providers.native import NativeStreams +from temporalio.streams.providers.redis import RedisStreams +from temporalio.streams.providers.workflow_streams import WorkflowStreamsProvider +from temporalio.testing import WorkflowEnvironment +from tests.contrib.external_workflow_streams.conftest import server_channel_support +from tests.helpers import new_worker + +# Defined once and shared by every case, the way an application shares them +# between its workflow, its activities and its backend. +OUT = topic("out", dict) +A = topic("a", dict) +B = topic("b", dict) +Y = topic("y", dict) +XY = topic("x:y", dict) + + +@dataclass +class ProviderCase: + """One provider under test, and what the cases may ask of it.""" + + name: str + provider: StreamProvider + client: Client | None = None + reports_positions: bool = True + """``append()`` returns where the records landed.""" + detects_divergent_retries: bool = True + """``append()`` compares a repeat's content with what it already holds.""" + encodes_bodies: bool = True + """The outside path runs each body through the client's data converter, so a + client with a converter of its own reads and writes another's records.""" + host: Callable[[str], Awaitable[None]] | None = None + """Starts the workflow that owns ``workflow_id``'s stream, when a store needs one.""" + task_queue: str | None = None + """Where the setup's worker runs the workflows below, when it has one.""" + truncate: Callable[[str, str, int], Awaitable[None]] | None = None + """Drops all but the newest records of a workflow's topic, standing in + for retention, or ``None`` when the provider offers no way to.""" + hosts_standalone_streams: bool = True + """The store holds a stream with an id of its own and no owner.""" + waits_for_standalone_creation: bool = False + """A read on a standalone stream id that does not exist yet parks until + the first write instead of raising ``StreamNotFoundError``.""" + bounds_standalone_bytes: bool = True + """A standalone stream's policy can bound the bytes it keeps.""" + refuses_appends_past_byte_cap: bool = False + """The byte bound refuses an append that would cross it, instead of + dropping the oldest records to make room.""" + trims_open_stream_by_age: bool = True + """A standalone stream drops records older than ``retention`` while it is + open, rather than keeping them that long after it closes.""" + wakes_by_notification: bool = False + """An outside append wakes a parked workflow reader through the server, + rather than the reader finding the record on a timer of its own.""" + wakes_by_linked_notification: bool = False + """The wake above reaches a workflow-owned stream's reader through the + channel linked to its workflow, so the reader subscribes to nothing.""" + + async def open( + self, + workflow_id: str, + *, + run_id: str | None = None, + client: Client | None = None, + ) -> StreamHandle: + if self.host is not None: + await self.host(workflow_id) + if client is not None: + # The explicit form, for a case that needs the handle to encode + # bodies through this client's data converter. + return self.provider.get_stream_handle(client, workflow_id, run_id=run_id) + if self.client is not None: + # A storage provider's setup registers the provider on the client, + # so the cases go through the accessor an application uses. + return self.client.get_stream_handle(workflow_id, run_id=run_id) + # Only the memory provider gets here, and it takes no client. + return self.provider.get_stream_handle( + None, # type: ignore[arg-type] + workflow_id, + run_id=run_id, + ) + + async def open_ref(self, ref: StreamRef) -> StreamHandle: + if self.client is not None: + return self.client.get_stream_handle(ref) + return open_ref(self.provider, None, ref) # type: ignore[arg-type] + + async def create_stream( + self, + stream_id: str, + *, + retention: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> StreamHandle: + if self.client is not None: + return await self.client.create_stream( + stream_id, + retention=retention, + max_records=max_records, + max_bytes=max_bytes, + ) + return await self.provider.create_standalone_stream( + None, # type: ignore[arg-type] + stream_id, + retention=retention, + max_records=max_records, + max_bytes=max_bytes, + ) + + async def open_standalone(self, stream_id: str) -> StreamHandle: + if self.client is not None: + return self.client.get_stream_handle(stream_id=stream_id) + return self.provider.get_standalone_stream_handle( + None, # type: ignore[arg-type] + stream_id, + ) + + +class RecordingDriver(StorageDriver): + """An in-memory external storage driver that counts what it was asked to hold.""" + + def __init__(self) -> None: + self.held: dict[str, bytes] = {} + self.stored = 0 + self.retrieved = 0 + + def name(self) -> str: + return "recording" + + async def store( + self, context: StorageDriverStoreContext, payloads: Sequence[Payload] + ) -> list[StorageDriverClaim]: + claims: list[StorageDriverClaim] = [] + for payload in payloads: + key = f"payload-{len(self.held)}" + self.held[key] = payload.SerializeToString() + self.stored += 1 + claims.append(StorageDriverClaim(claim_data={"key": key})) + return claims + + async def retrieve( + self, + context: StorageDriverRetrieveContext, + claims: Sequence[StorageDriverClaim], + ) -> list[Payload]: + self.retrieved += len(claims) + return [Payload.FromString(self.held[c.claim_data["key"]]) for c in claims] + + +class NonceCodec(PayloadCodec): + """A codec whose output differs on every call, as one that encrypts with a fresh nonce does.""" + + def __init__(self) -> None: + self.encoded = 0 + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + self.encoded += len(payloads) + return [ + Payload( + metadata={"encoding": b"binary/nonce"}, + data=os.urandom(16) + p.SerializeToString(), + ) + for p in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [Payload.FromString(p.data[16:]) for p in payloads] + + +def _client_with(client: Client, converter: DataConverter) -> Client: + # The same connection, carrying the converter the case wants bodies to + # pass through. + config = client.config() + config["data_converter"] = converter + return Client(**config) + + +async def _memory_case(_client: Client) -> AsyncIterator[ProviderCase]: + provider = MemoryStreams() + + async def truncate(workflow_id: str, topic: str, keep: int) -> None: + provider.truncate(workflow_id, topic, keep=keep) + + yield ProviderCase( + "memory", + provider, + truncate=truncate, + bounds_standalone_bytes=True, + refuses_appends_past_byte_cap=False, + trims_open_stream_by_age=True, + ) + provider.reset() + + +@workflow.defn +class StreamHost: + """Owns a stream and lingers, so outside code has a running workflow to address.""" + + def __init__(self) -> None: + self._released = False + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: self._released) + + +@workflow.defn +class TruncatingStreamHost: + """A stream host whose log an update can truncate, the way a workflow's retention would.""" + + def __init__(self) -> None: + self._released = False + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.update + def truncate(self, topic: str, keep: int) -> None: + stream = workflow_streams._instance().stream # pyright: ignore[reportPrivateUsage] + stream.truncate(workflow_streams.start_offset(stream, topic, keep)) + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: self._released) + + +@workflow.defn +class DefaultTopicAnswer: + """Reads one value on its default topic and answers on the same topic.""" + + @workflow.run + async def run(self) -> None: + reader = workflow.stream_reader(result_type=dict) + async for value in reader.values(): + workflow.stream_writer().publish({"answer": value["n"] * 2}) + reader.close() + + +async def _workflow_streams_case(client: Client) -> AsyncIterator[ProviderCase]: + # No STREAMS_LIVE gate: the store is the workflow's own History, which the + # test environment's server provides. + provider = WorkflowStreamsProvider(poll_cooldown=timedelta(milliseconds=20)) + # Registered once, on the client: the host's worker inherits it and the + # cases open handles through client.get_stream_handle. + config = client.config() + config["plugins"] = [provider] + client = Client(**config) + hosts: dict[str, WorkflowHandle[Any, Any]] = {} + async with new_worker(client, TruncatingStreamHost, DefaultTopicAnswer) as worker: + + async def host(workflow_id: str) -> None: + if workflow_id not in hosts: + hosts[workflow_id] = await client.start_workflow( + TruncatingStreamHost.run, + id=workflow_id, + task_queue=worker.task_queue, + ) + + async def truncate(workflow_id: str, topic: str, keep: int) -> None: + await hosts[workflow_id].execute_update( + TruncatingStreamHost.truncate, args=[topic, keep] + ) + + # A publish is an Update, so the workflow answers with the position + # and refuses a divergent repeat: both capabilities hold here. Bodies + # meet the codec and external storage at the transport's envelope, + # which the worker's converter has to match, so a case whose client + # carries a converter of its own is skipped; see the provider's + # module docstring. + yield ProviderCase( + "workflow_streams", + provider, + client, + encodes_bodies=False, + host=host, + task_queue=worker.task_queue, + truncate=truncate, + # Every log is a running workflow's state; a stream with no owner + # has no workflow to live in, so neither of its policies exists. + hosts_standalone_streams=False, + bounds_standalone_bytes=False, + trims_open_stream_by_age=False, + refuses_appends_past_byte_cap=False, + ) + for handle in hosts.values(): + await handle.terminate() + + +async def _native_case(client: Client) -> AsyncIterator[ProviderCase]: + # The store is a server built from the stream-carrying branch, which the + # test environment's own server is not; TEMPORAL_ADDRESS names it. + # Without it the cases would reach a server that has no stream service and + # fail on the wire, which says nothing about the provider. + address = os.environ.get("TEMPORAL_ADDRESS") + if not address: + pytest.skip( + "STREAMS_LIVE=native needs TEMPORAL_ADDRESS naming a server with the " + "stream service" + ) + client = await Client.connect( + address, namespace=os.environ.get("TEMPORAL_NAMESPACE", "default") + ) + provider = NativeStreams() + # Registered once, on the client: the host's worker inherits it and the + # cases open handles through client.get_stream_handle. + config = client.config() + config["plugins"] = [provider] + client = Client(**config) + hosts: dict[str, WorkflowHandle[Any, Any]] = {} + async with new_worker(client, StreamHost, DefaultTopicAnswer) as worker: + + async def host(workflow_id: str) -> None: + if workflow_id not in hosts: + hosts[workflow_id] = await client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + + # No truncate hook: a stream a workflow owns has no truncation call. + # BEGINNING on a truncated stream is covered on the stream client, + # whose standalone streams can be truncated. + yield ProviderCase( + "native", + provider, + client, + host=host, + task_queue=worker.task_queue, + waits_for_standalone_creation=True, + refuses_appends_past_byte_cap=True, + ) + for handle in hosts.values(): + await handle.terminate() + await provider.close() + + +async def _redis_case(client: Client) -> AsyncIterator[ProviderCase]: + # The store is a Redis the test environment does not start; the server + # is the environment's own unless TEMPORAL_ADDRESS names another. + address = os.environ.get("TEMPORAL_ADDRESS") + if address: + client = await Client.connect( + address, namespace=os.environ.get("TEMPORAL_NAMESPACE", "default") + ) + provider = RedisStreams( + url=os.environ.get("TEMPORAL_TEST_REDIS_URL") + or os.environ.get("AI198_REDIS_URL", "redis://127.0.0.1:6379"), + # A prefix per setup, because the store keeps what earlier runs wrote. + key_prefix=f"streams-conformance-{uuid.uuid4().hex}", + ) + # Registered once, on the client: the host's worker inherits it and the + # cases open handles through client.get_stream_handle. + config = client.config() + config["plugins"] = [provider] + client = Client(**config) + hosts: dict[str, WorkflowHandle[Any, Any]] = {} + async with new_worker(client, StreamHost, DefaultTopicAnswer) as worker: + + async def host(workflow_id: str) -> None: + if workflow_id not in hosts: + hosts[workflow_id] = await client.start_workflow( + StreamHost.run, id=workflow_id, task_queue=worker.task_queue + ) + + yield ProviderCase( + "redis", + provider, + client, + host=host, + task_queue=worker.task_queue, + # The refusal of a trimmed cursor lands on the first step on this + # provider, where the case wants it at the call; its own live module + # covers the trimmed floor. + truncate=None, + # A standalone stream's append script keeps a byte total per topic + # and trims by age on every append, so both bounds hold while the + # stream is open. + bounds_standalone_bytes=True, + trims_open_stream_by_age=True, + # An outside append notifies the stream's channel, addressed to + # the workflow that owns the stream; the reader's run is + # subscribed to it when the task that opened the read ends where + # the server has no linked kind, and listens by construction + # where it has. + wakes_by_notification=True, + wakes_by_linked_notification=True, + ) + for handle in hosts.values(): + await handle.terminate() + await provider.close() + + +SETUPS: dict[str, Callable[[Client], AsyncIterator[ProviderCase]]] = { + "memory": _memory_case, + "workflow_streams": _workflow_streams_case, +} +if os.environ.get("STREAMS_LIVE") == "native": + SETUPS["native"] = _native_case +if os.environ.get("STREAMS_LIVE") == "redis": + SETUPS["redis"] = _redis_case + +_CAPABILITIES = { + "reports_positions": lambda case: case.reports_positions, + "detects_divergent_retries": lambda case: case.detects_divergent_retries, + "encodes_bodies": lambda case: case.encodes_bodies, + "truncates": lambda case: case.truncate is not None, + "hosts_standalone_streams": lambda case: case.hosts_standalone_streams, + "wakes_by_notification": lambda case: case.wakes_by_notification, + "wakes_by_linked_notification": lambda case: case.wakes_by_linked_notification, +} + + +@pytest.fixture(params=sorted(SETUPS)) +async def case( + request: pytest.FixtureRequest, client: Client +) -> AsyncIterator[ProviderCase]: + async for provider_case in SETUPS[request.param](client): + for marker, supported in _CAPABILITIES.items(): + if request.node.get_closest_marker(marker) and not supported(provider_case): + pytest.skip(f"the {provider_case.name} provider does not {marker}") + yield provider_case + + +async def test_the_native_setup_skips_without_a_server_address( + monkeypatch: pytest.MonkeyPatch, client: Client +): + monkeypatch.delenv("TEMPORAL_ADDRESS", raising=False) + setup = _native_case(client) + with pytest.raises(pytest.skip.Exception, match="TEMPORAL_ADDRESS"): + await setup.__anext__() + + +def new_workflow_id() -> str: + # Unique per case, because a storage provider keeps what earlier cases + # wrote and the memory provider only happens to forget. + return f"wf-{uuid.uuid4().hex}" + + +async def take(records: Any, count: int, timeout: float = 5.0) -> list: + out: list = [] + + async def _collect() -> None: + async for record in records: + out.append(record) + if len(out) >= count: + return + + await asyncio.wait_for(_collect(), timeout) + return out + + +async def drain(records: Any, timeout: float = 5.0) -> list: + """Every record until the read ends on its own.""" + + async def _collect() -> list: + return [record async for record in records] + + return await asyncio.wait_for(_collect(), timeout) + + +def new_stream_id() -> str: + return f"stream-{uuid.uuid4().hex}" + + +async def test_append_read_roundtrip(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + assert (producer.producer_id, producer.attempt) == ("model", 1) + await producer.append({"id": "r1"}, {"id": "r2"}) + await producer.finish() + + records = await take(stream.read(topic=OUT), 3) + assert [r.kind for r in records] == [ + RecordKind.DATA, + RecordKind.DATA, + RecordKind.FINISH, + ] + assert [r.value for r in records[:2]] == [{"id": "r1"}, {"id": "r2"}] + assert records[2].value is None + assert all(r.producer_id == "model" and r.attempt == 1 for r in records) + assert [r.sequence for r in records] == [1, 2, 3] + assert all(r.topic == OUT.name for r in records) + + +async def test_raw_values_pass_through_untouched(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + payload = Payload(metadata={"encoding": b"binary/plain"}, data=b"\x00\x01raw") + producer = stream.producer(topic=OUT.name, producer_id="model", attempt=1) + await producer.append(RawValue(payload)) + + records = await take(stream.read(topic="out", result_type=RawValue), 1) + assert isinstance(records[0].value, RawValue) + assert records[0].value.payload == payload + + +@pytest.mark.reports_positions +async def test_retried_append_returns_the_original_position(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + first = stream.producer(topic=OUT, producer_id="model", attempt=1) + landed = await first.append({"id": "r1"}) + assert landed is not None + # The retry of the same attempt starts its sequence over and appends the + # same record. The provider stores it once and answers with where the + # original landed, so the retry can checkpoint the same position. + retry = stream.producer(topic=OUT, producer_id="model", attempt=1) + assert await retry.append({"id": "r1"}) == landed + # An empty call writes nothing and answers the same way. + assert await retry.append() == landed + + records = await take(stream.read(topic=OUT), 1) + assert records[0].value == {"id": "r1"} + assert records[0].cursor == landed + # The store holds exactly the one record: the newest position is its cursor. + assert await stream.latest(topic=OUT) == landed + + +async def test_retried_append_is_stored_once(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + first = stream.producer(topic=OUT, producer_id="model", attempt=1) + await first.append({"id": "r1"}) + retry = stream.producer(topic=OUT, producer_id="model", attempt=1) + await retry.append({"id": "r1"}) + await retry.append({"id": "r2"}) + + records = await take(stream.read(topic=OUT), 2) + assert [r.value for r in records] == [{"id": "r1"}, {"id": "r2"}] + + +@pytest.mark.detects_divergent_retries +async def test_a_divergent_retry_is_refused(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + first = stream.producer(topic=OUT, producer_id="model", attempt=1) + await first.append({"id": "r1"}) + + # Same producer, attempt and sequence, different content. The store has no + # way to know which of the two the reader was meant to see, so it says so + # rather than answering with the position of the one it kept. + retry = stream.producer(topic=OUT, producer_id="model", attempt=1) + with pytest.raises(StreamProducerError): + await retry.append({"id": "other"}) + + # And it wrote nothing: the producer that owns the sequence carries on + # past the original, with no second record wedged in front of it. + await first.append({"id": "r2"}) + records = await take(stream.read(topic=OUT), 2) + assert [r.value for r in records] == [{"id": "r1"}, {"id": "r2"}] + + +async def test_closing_a_read_early_releases_it(case: ProviderCase): + # A read with nothing left to hand over waits against the store. Closing + # the generator is how a caller that stops early says so, and it has to + # let go of whatever it parked instead of hanging on it. + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + await producer.append({"n": 1}) + + records = stream.read(topic=OUT) + assert (await asyncio.wait_for(records.__anext__(), 5.0)).value == {"n": 1} + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(records.__anext__(), 0.5) + await asyncio.wait_for(records.aclose(), 5.0) + + # The topic is untouched by the close: a new read still sees everything. + await producer.append({"n": 2}) + again = await take(stream.read(topic=OUT), 2) + assert [r.value for r in again] == [{"n": 1}, {"n": 2}] + + +async def test_new_attempt_supersedes_the_old_one(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + first = stream.producer(topic=OUT, producer_id="model", attempt=1) + await first.append({"text": "The capital of"}) + second = stream.producer(topic=OUT, producer_id="model", attempt=2) + await second.append({"text": "Paris is the capital"}) + + records = await take(stream.read(topic=OUT), 3) + assert records[0].kind is RecordKind.DATA and records[0].attempt == 1 + assert records[1].kind is RecordKind.SUPERSEDED + assert records[1].supersession == Supersession("model", 1, 2) + assert records[1].value is None + assert records[2].kind is RecordKind.DATA and records[2].attempt == 2 + + +async def test_a_superseded_record_resumes_to_the_triggering_record( + case: ProviderCase, +): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + first = stream.producer(topic=OUT, producer_id="model", attempt=1) + await first.append({"n": 1}) + second = stream.producer(topic=OUT, producer_id="model", attempt=2) + await second.append({"n": 2}) + + records = await take(stream.read(topic=OUT), 3) + superseded = records[1] + assert superseded.kind is RecordKind.SUPERSEDED + # The synthesized record sits at the position before the new attempt's + # first record, so a consumer that checkpoints it and restarts is handed + # that record rather than skipping it. + assert superseded.cursor == records[0].cursor + resumed = await take(stream.read(topic=OUT, after=superseded.cursor), 1) + assert resumed[0].kind is RecordKind.DATA + assert resumed[0].value == {"n": 2} + + +async def test_topics_are_addressed_by_name(case: ProviderCase): + # Two producers on two topics of the same workflow's stream: each read + # names its topic and sees only that topic's records. + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + on_a = stream.producer(topic=A, producer_id="tool-a", attempt=1) + await on_a.append({"n": 1}) + on_b = stream.producer(topic=B, producer_id="tool-b", attempt=1) + await on_b.append({"n": 2}) + + only_a = await take(stream.read(topic=A), 1) + assert [(r.topic, r.value) for r in only_a] == [("a", {"n": 1})] + only_b = await take(stream.read(topic=B), 1) + assert [(r.topic, r.value) for r in only_b] == [("b", {"n": 2})] + + +async def test_naming_no_topic_addresses_the_default_topic(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + assert await stream.latest() == BEGINNING + producer = stream.producer(producer_id="model", attempt=1) + await producer.append({"n": 1}) + await stream.producer(topic=OUT, producer_id="model", attempt=1).append({"n": 2}) + + records = await take(stream.read(), 1) + assert [(r.topic, r.value) for r in records] == [(DEFAULT_TOPIC, {"n": 1})] + assert await stream.latest() == records[0].cursor + # The default is an ordinary name, so naming it is the same topic. + named = await take(stream.read(topic=DEFAULT_TOPIC, result_type=dict), 1) + assert [r.value for r in named] == [{"n": 1}] + assert await stream.latest(topic=DEFAULT_TOPIC) == records[0].cursor + + +async def test_a_workflow_answers_on_its_default_topic(case: ProviderCase): + if case.client is None or case.task_queue is None: + pytest.skip( + f"the {case.name} setup runs no worker; test_streams_workflow covers " + "its workflow half" + ) + workflow_id = new_workflow_id() + handle = await case.client.start_workflow( + DefaultTopicAnswer.run, id=workflow_id, task_queue=case.task_queue + ) + stream = case.client.get_stream_handle(workflow_id) + await stream.producer(producer_id="client", attempt=1).append({"n": 21}) + await handle.result() + + # The outside producer and the workflow, each naming no topic, meet on + # one topic that a reader naming none sees in order. + records = await take(stream.read(), 2, timeout=30.0) + assert [(r.topic, r.producer_id, r.value) for r in records] == [ + (DEFAULT_TOPIC, "client", {"n": 21}), + (DEFAULT_TOPIC, "", {"answer": 42}), + ] + + +async def test_cursor_resumes_where_it_points(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}) + + records = await take(stream.read(topic=OUT), 3) + checkpoint = records[0].cursor + + # Resuming after a record hands back everything past it and nothing + # twice, without the reader ever advancing a cursor itself. + again = await take(stream.read(topic=OUT, after=checkpoint), 2) + assert [r.value for r in again] == [{"n": 2}, {"n": 3}] + + +@pytest.mark.reports_positions +async def test_append_cursor_names_the_last_record_of_the_batch(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + appended = await producer.append({"n": 1}, {"n": 2}, {"n": 3}) + assert appended is not None + then = await producer.append({"n": 4}) + + # A producer that resumes a reader after its own append must see only + # what came later, not the tail of the batch it just wrote. + records = await take(stream.read(topic=OUT, after=appended), 1) + assert [r.value for r in records] == [{"n": 4}] + assert records[0].cursor == then + assert await producer.append() == then + + +async def test_latest_positions_a_reader_at_the_end(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + assert await stream.latest(topic=OUT) == BEGINNING + + await producer.append({"n": 1}, {"n": 2}) + since = await stream.latest(topic=OUT) + await producer.append({"n": 3}) + + # A reader that positioned itself before the last append sees only what + # came after, which is how a client follows a turn it is about to start. + records = await take(stream.read(topic=OUT, after=since), 1) + assert [r.value for r in records] == [{"n": 3}] + + +async def test_topic_addresses_with_colons_do_not_share_a_store(case: ProviderCase): + # ("wf:x", "y") and ("wf", "x:y") differ only in where the colon sits. + base = new_workflow_id() + left = await case.open(f"{base}:x") + right = await case.open(base) + await left.producer(topic=Y, producer_id="l", attempt=1).append({"side": "left"}) + await right.producer(topic=XY, producer_id="r", attempt=1).append({"side": "right"}) + + only_left = await take(left.read(topic=Y), 1) + assert [r.value for r in only_left] == [{"side": "left"}] + assert await left.latest(topic=Y) == only_left[0].cursor + only_right = await take(right.read(topic=XY), 1) + assert [r.value for r in only_right] == [{"side": "right"}] + assert await right.latest(topic=XY) == only_right[0].cursor + + +async def test_a_foreign_cursor_is_refused_at_the_call(case: ProviderCase): + stream = await case.open(new_workflow_id()) + # Refused by read() itself, not by the first iteration of its generator, + # so the caller's except clause is where the mistake surfaces. + with pytest.raises(StreamCursorError): + stream.read(topic=OUT, after=Cursor("elsewhere:42")) + + +async def test_argument_mistakes_are_value_errors(case: ProviderCase): + stream = await case.open(new_workflow_id()) + with pytest.raises(ValueError): + stream.read(topic="") + with pytest.raises(ValueError): + stream.producer(topic="", producer_id="model", attempt=1) + # Outside an activity there is no identity to fall back on. + with pytest.raises(ValueError, match="producer_id is required"): + stream.producer(topic=OUT) + + +async def test_a_definition_carries_its_type_once(case: ProviderCase): + stream = await case.open(new_workflow_id()) + with pytest.raises(ValueError, match="already carries its type"): + stream.read(topic=OUT, result_type=dict) # type: ignore[call-overload] + with pytest.raises(ValueError): + topic("", dict) + # A string names a topic decided at runtime, and the hint rides the call. + assert await stream.latest(topic=OUT.name) == BEGINNING + + +async def test_last_n_starts_at_the_newest_records(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}) + + newest = await take(stream.read(topic=OUT, last=2), 2) + assert [r.value for r in newest] == [{"n": 3}, {"n": 4}] + # Fewer records than asked for is all of them, not an error. + everything = await take(stream.read(topic=OUT, last=100), 4) + assert [r.value for r in everything] == [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}] + # The cursors it yields are ordinary cursors, so a resume after one works. + again = await take(stream.read(topic=OUT, after=newest[0].cursor), 1) + assert [r.value for r in again] == [{"n": 4}] + + +async def test_last_n_counts_finish_records(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + await producer.finish() + + records = await take(stream.read(topic=OUT, last=2), 2) + assert [(r.kind, r.value) for r in records] == [ + (RecordKind.DATA, {"n": 2}), + (RecordKind.FINISH, None), + ] + + +async def test_end_reads_only_what_arrives_after_the_read_starts(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + await producer.append({"n": "old"}, {"n": "old"}) + + records = stream.read(topic=OUT, after=END) + first = asyncio.ensure_future(records.__anext__()) + # END resolves when the read starts, and nothing says when that was, so + # appends keep coming until the reader takes one. + try: + for _ in range(100): + await producer.append({"n": "new"}) + done, _ = await asyncio.wait({first}, timeout=0.1) + if done: + break + record = await asyncio.wait_for(first, 5) + finally: + await records.aclose() + assert record.value == {"n": "new"} + + +@pytest.mark.truncates +async def test_beginning_starts_at_the_oldest_record_still_held(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}) + before = await take(stream.read(topic=OUT), 1) + assert case.truncate is not None + await case.truncate(workflow_id, OUT.name, 2) + + # BEGINNING is the oldest record retained, not offset zero, which a + # truncated stream no longer holds. + records = await take(stream.read(topic=OUT), 2) + assert [r.value for r in records] == [{"n": 3}, {"n": 4}] + newest = await take(stream.read(topic=OUT, last=3), 2) + assert [r.value for r in newest] == [{"n": 3}, {"n": 4}] + # A cursor below the floor is refused. A provider that needs a round trip + # to know says so on the first step rather than on the call. + with pytest.raises(StreamCursorError): + await take(stream.read(topic=OUT, after=before[0].cursor), 1) + + +async def test_a_read_start_names_one_place(case: ProviderCase): + stream = await case.open(new_workflow_id()) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + appended = await producer.append({"n": 1}) + for last in (0, -1, True): + with pytest.raises(ValueError, match="positive"): + stream.read(topic=OUT, last=last) + if appended is not None: + with pytest.raises(ValueError, match="either after= or last="): + stream.read(topic=OUT, after=appended, last=1) + with pytest.raises(ValueError, match="either after= or last="): + stream.read(topic=OUT, after=END, last=1) + + +async def test_a_ref_names_the_stream_and_round_trips_as_data(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + ref = stream.ref(topic=OUT) + assert ref == StreamRef.for_workflow(workflow_id, topic="out") + assert (ref.kind, ref.run_id, ref.activity_id, ref.stream_id) == ( + "workflow", + None, + None, + None, + ) + # Without a topic the ref names the default topic, like every other call. + assert stream.ref().topic == DEFAULT_TOPIC + assert stream.ref().with_topic(A) == stream.ref(topic=A) + # A pinned handle hands out a pinned ref. + pinned = await case.open(workflow_id, run_id="run-1") + assert pinned.ref(topic=OUT).run_id == "run-1" + + # Plain data through the default converter, so it can be a workflow + # argument, an activity result or a Nexus operation input or result. + converter = DataConverter.default + [carried] = await converter.decode(await converter.encode([ref]), [StreamRef]) + assert carried == ref + + +async def test_an_owned_stream_cannot_be_closed_by_a_handle(case: ProviderCase): + # A workflow's stream ends with the workflow; close() is for a stream + # that stands alone. + stream = await case.open(new_workflow_id()) + with pytest.raises(ValueError, match="standalone"): + await stream.close() + + +@pytest.mark.encodes_bodies +async def test_a_body_above_the_threshold_is_offloaded_and_read_back( + case: ProviderCase, client: Client +): + driver = RecordingDriver() + converter = dataclasses.replace( + DataConverter.default, + external_storage=ExternalStorage(drivers=[driver], payload_size_threshold=256), + ) + workflow_id = new_workflow_id() + stream = await case.open( + workflow_id, client=_client_with(case.client or client, converter) + ) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + small = {"n": 1} + large = {"blob": "x" * 1024} + await producer.append(small) + await producer.append(large) + # Only the body over the threshold left the record; the small one stayed + # inline, as it would on any other payload the SDK sends. + assert driver.stored == 1 + + records = await take(stream.read(topic=OUT), 2) + assert [r.value for r in records] == [small, large] + assert driver.retrieved == 1 + + +@pytest.mark.detects_divergent_retries +@pytest.mark.encodes_bodies +async def test_a_retry_through_a_nondeterministic_codec_still_deduplicates( + case: ProviderCase, client: Client +): + codec = NonceCodec() + converter = dataclasses.replace(DataConverter.default, payload_codec=codec) + workflow_id = new_workflow_id() + stream = await case.open( + workflow_id, client=_client_with(case.client or client, converter) + ) + first = stream.producer(topic=OUT, producer_id="model", attempt=1) + landed = await first.append({"id": "r1"}) + assert codec.encoded == 1 + + # The codec produced different bytes for the retry. The provider matched + # it by the plaintext it converted, so it is the same append: stored + # once, answered with the original position. + retry = stream.producer(topic=OUT, producer_id="model", attempt=1) + again = await retry.append({"id": "r1"}) + if landed is not None: + assert again == landed + # And a retry that really does differ is still told apart. + divergent = stream.producer(topic=OUT, producer_id="model", attempt=1) + with pytest.raises(StreamProducerError): + await divergent.append({"id": "other"}) + + await first.append({"id": "r2"}) + records = await take(stream.read(topic=OUT), 2) + assert [r.value for r in records] == [{"id": "r1"}, {"id": "r2"}] + + +async def test_a_ref_opens_the_stream_it_names(case: ProviderCase): + workflow_id = new_workflow_id() + stream = await case.open(workflow_id) + await stream.producer(topic=A, producer_id="model", attempt=1).append({"n": 1}) + ref = stream.ref(topic=A) + + # The receiver names no topic: the ref carried it, so every call on the + # handle it opened addresses topic ``a`` of that workflow. + opened = await case.open_ref(ref) + records = await take(opened.read(result_type=dict), 1) + assert [(r.topic, r.value) for r in records] == [("a", {"n": 1})] + assert await opened.latest() == records[0].cursor + assert opened.ref() == ref + await opened.producer(producer_id="tool", attempt=1).append({"n": 2}) + assert [r.value for r in await take(stream.read(topic=A), 2)] == [ + {"n": 1}, + {"n": 2}, + ] + # Naming a topic on the opened handle addresses that topic instead. + assert await opened.latest(topic=B) == BEGINNING + assert opened.ref(topic=B) == stream.ref(topic=B) + + +@pytest.mark.hosts_standalone_streams +async def test_a_standalone_stream_is_read_from_another_handle(case: ProviderCase): + stream_id = new_stream_id() + created = await case.create_stream(stream_id) + producer = created.producer(topic=OUT, producer_id="writer", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + + # Any process reaches the stream by its id, or by a ref the creator + # handed out; nothing about the stream depends on who created it. + other = await case.open_standalone(stream_id) + records = await take(other.read(topic=OUT), 2) + assert [r.value for r in records] == [{"n": 1}, {"n": 2}] + assert await other.latest(topic=OUT) == records[1].cursor + assert other.ref(topic=OUT) == StreamRef.for_standalone(stream_id, topic="out") + via_ref = await case.open_ref(created.ref(topic=OUT)) + assert [r.value for r in await take(via_ref.read(), 2)] == [{"n": 1}, {"n": 2}] + + +@pytest.mark.hosts_standalone_streams +async def test_a_missing_standalone_stream_is_not_found(case: ProviderCase): + if case.waits_for_standalone_creation: + pytest.skip( + f"the {case.name} provider waits for a standalone stream to be created" + ) + # get_stream_handle(stream_id=) creates nothing: the stream has to have + # been created on purpose, and a use before that says so. + stream = await case.open_standalone(new_stream_id()) + with pytest.raises(StreamNotFoundError): + await stream.latest(topic=OUT) + with pytest.raises(StreamNotFoundError): + await take(stream.read(topic=OUT), 1) + with pytest.raises(StreamNotFoundError): + await stream.producer(topic=OUT, producer_id="writer", attempt=1).append( + {"n": 1} + ) + + +@pytest.mark.hosts_standalone_streams +async def test_creating_a_standalone_stream_is_idempotent_for_one_policy( + case: ProviderCase, +): + stream_id = new_stream_id() + first = await case.create_stream(stream_id, max_records=10) + # The same id and policy again is the same stream, not an error, so a + # retried create is harmless. + again = await case.create_stream(stream_id, max_records=10) + await first.producer(topic=OUT, producer_id="writer", attempt=1).append({"n": 1}) + assert [r.value for r in await take(again.read(topic=OUT), 1)] == [{"n": 1}] + # A different policy on an existing id is a mistake, not a change. + with pytest.raises(ValueError): + await case.create_stream(stream_id, max_records=5) + for bad in (dict(max_records=0), dict(max_bytes=-1), dict(retention=timedelta(0))): + with pytest.raises(ValueError): + await case.create_stream(new_stream_id(), **bad) # type: ignore[arg-type] + + +@pytest.mark.hosts_standalone_streams +async def test_closing_a_standalone_stream_ends_reads_and_refuses_appends( + case: ProviderCase, +): + stream_id = new_stream_id() + stream = await case.create_stream(stream_id) + producer = stream.producer(topic=OUT, producer_id="writer", attempt=1) + await producer.append({"n": 1}) + # A reader parked on the tail before the close has to learn of it. + other = await case.open_standalone(stream_id) + parked = asyncio.ensure_future(drain(other.read(topic=OUT), timeout=10)) + await asyncio.sleep(0.2) + await producer.append({"n": 2}) + + await stream.close() + assert [r.value for r in await parked] == [{"n": 1}, {"n": 2}] + # Sealed: the tail stays readable, and a read opened now ends by itself. + assert [r.value for r in await drain(stream.read(topic=OUT))] == [ + {"n": 1}, + {"n": 2}, + ] + with pytest.raises(StreamClosedError): + await producer.append({"n": 3}) + with pytest.raises(StreamClosedError): + await other.producer(topic=A, producer_id="late", attempt=1).append({"n": 3}) + # Closing again is not an error. + await stream.close() + + +@pytest.mark.hosts_standalone_streams +async def test_a_standalone_stream_honors_its_retention_policy(case: ProviderCase): + by_count = await case.create_stream(new_stream_id(), max_records=2) + producer = by_count.producer(topic=OUT, producer_id="writer", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}) + # BEGINNING is the oldest record still held, which the policy decided. + kept = await take(by_count.read(topic=OUT), 2) + assert [r.value for r in kept] == [{"n": 3}, {"n": 4}] + + if case.bounds_standalone_bytes and case.refuses_appends_past_byte_cap: + # The cap counts the stored record, body and metadata included, so it + # holds one of these and not two. + by_bytes = await case.create_stream(new_stream_id(), max_bytes=1000) + producer = by_bytes.producer(topic=OUT, producer_id="writer", attempt=1) + await producer.append({"n": 0, "blob": "x" * 500}) + # The second record would take the stream past its cap, so the store + # refuses it and keeps what it holds. + with pytest.raises((RPCError, StreamError)): + await producer.append({"n": 1, "blob": "x" * 500}) + kept = await take(by_bytes.read(topic=OUT), 1) + assert kept[0].value is not None and kept[0].value["n"] == 0 + elif case.bounds_standalone_bytes: + by_bytes = await case.create_stream(new_stream_id(), max_bytes=700) + producer = by_bytes.producer(topic=OUT, producer_id="writer", attempt=1) + for n in range(3): + await producer.append({"n": n, "blob": "x" * 500}) + kept = await take(by_bytes.read(topic=OUT), 1) + assert kept[0].value is not None and kept[0].value["n"] == 2 + else: + with pytest.raises(StreamUnsupportedError): + await case.create_stream(new_stream_id(), max_bytes=700) + + if not case.trims_open_stream_by_age: + return + by_age = await case.create_stream( + new_stream_id(), retention=timedelta(milliseconds=200) + ) + producer = by_age.producer(topic=OUT, producer_id="writer", attempt=1) + await producer.append({"n": "old"}) + await asyncio.sleep(0.3) + await producer.append({"n": "new"}) + # A store reclaims by age on its own schedule, and a coarse one may have + # aged the newer record out as well by the time it is looked at. What the + # policy decides is that the older record is no longer where BEGINNING + # starts. + kept = [] + for _ in range(50): + try: + kept = await take(by_age.read(topic=OUT), 1, timeout=1) + except asyncio.TimeoutError: + kept = [] + if not kept or kept[0].value == {"n": "new"}: + break + await asyncio.sleep(0.2) + assert [r.value for r in kept] in ([{"n": "new"}], []) + + +@workflow.defn +class PublishFromConstructor: + """Publishes once from its ``@workflow.init`` constructor and once from ``run``.""" + + @workflow.init + def __init__(self) -> None: + workflow.stream_writer(OUT).publish({"from": "init"}) + + @workflow.run + async def run(self) -> None: + workflow.stream_writer(OUT).publish({"from": "run"}) + + +async def test_a_publish_from_the_constructor_is_delivered( + case: ProviderCase, client: Client, env: WorkflowEnvironment +): + if env.supports_time_skipping and case.client is None: + pytest.skip("the memory provider polls on a timer, which time skipping spins") + # A storage provider's setup registers it on its client, which a worker + # inherits; the memory provider is handed to the worker directly. + worker_client = case.client or client + plugins = [] if case.client is not None else [case.provider] + workflow_id = new_workflow_id() + async with new_worker( + worker_client, PublishFromConstructor, plugins=plugins + ) as worker: + handle = await worker_client.start_workflow( + PublishFromConstructor.run, id=workflow_id, task_queue=worker.task_queue + ) + await asyncio.wait_for(handle.result(), 30) + # Read while the worker still polls: the workflow_streams provider + # answers reads from the workflow itself. + stream = case.provider.get_stream_handle(worker_client, workflow_id) + records = await take(stream.read(topic=OUT), 2, 30) + assert [r.value for r in records] == [{"from": "init"}, {"from": "run"}] + + +@workflow.defn +class ReadUntilFinished: + """Reads ``OUT`` until its producer finishes. + + Nothing but an outside append moves it, so what wakes it between appends + is the transport under test. + """ + + @workflow.run + async def run(self) -> list[Any]: + seen: list[Any] = [] + async for record in workflow.stream_reader(OUT): + if record.kind is RecordKind.FINISH: + break + seen.append(record.value) + return seen + + +@pytest.mark.wakes_by_notification +@pytest.mark.needs_channel_server +async def test_an_outside_producer_wakes_the_reader_through_the_channel( + case: ProviderCase, client: Client +): + """The channel path, on the public surface. + + The reader's run is subscribed to the stream's channel on the completion + that ends the task that opened the reader, after that task's marker, so + the task stays retained and parks as it would on a server without + channels. The producer's append notifies the channel, and the server + wakes the run with a Workflow Task whose scheduled event carries the + notification. History then holds the subscription and no Signal. + """ + worker_client = case.client or client + support = await server_channel_support(worker_client) + if support is ChannelSupport.NONE: + pytest.skip("the server does not implement notification channels") + if support is ChannelSupport.LINKED: + pytest.skip("a workflow-owned stream listens on its linked channel there") + handle = await _read_two_woken_from_outside(case, worker_client) + events = [e async for e in handle.fetch_history_events()] + assert _signalled(events) == [], ( + "a Signal woke the reader, so the channel was not the transport" + ) + subscribed = _subscribed(events) + assert len(subscribed) == 1, "the run subscribes once per channel" + assert _preceded_by_a_marker(events, _subscribed_event_index(events)), ( + "the subscription did not wait for the completion that ends the task" + ) + notified = _notified(events) + assert notified, "no Workflow Task was scheduled with a notification" + assert {n.channel for n in notified} == set(subscribed) + assert not any(n.HasField("linked_to") for n in notified) + + +@pytest.mark.wakes_by_linked_notification +@pytest.mark.needs_linked_server +async def test_an_outside_producer_wakes_the_reader_through_its_linked_channel( + case: ProviderCase, client: Client +): + """The linked kind, on the public surface. + + The stream's channel lives in the reading workflow's own state, so the run + subscribes to nothing; the producer's append notifies the channel by the + owner's id, and the server wakes the owner with a Workflow Task whose + scheduled event carries the notification naming it. History then holds + neither a Signal nor a subscription. + """ + worker_client = case.client or client + if await server_channel_support(worker_client) is not ChannelSupport.LINKED: + pytest.skip("the server does not serve channels linked to a workflow") + handle = await _read_two_woken_from_outside(case, worker_client) + events = [e async for e in handle.fetch_history_events()] + assert _signalled(events) == [], ( + "a Signal woke the reader, so the channel was not the transport" + ) + assert _subscribed(events) == [], "the owner is the listener by construction" + notified = _notified(events) + assert notified, "no Workflow Task was scheduled with a notification" + owners = {Execution.from_proto(n.linked_to) for n in notified} + assert {(owner.type, owner.business_id) for owner in owners} == { + (ExecutionType.WORKFLOW, handle.id) + } + assert len({n.channel for n in notified}) == 1 + + +@workflow.defn +class ReadTwoThenClose: + """Reads two values of ``OUT``, closes the reader short of ``FINISH``, waits. + + The timer after the close is what makes the close leave on a completion + the run survives: the channel has to leave on that completion, not with + the run. + """ + + @workflow.run + async def run(self) -> list[Any]: + reader = workflow.stream_reader(OUT) + seen: list[Any] = [] + async for record in reader: + seen.append(record.value) + if len(seen) == 2: + break + reader.close() + await workflow.sleep(1) + return seen + + +@workflow.defn +class ReadUntilFinishedThenClose: + """Reads ``OUT`` to its producer's ``FINISH``, closes the reader there, waits.""" + + @workflow.run + async def run(self) -> list[Any]: + reader = workflow.stream_reader(OUT) + seen: list[Any] = [] + async for record in reader: + if record.kind is RecordKind.FINISH: + break + seen.append(record.value) + reader.close() + await workflow.sleep(1) + return seen + + +@workflow.defn +class OpenAndCloseBesideTheRead: + """Opens and closes a reader on ``A`` in the task that opens the ``OUT`` reader.""" + + @workflow.run + async def run(self) -> list[Any]: + workflow.stream_reader(A).close() + seen: list[Any] = [] + async for record in workflow.stream_reader(OUT): + if record.kind is RecordKind.FINISH: + break + seen.append(record.value) + return seen + + +async def _independent_channels_or_skip(worker_client: Client) -> None: + support = await server_channel_support(worker_client) + if support is ChannelSupport.NONE: + pytest.skip("the server does not implement notification channels") + if support is ChannelSupport.LINKED: + pytest.skip("a workflow-owned stream listens on its linked channel there") + + +@pytest.mark.wakes_by_notification +@pytest.mark.needs_unsubscribe_server +async def test_a_reader_closed_short_of_finish_leaves_its_channel( + case: ProviderCase, client: Client +): + """Closing the reader ends the run's subscription on the completion that + leaves, after the progress marker and before the workflow's own command. + + The producer finishes only after the run has returned, so the reader + never saw ``FINISH``; it left because the workflow closed it. + """ + worker_client = case.client or client + await _independent_channels_or_skip(worker_client) + handle = await _read_two_woken_from_outside( + case, worker_client, ReadTwoThenClose, finish_before_result=False + ) + _assert_the_channel_left_after_the_marker( + [e async for e in handle.fetch_history_events()] + ) + + +@pytest.mark.wakes_by_notification +@pytest.mark.needs_unsubscribe_server +async def test_a_reader_closed_at_finish_leaves_its_channel( + case: ProviderCase, client: Client +): + """The same leaving, on the completion that consumed the producer's ``FINISH``.""" + worker_client = case.client or client + await _independent_channels_or_skip(worker_client) + handle = await _read_two_woken_from_outside( + case, worker_client, ReadUntilFinishedThenClose + ) + _assert_the_channel_left_after_the_marker( + [e async for e in handle.fetch_history_events()] + ) + + +@pytest.mark.wakes_by_notification +@pytest.mark.needs_unsubscribe_server +async def test_a_channel_opened_and_closed_in_one_task_is_never_subscribed( + case: ProviderCase, client: Client +): + """Only the latest report of a task counts, so a reader that came and went + inside it costs the server nothing: no subscription, no unsubscription.""" + worker_client = case.client or client + await _independent_channels_or_skip(worker_client) + handle = await _read_two_woken_from_outside( + case, worker_client, OpenAndCloseBesideTheRead + ) + events = [e async for e in handle.fetch_history_events()] + subscribed = _subscribed(events) + assert len(subscribed) == 1, "only the reader that stayed open subscribes" + assert {n.channel for n in _notified(events)} == set(subscribed) + assert _unsubscribed(events) == [], "the run ended with its reader open" + + +def _subscribed_event_index(events: Sequence[Any]) -> int: + [index] = [ + i + for i, e in enumerate(events) + if e.HasField("workflow_notification_channel_subscribed_event_attributes") + ] + return index + + +def _preceded_by_a_marker(events: Sequence[Any], index: int) -> bool: + """Whether the event at ``index`` follows the progress marker of its completion. + + Core issues the channel commands after the external stream marker, so an + event right after a marker landed on the completion that ended a task + rather than on a task of its own. + """ + return events[index - 1].HasField("marker_recorded_event_attributes") + + +def _assert_the_channel_left_after_the_marker(events: Sequence[Any]) -> None: + [channel] = _subscribed(events) + assert _unsubscribed(events) == [channel], "the channel leaves once" + [(index, leaving)] = [ + (i, e) + for i, e in enumerate(events) + if e.HasField("workflow_notification_channel_unsubscribed_event_attributes") + ] + joined = events[_subscribed_event_index(events)] + attributes = leaving.workflow_notification_channel_unsubscribed_event_attributes + assert attributes.subscribed_event_id == joined.event_id + assert _preceded_by_a_marker(events, index), ( + "the unsubscribe follows the progress marker of the leaving completion" + ) + assert events[index + 1].HasField("timer_started_event_attributes"), ( + "the leaving completion carried the workflow's own command after it" + ) + assert not any( + e.workflow_task_scheduled_event_attributes.notifications + for e in events[index + 1 :] + if e.HasField("workflow_task_scheduled_event_attributes") + ), "a notification reached the run after it left the channel" + + +async def _read_two_woken_from_outside( + case: ProviderCase, + worker_client: Client, + workflow_class: Any = ReadUntilFinished, + *, + finish_before_result: bool = True, +): + """Runs a reader with two appends spaced past its idle timeout. + + ``finish_before_result`` says whether the producer's ``FINISH`` is what + lets the workflow return, or is written only once it has. + """ + plugins = [] if case.client is not None else [case.provider] + workflow_id = new_workflow_id() + async with new_worker(worker_client, workflow_class, plugins=plugins) as worker: + handle = await worker_client.start_workflow( + workflow_class.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = case.provider.get_stream_handle(worker_client, workflow_id) + producer = stream.producer(topic=OUT, producer_id="model", attempt=1) + # Spaced past the reader's idle timeout, so the reader parks between + # appends and only a wake from outside can move it. + for n in (1, 2): + await producer.append({"n": n}) + await asyncio.sleep(2) + if finish_before_result: + await producer.finish() + assert await asyncio.wait_for(handle.result(), 60) == [{"n": 1}, {"n": 2}] + if not finish_before_result: + await producer.finish() + return handle + + +def _signalled(events: Sequence[Any]) -> list[Any]: + return [ + e for e in events if e.HasField("workflow_execution_signaled_event_attributes") + ] + + +def _subscribed(events: Sequence[Any]) -> list[str]: + return [ + e.workflow_notification_channel_subscribed_event_attributes.channel + for e in events + if e.HasField("workflow_notification_channel_subscribed_event_attributes") + ] + + +def _unsubscribed(events: Sequence[Any]) -> list[str]: + return [ + e.workflow_notification_channel_unsubscribed_event_attributes.channel + for e in events + if e.HasField("workflow_notification_channel_unsubscribed_event_attributes") + ] + + +def _notified(events: Sequence[Any]) -> list[Any]: + return [ + notification + for e in events + if e.HasField("workflow_task_scheduled_event_attributes") + for notification in e.workflow_task_scheduled_event_attributes.notifications + ] diff --git a/tests/streams/test_streams_internals.py b/tests/streams/test_streams_internals.py new file mode 100644 index 0000000000..52020fb41b --- /dev/null +++ b/tests/streams/test_streams_internals.py @@ -0,0 +1,299 @@ +"""Unit tests for the pieces under ``temporalio.streams`` that no provider owns. + +The wire format, the supersession policy, the store key, the cursor prefix and +the plugin registration are shared by every provider and implemented once, so +they are tested once, here, against the private modules. What a provider owes +is in ``test_streams_conformance``; keeping the two apart is what makes that +file answerable by a new provider. +""" + +from __future__ import annotations + +import dataclasses +import os +from collections.abc import Sequence + +import pytest + +from temporalio.api.common.v1 import Payload +from temporalio.client import ClientConfig +from temporalio.converter import ( + DataConverter, + ExternalStorage, + PayloadCodec, + StorageDriver, + StorageDriverClaim, + StorageDriverRetrieveContext, + StorageDriverStoreContext, +) +from temporalio.streams import ( + BEGINNING, + CONTENT_HASH_KEY, + Cursor, + RecordKind, + StreamCursorError, + StreamRef, + Supersession, + _ids, + _wire, + content_fingerprint, + content_hash, + decode_body, + encode_body, +) +from temporalio.streams._policy import AttemptTracker +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.worker import ReplayerConfig, WorkerConfig + + +def test_record_roundtrips_through_the_wire(): + converter = DataConverter.default.payload_converter + wire = _wire.to_wire( + converter, + topic="decisions", + kind=RecordKind.DATA, + value={"n": 1}, + producer_id="model", + attempt=3, + sequence=7, + ) + parsed = _wire.WireRecord.FromString(wire.SerializeToString()) + record = _wire.from_wire(converter, Cursor("memory:0"), parsed, dict) + assert ( + record.kind, + record.topic, + record.producer_id, + record.attempt, + record.sequence, + record.value, + ) == (RecordKind.DATA, "decisions", "model", 3, 7, {"n": 1}) + assert record.supersession is None + finish = _wire.to_wire(converter, topic="decisions", kind=RecordKind.FINISH) + assert not finish.HasField("body") + assert _wire.from_wire(converter, Cursor("memory:1"), finish, dict).value is None + + +def test_a_stored_supersession_is_not_a_record(): + converter = DataConverter.default.payload_converter + wire = _wire.WireRecord(topic="t", kind=int(RecordKind.SUPERSEDED)) # type: ignore[arg-type] + with pytest.raises(ValueError, match="synthesized"): + _wire.from_wire(converter, Cursor("memory:0"), wire, None) + + +def test_an_unset_kind_is_read_as_data(): + converter = DataConverter.default.payload_converter + wire = _wire.WireRecord(topic="t", body=converter.to_payloads([{"n": 1}])[0]) + record = _wire.from_wire(converter, Cursor("memory:0"), wire, dict) + assert record.kind is RecordKind.DATA + assert record.value == {"n": 1} + + +def test_supersession_is_synthesized_from_observations(): + attempts = AttemptTracker() + assert attempts.note("model", 1, topic="t", previous=BEGINNING) is None + superseded = attempts.note("model", 2, topic="t", previous=Cursor("memory:0")) + assert superseded is not None + assert superseded.kind is RecordKind.SUPERSEDED + assert superseded.supersession == Supersession("model", 1, 2) + assert superseded.value is None + # Positioned before the triggering record, so a resume after it delivers + # that record next. + assert superseded.cursor == Cursor("memory:0") + # The same attempt again is not a new generation. + assert attempts.note("model", 2, topic="t", previous=Cursor("memory:1")) is None + + +def test_an_attempt_that_goes_backwards_is_said_rather_than_passed_off(): + # Attempts only rise on one producer, so a lower one means the store handed + # two generations back out of order. Yielded as data with no signal, a + # consumer renders the stale generation as the current answer. + said: list[str] = [] + attempts = AttemptTracker(said.append) + assert attempts.note("model", 2, topic="t", previous=BEGINNING) is None + assert attempts.note("model", 1, topic="t", previous=Cursor("memory:3")) is None + assert len(said) == 1 + assert "attempt 1" in said[0] and "behind attempt 2" in said[0] + assert "model" in said[0] + + +def test_a_repeat_of_the_current_attempt_is_not_worth_saying(): + said: list[str] = [] + attempts = AttemptTracker(said.append) + attempts.note("model", 1, topic="t", previous=BEGINNING) + assert attempts.note("model", 1, topic="t", previous=Cursor("memory:1")) is None + assert said == [] + + +def test_topic_keys_cannot_collide(): + # A colon in a workflow id must not make two addresses one key. + assert _ids.topic_key("a:b", "c") != _ids.topic_key("a", "b:c") + assert _ids.topic_key("a%3Ab", "c") != _ids.topic_key("a:b", "c") + assert _ids.topic_key("wf", "inputs") == "wf:inputs" + + +def test_cursors_name_their_provider(): + assert _wire.cursor_position(BEGINNING, provider="memory") is None + assert _wire.cursor_position(Cursor("memory:42"), provider="memory") == "42" + with pytest.raises(StreamCursorError): + _wire.cursor_position(Cursor("redis:1700000000000-0"), provider="memory") + + +def test_registering_a_provider_twice_is_refused(): + # There is one slot on each of the three, and a user who passes a provider + # by hand and a provider plugin, or two provider plugins, meant both. + first, second = MemoryStreams(), MemoryStreams() + with pytest.raises(ValueError, match="already registered"): + second.configure_client(ClientConfig(stream_provider=first)) # type: ignore[typeddict-item] + with pytest.raises(ValueError, match="already registered"): + second.configure_worker(WorkerConfig(stream_provider=first)) # type: ignore[typeddict-item] + with pytest.raises(ValueError, match="already registered"): + second.configure_replayer(ReplayerConfig(stream_provider=first)) # type: ignore[typeddict-item] + + +def test_registering_the_same_provider_twice_is_fine(): + # A worker built from a client that already carries the plugin configures + # it again with the same object, which is not a conflict. + provider = MemoryStreams() + config = provider.configure_client(ClientConfig(stream_provider=provider)) # type: ignore[typeddict-item] + assert config.get("stream_provider") is provider + assert provider.configure_client(ClientConfig()).get("stream_provider") is provider # type: ignore[typeddict-item] + + +class _HoldEverything(StorageDriver): + """A driver that keeps every payload it is handed, in memory.""" + + def __init__(self) -> None: + self.held: list[bytes] = [] + + def name(self) -> str: + return "hold" + + async def store( + self, context: StorageDriverStoreContext, payloads: Sequence[Payload] + ) -> list[StorageDriverClaim]: + claims = [] + for payload in payloads: + claims.append(StorageDriverClaim(claim_data={"i": str(len(self.held))})) + self.held.append(payload.SerializeToString()) + return claims + + async def retrieve( + self, + context: StorageDriverRetrieveContext, + claims: Sequence[StorageDriverClaim], + ) -> list[Payload]: + return [Payload.FromString(self.held[int(c.claim_data["i"])]) for c in claims] + + +class _NonceCodec(PayloadCodec): + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [ + Payload( + metadata={"encoding": b"binary/nonce"}, + data=os.urandom(8) + p.SerializeToString(), + ) + for p in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [Payload.FromString(p.data[8:]) for p in payloads] + + +async def test_encode_body_stamps_the_plaintext_hash_and_offloads_the_body(): + driver = _HoldEverything() + converter = dataclasses.replace( + DataConverter.default, + payload_codec=_NonceCodec(), + external_storage=ExternalStorage(drivers=[driver], payload_size_threshold=0), + ) + wire = _wire.to_wire( + converter.payload_converter, topic="t", kind=RecordKind.DATA, value={"n": 1} + ) + plaintext = Payload() + plaintext.CopyFrom(wire.body) + + await encode_body(converter, wire) + # The hash is over what the converter produced, not over what the codec + # or the driver made of it, and it rides the record where the store can + # read it without the plaintext. + stamped = wire.metadata[CONTENT_HASH_KEY] + assert stamped.metadata["encoding"] == b"binary/plain" + assert stamped.data.decode() == content_hash(plaintext) + assert len(stamped.data) == 64 + # With a threshold of zero the body was offloaded: the record holds the + # claim and the driver holds the coded payload. + assert wire.body != plaintext + assert len(wire.body.external_payloads) == 1 + assert len(driver.held) == 1 + + await decode_body(converter, wire) + assert wire.body == plaintext + assert wire.metadata[CONTENT_HASH_KEY] == stamped + + # A record without a body has nothing to hash or offload. + finish = _wire.to_wire( + converter.payload_converter, topic="t", kind=RecordKind.FINISH + ) + await encode_body(converter, finish) + assert CONTENT_HASH_KEY not in finish.metadata + assert len(driver.held) == 1 + + +async def test_content_fingerprint_is_taken_before_the_codec(): + converter = dataclasses.replace(DataConverter.default, payload_codec=_NonceCodec()) + plain = converter.payload_converter + + def batch(*values: dict) -> list[_wire.WireRecord]: + return [ + _wire.to_wire(plain, topic="t", kind=RecordKind.DATA, value=v, sequence=i) + for i, v in enumerate(values, 1) + ] + + first, retry = batch({"n": 1}, {"n": 2}), batch({"n": 1}, {"n": 2}) + before = content_fingerprint(first) + assert before == content_fingerprint(retry) + # Different content, and the same content split differently, both differ. + assert before != content_fingerprint(batch({"n": 1}, {"n": 3})) + assert before != content_fingerprint(batch({"n": 1}) + batch({"n": 2})) + + for record in first + retry: + await encode_body(converter, record) + # The codec made the two batches' bytes differ; the identity taken first + # is what lets a store still recognise the retry. + assert first[0].body != retry[0].body + assert content_fingerprint(first) != content_fingerprint(retry) + # Decoding gives the converted bodies back; the hash stays stamped on the + # record, which is why the identity is taken before encoding, not after. + for record in first: + await decode_body(converter, record) + assert [r.body for r in first] == [r.body for r in batch({"n": 1}, {"n": 2})] + assert all(CONTENT_HASH_KEY in r.metadata for r in first) + + +async def test_a_stream_ref_names_one_owner_and_travels_as_json(): + workflow = StreamRef.for_workflow("wf", run_id="r", topic="out") + activity = StreamRef.for_activity("act", workflow_id="wf", topic="progress") + standalone = StreamRef.for_standalone("shared") + assert workflow == StreamRef("workflow", "out", workflow_id="wf", run_id="r") + assert activity.kind == "activity" and activity.activity_id == "act" + assert standalone == StreamRef("standalone", "output", stream_id="shared") + assert standalone.with_topic("x").topic == "x" + + for bad in ( + dict(kind="workflow"), + dict(kind="workflow", workflow_id="wf", stream_id="s"), + dict(kind="activity", workflow_id="wf"), + dict(kind="standalone", stream_id="s", workflow_id="wf"), + dict(kind="standalone"), + dict(kind="nexus", stream_id="s"), + dict(kind="workflow", workflow_id="wf", topic=""), + ): + with pytest.raises(ValueError): + StreamRef(**bad) # type: ignore[arg-type] + + converter = DataConverter.default + for ref in (workflow, activity, standalone): + [carried] = await converter.decode(await converter.encode([ref]), [StreamRef]) + assert carried == ref + payload = (await converter.encode([standalone]))[0] + assert payload.metadata["encoding"] == b"json/plain" diff --git a/tests/streams/test_streams_workflow.py b/tests/streams/test_streams_workflow.py new file mode 100644 index 0000000000..01d0e05c56 --- /dev/null +++ b/tests/streams/test_streams_workflow.py @@ -0,0 +1,654 @@ +"""Workflow-side conformance for the stream contract. + +Runs the reader and writer inside a real workflow on the memory provider +with a warm cache, and states the two rules about Workflow Tasks as tests: a +publish commits with its task (rule 1), and reads are recorded observations +that replay re-supplies (rule 2). The memory provider keeps neither and says +so in its docstring, so those two are strict expected failures here. A +storage provider that runs this module turns them into passes; that is the +measurement they exist for. + +The rest is what the portable surface promises on every provider: the +lifecycle hooks the worker calls, one subscription per topic per run, a read +that ends when the chain closes, a handle that follows continue-as-new, and +a topic shared by the workflow and an outside producer. +""" + +from __future__ import annotations + +import asyncio +import uuid +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.client import Client +from temporalio.streams import ( + BEGINNING, + DEFAULT_TOPIC, + END, + Cursor, + ReadSource, + RecordKind, + StreamCursorError, + WriteSink, + topic, +) +from temporalio.streams.providers.memory import MemoryStreams +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Replayer +from tests.helpers import new_worker +from tests.streams.test_streams_conformance import take + +INPUTS = topic("inputs", dict) +DECISIONS = topic("decisions", dict) + + +@pytest.fixture +def provider(env: WorkflowEnvironment): # pyright: ignore[reportUnusedFunction] + if env.supports_time_skipping: + pytest.skip( + "the memory provider polls on a timer, which time skipping turns into a spin" + ) + streams = MemoryStreams() + yield streams + streams.reset() + + +@workflow.defn +class ContractLoop: + """Reads ``inputs``, publishes a decision per value, reports control records.""" + + @workflow.run + async def run(self) -> list[dict[str, Any]]: + inputs = workflow.stream_reader(INPUTS) + decisions = workflow.stream_writer(DECISIONS) + trace: list[dict[str, Any]] = [] + try: + async for record in inputs: + if record.kind is RecordKind.SUPERSEDED: + assert record.supersession is not None + trace.append( + { + "kind": "superseded", + "replaced": record.supersession.previous_attempt, + "attempt": record.supersession.attempt, + } + ) + decisions.publish( + {"retracting_attempt": record.supersession.previous_attempt} + ) + continue + if record.kind is RecordKind.FINISH: + trace.append({"kind": "finish", "producer": record.producer_id}) + break + assert record.value is not None + decisions.publish({"decided": record.value["n"]}) + trace.append( + { + "kind": "decision", + "n": record.value["n"], + "attempt": record.attempt, + } + ) + finally: + inputs.close() + decisions.finish() + # Twice on purpose: a finished topic stays finished, with one marker. + decisions.finish() + return trace + + +async def _run_the_loop( + client: Client, provider: MemoryStreams +) -> tuple[Any, list[dict[str, Any]]]: + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, ContractLoop, plugins=[provider]) as worker: + handle = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + first = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await first.append({"n": 1}, {"n": 2}) + second = stream.producer(topic=INPUTS, producer_id="model", attempt=2) + await second.append({"n": 3}) + await second.finish() + trace = await handle.result() + return handle, trace + + +async def test_workflow_reads_decides_and_publishes( + client: Client, provider: MemoryStreams +): + handle, trace = await _run_the_loop(client, provider) + assert trace == [ + {"kind": "decision", "n": 1, "attempt": 1}, + {"kind": "decision", "n": 2, "attempt": 1}, + {"kind": "superseded", "replaced": 1, "attempt": 2}, + {"kind": "decision", "n": 3, "attempt": 2}, + {"kind": "finish", "producer": "model"}, + ] + + # The outside view of what the workflow published, on its own topic. + stream = provider.get_stream_handle(client, handle.id) + records = await take(stream.read(topic=DECISIONS), 5) + assert [(r.kind, r.value) for r in records] == [ + (RecordKind.DATA, {"decided": 1}), + (RecordKind.DATA, {"decided": 2}), + (RecordKind.DATA, {"retracting_attempt": 1}), + (RecordKind.DATA, {"decided": 3}), + (RecordKind.FINISH, None), + ] + assert all(r.producer_id == "" and r.topic == DECISIONS.name for r in records) + # The second finish() wrote nothing: the marker is the newest record. + assert await stream.latest(topic=DECISIONS) == records[-1].cursor + + +async def test_read_ends_when_the_workflow_closes_and_the_tail_is_delivered( + client: Client, provider: MemoryStreams +): + handle, _ = await _run_the_loop(client, provider) + stream = provider.get_stream_handle(client, handle.id) + + async def read_everything() -> list[Any]: + return [r.value async for r in stream.read(topic=DECISIONS)] + + # No count and no early break: the read ends by itself once the workflow + # is closed and everything it retained has been handed over. + values = await asyncio.wait_for(read_everything(), 30) + assert values == [ + {"decided": 1}, + {"decided": 2}, + {"retracting_attempt": 1}, + {"decided": 3}, + None, + ] + + +@pytest.mark.xfail( + strict=True, + reason=( + "rule 2: reading is a recorded observation, so replaying the history " + "with the store gone must re-supply the same records; the memory " + "provider reads live process memory instead" + ), +) +async def test_replay_without_the_store_resupplies_the_records( + client: Client, provider: MemoryStreams +): + handle, _ = await _run_the_loop(client, provider) + history = await handle.fetch_history() + + provider.reset() + replayer = Replayer(workflows=[ContractLoop], plugins=[provider]) + await replayer.replay_workflow(history) + + +# Run ids whose first workflow task already failed, shared with the workflow +# thread so the retry can tell it is the retry. Outside the sandbox on +# purpose: the sandbox re-imports this module per run and would hide the set. +_failed_once: set[str] = set() + + +@workflow.defn(sandboxed=False) +class PublishThenFail: + """Publishes, then fails its first workflow task; the retry publishes again.""" + + @workflow.run + async def run(self) -> None: + decisions = workflow.stream_writer(DECISIONS) + run_id = workflow.info().run_id + committed = run_id in _failed_once + decisions.publish({"committed": committed}) + if not committed: + _failed_once.add(run_id) + raise RuntimeError("the first task fails after publishing") + decisions.finish() + + +@pytest.mark.xfail( + strict=True, + reason=( + "rule 1: a publish commits with its workflow task, so no reader sees " + "a record from a task that failed; the memory provider makes it " + "visible at publish time" + ), +) +async def test_a_failed_task_publishes_nothing(client: Client, provider: MemoryStreams): + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, PublishThenFail, plugins=[provider]) as worker: + handle = await client.start_workflow( + PublishThenFail.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + records = await take(stream.read(topic=DECISIONS), 2, timeout=30) + await handle.result() + assert [(r.kind, r.value) for r in records] == [ + (RecordKind.DATA, {"committed": True}), + (RecordKind.FINISH, None), + ] + + +@workflow.defn +class Relay: + """Publishes one record per run and continues as new once.""" + + @workflow.run + async def run(self, run: int) -> None: + decisions = workflow.stream_writer(DECISIONS) + decisions.publish({"run": run}) + if run == 0: + workflow.continue_as_new(run + 1) + decisions.finish() + + +class _RecordingHalf: + """A workflow half that logs the hooks the worker calls, then delegates.""" + + def __init__(self, inner: Any, calls: list[tuple[str, str]]) -> None: + self._inner = inner + self._calls = calls + + def open_reader(self, topic: str, *, after: Cursor) -> ReadSource: + return self._inner.open_reader(topic, after=after) + + def open_writer(self, topic: str) -> WriteSink: + return self._inner.open_writer(topic) + + def on_workflow_start(self) -> None: + self._calls.append(("start", workflow.info().run_id)) + + async def on_workflow_finish(self) -> None: + self._calls.append(("finish", workflow.info().run_id)) + + +class HookedMemory(MemoryStreams): + """The memory provider with its lifecycle hooks made visible.""" + + def __init__(self) -> None: + super().__init__() + self.calls: list[tuple[str, str]] = [] + + def workflow_provider(self) -> Any: + return _RecordingHalf(super().workflow_provider(), self.calls) + + +@pytest.mark.usefixtures("provider") +async def test_the_worker_calls_the_lifecycle_hooks_around_every_run(client: Client): + hooked = HookedMemory() + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, Relay, plugins=[hooked]) as worker: + handle = await client.start_workflow( + Relay.run, 0, id=workflow_id, task_queue=worker.task_queue + ) + await handle.result() + # Start before the function, finish after it, on both runs: the finish + # hook runs on the continue-as-new exit too, so a provider that parked + # something against the first run can let go before the successor starts. + kinds = [kind for kind, _ in hooked.calls] + assert kinds == ["start", "finish", "start", "finish"] + runs = [run_id for _, run_id in hooked.calls] + assert runs[0] == runs[1] and runs[2] == runs[3] and runs[0] != runs[2] + + +async def test_a_handle_without_a_run_id_reads_across_continue_as_new( + client: Client, provider: MemoryStreams +): + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, Relay, plugins=[provider]) as worker: + handle = await client.start_workflow( + Relay.run, 0, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + + async def read_everything() -> list[Any]: + return [(r.kind, r.value) async for r in stream.read(topic=DECISIONS)] + + records = await asyncio.wait_for(read_everything(), 30) + await handle.result() + # The chain is followed: the successor's records arrive on the same read, + # and the read ends only when the last run of the chain is closed. + assert records == [ + (RecordKind.DATA, {"run": 0}), + (RecordKind.DATA, {"run": 1}), + (RecordKind.FINISH, None), + ] + + +@workflow.defn +class SharedReaders: + """Opens the same topic twice and pulls from both readers in turn.""" + + @workflow.run + async def run(self) -> list[Any]: + first = workflow.stream_reader(INPUTS) + second = workflow.stream_reader(INPUTS) + trace: list[Any] = ["shared" if first is second else "separate"] + try: + workflow.stream_reader(INPUTS, after=Cursor("memory:0")) + except ValueError: + trace.append("after-rejected") + try: + workflow.stream_reader(INPUTS.name, result_type=list) + except ValueError: + trace.append("type-rejected") + trace.append((await first.__anext__()).value) + trace.append((await second.__anext__()).value) + first.close() + return trace + + +async def test_a_second_reader_on_a_topic_shares_the_subscription( + client: Client, provider: MemoryStreams +): + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, SharedReaders, plugins=[provider]) as worker: + handle = await client.start_workflow( + SharedReaders.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + await stream.producer(topic=INPUTS, producer_id="model", attempt=1).append( + {"n": 1}, {"n": 2} + ) + assert await handle.result() == [ + "shared", + "after-rejected", + "type-rejected", + {"n": 1}, + {"n": 2}, + ] + + +@workflow.defn +class FinishThenPublish: + """Finishes a topic on one writer and publishes on a second one.""" + + @workflow.run + async def run(self) -> str: + workflow.stream_writer(DECISIONS).finish() + # A fresh writer object, the same topic. The marker is already there, + # so this publish would land after the end of the topic. + try: + workflow.stream_writer(DECISIONS).publish({"after": "finish"}) + except ValueError as error: + return str(error) + return "published" + + +async def test_a_finished_topic_stays_finished_across_writers( + client: Client, provider: MemoryStreams +): + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, FinishThenPublish, plugins=[provider]) as worker: + handle = await client.start_workflow( + FinishThenPublish.run, id=workflow_id, task_queue=worker.task_queue + ) + assert "already finished" in await handle.result() + stream = provider.get_stream_handle(client, workflow_id) + + async def read_everything() -> list[Any]: + return [(r.kind, r.value) async for r in stream.read(topic=DECISIONS)] + + records = await asyncio.wait_for(read_everything(), 30) + # The marker is all that landed: the publish behind it never reached the + # topic, which is the point of the guard reading like a topic-wide rule. + assert records == [(RecordKind.FINISH, None)] + + +@workflow.defn +class NoStreams: + """Touches no stream at all, on a worker that has a provider.""" + + @workflow.run + async def run(self) -> str: + await asyncio.sleep(0) + return "done" + + +@pytest.mark.usefixtures("provider") +async def test_a_workflow_that_touches_no_stream_runs_unchanged(client: Client): + hooked = HookedMemory() + async with new_worker(client, NoStreams, plugins=[hooked]) as worker: + result = await client.execute_workflow( + NoStreams.run, + id=f"streams-wf-{uuid.uuid4().hex}", + task_queue=worker.task_queue, + ) + assert result == "done" + # The interceptor is installed per worker, not per workflow, so the hooks + # still bracket a run that never opened a reader or a writer. A provider's + # hooks therefore have to be cheap and safe on a workflow that uses none. + assert [kind for kind, _ in hooked.calls] == ["start", "finish"] + + +@workflow.defn +class ForeignCursor: + """Resumes from a cursor another provider minted.""" + + @workflow.run + async def run(self) -> str: + try: + workflow.stream_reader(INPUTS, after=Cursor("elsewhere:1")) + except StreamCursorError: + return "refused" + return "accepted" + + +async def test_the_workflow_reader_refuses_a_foreign_cursor( + client: Client, provider: MemoryStreams +): + async with new_worker(client, ForeignCursor, plugins=[provider]) as worker: + result = await client.execute_workflow( + ForeignCursor.run, + id=f"streams-wf-{uuid.uuid4().hex}", + task_queue=worker.task_queue, + ) + assert result == "refused" + + +@workflow.defn +class NoProvider: + """Opens a stream on a worker that has no provider.""" + + @workflow.run + async def run(self) -> str: + try: + workflow.stream_reader(INPUTS) + except RuntimeError as error: + return str(error) + return "opened" + + +async def test_a_worker_without_a_provider_says_so(client: Client): + async with new_worker(client, NoProvider) as worker: + result = await client.execute_workflow( + NoProvider.run, + id=f"streams-wf-{uuid.uuid4().hex}", + task_queue=worker.task_queue, + ) + assert "no stream provider is configured" in result + + +@workflow.defn +class OneLine: + """Publishes one record and finishes the topic.""" + + @workflow.run + async def run(self) -> None: + decisions = workflow.stream_writer(DECISIONS) + decisions.publish({"from": "workflow"}) + decisions.finish() + + +async def test_an_outside_producer_and_the_workflow_share_a_topic( + client: Client, provider: MemoryStreams +): + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, OneLine, plugins=[provider]) as worker: + stream = provider.get_stream_handle(client, workflow_id) + await stream.producer(topic=DECISIONS, producer_id="tool", attempt=1).append( + {"from": "producer"} + ) + handle = await client.start_workflow( + OneLine.run, id=workflow_id, task_queue=worker.task_queue + ) + await handle.result() + + async def read_everything() -> list[Any]: + return [r async for r in stream.read(topic=DECISIONS)] + + records = await asyncio.wait_for(read_everything(), 30) + # Both writers land on one topic in one order, each under its own + # identity: the producer's records carry its id, the workflow's carry none. + assert [(r.producer_id, r.kind, r.value) for r in records] == [ + ("tool", RecordKind.DATA, {"from": "producer"}), + ("", RecordKind.DATA, {"from": "workflow"}), + ("", RecordKind.FINISH, None), + ] + + +@workflow.defn +class PublishFromQuery: + """Publishes from a query handler, which commits nothing.""" + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: False) + + @workflow.query + def peek(self) -> str: + workflow.stream_writer(DECISIONS).publish({"from": "query"}) + return "unreachable" + + +async def test_a_publish_from_a_query_handler_is_refused_at_the_call( + client: Client, provider: MemoryStreams +): + # A query is answered on a task that carries the answer alone, so the record + # could only ever be dropped on the way out. The call says so instead. + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, PublishFromQuery, plugins=[provider]) as worker: + handle = await client.start_workflow( + PublishFromQuery.run, id=workflow_id, task_queue=worker.task_queue + ) + try: + with pytest.raises(Exception) as caught: + await handle.query(PublishFromQuery.peek) + assert "publish to a stream" in str(caught.value) + + stream = provider.get_stream_handle(client, workflow_id) + assert await stream.latest(topic=DECISIONS) == BEGINNING + finally: + await handle.terminate() + + +@workflow.defn +class DefaultTopicEcho: + """Reads one value on its default topic and answers on the same topic.""" + + @workflow.run + async def run(self) -> None: + reader = workflow.stream_reader(result_type=dict) + writer = workflow.stream_writer() + async for value in reader.values(): + writer.publish({"echo": value["n"] * 2}) + reader.close() + writer.finish() + + +async def test_a_workflow_reads_and_writes_its_default_topic( + client: Client, provider: MemoryStreams +): + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, DefaultTopicEcho, plugins=[provider]) as worker: + stream = provider.get_stream_handle(client, workflow_id) + await stream.producer(producer_id="client", attempt=1).append({"n": 21}) + handle = await client.start_workflow( + DefaultTopicEcho.run, id=workflow_id, task_queue=worker.task_queue + ) + await handle.result() + + async def read_everything() -> list[Any]: + return [r async for r in stream.read(topic=DEFAULT_TOPIC)] + + records = await asyncio.wait_for(read_everything(), 30) + assert [(r.topic, r.kind, r.value) for r in records] == [ + (DEFAULT_TOPIC, RecordKind.DATA, {"n": 21}), + (DEFAULT_TOPIC, RecordKind.DATA, {"echo": 42}), + (DEFAULT_TOPIC, RecordKind.FINISH, None), + ] + + +@workflow.defn +class NewestTwo: + """Starts at the newest two records on ``inputs`` and returns their values.""" + + @workflow.run + async def run(self) -> list[Any]: + reader = workflow.stream_reader(INPUTS, last=2) + values: list[Any] = [] + async for value in reader.values(): + values.append(value["n"]) + if len(values) == 2: + reader.close() + return values + + +async def test_a_workflow_reader_starts_at_the_last_n_records( + client: Client, provider: MemoryStreams +): + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, NewestTwo, plugins=[provider]) as worker: + stream = provider.get_stream_handle(client, workflow_id) + await stream.producer(topic=INPUTS, producer_id="tool", attempt=1).append( + {"n": 1}, {"n": 2}, {"n": 3}, {"n": 4} + ) + handle = await client.start_workflow( + NewestTwo.run, id=workflow_id, task_queue=worker.task_queue + ) + assert await handle.result() == [3, 4] + + +@workflow.defn +class FromNow: + """Follows ``inputs`` from when it subscribes and returns the first value.""" + + @workflow.run + async def run(self) -> Any: + reader = workflow.stream_reader(INPUTS, after=END) + async for value in reader.values(): + reader.close() + return value["n"] + return None + + +async def test_a_workflow_reader_at_end_skips_what_was_there( + client: Client, provider: MemoryStreams +): + workflow_id = f"streams-wf-{uuid.uuid4().hex}" + async with new_worker(client, FromNow, plugins=[provider]) as worker: + stream = provider.get_stream_handle(client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + await producer.append({"n": "old"}) + handle = await client.start_workflow( + FromNow.run, id=workflow_id, task_queue=worker.task_queue + ) + result = asyncio.ensure_future(handle.result()) + # The subscription starts when the workflow runs, which the test does + # not observe, so appends keep coming until the workflow takes one. + for _ in range(150): + await producer.append({"n": "new"}) + done, _ = await asyncio.wait({result}, timeout=0.2) + if done: + break + assert await asyncio.wait_for(result, 10) == "new" + + +def test_a_workflow_reader_start_names_one_place(): + # Checked before the reader needs a running workflow, so a mistake says + # what it is rather than that there is no workflow. + with pytest.raises(ValueError, match="either after= or last="): + workflow.stream_reader(INPUTS, after=END, last=1) + with pytest.raises(ValueError, match="positive"): + workflow.stream_reader(INPUTS, last=0) diff --git a/tests/streams/test_workflow_streams_provider.py b/tests/streams/test_workflow_streams_provider.py new file mode 100644 index 0000000000..218c55fdcd --- /dev/null +++ b/tests/streams/test_workflow_streams_provider.py @@ -0,0 +1,1341 @@ +"""Conformance for the workflow_streams provider on the test environment's server. + +Runs the interface loop over the shipped Option 0 transport: an outside +producer appends through the publish Update, or the shipped publish Signal +where a workflow has no Update, the workflow reads and republishes through +its own state, and an outside reader follows the poll Update while the run is +open and the tail Query once it has closed. The outside-surface cases shared +by every provider run from ``test_streams_conformance``; this file covers +what the transport adds. +""" + +from __future__ import annotations + +import asyncio +import base64 +import dataclasses +import uuid +from collections.abc import AsyncIterator +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import activity, workflow +from temporalio.api.common.v1 import Payload, WorkflowExecution +from temporalio.api.workflowservice.v1 import ResetWorkflowExecutionRequest +from temporalio.client import ( + Client, + WorkflowExecutionStatus, + WorkflowQueryFailedError, + WorkflowUpdateFailedError, +) +from temporalio.contrib.workflow_streams import PublishInput, WorkflowStream +from temporalio.converter import DataConverter, ExternalStorage +from temporalio.exceptions import ApplicationError +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import ( + BEGINNING, + CONTENT_HASH_KEY, + DEFAULT_TOPIC, + END, + Cursor, + RecordKind, + StreamCursorError, + StreamError, + StreamNotFoundError, + StreamProducerError, + StreamRef, + StreamUnsupportedError, + Supersession, + content_hash, +) +from temporalio.streams._wire import WireRecord +from temporalio.streams.providers import workflow_streams +from temporalio.streams.providers.workflow_streams import ( + WorkflowStreamsActivityHandle, + WorkflowStreamsHandle, + WorkflowStreamsProducer, + WorkflowStreamsProvider, + _InstanceStream, + _PublishResult, +) +from temporalio.worker._workflow_instance import QUERY_HANDLER_NOT_FOUND +from tests.helpers import new_worker + +INPUTS = "inputs" +DECISIONS = "decisions" + + +@pytest.fixture +def provider() -> WorkflowStreamsProvider: + return WorkflowStreamsProvider(poll_cooldown=timedelta(milliseconds=20)) + + +@workflow.defn +class EchoLoop: + """Reads ``inputs``, echoes each value onto ``decisions``, ends on FINISH. + + Lingers until released so a reader can follow it while it runs; whoever + arrives after it closed is served by the tail Query instead. + """ + + def __init__(self) -> None: + self._released = False + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self) -> int: + inputs = workflow.stream_reader(INPUTS, result_type=dict) + decisions = workflow.stream_writer(DECISIONS) + seen = 0 + async for record in inputs: + if record.kind is RecordKind.FINISH: + break + if record.kind is not RecordKind.DATA: + continue + assert record.value is not None + seen += 1 + decisions.publish({"echo": record.value["n"]}) + decisions.finish() + await workflow.wait_condition(lambda: self._released) + return seen + + +async def take(records: Any, count: int, timeout: float = 30.0) -> list: + out: list = [] + + async def _collect() -> None: + async for record in records: + out.append(record) + if len(out) >= count: + return + + await asyncio.wait_for(_collect(), timeout) + return out + + +async def _feed( + provider: WorkflowStreamsProvider, client: Client, workflow_id: str +) -> None: + stream = provider.get_stream_handle(client, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + await producer.append({"n": 3}) + await producer.finish() + + +ECHOED = [RecordKind.DATA, RecordKind.DATA, RecordKind.DATA, RecordKind.FINISH] + + +async def test_interface_loop_over_workflow_streams( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, EchoLoop, plugins=[provider]) as worker: + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + await _feed(provider, client, workflow_id) + + stream = provider.get_stream_handle(client, workflow_id) + records = await take(stream.read(topic=DECISIONS, result_type=dict), 4) + assert [r.kind for r in records] == ECHOED + assert [r.value["echo"] for r in records[:3]] == [1, 2, 3] + assert all(r.topic == DECISIONS and r.producer_id == "" for r in records) + + await handle.signal(EchoLoop.release) + assert await handle.result() == 3 + + +async def test_retried_producer_dedupes_and_new_attempt_supersedes( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, EchoLoop, plugins=[provider]) as worker: + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + + first = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + # The publish Update answers with where the batch landed. + landed = await first.append({"n": 1}) + assert landed is not None + # The retry of the same attempt re-sends its first batch and is + # answered with the same position. + retry = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + assert await retry.append({"n": 1}) == landed + second = stream.producer(topic=INPUTS, producer_id="model", attempt=2) + await second.append({"n": 2}) + + records = await take(stream.read(topic=INPUTS, result_type=dict), 3) + assert records[0].kind is RecordKind.DATA and records[0].attempt == 1 + assert records[0].cursor == landed + assert records[1].kind is RecordKind.SUPERSEDED + assert records[1].supersession == Supersession("model", 1, 2) + assert records[2].kind is RecordKind.DATA and records[2].attempt == 2 + assert all(r.topic == INPUTS for r in records) + + # A sequence behind the producer's most recent one is stale, and a + # refusal is typed rather than a silent drop. + await second.append({"n": 3}) + stale = stream.producer(topic=INPUTS, producer_id="model", attempt=2) + with pytest.raises(StreamProducerError, match="most recent"): + await stale.append({"n": "other"}) + assert stale.transport == "update" + + await second.finish() + await handle.signal(EchoLoop.release) + await handle.result() + + +async def test_cold_cache_serves_each_record_once( + client: Client, provider: WorkflowStreamsProvider +): + # Every task rebuilds the workflow from history, so the stream object + # has to belong to the instance that is running: a stale one would carry + # the previous instance's log and hand out every record twice. + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker( + client, EchoLoop, plugins=[provider], max_cached_workflows=0 + ) as worker: + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + await _feed(provider, client, workflow_id) + + stream = provider.get_stream_handle(client, workflow_id) + records = await take(stream.read(topic=DECISIONS, result_type=dict), 4) + assert [r.kind for r in records] == ECHOED + assert [r.value["echo"] for r in records[:3]] == [1, 2, 3] + arrived = await take(stream.read(topic=INPUTS, result_type=dict), 4) + assert [r.kind for r in arrived] == ECHOED + assert [r.value["n"] for r in arrived[:3]] == [1, 2, 3] + + await handle.signal(EchoLoop.release) + assert await handle.result() == 3 + + +async def test_a_closed_run_serves_its_tail_by_query_and_the_read_ends( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, EchoLoop, plugins=[provider]) as worker: + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + await _feed(provider, client, workflow_id) + await handle.signal(EchoLoop.release) + assert await handle.result() == 3 + + # Nothing polled while the run was open. The poll Update is gone with + # the run, so everything below arrives through the tail Query, and + # the read ends by itself once the tail is delivered. + stream = provider.get_stream_handle(client, workflow_id) + + async def read_everything() -> list[Any]: + return [r async for r in stream.read(topic=DECISIONS, result_type=dict)] + + records = await asyncio.wait_for(read_everything(), 30) + assert [r.kind for r in records] == ECHOED + assert [r.value["echo"] for r in records[:3]] == [1, 2, 3] + + checkpoint = records[1].cursor + again = await take( + stream.read(topic=DECISIONS, result_type=dict, after=checkpoint), 2 + ) + assert [r.value["echo"] for r in again[:1]] == [3] + assert again[1].kind is RecordKind.FINISH + + +async def test_a_bounded_read_is_cancelled_within_its_timeout( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, EchoLoop, plugins=[provider]) as worker: + handle = await client.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + + async def read_forever() -> None: + async for _ in stream.read(topic=DECISIONS, result_type=dict): + pass + + # The run is open and nothing is published, so the read parks on the + # poll Update. The bound has to come out as a timeout rather than + # vanish into a resubscribe. + started = asyncio.get_running_loop().time() + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(read_forever(), timeout=1) + assert asyncio.get_running_loop().time() - started < 10 + # A reader task cancelled outright ends the same way. + task = asyncio.create_task(read_forever()) + await asyncio.sleep(0.2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + await stream.producer(topic=INPUTS, producer_id="model", attempt=1).finish() + await handle.signal(EchoLoop.release) + assert await handle.result() == 0 + + +@workflow.defn +class Truncating: + """Publishes on two topics and truncates its log when told to.""" + + def __init__(self) -> None: + # Constructed here so the shipped signal handler is registered before + # the provider looks for it, the way a migrating application holds it. + self._stream = WorkflowStream() + self._released = False + + @workflow.signal + def truncate_to(self, offset: int) -> None: + self._stream.truncate(offset) + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self) -> None: + decisions = workflow.stream_writer(DECISIONS) + other = workflow.stream_writer(INPUTS) + decisions.publish({"n": 0}) + other.publish({"side": "inputs"}) + decisions.publish({"n": 1}) + await workflow.wait_condition(lambda: self._released) + + +async def test_a_truncated_position_is_refused_rather_than_restarted( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, Truncating, plugins=[provider]) as worker: + handle = await client.start_workflow( + Truncating.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + first = await take(stream.read(topic=DECISIONS, result_type=dict), 1) + assert first[0].value == {"n": 0} + + # Everything the reader's cursor names is dropped from the log. + await handle.signal(Truncating.truncate_to, 3) + resumed = stream.read(topic=DECISIONS, result_type=dict, after=first[0].cursor) + # Starting over would hand back records the caller already handled, + # and only the caller can decide to do that. + with pytest.raises(StreamCursorError): + await take(resumed, 1, timeout=30) + + await handle.signal(Truncating.release) + await handle.result() + + +async def test_latest_names_the_newest_record_on_the_topic_asked_for( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, Truncating, plugins=[provider]) as worker: + handle = await client.start_workflow( + Truncating.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + decisions = await take(stream.read(topic=DECISIONS, result_type=dict), 2) + + # One log orders both topics and the newest item on it belongs to + # `inputs`, so a log-global answer would be the wrong cursor here. + assert await stream.latest(topic=DECISIONS) == decisions[-1].cursor + inputs = await take(stream.read(topic=INPUTS, result_type=dict), 1) + assert await stream.latest(topic=INPUTS) == inputs[0].cursor + assert await stream.latest(topic="never-written") == BEGINNING + + await handle.signal(Truncating.release) + await handle.result() + + +async def test_the_tail_query_pages_instead_of_answering_in_one_blob(): + # The workflow-side half of the tail, driven directly: a Query response + # has to fit the server's blob limit, so a log larger than the cap comes + # back a page at a time with a position to resume from. + big = Payload(metadata={"encoding": b"binary/plain"}, data=b"x" * 400_000) + items = [(offset, DECISIONS, big) for offset in range(6)] + + class _Log: + next_offset = len(items) + + def items_from(self, offset: int) -> list[Any]: + return [item for item in items if item[0] >= offset] + + instance = _InstanceStream.__new__(_InstanceStream) + instance._stream = _Log() # type: ignore[assignment] # pyright: ignore[reportPrivateUsage] + + seen: list[int] = [] + offset, more = 0, True + while more: + page = instance._tail(offset, DECISIONS) + assert page["items"], "a page that fits nothing would never finish" + seen.extend(item["offset"] for item in page["items"]) + offset, more = page["next_offset"], page["more_ready"] + assert seen == list(range(6)) + + +@workflow.defn +class Relay: + """Publishes one record per run and continues as new once.""" + + @workflow.run + async def run(self, run: int) -> None: + decisions = workflow.stream_writer(DECISIONS) + decisions.publish({"run": run}) + if run == 0: + workflow.continue_as_new(run + 1) + decisions.finish() + + +async def test_a_handle_without_a_run_id_reads_across_continue_as_new( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, Relay, plugins=[provider]) as worker: + handle = await client.start_workflow( + Relay.run, 0, id=workflow_id, task_queue=worker.task_queue + ) + await handle.result() + first_run = handle.first_execution_run_id + assert first_run is not None + + chain = provider.get_stream_handle(client, workflow_id) + + async def read_everything(stream: Any) -> list[Any]: + return [ + (r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + + # Each run keeps its own log, so following the chain means reading + # the first run to its close and then the successor from its start. + assert await asyncio.wait_for(read_everything(chain), 30) == [ + (RecordKind.DATA, {"run": 0}), + (RecordKind.DATA, {"run": 1}), + (RecordKind.FINISH, None), + ] + pinned = provider.get_stream_handle(client, workflow_id, run_id=first_run) + assert await asyncio.wait_for(read_everything(pinned), 30) == [ + (RecordKind.DATA, {"run": 0}), + ] + # A cursor from the first run resumes into the successor. + records = await take(chain.read(topic=DECISIONS, result_type=dict), 1) + resumed = await asyncio.wait_for( + asyncio.ensure_future( + _values( + chain.read( + topic=DECISIONS, result_type=dict, after=records[0].cursor + ) + ) + ), + 30, + ) + assert resumed == [{"run": 1}, None] + + +@workflow.defn +class Rolling: + """Publishes what it is sent, continues as new onto ``successor_queue`` once.""" + + def __init__(self) -> None: + self._sent: list[int] = [] + self._roll_to = "" + self._released = False + + @workflow.signal + def emit(self, n: int) -> None: + self._sent.append(n) + + @workflow.signal + def roll(self, successor_queue: str) -> None: + self._roll_to = successor_queue + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self, run: int) -> None: + decisions = workflow.stream_writer(DECISIONS) + while True: + await workflow.wait_condition( + lambda: bool(self._sent or self._roll_to or self._released) + ) + while self._sent: + decisions.publish({"run": run, "n": self._sent.pop(0)}) + if self._roll_to: + workflow.continue_as_new(run + 1, task_queue=self._roll_to) + if self._released: + decisions.finish() + return + + +@workflow.defn +class Idle: + def __init__(self) -> None: + self._released = False + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: self._released) + + +async def _collect_all(records: Any, into: list[Any]) -> None: + async for record in records: + into.append((record.kind, record.value)) + + +async def test_a_live_read_without_a_run_id_follows_continue_as_new( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + successor_queue = f"streams-ws-successor-{uuid.uuid4().hex}" + async with new_worker(client, Rolling, plugins=[provider]) as worker: + handle = await client.start_workflow( + Rolling.run, 0, id=workflow_id, task_queue=worker.task_queue + ) + seen: list[Any] = [] + reader = asyncio.create_task( + _collect_all( + provider.get_stream_handle(client, workflow_id).read( + topic=DECISIONS, result_type=dict + ), + seen, + ) + ) + await handle.signal(Rolling.emit, 1) + await handle.signal(Rolling.emit, 2) + await assert_eventually_len(seen, 2, reader) + + # The successor runs on a queue nobody polls yet, so the reader's + # first poll on it waits for the successor's first task and lands in + # it, ahead of the hook that registers the handler. + await handle.signal(Rolling.roll, successor_queue) + successor = client.get_workflow_handle(workflow_id) + while (await successor.describe()).run_id == handle.result_run_id: + await asyncio.sleep(0.05) + await asyncio.sleep(1) + assert not reader.done() + async with new_worker( + client, Rolling, plugins=[provider], task_queue=successor_queue + ): + await successor.signal(Rolling.emit, 3) + await successor.signal(Rolling.emit, 4) + await assert_eventually_len(seen, 4, reader) + await successor.signal(Rolling.release) + await asyncio.wait_for(reader, 30) + await successor.result() + + assert seen == [ + (RecordKind.DATA, {"run": 0, "n": 1}), + (RecordKind.DATA, {"run": 0, "n": 2}), + (RecordKind.DATA, {"run": 1, "n": 3}), + (RecordKind.DATA, {"run": 1, "n": 4}), + (RecordKind.FINISH, None), + ] + + +async def test_a_poll_that_arrives_before_the_first_task_is_retried( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + task_queue = f"streams-ws-{uuid.uuid4().hex}" + handle = await client.start_workflow( + Rolling.run, 0, id=workflow_id, task_queue=task_queue + ) + seen: list[Any] = [] + reader = asyncio.create_task( + _collect_all( + provider.get_stream_handle(client, workflow_id).read( + topic=DECISIONS, result_type=dict + ), + seen, + ) + ) + # No worker yet, so the poll is delivered in the run's first task. + await asyncio.sleep(1) + async with new_worker(client, Rolling, plugins=[provider], task_queue=task_queue): + await handle.signal(Rolling.emit, 1) + await assert_eventually_len(seen, 1, reader) + await handle.signal(Rolling.release) + await asyncio.wait_for(reader, 30) + await handle.result() + assert seen == [(RecordKind.DATA, {"run": 0, "n": 1}), (RecordKind.FINISH, None)] + + +async def _reset_at_last_completed_task( + client: Client, workflow_id: str, run_id: str +) -> str: + """Reset ``run_id`` at its last completed task; the id of the run reset into.""" + completion_id = 0 + events = client.get_workflow_handle( + workflow_id, run_id=run_id + ).fetch_history_events() + async for event in events: + if event.HasField("workflow_task_completed_event_attributes"): + completion_id = event.event_id + assert completion_id + try: + answer = await client.workflow_service.reset_workflow_execution( + ResetWorkflowExecutionRequest( + namespace=client.namespace, + workflow_execution=WorkflowExecution( + workflow_id=workflow_id, run_id=run_id + ), + reason="re-run from the last completed task", + workflow_task_finish_event_id=completion_id, + request_id=uuid.uuid4().hex, + ) + ) + except RPCError as error: + if error.status != RPCStatusCode.UNIMPLEMENTED: + raise + # The Java time-skipping test server has no reset; the real server + # does, and this never skips there. + pytest.skip("this test server does not implement ResetWorkflowExecution") + return answer.run_id + + +async def _collect_records(records: Any, into: list[Any]) -> None: + async for record in records: + into.append(record) + + +def _run_and_offset(record: Any) -> tuple[str, int]: + _, run_id, offset = record.cursor.token.split(":") + return run_id, int(offset) + + +async def test_a_live_read_without_a_run_id_follows_a_reset( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, Rolling, plugins=[provider]) as worker: + handle = await client.start_workflow( + Rolling.run, 0, id=workflow_id, task_queue=worker.task_queue + ) + base_run = handle.result_run_id + assert base_run is not None + seen: list[Any] = [] + reader = asyncio.create_task( + _collect_records( + provider.get_stream_handle(client, workflow_id).read( + topic=DECISIONS, result_type=dict + ), + seen, + ) + ) + await handle.signal(Rolling.emit, 1) + await handle.signal(Rolling.emit, 2) + await assert_eventually_len(seen, 2, reader) + + # The base run is closed by the reset with nothing in its own History + # to say so; the reader learns where it went from describe. The reset + # run replays the base run's History up to the last completed task, + # so its log holds the same two records at the same offsets, and the + # read carries on from the position it had reached. + reset_run = await _reset_at_last_completed_task(client, workflow_id, base_run) + assert reset_run != base_run + current = client.get_workflow_handle(workflow_id) + assert (await current.describe()).run_id == reset_run + await current.signal(Rolling.emit, 3) + await current.signal(Rolling.emit, 4) + await assert_eventually_len(seen, 4, reader) + await current.signal(Rolling.release) + await asyncio.wait_for(reader, 30) + await current.result() + + assert [(r.kind, r.value) for r in seen] == [ + (RecordKind.DATA, {"run": 0, "n": 1}), + (RecordKind.DATA, {"run": 0, "n": 2}), + (RecordKind.DATA, {"run": 0, "n": 3}), + (RecordKind.DATA, {"run": 0, "n": 4}), + (RecordKind.FINISH, None), + ] + # Two records from the base run, then the reset run's, whose offsets + # continue where the base run's log stood at the reset point. + assert [_run_and_offset(r) for r in seen] == [ + (base_run, 0), + (base_run, 1), + (reset_run, 2), + (reset_run, 3), + (reset_run, 4), + ] + # A handle pinned to the base run ends with it. Both closed runs are + # served by the tail Query, which needs the worker still up. + pinned = provider.get_stream_handle(client, workflow_id, run_id=base_run) + pinned_records: list[Any] = [] + await asyncio.wait_for( + _collect_records(pinned.read(topic=DECISIONS), pinned_records), 30 + ) + assert [_run_and_offset(r) for r in pinned_records] == [ + (base_run, 0), + (base_run, 1), + ] + # BEGINNING on the chain starts at the base run, whose start event the + # reset run copied, and a resume from a base run cursor crosses over. + chain = provider.get_stream_handle(client, workflow_id) + resumed = await take(chain.read(topic=DECISIONS, after=seen[1].cursor), 3) + assert [_run_and_offset(r) for r in resumed] == [ + (reset_run, 2), + (reset_run, 3), + (reset_run, 4), + ] + + +@workflow.defn +class ShippedOnly: + """Holds the shipped stream object alone, as a workflow on a worker without the provider.""" + + def __init__(self) -> None: + self._stream = WorkflowStream() + self._released = False + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self) -> None: + await workflow.wait_condition(lambda: self._released) + + +async def test_a_workflow_without_the_publish_update_is_appended_to_by_signal( + client: Client, provider: WorkflowStreamsProvider +): + # The worker has no provider, so the workflow serves the shipped Signal + # and nothing else. The producer learns that from two rejections and + # falls back, and the batches land in the shipped log all the same. + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, ShippedOnly) as worker: + handle = await client.start_workflow( + ShippedOnly.run, id=workflow_id, task_queue=worker.task_queue + ) + producer = provider.get_stream_handle(client, workflow_id).producer( + topic=INPUTS, producer_id="model", attempt=1 + ) + assert await producer.append({"n": 1}) is None + assert producer.transport == "signal" + await producer.append({"n": 2}, {"n": 3}) + assert ( + await handle.query("__temporal_workflow_stream_offset", result_type=int) + == 3 + ) + await handle.signal(ShippedOnly.release) + await handle.result() + + +async def test_a_running_workflow_without_the_provider_fails_the_read_clearly( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(client, Idle) as worker: + handle = await client.start_workflow( + Idle.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(client, workflow_id) + with pytest.raises(StreamError, match="provider is not installed"): + await take(stream.read(topic=DECISIONS), 1) + await handle.signal(Idle.release) + await handle.result() + + +async def assert_eventually_len( + items: list[Any], count: int, reader: asyncio.Task[None] +) -> None: + async def _wait() -> None: + while len(items) < count: + if reader.done(): + # A read that failed surfaces its error instead of a timeout. + reader.result() + raise AssertionError(f"the read ended early with {items}") + await asyncio.sleep(0.05) + + await asyncio.wait_for(_wait(), 30) + + +async def _values(records: Any) -> list[Any]: + return [r.value async for r in records] + + +class _FlakyHandle: + """A workflow handle whose first Signal is accepted and then reported failed.""" + + id = "flaky" + + def __init__(self) -> None: + self.sent: list[PublishInput] = [] + self._fail_next = True + + async def signal(self, name: str, arg: PublishInput) -> None: + del name + self.sent.append(arg) + if self._fail_next: + self._fail_next = False + raise ConnectionResetError( + "the server accepted the signal, the reply was lost" + ) + + +def _wires(publish: PublishInput) -> list[WireRecord]: + out = [] + for entry in publish.items: + payload = Payload.FromString(base64.b64decode(entry.data)) + out.append(WireRecord.FromString(payload.data)) + return out + + +def _sequences(sent: list[PublishInput]) -> list[tuple[int, list[int]]]: + return [ + (publish.sequence, [wire.sequence for wire in _wires(publish)]) + for publish in sent + ] + + +def _producer(handle: Any, transport: Any = "signal") -> WorkflowStreamsProducer: + return WorkflowStreamsProducer( + handle, + DataConverter.default.payload_converter, + INPUTS, + "model", + 1, + transport=transport, + retry_cooldown=timedelta(0), + ) + + +async def test_a_retried_append_after_an_ambiguous_failure_writes_once(): + # The Signal transport: what a producer that fell back to it does. + handle = _FlakyHandle() + producer = _producer(handle) + with pytest.raises(ConnectionResetError): + await producer.append({"n": 1}) + assert await producer.append({"n": 1}) is None + await producer.append({"n": 2}) + # The retry carries the same signal sequence and the same record + # sequence as the failed send, so the shipped dedupe drops the copy; the + # batch after it continues the numbering. + assert _sequences(handle.sent) == [(2, [1]), (2, [1]), (3, [2])] + assert all(publish.publisher_id == "model#1" for publish in handle.sent) + + +async def test_a_batch_whose_signal_failed_goes_out_before_the_next_one(): + handle = _FlakyHandle() + producer = _producer(handle) + with pytest.raises(ConnectionResetError): + await producer.append({"n": 1}) + await producer.append({"n": 2}, {"n": 3}) + await producer.finish() + assert _sequences(handle.sent) == [(2, [1]), (2, [1]), (4, [2, 3]), (5, [4])] + assert _wires(handle.sent[-1])[0].kind == int(RecordKind.FINISH) + + +class _UpdateHandle: + """A workflow handle serving the publish Update as the server and workflow would. + + Answers a repeated update id from the first outcome, the way the server + does, positions each new batch at the head of a log, the way the + workflow does, and loses the reply of the first call when told to. + Without a handler it rejects every Update the way a workflow whose + worker predates it does, and takes Signals. + """ + + id = "update" + + def __init__(self, *, lose_first_reply: bool = False, handler: bool = True): + self.sent: list[tuple[str, PublishInput]] = [] + self.signalled: list[PublishInput] = [] + self._outcomes: dict[str, _PublishResult] = {} + self._head = 0 + self._lose = lose_first_reply + self._handler = handler + + async def execute_update( + self, name: str, arg: PublishInput, *, id: str, result_type: Any + ) -> _PublishResult: + del name, result_type + self.sent.append((id, arg)) + if not self._handler: + raise WorkflowUpdateFailedError( + ApplicationError( + f"Update handler for 'x' {QUERY_HANDLER_NOT_FOUND}, known updates: []" + ) + ) + answer = self._outcomes.get(id) + if answer is None: + answer = _PublishResult("run", self._head + len(arg.items) - 1) + self._head += len(arg.items) + self._outcomes[id] = answer + if self._lose: + self._lose = False + raise ConnectionResetError("the server took the update, the reply was lost") + return answer + + async def signal(self, name: str, arg: PublishInput) -> None: + del name + self.signalled.append(arg) + + +def _at(offset: int) -> Cursor: + return Cursor(f"workflow_streams:run:{offset}") + + +async def test_a_retried_update_after_a_lost_reply_is_answered_from_its_id(): + handle = _UpdateHandle(lose_first_reply=True) + producer = _producer(handle, "update") + assert await producer.append() is None + with pytest.raises(ConnectionResetError): + await producer.append({"n": 1}) + # The retry is the same Update: same producer, sequence and content make + # the same id, so the server answers with the outcome it already holds + # and the log takes the batch once. The batch after it is a new one. + assert await producer.append({"n": 1}) == _at(0) + assert await producer.append({"n": 2}) == _at(1) + assert await producer.append() == _at(1) + ids = [id for id, _ in handle.sent] + assert ids[0] == ids[1] != ids[2] + assert _sequences([arg for _, arg in handle.sent]) == [(2, [1]), (2, [1]), (3, [2])] + + +async def test_a_batch_whose_update_failed_goes_out_before_the_next_one(): + handle = _UpdateHandle(lose_first_reply=True) + producer = _producer(handle, "update") + with pytest.raises(ConnectionResetError): + await producer.append({"n": 1}) + # The pending batch lands first, at offset 0, so the new one follows it. + assert await producer.append({"n": 2}, {"n": 3}) == _at(2) + await producer.finish() + sent = [arg for _, arg in handle.sent] + assert _sequences(sent) == [(2, [1]), (2, [1]), (4, [2, 3]), (5, [4])] + assert _wires(sent[-1])[0].kind == int(RecordKind.FINISH) + + +async def test_a_producer_falls_back_to_the_signal_without_a_publish_update(): + handle = _UpdateHandle(handler=False) + producer = _producer(handle, "update") + # Rejected twice across a task boundary means the workflow's worker + # predates the Update; the batch goes by Signal, and so does every later + # one, without asking again. + assert await producer.append({"n": 1}) is None + assert producer.transport == "signal" + assert await producer.append({"n": 2}) is None + assert len(handle.sent) == 2 + assert _sequences(handle.signalled) == [(2, [1]), (3, [2])] + + +class _Description: + run_id = "the-only-run" + status = WorkflowExecutionStatus.COMPLETED + + +class _StubHandle: + """A workflow handle that answers describe and fails whatever the test names.""" + + id = "stub" + run_id = "the-only-run" + + def __init__( + self, + *, + query_error: BaseException | None = None, + events_error: BaseException | None = None, + ) -> None: + self._query_error = query_error + self._events_error = events_error + + async def describe(self) -> Any: + return _Description() + + async def start_update(self, *args: Any, **kwargs: Any) -> Any: + del args, kwargs + # The run is closed, so its poll Update is gone with it and the read + # goes on to the tail Query, which is what these cases are about. + raise RPCError("no poll update", RPCStatusCode.NOT_FOUND, b"") + + async def query(self, *args: Any, **kwargs: Any) -> Any: + del args, kwargs + assert self._query_error is not None + raise self._query_error + + def fetch_history_events(self, *args: Any, **kwargs: Any) -> Any: + del args, kwargs + error = self._events_error + + async def _events() -> AsyncIterator[Any]: + events: tuple[Any, ...] = () + for event in events: + yield event + if error is not None: + raise error + + return _events() + + +class _OneHandleClient: + """A client that answers every handle request with the same handle.""" + + data_converter = DataConverter.default + + def __init__(self, handle: Any) -> None: + self._handle = handle + + def get_workflow_handle( + self, workflow_id: str, *, run_id: str | None = None + ) -> Any: + del workflow_id, run_id + return self._handle + + +def _handle_over(stub: _StubHandle) -> WorkflowStreamsHandle: + return WorkflowStreamsHandle( + _OneHandleClient(stub), # type: ignore[arg-type] + "wf", + "the-only-run", + timedelta(0), + ) + + +async def test_a_failing_tail_query_comes_back_as_a_stream_error(): + # The handler being absent is the one benign case; anything else is a + # real failure, and the interface says a caller catches stream conditions + # by meaning rather than by the client's own exception types. + stub = _StubHandle(query_error=WorkflowQueryFailedError("the workflow rejected it")) + with pytest.raises(StreamError): + await take(_handle_over(stub).read(topic=DECISIONS, result_type=dict), 1) + + +async def test_a_missing_tail_handler_ends_the_read_instead_of_failing_it(): + stub = _StubHandle( + query_error=WorkflowQueryFailedError( + f"Query handler for 'x' {QUERY_HANDLER_NOT_FOUND}, known queries: []" + ) + ) + # The workflow never opened a stream through this provider, so the run + # holds no tail and the read is simply over. + assert [r async for r in _handle_over(stub).read(topic=DECISIONS)] == [] + + +async def test_a_failing_latest_query_comes_back_as_a_stream_error(): + stub = _StubHandle(query_error=WorkflowQueryFailedError("the workflow rejected it")) + with pytest.raises(StreamError): + await _handle_over(stub).latest(topic=DECISIONS) + + +async def test_a_missing_run_leaves_the_successor_lookup_as_a_stream_error(): + stub = _StubHandle(events_error=RPCError("gone", RPCStatusCode.NOT_FOUND, b"")) + with pytest.raises(StreamNotFoundError): + await _handle_over(stub)._successor(stub) # type: ignore[arg-type] + + +class _PlainHandle: + """A workflow handle that accepts every Signal and remembers it.""" + + id = "plain" + + def __init__(self) -> None: + self.sent: list[PublishInput] = [] + + async def signal(self, name: str, arg: PublishInput) -> None: + del name + self.sent.append(arg) + + +async def test_the_dedupe_sequence_names_where_the_records_end(): + # The shipped handler drops a batch whose sequence it has already passed, + # so the sequence has to say how far this producer's records reach. A + # count of signals does not: a retry that batches its records differently + # from the send it repeats then carries a sequence the workflow has not + # seen, and the records it already holds go in a second time. + first = _PlainHandle() + original = _producer(first) # type: ignore[arg-type] + await original.append({"n": 1}, {"n": 2}) + + second = _PlainHandle() + retry = _producer(second) # type: ignore[arg-type] + await retry.append({"n": 1}) + await retry.append({"n": 2}) + await retry.append({"n": 3}) + + # The original ended at record 2, so its sequence is 3. Neither half of + # the retry's re-split reaches past it, and only the new record does. + assert _sequences(first.sent) == [(3, [1, 2])] + assert _sequences(second.sent) == [(2, [1]), (3, [2]), (4, [3])] + + +@workflow.defn +class StartsWhenTold: + """Opens a reader on ``inputs`` at the start a signal names and returns what it read.""" + + def __init__(self) -> None: + self._start: str | None = None + + @workflow.signal + def begin(self, start: str) -> None: + self._start = start + + @workflow.run + async def run(self) -> list[Any]: + await workflow.wait_condition(lambda: self._start is not None) + if self._start == "end": + reader = workflow.stream_reader(INPUTS, result_type=dict, after=END) + want = 1 + else: + reader = workflow.stream_reader(INPUTS, result_type=dict, last=2) + want = 2 + values: list[Any] = [] + async for value in reader.values(): + values.append(value["n"]) + if len(values) == want: + break + return values + + +async def test_a_workflow_reader_starts_at_end_or_the_newest_records( + client: Client, provider: WorkflowStreamsProvider +): + # The log is workflow state, so both starts resolve against it on the + # workflow thread; a cold cache replays every task and has to land the + # reader on the same offset each time. + async with new_worker( + client, StartsWhenTold, plugins=[provider], max_cached_workflows=0 + ) as worker: + for start, expected in (("last", [3, 4]), ("end", ["new"])): + handle = await client.start_workflow( + StartsWhenTold.run, + id=f"ws-start-{uuid.uuid4().hex}", + task_queue=worker.task_queue, + ) + stream = provider.get_stream_handle(client, handle.id) + producer = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}) + await handle.signal(StartsWhenTold.begin, start) + result = asyncio.ensure_future(handle.result()) + if start == "end": + for _ in range(150): + try: + await producer.append({"n": "new"}) + except StreamNotFoundError: + # The Update returns once the workflow took the + # batch, and reading it is what closes the workflow, + # so the next append can find it gone before the + # result has been noticed. + break + done, _ = await asyncio.wait({result}, timeout=0.2) + if done: + break + assert await asyncio.wait_for(result, 30) == expected + + +async def test_a_standalone_activity_stream_is_refused( + client: Client, provider: WorkflowStreamsProvider +): + # The log lives inside a running workflow, so an activity outside any + # workflow has no place to put a stream of its own here. The refusal is + # the documented error, not an AttributeError or a stream silently put + # somewhere else. An activity a workflow scheduled is served. + with pytest.raises(StreamUnsupportedError, match="standalone"): + provider.get_activity_stream_handle(client, "act") + assert isinstance( + provider.get_activity_stream_handle(client, "act", workflow_id="wf"), + WorkflowStreamsActivityHandle, + ) + + +async def test_a_standalone_stream_is_refused( + client: Client, provider: WorkflowStreamsProvider +): + # Every log is a running workflow's state; a stream with no owner has no + # workflow to live in. Both the create and the lookup say so. + with pytest.raises(StreamUnsupportedError, match="standalone"): + await provider.create_standalone_stream(client, "shared") + with pytest.raises(StreamUnsupportedError, match="standalone"): + provider.get_standalone_stream_handle(client, "shared") + + +async def test_a_handle_names_its_stream_as_a_ref( + client: Client, provider: WorkflowStreamsProvider +): + workflow_stream = provider.get_stream_handle(client, "wf", run_id="run-1") + assert workflow_stream.ref(topic=INPUTS) == StreamRef.for_workflow( + "wf", run_id="run-1", topic=INPUTS + ) + own = provider.get_activity_stream_handle(client, "act", workflow_id="wf") + assert own.ref(topic=TOKENS) == StreamRef.for_activity( + "act", workflow_id="wf", topic=TOKENS + ) + assert own.ref().topic == DEFAULT_TOPIC + # An owned stream ends with its owner; only a standalone one closes. + with pytest.raises(ValueError, match="standalone"): + await own.close() + + +def test_a_record_carries_the_plaintext_hash_the_workflow_dedupes_by(): + handle = _UpdateHandle() + producer = _producer(handle, "update") + entries, _ = producer._entries( + [(RecordKind.DATA, {"n": 1}), (RecordKind.FINISH, None)] + ) # pyright: ignore[reportPrivateUsage] + data, finish = _wires(PublishInput(items=entries)) + # A DATA record is stamped with the hash of its converted body, where the + # workflow can read it without the body; FINISH has nothing to hash. + assert data.metadata[CONTENT_HASH_KEY].data.decode() == content_hash(data.body) + assert CONTENT_HASH_KEY not in finish.metadata + + # The workflow's identity for a batch is those hashes, so a body whose + # bytes a codec changed is still the same batch, and a different value + # is not. + def batch(*values: dict) -> PublishInput: + made, _ = _producer(handle, "update")._entries( # pyright: ignore[reportPrivateUsage] + [(RecordKind.DATA, value) for value in values] + ) + return PublishInput(items=made, publisher_id="model#1", sequence=3) + + same, recoded, other = batch({"n": 1}), batch({"n": 1}), batch({"n": 2}) + record = _wires(recoded)[0] + record.body.data = b"\x00" + record.body.data + recoded.items[0].data = base64.b64encode( + Payload( + metadata={"encoding": b"binary/plain"}, data=record.SerializeToString() + ).SerializeToString() + ).decode("ascii") + assert workflow_streams._content(same) == workflow_streams._content(recoded) # pyright: ignore[reportPrivateUsage] + assert workflow_streams._content(same) != workflow_streams._content(other) # pyright: ignore[reportPrivateUsage] + + +async def test_external_storage_applies_at_the_envelope_the_workflow_reads_through( + client: Client, provider: WorkflowStreamsProvider +): + # Worker and clients share one converter, as a deployment's do. A batch + # above the threshold leaves as a claim on the Update argument, the + # worker redeems it, the workflow reads the value, and the outside + # reader gets the poll response the same way. + from tests.streams.test_streams_conformance import RecordingDriver + + driver = RecordingDriver() + converter = dataclasses.replace( + DataConverter.default, + external_storage=ExternalStorage(drivers=[driver], payload_size_threshold=512), + ) + config = client.config() + config["data_converter"] = converter + shared = Client(**config) + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker(shared, EchoLoop, plugins=[provider]) as worker: + handle = await shared.start_workflow( + EchoLoop.run, id=workflow_id, task_queue=worker.task_queue + ) + stream = provider.get_stream_handle(shared, workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}) + assert driver.stored == 0 + await producer.append({"n": 2, "blob": "x" * 4096}) + assert driver.stored == 1 + await producer.finish() + + records = await take(stream.read(topic=DECISIONS, result_type=dict), 3) + assert [r.value.get("echo") for r in records[:2]] == [1, 2] + assert records[2].kind is RecordKind.FINISH + assert driver.retrieved >= 1 + + await handle.signal(EchoLoop.release) + assert await handle.result() == 2 + + +TOKENS = "tokens" + + +@activity.defn +async def stream_then_finish(count: int) -> None: + producer = activity.stream_handle(scope="activity").producer(topic=TOKENS) + for n in range(count): + await producer.append({"n": n}) + # Long enough for a reader polling every few milliseconds to see this + # activity pending before it finishes. + await asyncio.sleep(1) + await producer.finish() + + +@workflow.defn +class RunsAnActivityThenLingers: + """Runs the streaming activity by a fixed id, then waits to be released.""" + + def __init__(self) -> None: + self._released = False + + @workflow.signal + def release(self) -> None: + self._released = True + + @workflow.run + async def run(self, count: int) -> None: + await workflow.execute_activity( + stream_then_finish, + count, + activity_id="streamer", + start_to_close_timeout=timedelta(seconds=30), + ) + await workflow.wait_condition(lambda: self._released) + + +async def test_an_activity_read_ends_with_the_activity_while_the_workflow_runs( + client: Client, provider: WorkflowStreamsProvider +): + workflow_id = f"streams-ws-{uuid.uuid4().hex}" + async with new_worker( + client, + RunsAnActivityThenLingers, + activities=[stream_then_finish], + plugins=[provider], + ) as worker: + handle = await client.start_workflow( + RunsAnActivityThenLingers.run, + 2, + id=workflow_id, + task_queue=worker.task_queue, + ) + own = provider.get_activity_stream_handle( + client, "streamer", workflow_id=workflow_id + ) + + async def read_everything() -> list[Any]: + return [r async for r in own.read(topic=TOKENS, result_type=dict)] + + records = await asyncio.wait_for(read_everything(), 30) + assert [(r.kind, r.value) for r in records] == [ + (RecordKind.DATA, {"n": 0}), + (RecordKind.DATA, {"n": 1}), + (RecordKind.FINISH, None), + ] + # The producer is the activity, and the record carries the plain + # topic name; the reserved name is the log's business. + assert all(r.producer_id == "streamer" and r.attempt == 1 for r in records) + assert all(r.topic == TOKENS for r in records) + # The activity's end ended the read: the workflow is still running. + assert (await handle.describe()).status == WorkflowExecutionStatus.RUNNING + + # The workflow's topic of the same name is another stream, and the + # reserved name cannot be reached as a workflow topic. + workflow_stream = provider.get_stream_handle(client, workflow_id) + assert await workflow_stream.latest(topic=TOKENS) == BEGINNING + with pytest.raises(ValueError, match="reserved"): + workflow_stream.read(topic="activity/streamer/tokens") + with pytest.raises(ValueError, match="reserved"): + workflow_stream.producer(topic="activity/x", producer_id="p", attempt=1) + + await handle.signal(RunsAnActivityThenLingers.release) + await handle.result() diff --git a/tests/test_client_stream.py b/tests/test_client_stream.py new file mode 100644 index 0000000000..779d1b1882 --- /dev/null +++ b/tests/test_client_stream.py @@ -0,0 +1,270 @@ +"""End-to-end tests for the server-side stream client. + +These need a Temporal server built from the AI-198 branch, because the stream +service does not exist on a released server. Point them at one: + + TEMPORAL_STREAM_TARGET=127.0.0.1:7333 uv run pytest tests/test_client_stream.py + +Skipped otherwise, rather than silently passing against a server that has no +idea what a stream is. +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from collections.abc import AsyncIterator + +import pytest + +from temporalio.api.common.v1 import Payload +from temporalio.api.stream.v1 import ( + StreamRecord, + StreamRecordKind, + StreamStartPosition, +) +from temporalio.client_stream import StreamClient, StreamHandle +from temporalio.service import RPCError +from temporalio.streams import ( + StreamCursorError, + StreamNotFoundError, + StreamProducerError, +) + +TARGET = os.environ.get("TEMPORAL_STREAM_TARGET") + +pytestmark = pytest.mark.skipif( + not TARGET, + reason="set TEMPORAL_STREAM_TARGET to a server with the stream service", +) + + +def rec(body: bytes, topic: str = "") -> StreamRecord: + return StreamRecord( + body=Payload(data=body, metadata={"encoding": b"binary/plain"}), topic=topic + ) + + +def data(entries: list) -> list[bytes]: + return [entry.record.body.data for entry in entries] + + +@pytest.fixture +async def streams() -> AsyncIterator[StreamClient]: + client = StreamClient.connect( + TARGET or "", os.environ.get("TEMPORAL_NAMESPACE", "default") + ) + try: + yield client + finally: + await client.close() + + +@pytest.fixture +async def stream(streams: StreamClient) -> StreamHandle: + return await streams.create("py-test-" + uuid.uuid4().hex[:8], max_items=1000) + + +async def test_append_returns_the_offset_it_landed_at(stream: StreamHandle) -> None: + first = await stream.append(rec(b"alpha"), rec(b"beta")) + assert (first.first_offset, first.next_offset, first.count) == (0, 2, 2) + assert not first.deduplicated + assert (await stream.append(rec(b"gamma"))).first_offset == 2 + + +async def test_read_from_an_offset(stream: StreamHandle) -> None: + await stream.append(rec(b"alpha"), rec(b"beta"), rec(b"gamma")) + + entries, next_offset = await stream.read() + assert data(entries) == [b"alpha", b"beta", b"gamma"] + assert [entry.offset for entry in entries] == [0, 1, 2] + assert next_offset == 3 + + entries, next_offset = await stream.read(from_offset=1) + assert data(entries) == [b"beta", b"gamma"] + assert next_offset == 3 + + +async def test_read_past_the_end_returns_nothing(stream: StreamHandle) -> None: + await stream.append(rec(b"alpha")) + + entries, next_offset = await stream.read(from_offset=1) + assert entries == [] + assert next_offset == 1, "a caught-up reader keeps its cursor" + + +# Without an identity the append is at-least-once, so this is the only way a +# producer that retries can avoid writing the same record twice. +async def test_append_is_idempotent_for_a_named_producer(stream: StreamHandle) -> None: + first = await stream.append(rec(b"once"), producer_id="p1", sequence=1) + retry = await stream.append(rec(b"once"), producer_id="p1", sequence=1) + assert first.first_offset == retry.first_offset == 0 + assert not first.deduplicated + # The retry is told it wrote nothing and where the original landed. + assert retry.deduplicated + assert retry.next_offset == first.next_offset + + entries, _ = await stream.read() + assert data(entries) == [b"once"] + + +async def test_topics_filter_a_read(stream: StreamHandle) -> None: + await stream.append(rec(b"tok", topic="tokens")) + await stream.append(rec(b"prog", topic="progress")) + await stream.append(rec(b"tok2", topic="tokens")) + + entries, next_offset = await stream.read(topics=["tokens"]) + assert data(entries) == [b"tok", b"tok2"] + assert [entry.offset for entry in entries] == [0, 2] + assert next_offset == 3, "the cursor advances past filtered-out offsets too" + + +# The record comes back as it went in: kind, producer identity and sequence +# are the server's to store, not to interpret. +async def test_the_record_roundtrips_field_for_field(stream: StreamHandle) -> None: + sent = rec(b"x", topic="t") + sent.producer_id = "model" + sent.attempt = 2 + sent.sequence = 7 + sent.kind = StreamRecordKind.STREAM_RECORD_KIND_DATA + sent.metadata["trace"].CopyFrom(Payload(data=b"abc")) + finish = StreamRecord( + topic="t", kind=StreamRecordKind.STREAM_RECORD_KIND_FINISH, producer_id="model" + ) + await stream.append(sent, finish) + + entries, _ = await stream.read() + got = entries[0].record + assert (got.topic, got.producer_id, got.attempt, got.sequence) == ( + "t", + "model", + 2, + 7, + ) + assert got.kind == StreamRecordKind.STREAM_RECORD_KIND_DATA + assert got.metadata["trace"].data == b"abc" + assert entries[1].record.kind == StreamRecordKind.STREAM_RECORD_KIND_FINISH + assert not entries[1].record.HasField("body") + + +async def test_an_unset_kind_reads_back_as_data(stream: StreamHandle) -> None: + await stream.append(rec(b"x")) + entries, _ = await stream.read() + assert entries[0].record.kind == StreamRecordKind.STREAM_RECORD_KIND_DATA + + +# A closed stream stays readable, which is what removes the shutdown handshake +# between producer and consumer. +async def test_follow_drains_then_ends_when_the_stream_closes( + stream: StreamHandle, +) -> None: + received: list[bytes] = [] + + async def follower() -> None: + async for entry in stream.follow(): + received.append(entry.record.body.data) + + task = asyncio.create_task(follower()) + await stream.append(rec(b"one")) + await asyncio.sleep(0.2) + await stream.append(rec(b"two")) + await asyncio.sleep(0.2) + await stream.close() + + await asyncio.wait_for(task, timeout=30) + assert received == [b"one", b"two"] + + +async def test_follow_started_late_still_sees_everything( + stream: StreamHandle, +) -> None: + await stream.append(rec(b"one"), rec(b"two")) + await stream.close() + + received = [entry.record.body.data async for entry in stream.follow()] + assert received == [b"one", b"two"] + + +async def test_describe_reports_the_frontier(stream: StreamHandle) -> None: + await stream.append(rec(b"one"), rec(b"two")) + + state = await stream.describe() + assert state.head_offset == 2 + assert not state.closed + + await stream.close() + assert (await stream.describe()).closed + + +# The transport's own exception never escapes: a missing stream is the SDK's +# not-found error and anything else is its RPC error. +async def test_failures_surface_as_sdk_errors(streams: StreamClient) -> None: + missing = streams.get("py-test-missing-" + uuid.uuid4().hex[:8]) + with pytest.raises(StreamNotFoundError): + await missing.describe() + with pytest.raises((StreamNotFoundError, RPCError)): + await missing.append(rec(b"x")) + + +# A producer that asked to be deduplicated and could not be is a condition of +# its own, not a bare argument error: the store already holds that sequence. +async def test_a_producer_conflict_is_a_stream_producer_error( + stream: StreamHandle, +) -> None: + await stream.append(rec(b"one"), producer_id="p1", sequence=0) + # Same producer and sequence, different content. The server cannot know + # which of the two the reader was meant to see. + with pytest.raises(StreamProducerError, match="different content"): + await stream.append(rec(b"other"), producer_id="p1", sequence=0) + # And a sequence behind the one it accepted last. + await stream.append(rec(b"two"), producer_id="p1", sequence=1) + with pytest.raises(StreamProducerError, match="stale producer sequence"): + await stream.append(rec(b"three"), producer_id="p1", sequence=0) + + entries, _ = await stream.read() + assert data(entries) == [b"one", b"two"] + + +async def test_earliest_reads_from_the_floor_of_a_truncated_stream( + stream: StreamHandle, +) -> None: + await stream.append(rec(b"a"), rec(b"b"), rec(b"c"), rec(b"d")) + await stream.truncate(2) + + # Offset zero is what a reader with no position used to send, and a + # truncated stream no longer holds it. + with pytest.raises(StreamCursorError, match="truncated"): + await stream.read(from_offset=0) + entries, next_offset = await stream.read(start=StreamStartPosition(earliest=True)) + assert data(entries) == [b"c", b"d"] + assert [entry.offset for entry in entries] == [2, 3] + assert next_offset == 4 + + +async def test_last_n_and_tail_resolve_on_the_first_read(stream: StreamHandle) -> None: + await stream.append(rec(b"a"), rec(b"b"), rec(b"c")) + + entries, _ = await stream.read(start=StreamStartPosition(last_n=2)) + assert data(entries) == [b"b", b"c"] + entries, next_offset = await stream.read(start=StreamStartPosition(tail=True)) + assert entries == [] + assert next_offset == 3 + with pytest.raises(RPCError): + await stream.read(from_offset=1, start=StreamStartPosition(earliest=True)) + + +async def test_follow_from_the_tail_skips_what_was_there(stream: StreamHandle) -> None: + await stream.append(rec(b"old")) + following = stream.follow(start=StreamStartPosition(tail=True)) + first = asyncio.ensure_future(following.__anext__()) + try: + for _ in range(100): + await stream.append(rec(b"new")) + done, _ = await asyncio.wait({first}, timeout=0.1) + if done: + break + entry = await asyncio.wait_for(first, 10) + finally: + first.cancel() + assert entry.record.body.data == b"new" diff --git a/tests/test_client_stream_connection.py b/tests/test_client_stream_connection.py new file mode 100644 index 0000000000..73e4c5314b --- /dev/null +++ b/tests/test_client_stream_connection.py @@ -0,0 +1,216 @@ +"""The stream channel is the client's connection, opened again with ``grpcio``. + +sdk-core does not know the stream service, so its channel cannot be shared. +What can be shared is the configuration: these pin that a TLS-configured +client yields a secure channel with the same material, that an API key rides +as the bearer header along with the client's other headers, and that the +client's ``retry_config`` is what the shared stream client retries under. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import grpc +import grpc.aio +import pytest + +import temporalio.api.streamservice.v1 as stream +from temporalio import client_stream +from temporalio.client_stream import Connection, StreamClient +from temporalio.service import ( + ConnectConfig, + HttpConnectProxyConfig, + KeepAliveConfig, + RetryConfig, + TLSConfig, + __version__, +) + +# The name the vendored stubs call the service by, server-internal as it is. +_SERVICE = "temporal.server.chasm.lib.stream.proto.v1.StreamService" + + +def _fake_client(config: ConnectConfig, namespace: str = "ns") -> Any: + return SimpleNamespace( + service_client=SimpleNamespace(config=config), namespace=namespace + ) + + +def test_a_plaintext_client_yields_an_insecure_channel( + monkeypatch: pytest.MonkeyPatch, +) -> None: + opened: dict[str, Any] = {} + + def insecure_channel(target: str, **kwargs: Any) -> str: + opened.update(target=target, **kwargs) + return "channel" + + monkeypatch.setattr(grpc.aio, "insecure_channel", insecure_channel) + connection = Connection.from_config(ConnectConfig(target_host="localhost:7233")) + assert not connection.secure + assert connection.channel() == "channel" + assert opened["target"] == "localhost:7233" + keep_alive = KeepAliveConfig.default + assert ("grpc.keepalive_time_ms", keep_alive.interval_millis) in opened["options"] + assert ("grpc.keepalive_timeout_ms", keep_alive.timeout_millis) in opened["options"] + + +def test_a_tls_client_yields_a_secure_channel_with_its_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + made: dict[str, Any] = {} + opened: dict[str, Any] = {} + + def ssl_channel_credentials(**kwargs: Any) -> str: + made.update(kwargs) + return "credentials" + + def secure_channel(target: str, credentials: Any, **kwargs: Any) -> str: + opened.update(target=target, credentials=credentials, **kwargs) + return "channel" + + monkeypatch.setattr(grpc, "ssl_channel_credentials", ssl_channel_credentials) + monkeypatch.setattr(grpc.aio, "secure_channel", secure_channel) + + connection = Connection.from_config( + ConnectConfig( + target_host="cloud.example:7233", + tls=TLSConfig( + server_root_ca_cert=b"root", + client_cert=b"cert", + client_private_key=b"key", + domain="cloud.example", + verification_server_name="pinned.test", + ), + ) + ) + assert connection.secure + assert connection.channel() == "channel" + assert made == { + "root_certificates": b"root", + "private_key": b"key", + "certificate_chain": b"cert", + } + assert opened["target"] == "cloud.example:7233" + assert opened["credentials"] == "credentials" + assert ("grpc.ssl_target_name_override", "pinned.test") in opened["options"] + assert ("grpc.default_authority", "cloud.example") in opened["options"] + + +def test_tls_is_on_by_default_with_an_api_key_and_off_when_refused() -> None: + with_key = Connection.from_config( + ConnectConfig(target_host="cloud.example:7233", api_key="secret") + ) + assert with_key.secure + assert with_key.server_root_ca_cert is None, "system roots" + refused = Connection.from_config( + ConnectConfig(target_host="localhost:7233", api_key="secret", tls=False) + ) + assert not refused.secure + assert ("authorization", "Bearer secret") in refused.headers + + +def test_a_scheme_in_the_target_decides_and_is_dropped() -> None: + connection = Connection.from_config( + ConnectConfig(target_host="https://cloud.example:7233") + ) + assert connection.secure + assert connection.target == "cloud.example:7233" + + +def test_a_proxy_and_the_clients_own_authorization_carry_over() -> None: + connection = Connection.from_config( + ConnectConfig( + target_host="localhost:7233", + api_key="ignored", + tls=False, + rpc_metadata={"Authorization": "Custom token", "x-tenant": "t1"}, + http_connect_proxy_config=HttpConnectProxyConfig( + target_host="proxy:3128", basic_auth=("user", "pass") + ), + ) + ) + # Core leaves a caller's own authorization header alone. + assert ("Authorization", "Custom token") in connection.headers + assert ("x-tenant", "t1") in connection.headers + assert not any(key == "authorization" for key, _ in connection.headers) + assert connection.http_proxy == "http://user:pass@proxy:3128" + + +class _Recorder: + """Answers describe and keeps the headers it arrived with.""" + + def __init__(self) -> None: + self.metadata: dict[str, str | bytes] = {} + + async def describe( + self, _request: Any, context: Any + ) -> stream.DescribeStreamResponse: + self.metadata = dict(context.invocation_metadata() or ()) + return stream.DescribeStreamResponse( + frontend_response=stream.DescribeStreamOutput( + state=stream.StreamState(head_offset=3) + ) + ) + + def register(self, server: grpc.aio.Server) -> None: + # The generated registration wants the whole servicer; one method is + # enough to see the headers. + handler: Any = grpc.unary_unary_rpc_method_handler( + self.describe, + request_deserializer=stream.DescribeStreamRequest.FromString, + response_serializer=stream.DescribeStreamResponse.SerializeToString, + ) + server.add_generic_rpc_handlers( + ( + grpc.method_handlers_generic_handler( + _SERVICE, {"DescribeStream": handler} + ), + ) + ) + + +async def test_an_api_key_client_sends_the_header_on_every_call() -> None: + recorder = _Recorder() + server = grpc.aio.server() + recorder.register(server) + port = server.add_insecure_port("127.0.0.1:0") + await server.start() + config = ConnectConfig( + target_host=f"127.0.0.1:{port}", + api_key="secret", + tls=False, + rpc_metadata={"x-tenant": "t1"}, + ) + streams = StreamClient.for_connection(Connection.from_config(config), "ns") + try: + state = await streams.get("s").describe() + assert state.head_offset == 3 + assert recorder.metadata["authorization"] == "Bearer secret" + assert recorder.metadata["x-tenant"] == "t1" + assert recorder.metadata["client-name"] == "temporal-python" + assert recorder.metadata["client-version"] == __version__ + finally: + await streams.close() + await server.stop(None) + + +async def test_the_shared_client_retries_under_the_clients_config() -> None: + retry = RetryConfig(max_retries=3) + client = _fake_client( + ConnectConfig(target_host="localhost:7233", retry_config=retry), "ns" + ) + try: + shared = client_stream.shared_client(client) + assert shared._retry_config is retry + assert shared is client_stream.shared_client(client), "one channel per key" + # A client with other credentials to the same host is not the same channel. + other = _fake_client( + ConnectConfig(target_host="localhost:7233", api_key="k", tls=False), "ns" + ) + assert client_stream.shared_client(other) is not shared + assert client_stream.shared_key(other) != client_stream.shared_key(client) + finally: + await client_stream.close_shared_clients() diff --git a/tests/test_client_stream_errors.py b/tests/test_client_stream_errors.py new file mode 100644 index 0000000000..7b379f4c77 --- /dev/null +++ b/tests/test_client_stream_errors.py @@ -0,0 +1,114 @@ +"""What a refused stream call raises. + +The service carries no typed detail for its refusals, so a typed one arrives +as a ``FAILED_PRECONDITION`` with a reason token at the front of the message. +These pin the mapping: each token to its error, the phrases an older server +sends to the same errors, and an unrelated message on the same code to a +plain ``RPCError`` that keeps the code. +""" + +from __future__ import annotations + +import grpc +import pytest + +from temporalio.client_stream import translate_error +from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams import ( + StreamClosedError, + StreamCursorError, + StreamNotFoundError, + StreamProducerError, +) + + +@pytest.mark.parametrize( + "details", + [ + "STREAM_PRODUCER_CONFLICT: producer sequence 3 already used with different content", + "STREAM_PRODUCER_STALE_SEQUENCE: stale producer sequence 2, last accepted for " + 'producer "p" is 3', + ], +) +def test_a_producer_refusal_is_a_producer_error(details: str) -> None: + error = translate_error(grpc.StatusCode.FAILED_PRECONDITION, details) + assert isinstance(error, StreamProducerError) + assert str(error) == details + + +def test_an_older_servers_producer_refusal_is_a_producer_error() -> None: + # Before the tokens the refusal was an INVALID_ARGUMENT with the phrase. + details = 'stale producer sequence 2, last accepted for producer "p" is 3' + error = translate_error(grpc.StatusCode.INVALID_ARGUMENT, details) + assert isinstance(error, StreamProducerError) + + +@pytest.mark.parametrize( + "details", + [ + "STREAM_CURSOR_BELOW_FLOOR: offset 2 is below the stream's floor of 5", + "offset 2 is below the stream's floor of 5", + ], +) +def test_a_read_below_the_floor_is_a_cursor_error(details: str) -> None: + error = translate_error(grpc.StatusCode.FAILED_PRECONDITION, details) + assert isinstance(error, StreamCursorError) + assert str(error) == details + + +@pytest.mark.parametrize( + "details", + ["STREAM_CLOSED: stream is closed", "stream is closed"], +) +def test_an_append_on_a_sealed_stream_is_a_closed_error(details: str) -> None: + error = translate_error(grpc.StatusCode.FAILED_PRECONDITION, details) + assert isinstance(error, StreamClosedError) + assert str(error) == details + + +def test_a_create_with_another_policy_is_a_value_error() -> None: + error = translate_error( + grpc.StatusCode.FAILED_PRECONDITION, + "STREAM_POLICY_MISMATCH: stream exists keeping 10 records, asked for 5", + ) + assert type(error) is ValueError + + +def test_not_found_is_a_not_found_error() -> None: + error = translate_error(grpc.StatusCode.NOT_FOUND, "no stream with id 's'") + assert isinstance(error, StreamNotFoundError) + + +def test_an_unrelated_message_keeps_its_code() -> None: + # The same codes with another message are the server's ordinary refusals. + for code, expected in [ + (grpc.StatusCode.INVALID_ARGUMENT, RPCStatusCode.INVALID_ARGUMENT), + (grpc.StatusCode.FAILED_PRECONDITION, RPCStatusCode.FAILED_PRECONDITION), + (grpc.StatusCode.UNAVAILABLE, RPCStatusCode.UNAVAILABLE), + ]: + error = translate_error(code, "no records to append", b"raw") + assert type(error) is RPCError + assert error.status == expected + assert error.raw_grpc_status == b"raw" + assert str(error) == "no records to append" + + +def test_a_token_needs_its_code_and_its_separator() -> None: + # A token on another code is prose, and so is one with no ": " after it. + error = translate_error( + grpc.StatusCode.INVALID_ARGUMENT, "STREAM_PRODUCER_CONFLICT: elsewhere" + ) + assert type(error) is RPCError + error = translate_error( + grpc.StatusCode.FAILED_PRECONDITION, "STREAM_CURSOR_BELOW_FLOOR" + ) + assert type(error) is RPCError + error = translate_error( + grpc.StatusCode.FAILED_PRECONDITION, "STREAM_SOMETHING_ELSE: unknown token" + ) + assert type(error) is RPCError + + +def test_an_empty_message_reads_as_the_code() -> None: + error = translate_error(grpc.StatusCode.UNAVAILABLE, "") + assert str(error) == "UNAVAILABLE" diff --git a/tests/test_client_stream_owner.py b/tests/test_client_stream_owner.py new file mode 100644 index 0000000000..a19e8dbc87 --- /dev/null +++ b/tests/test_client_stream_owner.py @@ -0,0 +1,90 @@ +"""Which owner an owned-stream call names on the wire. + +A workflow owner goes out in the workflow fields, which a server without +owner support still routes on. An activity owner needs the owner reference: +a standalone activity is routed on its own id, and an activity a workflow +scheduled is routed on the workflow. +""" + +from __future__ import annotations + +from typing import Any + +import temporalio.api.streamservice.v1 as stream +from temporalio.api.stream.v1 import StreamRecord, StreamStartPosition +from temporalio.client_stream import StreamClient + + +class _Recorder: + """Stands in for the stub and answers every call with an empty response.""" + + def __init__(self) -> None: + self.requests: list[Any] = [] + + def __getattr__(self, method: str) -> Any: + responses = { + "AddWorkflowMessages": stream.AddWorkflowMessagesResponse, + "PollWorkflowMessages": stream.PollWorkflowMessagesResponse, + "DescribeWorkflowStream": stream.DescribeWorkflowStreamResponse, + } + + async def call(request: Any, **_: Any) -> Any: + self.requests.append(request.frontend_request) + return responses[method]() + + return call + + +def _client(recorder: _Recorder) -> StreamClient: + client = StreamClient.__new__(StreamClient) + client._stub = recorder + client._namespace = "ns" + client._retry_config = None + return client + + +async def _every_call(handle: Any) -> None: + await handle.append(StreamRecord()) + await handle.poll(wait=False) + await handle.describe() + + +async def test_a_workflow_owner_uses_the_workflow_fields(): + recorder = _Recorder() + await _every_call(_client(recorder).workflow_stream("wf", owner_run_id="run")) + for request in recorder.requests: + assert (request.workflow_id, request.owner_run_id) == ("wf", "run") + assert not request.HasField("owner") + + +async def test_a_standalone_activity_is_its_own_owner(): + recorder = _Recorder() + await _every_call(_client(recorder).activity_stream("act", run_id="run")) + for request in recorder.requests: + assert request.owner == stream.StreamOwner( + kind=stream.STREAM_OWNER_KIND_ACTIVITY, id="act", run_id="run" + ) + assert request.workflow_id == "" + + +async def test_a_workflow_activity_is_reached_through_its_workflow(): + recorder = _Recorder() + await _every_call( + _client(recorder).activity_stream("act", "reasoning", workflow_id="wf") + ) + for request in recorder.requests: + assert request.owner == stream.StreamOwner( + kind=stream.STREAM_OWNER_KIND_WORKFLOW_ACTIVITY, id="wf", activity_id="act" + ) + assert request.stream_name == "reasoning" + + +async def test_a_first_read_carries_its_start_position(): + recorder = _Recorder() + handle = _client(recorder).workflow_stream("wf") + await handle.read(start=StreamStartPosition(last_n=3)) + await handle.read(from_offset=4) + first, later = recorder.requests + assert first.start_position == StreamStartPosition(last_n=3) + assert first.from_offset == 0 + assert not later.HasField("start_position") diff --git a/tests/test_client_stream_records.py b/tests/test_client_stream_records.py new file mode 100644 index 0000000000..261d0b74fb --- /dev/null +++ b/tests/test_client_stream_records.py @@ -0,0 +1,50 @@ +"""How a record crosses between the public shape and the stored one. + +No server: what this pins is that the two converters carry every field the +record declares, so a field added to ``StreamRecord`` is not dropped in +silence on the way out or the way back. +""" + +from __future__ import annotations + +import pytest + +from temporalio.api.common.v1 import Payload +from temporalio.api.stream.v1 import StreamRecord, StreamRecordKind +from temporalio.client_stream import _copy_by_name, _to_public, _to_service + + +def test_a_record_crosses_both_ways_without_dropping_a_field() -> None: + # Copied by descriptor rather than field by field, so a field added to + # StreamRecord crosses without anybody adding a line to the converters. + sent = StreamRecord( + topic="t", + kind=StreamRecordKind.STREAM_RECORD_KIND_FINISH, + producer_id="model", + attempt=3, + sequence=7, + ) + sent.body.CopyFrom(Payload(data=b"x", metadata={"encoding": b"binary/plain"})) + sent.metadata["trace"].CopyFrom(Payload(data=b"abc")) + + entry = _to_public(_to_service(sent)) + assert entry.record == sent + assert {f.name for f, _ in sent.ListFields()} == { + f.name for f, _ in entry.record.ListFields() + } + # Every field the public record declares is carried, not only the ones + # the converters happened to name. + crossed = {f.name for f, _ in _to_service(sent).ListFields()} - {"offset"} + assert crossed == {f.name for f in StreamRecord.DESCRIPTOR.fields} + + +def test_a_field_with_no_counterpart_is_refused_rather_than_dropped() -> None: + # The stored shape carries an offset the public record has no room for. + # Copying it across has to say so rather than leave it behind, because + # the same silence is what loses a field somebody adds later. + stored = _to_service(StreamRecord(topic="t")) + stored.offset = 12 + with pytest.raises(ValueError, match="offset"): + _copy_by_name(stored, StreamRecord(), skip=frozenset()) + # Named explicitly, the offset is the store's and rides on the entry. + assert _to_public(stored).offset == 12 diff --git a/tests/test_client_stream_retry.py b/tests/test_client_stream_retry.py new file mode 100644 index 0000000000..8baa2939b3 --- /dev/null +++ b/tests/test_client_stream_retry.py @@ -0,0 +1,257 @@ +"""What the stream client retries, and what it raises at once. + +The stream service is reached over a channel of the client's own, outside +sdk-core, so the retry Core gives every other call is reproduced here. These +pin its edges with a scripted stub: a throttled read or numbered append goes +again, an append the server could not tell from its repeat does not, a code +Core would not retry is raised at once, and cancelling the caller lands during +the wait between attempts. +""" + +from __future__ import annotations + +import asyncio +from collections import defaultdict +from types import SimpleNamespace +from typing import Any + +import grpc +import grpc.aio +import pytest + +import temporalio.api.streamservice.v1 as stream +import temporalio.converter +from temporalio import client_stream +from temporalio.api.common.v1 import GrpcStatus +from temporalio.api.enums.v1 import ResourceExhaustedCause +from temporalio.api.errordetails.v1 import ResourceExhaustedFailure +from temporalio.api.stream.v1 import StreamRecord +from temporalio.client_stream import WorkflowStreamHandle, _to_service +from temporalio.service import RetryConfig, RPCError, RPCStatusCode +from temporalio.streams import StreamNotFoundError +from temporalio.streams._record import RecordKind +from temporalio.streams._wire import to_wire +from temporalio.streams.providers.native import NativeStreamHandle + +FAST = RetryConfig( + initial_interval_millis=1, + max_interval_millis=2, + max_elapsed_time_millis=5000, + max_retries=5, +) + + +@pytest.fixture(autouse=True) +def fast_throttle(monkeypatch: pytest.MonkeyPatch) -> None: + # The floor under a throttled wait is a second, which is right for a + # caller and wrong for a test. + monkeypatch.setattr( + client_stream, + "_THROTTLE", + RetryConfig( + initial_interval_millis=1, + max_interval_millis=2, + max_elapsed_time_millis=None, + max_retries=0, + ), + ) + + +class _Stub: + """Answers each method from a script of exceptions and responses, in order.""" + + def __init__(self, **scripts: list[Any]) -> None: + self.calls: dict[str, int] = defaultdict(int) + self._scripts = scripts + + def __getattr__(self, method: str) -> Any: + script = self._scripts[method] + + async def call(_request: Any, **_: Any) -> Any: + self.calls[method] += 1 + outcome = script.pop(0) + if isinstance(outcome, BaseException): + raise outcome + return outcome + + return call + + +def _error( + code: grpc.StatusCode, + details: str = "", + *, + cause: ResourceExhaustedCause.ValueType | None = None, +) -> grpc.aio.AioRpcError: + trailing = grpc.aio.Metadata() + if cause is not None: + status = GrpcStatus(code=code.value[0], message=details) + status.details.add().Pack(ResourceExhaustedFailure(cause=cause)) + trailing.add("grpc-status-details-bin", status.SerializeToString()) + return grpc.aio.AioRpcError(code, grpc.aio.Metadata(), trailing, details=details) + + +def _throttled() -> grpc.aio.AioRpcError: + return _error( + grpc.StatusCode.RESOURCE_EXHAUSTED, + "service rate limit exceeded", + cause=ResourceExhaustedCause.RESOURCE_EXHAUSTED_CAUSE_RPS_LIMIT, + ) + + +def _page(*records: stream.StreamRecord) -> stream.PollWorkflowMessagesResponse: + return stream.PollWorkflowMessagesResponse( + frontend_response=stream.PollMessagesOutput( + records=list(records), + next_offset=len(records), + head_offset=len(records), + closed=True, + run_id="run", + ) + ) + + +def _appended() -> stream.AddWorkflowMessagesResponse: + return stream.AddWorkflowMessagesResponse( + frontend_response=stream.AddMessagesOutput( + first_offset=0, next_offset=1, count=1 + ) + ) + + +def _handle(stub: _Stub, retry: RetryConfig = FAST) -> WorkflowStreamHandle: + return WorkflowStreamHandle(stub, "ns", "wf", "topic", "run", retry_config=retry) + + +async def test_a_throttled_poll_is_read_again() -> None: + stub = _Stub(PollWorkflowMessages=[_throttled(), _page()]) + page = await _handle(stub).poll() + assert page.closed + assert stub.calls["PollWorkflowMessages"] == 2 + + +async def test_a_throttled_numbered_append_goes_again() -> None: + stub = _Stub(AddWorkflowMessages=[_throttled(), _appended()]) + appended = await _handle(stub).append(StreamRecord(), producer_id="p", sequence=1) + assert appended.next_offset == 1 + assert stub.calls["AddWorkflowMessages"] == 2 + + +async def test_a_numbered_append_survives_an_ambiguous_failure() -> None: + # The server holds the producer's sequence, so whichever attempt landed, + # the repeat comes back with the original offsets. + stub = _Stub(AddWorkflowMessages=[_error(grpc.StatusCode.UNAVAILABLE), _appended()]) + await _handle(stub).append(StreamRecord(), producer_id="p", sequence=1) + assert stub.calls["AddWorkflowMessages"] == 2 + + +async def test_an_unnumbered_append_is_not_made_again_after_an_ambiguous_failure() -> ( + None +): + stub = _Stub(AddWorkflowMessages=[_error(grpc.StatusCode.UNAVAILABLE), _appended()]) + with pytest.raises(RPCError) as raised: + await _handle(stub).append(StreamRecord()) + assert raised.value.status == RPCStatusCode.UNAVAILABLE + assert stub.calls["AddWorkflowMessages"] == 1 + + +async def test_an_unnumbered_append_goes_again_after_a_refusal() -> None: + # Throttling is answered before the handler runs, so nothing landed. + stub = _Stub(AddWorkflowMessages=[_throttled(), _appended()]) + await _handle(stub).append(StreamRecord()) + assert stub.calls["AddWorkflowMessages"] == 2 + + +async def test_a_code_core_would_not_retry_is_raised_at_once() -> None: + stub = _Stub( + PollWorkflowMessages=[_error(grpc.StatusCode.INVALID_ARGUMENT, "bad"), _page()] + ) + with pytest.raises(RPCError) as raised: + await _handle(stub).poll() + assert raised.value.status == RPCStatusCode.INVALID_ARGUMENT + assert stub.calls["PollWorkflowMessages"] == 1 + + stub = _Stub(PollWorkflowMessages=[_error(grpc.StatusCode.NOT_FOUND), _page()]) + with pytest.raises(StreamNotFoundError): + await _handle(stub).poll() + assert stub.calls["PollWorkflowMessages"] == 1 + + +async def test_the_budget_is_bounded() -> None: + stub = _Stub(PollWorkflowMessages=[_throttled() for _ in range(10)]) + with pytest.raises(RPCError) as raised: + await _handle( + stub, RetryConfig(initial_interval_millis=1, max_retries=3) + ).poll() + assert raised.value.status == RPCStatusCode.RESOURCE_EXHAUSTED + assert stub.calls["PollWorkflowMessages"] == 3 + + +async def test_a_full_stream_is_not_waited_out() -> None: + full = _error( + grpc.StatusCode.RESOURCE_EXHAUSTED, + "stream holds 10 of its budget of 10 records", + cause=ResourceExhaustedCause.RESOURCE_EXHAUSTED_CAUSE_PERSISTENCE_STORAGE_LIMIT, + ) + stub = _Stub(AddWorkflowMessages=[full, _appended()]) + with pytest.raises(RPCError) as raised: + await _handle(stub).append(StreamRecord(), producer_id="p", sequence=1) + assert raised.value.status == RPCStatusCode.RESOURCE_EXHAUSTED + assert stub.calls["AddWorkflowMessages"] == 1 + + too_large = _error( + grpc.StatusCode.RESOURCE_EXHAUSTED, + "grpc: received message larger than max (5 vs. 4)", + ) + stub = _Stub(AddWorkflowMessages=[too_large, _appended()]) + with pytest.raises(RPCError): + await _handle(stub).append(StreamRecord(), producer_id="p", sequence=1) + assert stub.calls["AddWorkflowMessages"] == 1 + + +async def test_cancelling_the_caller_lands_during_the_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + client_stream, + "_THROTTLE", + RetryConfig(initial_interval_millis=60_000, max_elapsed_time_millis=None), + ) + stub = _Stub(PollWorkflowMessages=[_throttled(), _page()]) + task = asyncio.ensure_future(_handle(stub).poll()) + for _ in range(10): + await asyncio.sleep(0) + assert stub.calls["PollWorkflowMessages"] == 1, "parked in the wait" + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert stub.calls["PollWorkflowMessages"] == 1 + + +async def test_a_native_read_survives_a_throttled_poll( + monkeypatch: pytest.MonkeyPatch, +) -> None: + converter = temporalio.converter.default() + record = _to_service( + to_wire( + converter.payload_converter, + topic="topic", + kind=RecordKind.DATA, + value="hello", + producer_id="p", + sequence=1, + ) + ) + stub = _Stub(PollWorkflowMessages=[_throttled(), _page(record)]) + client: Any = SimpleNamespace(data_converter=converter) + handle = NativeStreamHandle(client, "wf", "run") + + def open_stream(_topic: str, _run_id: str) -> WorkflowStreamHandle: + return _handle(stub) + + monkeypatch.setattr(handle, "_stream", open_stream) + + values = [item.value async for item in handle.read(topic="topic")] + + assert values == ["hello"] + assert stub.calls["PollWorkflowMessages"] == 2 diff --git a/tests/test_client_stream_sharing.py b/tests/test_client_stream_sharing.py new file mode 100644 index 0000000000..55fd3e7e33 --- /dev/null +++ b/tests/test_client_stream_sharing.py @@ -0,0 +1,57 @@ +"""Who owns a shared channel, and who may close it. + +One channel is shared per loop, connection and namespace, so two providers in +one process can be using the same one. Closing a provider has to leave the +other's channels alone. +""" + +from __future__ import annotations + +import asyncio + +from temporalio import client_stream +from temporalio.client_stream import Connection, SharedKey +from temporalio.streams.providers.native import NativeStreams + + +class _FakeClient: + """Stands in for a StreamClient, which would want a real channel.""" + + def __init__(self) -> None: + self.closed = False + + async def close(self) -> None: + self.closed = True + + +def _key(target: str, namespace: str) -> SharedKey: + return Connection(target=target, secure=False), namespace + + +def _put(key: SharedKey) -> _FakeClient: + fake = _FakeClient() + per_loop = client_stream._shared.setdefault(asyncio.get_running_loop(), {}) + per_loop[key] = fake # type: ignore[assignment] + return fake + + +async def test_closing_named_clients_leaves_the_others_open() -> None: + mine = _put(_key("host-a:7233", "ns")) + theirs = _put(_key("host-b:7233", "ns")) + await client_stream.close_shared_clients(_key("host-a:7233", "ns")) + assert mine.closed + assert not theirs.closed, "another provider is still reading through it" + await client_stream.close_shared_clients() + assert theirs.closed + + +async def test_a_provider_closes_only_what_its_own_handles_opened() -> None: + mine = _put(_key("host-a:7233", "ns")) + theirs = _put(_key("host-b:7233", "ns")) + + provider = NativeStreams() + provider._opened.add(_key("host-a:7233", "ns")) + await provider.close() + assert mine.closed + assert not theirs.closed + await client_stream.close_shared_clients() diff --git a/tests/testing/test_workflow.py b/tests/testing/test_workflow.py index b47d0ac052..15ef1e6cb8 100644 --- a/tests/testing/test_workflow.py +++ b/tests/testing/test_workflow.py @@ -174,6 +174,51 @@ async def test_workflow_env_time_skipping_disabled(): assert monotonic() - start > 2.5 +@activity.defn +async def echo(value: str) -> str: + return value + + +@workflow.defn +class TwoActivitiesWorkflow: + @workflow.run + async def run(self, value: str) -> str: + for _ in range(2): + value = await workflow.execute_activity( + echo, value, start_to_close_timeout=timedelta(seconds=15) + ) + return value + + +async def test_workflow_env_time_skipping_concurrent_result_waiters(): + # The test server holds one time-skipping lock per in-flight task and counts + # one unlock per request, so concurrent result() waiters have to share one + # unlock. Otherwise the clock skips as soon as the first workflow completes + # and times out the task the other workflow still has in flight. + async with await WorkflowEnvironment.start_time_skipping() as env: + async with new_worker( + env.client, TwoActivitiesWorkflow, activities=[echo] + ) as worker: + handles = [ + await env.client.start_workflow( + TwoActivitiesWorkflow.run, + value, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + for value in ("first", "second") + ] + assert ["first", "second"] == await asyncio.gather( + *(handle.result() for handle in handles) + ) + for handle in handles: + async for event in handle.fetch_history_events(): + assert not event.HasField("workflow_task_timed_out_event_attributes") + assert not event.HasField( + "workflow_execution_timed_out_event_attributes" + ) + + @workflow.defn class AssertFailWorkflow: @workflow.run diff --git a/tests/worker/test_replayer.py b/tests/worker/test_replayer.py index 52fed74165..ddce04826c 100644 --- a/tests/worker/test_replayer.py +++ b/tests/worker/test_replayer.py @@ -8,8 +8,12 @@ from pathlib import Path from typing import Any +import google.protobuf.json_format import pytest +import temporalio.api.common.v1 +import temporalio.api.history.v1 +import temporalio.api.stream.v1 import temporalio.worker._workflow_instance from temporalio import activity, workflow from temporalio.client import Client, WorkflowFailureError, WorkflowHistory @@ -121,6 +125,48 @@ async def test_replayer_workflow_complete(client: Client) -> None: ) +def test_workflow_history_json_carries_stream_slices_only_when_present() -> None: + """The JSON shape is unchanged without slices, and round-trips them when present.""" + with Path(__file__).with_name("test_replayer_complete_history.json").open("r") as f: + history = WorkflowHistory.from_json("fake", f.read()) + assert list(history.stream_slices) == [] + # Byte for byte what the history proto's JSON was before slices existed. + assert history.to_json() == google.protobuf.json_format.MessageToJson( + temporalio.api.history.v1.History(events=history.events) + ) + assert "streamSlices" not in history.to_json_dict() + + record = temporalio.api.stream.v1.StreamRecord( + body=temporalio.api.common.v1.Payload( + data=b'{"n": 1}', metadata={"encoding": b"json/plain"} + ), + topic="inputs", + kind=temporalio.api.stream.v1.StreamRecordKind.STREAM_RECORD_KIND_DATA, + producer_id="model", + attempt=1, + sequence=0, + ) + stream_slice = temporalio.api.stream.v1.StreamSlice( + stream_id="inputs", + run_id="run-1", + from_offset=0, + to_offset=1, + records=[record], + workflow_task_completed_event_id=4, + ) + bundle = WorkflowHistory("fake", history.events, [stream_slice]) + + text = bundle.to_json() + assert "streamSlices" in text + restored = WorkflowHistory.from_json("fake", text) + assert list(restored.stream_slices) == [stream_slice] + assert list(restored.events) == list(history.events) + # The dict form reads back the same, and a plain export stays plain. + from_dict = WorkflowHistory.from_json("fake", bundle.to_json_dict()) + assert list(from_dict.stream_slices) == [stream_slice] + assert WorkflowHistory("fake", restored.events).to_json() == history.to_json() + + @pytest.mark.skipif(sys.version_info < (3, 12), reason="Skipping for < 3.12") async def test_replayer_workflow_complete_json() -> None: # See `test_replayer_workflow_complete` for full skip description. diff --git a/tests/worker/test_stream_resupply.py b/tests/worker/test_stream_resupply.py new file mode 100644 index 0000000000..f7dcf3f786 --- /dev/null +++ b/tests/worker/test_stream_resupply.py @@ -0,0 +1,150 @@ +"""A task whose re-supplied ranges stop short is made whole before it runs. + +The server re-supplies a replaying workflow's recorded ranges within a budget. +These pin the worker's half: a ``DeliverStreamRecords`` job with fewer records +than its range spans has the rest fetched from the stream service, resolved as +the server resolves a subscribed name, while a whole job and an empty +observation are left alone and a range the stream no longer holds fails the +task rather than let it run on less input. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest + +from temporalio import client_stream +from temporalio.api.common.v1 import Payload +from temporalio.api.stream.v1 import StreamRecord +from temporalio.bridge.proto.workflow_activation import WorkflowActivation +from temporalio.client_stream import Page, StreamEntry +from temporalio.streams import StreamNotFoundError +from temporalio.worker._stream_ranges import fill_short_stream_ranges + +# The fill-in only hands the client to the shared-channel lookup, which the +# tests replace. +_CLIENT: Any = SimpleNamespace() + + +def _record(offset: int) -> StreamRecord: + return StreamRecord(topic="inputs", body=Payload(data=f"r{offset}".encode())) + + +class _FakeStream: + """One stream's records by offset, answering polls as the service does.""" + + def __init__(self, held: dict[int, StreamRecord]) -> None: + self.held = held + self.polls: list[tuple[int, int]] = [] + + async def poll(self, *, from_offset: int, max_records: int, wait: bool) -> Page: + assert not wait + self.polls.append((from_offset, max_records)) + if not self.held: + raise StreamNotFoundError("no such stream") + entries = [ + StreamEntry(record=self.held[o], offset=o) + for o in range(from_offset, from_offset + max_records) + if o in self.held + ] + head = max(self.held) + 1 + return Page( + entries=entries, + next_offset=from_offset + len(entries), + head_offset=head, + closed=False, + ) + + +class _FakeStreams: + """The stream client: an owned stream per (workflow, name) and standalone ones by id.""" + + def __init__( + self, owned: dict[str, _FakeStream], standalone: dict[str, _FakeStream] + ) -> None: + self.owned = owned + self.standalone = standalone + + def workflow_stream( + self, workflow_id: str, name: str, *, owner_run_id: str + ) -> _FakeStream: + assert (workflow_id, owner_run_id) == ("wf", "run") + return self.owned.get(name, _FakeStream({})) + + def get(self, stream_id: str) -> _FakeStream: + return self.standalone.get(stream_id, _FakeStream({})) + + +def _activation(*jobs: tuple[str, int, int, int]) -> WorkflowActivation: + """An activation with one delivery per ``(stream, from, to, records present)``.""" + act = WorkflowActivation(run_id="run") + for stream_id, from_offset, to_offset, present in jobs: + job = act.jobs.add() + job.deliver_stream_records.stream_id = stream_id + job.deliver_stream_records.from_offset = from_offset + job.deliver_stream_records.to_offset = to_offset + job.deliver_stream_records.records.extend( + _record(o) for o in range(from_offset, from_offset + present) + ) + return act + + +@pytest.fixture +def streams(monkeypatch: pytest.MonkeyPatch) -> _FakeStreams: + fake = _FakeStreams( + owned={"inputs": _FakeStream({o: _record(o) for o in range(10)})}, + standalone={"shared": _FakeStream({o: _record(o) for o in range(5)})}, + ) + monkeypatch.setattr(client_stream, "shared_client", lambda _client: fake) + return fake + + +def _bodies(act: WorkflowActivation, index: int = 0) -> list[bytes]: + return [r.body.data for r in act.jobs[index].deliver_stream_records.records] + + +async def test_a_short_job_is_filled_from_the_owned_stream( + streams: _FakeStreams, +) -> None: + act = _activation(("inputs", 2, 7, 2)) + await fill_short_stream_ranges(act, "wf", _CLIENT) + assert _bodies(act) == [b"r2", b"r3", b"r4", b"r5", b"r6"] + # Only the missing tail was asked for. + assert streams.owned["inputs"].polls == [(4, 3)] + + +@pytest.mark.usefixtures("streams") +async def test_a_name_the_workflow_does_not_own_is_a_standalone_stream() -> None: + act = _activation(("shared", 0, 5, 1)) + await fill_short_stream_ranges(act, "wf", _CLIENT) + assert _bodies(act) == [b"r0", b"r1", b"r2", b"r3", b"r4"] + + +async def test_whole_and_empty_jobs_are_left_alone(streams: _FakeStreams) -> None: + act = _activation(("inputs", 0, 3, 3), ("inputs", 3, 3, 0)) + await fill_short_stream_ranges(act, "wf", _CLIENT) + assert _bodies(act, 0) == [b"r0", b"r1", b"r2"] + assert _bodies(act, 1) == [] + assert streams.owned["inputs"].polls == [] + + +@pytest.mark.usefixtures("streams") +async def test_a_range_the_stream_no_longer_holds_fails_loudly() -> None: + act = _activation(("inputs", 8, 12, 1)) + with pytest.raises( + StreamNotFoundError, match="no longer holds offsets \\[9, 12\\)" + ): + await fill_short_stream_ranges(act, "wf", _CLIENT) + + +async def test_no_client_is_needed_when_nothing_is_short( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def refuse(_client: Any) -> Any: + raise AssertionError("no channel should be opened") + + monkeypatch.setattr(client_stream, "shared_client", refuse) + act = _activation(("inputs", 0, 2, 2)) + await fill_short_stream_ranges(act, "wf", _CLIENT) diff --git a/tests/worker/test_workflow_stream.py b/tests/worker/test_workflow_stream.py new file mode 100644 index 0000000000..b4dd6107be --- /dev/null +++ b/tests/worker/test_workflow_stream.py @@ -0,0 +1,353 @@ +"""In-workflow consumption and publication of a server-side stream. + +Ranges arrive on Workflow Tasks and only the offsets they covered are written to +History, so replay is served by the server reading the stream again. These tests +drive the buffering, the workflow-facing read and the per-task publish directly, +which is the part this SDK owns; the delivery decision itself lives in the +server and sdk-core. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +import temporalio.api.common.v1 +import temporalio.api.stream.v1 as api_stream +from temporalio.bridge.proto.workflow_commands import WorkflowCommand +from temporalio.worker._workflow_instance import ( + _MAX_STREAM_BATCH_BYTES, + _MAX_STREAM_RECORD_BYTES, + _MAX_STREAM_RECORDS_PER_BATCH, + _StreamBuffer, + _WorkflowInstanceImpl, +) +from temporalio.workflow import ReadOnlyContextError + + +def record(body: bytes, topic: str = "") -> api_stream.StreamRecord: + return api_stream.StreamRecord( + body=temporalio.api.common.v1.Payload(data=body), topic=topic + ) + + +def bodies(delivered: list[Any]) -> list[bytes]: + return [item.record.body.data for item in delivered] + + +async def test_buffer_hands_over_in_order() -> None: + buffer = _StreamBuffer() + buffer.extend([record(b"one"), record(b"two")]) + + assert bodies(buffer.take()) == [b"one", b"two"] + assert len(buffer) == 0 + + +# A reader that arrives before the data must not miss it, and one that arrives +# after must not block: the range is delivered once and never resent. +async def test_buffer_wakes_a_waiting_reader() -> None: + buffer = _StreamBuffer() + waiter = buffer.wait_future() + assert not waiter.done() + + buffer.extend([record(b"late")]) + await asyncio.wait_for(waiter, timeout=1) + assert bodies(buffer.take()) == [b"late"] + + +# An empty range is still a delivery the server recorded, but there is nothing +# to hand a reader, so it must not wake one into returning nothing. +async def test_empty_range_does_not_wake_a_reader() -> None: + buffer = _StreamBuffer() + waiter = buffer.wait_future() + + buffer.extend([]) + + assert not waiter.done() + assert len(buffer) == 0 + + +async def test_buffer_keeps_data_delivered_before_anyone_reads() -> None: + buffer = _StreamBuffer() + buffer.extend([record(b"early")]) + + # No waiting: the data is already there. + assert len(buffer) == 1 + waiter = buffer.wait_future() + assert not waiter.done(), "a fresh waiter is only resolved by new data" + assert bodies(buffer.take()) == [b"early"] + + +class _ReadOnlyStub: + """Enough of the workflow instance to drive the real read. + + The cap lives inside ``workflow_read_stream_records``, so a test that + reimplements it proves nothing about the code that ships. Borrowing the + method off the real class is what keeps the test on the shipped path. + """ + + workflow_read_stream_records = _WorkflowInstanceImpl.workflow_read_stream_records + + def __init__(self, read_only: bool = False) -> None: + self._stream_buffers: dict[str, _StreamBuffer] = {} + self._read_only = read_only + + def _assert_not_read_only(self, action: str) -> None: + if self._read_only: + raise ReadOnlyContextError(f"cannot {action} in a read-only context") + + async def read(self, stream: str, max_records: int) -> list[bytes]: + instance: Any = self + return bodies( + await _WorkflowInstanceImpl.workflow_read_stream_records( + instance, stream, max_records + ) + ) + + +@pytest.mark.parametrize("max_records", [1, 2, 5]) +async def test_read_respects_a_cap_without_losing_the_tail(max_records: int) -> None: + stub = _ReadOnlyStub() + expected = [b"a", b"b", b"c"] + stub._stream_buffers["s"] = _StreamBuffer() + stub._stream_buffers["s"].extend([record(b) for b in expected]) + + got = await stub.read("s", max_records) + + assert got == expected[:max_records] + # Whatever the cap left behind has to still be there: nothing resends it. + assert len(stub._stream_buffers["s"]) == max(0, len(expected) - max_records) + + if len(stub._stream_buffers["s"]): + rest = await stub.read("s", 0) + assert got + rest == expected + else: + assert got == expected + + +# A query activation carries no ranges, so a read there would wait on a future +# nothing can resolve and the query would time out saying nothing. +async def test_read_is_refused_in_a_read_only_context() -> None: + stub = _ReadOnlyStub(read_only=True) + stub._stream_buffers["s"] = _StreamBuffer() + stub._stream_buffers["s"].extend([record(b"a")]) + + with pytest.raises(ReadOnlyContextError, match="read stream"): + await stub.read("s", 0) + # Refused before the buffer was touched, so the range is still there for + # the task that is allowed to read it. + assert len(stub._stream_buffers["s"]) == 1 + + +# A range is recorded as consumed once and never resent, so the buffer is the +# only place a repeated, skipped or mis-sized delivery can still be noticed. +async def test_ranges_have_to_abut_the_last_one() -> None: + buffer = _StreamBuffer("s") + buffer.extend([record(b"a"), record(b"b")], 0, 2) + # An empty range moves the expectation too: the server recorded it. + buffer.extend([], 2, 2) + buffer.extend([record(b"c")], 2, 3) + assert [item.offset for item in buffer.take()] == [0, 1, 2] + + with pytest.raises(RuntimeError, match=r"\[2, 3\).*ended at 3"): + buffer.extend([record(b"c")], 2, 3) + with pytest.raises(RuntimeError, match=r"\[5, 6\).*ended at 3"): + buffer.extend([record(b"f")], 5, 6) + + +async def test_a_range_has_to_carry_as_many_records_as_it_spans() -> None: + buffer = _StreamBuffer("s") + with pytest.raises(RuntimeError, match=r"2 records for offsets \[0, 1\)"): + buffer.extend([record(b"a"), record(b"b")], 0, 1) + assert len(buffer) == 0 + + +class _Completion: + def __init__(self) -> None: + self.commands: list[WorkflowCommand] = [] + + +class _Successful: + def __init__(self) -> None: + self.successful = _Completion() + + +class _CommandStub: + """Drives the real publish path and keeps the commands it issued. + + The completion's command list is a plain list here, which supports the + same ``insert`` the protobuf container does. + """ + + workflow_append_stream_records = ( + _WorkflowInstanceImpl.workflow_append_stream_records + ) + _flush_stream_appends = _WorkflowInstanceImpl._flush_stream_appends + + def __init__(self) -> None: + self._stream_appends: dict[str, list[api_stream.StreamRecord]] = {} + self._current_completion = _Successful() + + def _assert_not_read_only(self, _action: str) -> None: + pass + + @property + def commands(self) -> list[WorkflowCommand]: + return self._current_completion.successful.commands + + def publish(self, *records: api_stream.StreamRecord, stream_name: str = "") -> None: + instance: Any = self + _WorkflowInstanceImpl.workflow_append_stream_records( + instance, stream_name, list(records) + ) + + def flush(self) -> None: + instance: Any = self + _WorkflowInstanceImpl._flush_stream_appends(instance) + + +# A task's publishes on one stream become one command, whatever their number, +# because the event the command produces is what bounds a workflow's History. +def test_a_tasks_publishes_on_one_stream_become_one_command() -> None: + stub = _CommandStub() + stub.publish(record(b"a"), record(b"b")) + stub.publish(record(b"c")) + stub.publish(record(b"d"), stream_name="other") + assert stub.commands == [] + + stub.flush() + + by_stream = { + command.append_stream_records.stream_name: [ + r.body.data for r in command.append_stream_records.records + ] + for command in stub.commands + } + assert by_stream == {"": [b"a", b"b", b"c"], "other": [b"d"]} + # Flushed once: a second flush has nothing left to say. + stub.flush() + assert len(stub.commands) == 2 + + +# The workflow is the producer of what it publishes, whatever the caller set. +def test_the_workflows_records_carry_no_producer() -> None: + stub = _CommandStub() + stub.publish(api_stream.StreamRecord(producer_id="someone", attempt=3)) + stub.flush() + assert stub.commands[0].append_stream_records.records[0].producer_id == "" + + +# The server accepts nothing after a command that ends the run, so the +# publishes have to go ahead of it. +def test_publishes_are_flushed_ahead_of_the_completion_command() -> None: + stub = _CommandStub() + stub.publish(record(b"a")) + done = WorkflowCommand() + done.complete_workflow_execution.SetInParent() + stub.commands.append(done) + + stub.flush() + + assert [c.WhichOneof("variant") for c in stub.commands] == [ + "append_stream_records", + "complete_workflow_execution", + ] + + +# The server refuses an oversized record, and a refused command is reissued on +# every replay, so the limit has to be applied before the record is buffered. +def test_a_record_over_the_server_limit_is_refused_before_the_command() -> None: + stub = _CommandStub() + stub.publish(record(b"x" * (_MAX_STREAM_RECORD_BYTES - 16))) + with pytest.raises(ValueError, match=f"{_MAX_STREAM_RECORD_BYTES} bytes"): + stub.publish(record(b"x" * (_MAX_STREAM_RECORD_BYTES + 1))) + stub.flush() + assert len(stub.commands) == 1 + + +# A task that publishes more than one batch holds is split into commands the +# server accepts, by count and by bytes, rather than refused. +def test_a_task_over_the_batch_limits_is_split_into_commands() -> None: + stub = _CommandStub() + stub.publish(*(record(b"x") for _ in range(_MAX_STREAM_RECORDS_PER_BATCH + 1))) + stub.flush() + assert [len(c.append_stream_records.records) for c in stub.commands] == [ + _MAX_STREAM_RECORDS_PER_BATCH, + 1, + ] + + stub = _CommandStub() + # Two records that fit one batch together, then one whose framing alone + # overflows the room they leave. + big = record(b"x" * (_MAX_STREAM_RECORD_BYTES - 16)) + room = _MAX_STREAM_BATCH_BYTES - 2 * big.ByteSize() + stub.publish(big, big, record(b"y" * room)) + stub.flush() + assert [len(c.append_stream_records.records) for c in stub.commands] == [2, 1] + + +async def test_a_closed_buffer_keeps_nothing_and_still_checks_continuity() -> None: + # There is no unsubscribe command, so the server keeps delivering for the + # life of the run. A reader that closed would otherwise grow the instance + # for the rest of it. + buffer = _StreamBuffer("s") + buffer.extend([record(b"one")], 0, 1) + assert len(buffer) == 1 + + buffer.close() + assert buffer.closed + assert len(buffer) == 0, "what it held is let go of, not kept for nobody" + + buffer.extend([record(b"two"), record(b"three")], 1, 3) + assert len(buffer) == 0 + # Continuity is still tracked across what it dropped, so a range that + # repeats or skips is caught rather than passing unnoticed. + with pytest.raises(RuntimeError, match="last range ended at 3"): + buffer.extend([record(b"four")], 9, 10) + buffer.extend([record(b"four")], 3, 4) + + +async def test_closing_a_buffer_wakes_a_reader_parked_on_it() -> None: + buffer = _StreamBuffer("s") + waiter = buffer.wait_future() + buffer.close() + # Woken rather than left parked: the reader has to unwind, and nothing + # will ever arrive for it again. + await asyncio.wait_for(waiter, 5) + assert waiter.done() + + +async def test_the_continuity_failure_says_what_to_do_about_it() -> None: + buffer = _StreamBuffer("s") + buffer.extend([record(b"one")], 0, 1) + with pytest.raises(RuntimeError) as failed: + buffer.extend([record(b"two")], 5, 6) + # The task fails and keeps failing, so the message has to name the way out. + assert "Reset the workflow" in str(failed.value) + + +def test_the_raw_workflow_stream_api_is_not_public() -> None: + # A second workflow surface taking raw stream ids and raw protos, beside + # the typed one, is not what an application should reach for. It stays + # reachable under its private name, which is what the provider and the + # contrib surface use. + import temporalio.workflow as wf + + for name in ( + "subscribe_stream", + "append_stream_records", + "read_stream_records", + "DeliveredStreamRecord", + ): + assert name not in wf.__all__ + assert not hasattr(wf, name) + for name in ( + "_subscribe_stream", + "_append_stream_records", + "_read_stream_records", + "_close_stream_records", + "_DeliveredStreamRecord", + ): + assert hasattr(wf, name) diff --git a/tests/worker/test_workflow_stream_e2e.py b/tests/worker/test_workflow_stream_e2e.py new file mode 100644 index 0000000000..b05fd653a8 --- /dev/null +++ b/tests/worker/test_workflow_stream_e2e.py @@ -0,0 +1,1181 @@ +"""The native provider inside a real workflow, against a server that has streams. + +The two rules the memory provider cannot keep are the measurement here: a +publish commits with its Workflow Task and never lands if the task fails, and +a read is a recorded observation the server re-supplies on replay. Needs a +Temporal server built from the AI-198 branch, because neither the stream +service nor the commands exist on a released one: + + TEMPORAL_STREAM_TARGET=127.0.0.1:7333 uv run pytest tests/worker/test_workflow_stream_e2e.py + +Skipped otherwise, rather than passing against a server that has no idea what a +stream is. +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from collections.abc import Sequence +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.api.common.v1 import Payload, WorkflowExecution +from temporalio.api.enums.v1 import EventType, WorkflowTaskFailedCause +from temporalio.api.history.v1 import HistoryEvent +from temporalio.api.stream.v1 import StreamRange, StreamRecord +from temporalio.api.workflowservice.v1 import ResetWorkflowExecutionRequest +from temporalio.client import Client, WorkflowExecutionStatus, WorkflowHistory +from temporalio.client_stream import StreamClient +from temporalio.converter import DataConverter, PayloadCodec +from temporalio.streams import END, RecordKind, StreamNotFoundError +from temporalio.streams.providers.native import NativeStreams +from temporalio.worker import Replayer, Worker +from temporalio.workflow import NondeterminismError +from tests.streams.test_streams_conformance import take + +TARGET = os.environ.get("TEMPORAL_STREAM_TARGET") + +pytestmark = pytest.mark.skipif( + not TARGET, + reason="set TEMPORAL_STREAM_TARGET to a server with the stream commands", +) + +INPUTS = "inputs" +DECISIONS = "decisions" + +EVENT_STREAM_SUBSCRIBED = EventType.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED +EVENT_STREAM_RECORDS_APPENDED = EventType.EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED + + +async def _connect(provider: NativeStreams | None = None) -> Client: + # Registered once, on the client: the worker inherits it and the tests + # open handles through client.get_stream_handle. The contrib tests below + # need no provider. + return await Client.connect(TARGET or "", plugins=[provider] if provider else []) + + +async def _event_counts(client: Client, workflow_id: str) -> dict[Any, int]: + counts = { + EVENT_STREAM_RECORDS_APPENDED: 0, + EVENT_STREAM_SUBSCRIBED: 0, + EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED: 0, + } + async for event in client.get_workflow_handle(workflow_id).fetch_history_events(): + if event.event_type in counts: + counts[event.event_type] += 1 + return counts + + +@workflow.defn +class ContractLoop: + """Reads ``inputs``, publishes a decision per value, reports control records.""" + + def __init__(self) -> None: + self._trace: list[dict[str, Any]] = [] + + @workflow.query + def trace(self) -> list[dict[str, Any]]: + """What the loop has decided so far, for a query against replayed state.""" + return self._trace + + @workflow.run + async def run(self) -> list[dict[str, Any]]: + inputs = workflow.stream_reader(INPUTS, result_type=dict) + decisions = workflow.stream_writer(DECISIONS) + trace = self._trace + async for record in inputs: + if record.kind is RecordKind.SUPERSEDED: + assert record.supersession is not None + trace.append( + { + "kind": "superseded", + "replaced": record.supersession.previous_attempt, + } + ) + decisions.publish( + {"retracting_attempt": record.supersession.previous_attempt} + ) + continue + if record.kind is RecordKind.FINISH: + trace.append({"kind": "finish", "producer": record.producer_id}) + break + assert record.value is not None + decisions.publish({"decided": record.value["n"]}) + trace.append({"kind": "decision", "n": record.value["n"]}) + decisions.finish() + return trace + + +async def test_the_interface_loop_runs_on_the_server_with_a_cold_cache() -> None: + """Rule 2 on the native provider: every task replays from the server. + + With the cache off, each Workflow Task rebuilds the workflow from History + and the server re-supplies the ranges earlier tasks consumed, so the loop + completing at all means the same records came back in the same order. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "loop-tq-" + uuid.uuid4().hex[:8] + workflow_id = "loop-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[ContractLoop], + max_cached_workflows=0, + ): + handle = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + first = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await first.append({"n": 1}, {"n": 2}) + second = stream.producer(topic=INPUTS, producer_id="model", attempt=2) + await second.append({"n": 3}) + await second.finish() + trace = await asyncio.wait_for(handle.result(), 60) + + assert trace == [ + {"kind": "decision", "n": 1}, + {"kind": "decision", "n": 2}, + {"kind": "superseded", "replaced": 1}, + {"kind": "decision", "n": 3}, + {"kind": "finish", "producer": "model"}, + ] + + # The read ends by itself: the workflow is closed and the tail + # delivered, with the workflow's own records carrying no producer. + async def read_everything() -> list[Any]: + return [ + (r.producer_id, r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + + assert await asyncio.wait_for(read_everything(), 60) == [ + ("", RecordKind.DATA, {"decided": 1}), + ("", RecordKind.DATA, {"decided": 2}), + ("", RecordKind.DATA, {"retracting_attempt": 1}), + ("", RecordKind.DATA, {"decided": 3}), + ("", RecordKind.FINISH, None), + ] + counts = await _event_counts(client, workflow_id) + assert counts[EVENT_STREAM_SUBSCRIBED] == 1 + # Several tasks published, one event each; the loop spanned more than + # one task or the cold cache proved nothing. + assert counts[EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED] >= 2 + assert ( + 1 + <= counts[EVENT_STREAM_RECORDS_APPENDED] + <= counts[EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED] + ) + finally: + await provider.close() + + +# Run ids whose first workflow task already failed, shared with the workflow +# thread so the retry can tell it is the retry. Outside the sandbox on +# purpose: the sandbox re-imports this module per run and would hide the set. +_failed_once: set[str] = set() + + +@workflow.defn(sandboxed=False) +class PublishThenFail: + """Publishes, then fails its first workflow task; the retry publishes again.""" + + @workflow.run + async def run(self) -> None: + decisions = workflow.stream_writer(DECISIONS) + run_id = workflow.info().run_id + committed = run_id in _failed_once + decisions.publish({"committed": committed}) + if not committed: + _failed_once.add(run_id) + raise RuntimeError("the first task fails after publishing") + decisions.finish() + + +async def test_a_failed_task_publishes_nothing() -> None: + """Rule 1 on the native provider: the server applies the command with the task.""" + provider = NativeStreams() + client = await _connect(provider) + task_queue = "fail-tq-" + uuid.uuid4().hex[:8] + workflow_id = "fail-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishThenFail], + ): + handle = await client.start_workflow( + PublishThenFail.run, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + records = await take( + stream.read(topic=DECISIONS, result_type=dict), 2, timeout=60 + ) + await handle.result() + assert [(r.kind, r.value) for r in records] == [ + (RecordKind.DATA, {"committed": True}), + (RecordKind.FINISH, None), + ] + finally: + await provider.close() + + +@workflow.defn +class Relay: + """Publishes one record per run and continues as new once.""" + + @workflow.run + async def run(self, run: int) -> None: + decisions = workflow.stream_writer(DECISIONS) + decisions.publish({"run": run}) + if run == 0: + workflow.continue_as_new(run + 1) + decisions.finish() + + +async def test_a_handle_without_a_run_id_reads_across_continue_as_new() -> None: + provider = NativeStreams() + client = await _connect(provider) + task_queue = "relay-tq-" + uuid.uuid4().hex[:8] + workflow_id = "relay-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker(client, task_queue=task_queue, workflows=[Relay]): + handle = await client.start_workflow( + Relay.run, 0, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + + async def read_everything() -> list[Any]: + return [ + (r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + + records = await asyncio.wait_for(read_everything(), 60) + await handle.result() + # The chain is followed: the successor's record arrives on the same + # read, each cursor names its run, and the read ends with the chain. + assert records == [ + (RecordKind.DATA, {"run": 0}), + (RecordKind.DATA, {"run": 1}), + (RecordKind.FINISH, None), + ] + # Pinned to the last run, a handle sees that run's stream alone. + last_run = (await client.get_workflow_handle(workflow_id).describe()).run_id + pinned = client.get_stream_handle(workflow_id, run_id=last_run) + only_last = [ + r.value async for r in pinned.read(topic=DECISIONS, result_type=dict) + ] + assert only_last == [{"run": 1}, None] + finally: + await provider.close() + + +@workflow.defn +class PublishAndRead: + """Publishes through the low-level surface and reads its own records back.""" + + @workflow.run + async def run(self) -> list[str]: + workflow._append_stream_records( + [_record(b"alpha", "progress"), _record(b"beta", "progress")], + stream_name="output", + ) + workflow._append_stream_records([_record(b"gamma")], stream_name="output") + # A name this workflow has not written yet still names a stream it + # owns, so subscribing creates the one the later publish lands in. + workflow._subscribe_stream("output") + + received: list[str] = [] + while len(received) < 3: + for item in await workflow._read_stream_records("output"): + received.append(item.record.body.data.decode()) + return received + + +def _record(body: bytes, topic: str = "") -> StreamRecord: + return StreamRecord( + body=Payload(data=body, metadata={"encoding": b"binary/plain"}), topic=topic + ) + + +async def test_a_tasks_publishes_become_one_event() -> None: + client = await _connect() + task_queue = "publish-tq-" + uuid.uuid4().hex[:8] + workflow_id = "publish-wf-" + uuid.uuid4().hex[:8] + + async with Worker( + client, + task_queue=task_queue, + workflows=[PublishAndRead], + # Every task after the first is a replay, so completing at all means the + # reissued publish matched the event the first run wrote. + max_cached_workflows=0, + ): + result = await asyncio.wait_for( + client.execute_workflow( + PublishAndRead.run, id=workflow_id, task_queue=task_queue + ), + timeout=60, + ) + assert result == ["alpha", "beta", "gamma"] + + counts = await _event_counts(client, workflow_id) + # Two calls in one task carrying three records, so one event: the event is + # per task and stream, which is what makes publishing often free. + assert counts[EVENT_STREAM_RECORDS_APPENDED] == 1 + assert counts[EVENT_STREAM_SUBSCRIBED] == 1 + + +@workflow.defn +class ConsumeAcrossTasks: + """Reads a standalone stream over several Workflow Tasks, then reports what it saw. + + The turn structure is the point: each read that finds nothing blocks, + which ends a Workflow Task, so the run spans several. With the cache on, + every task after the first is sticky. + """ + + @workflow.run + async def run(self, stream_id: str, expected: int) -> list[str]: + workflow._subscribe_stream(stream_id) + seen: list[str] = [] + while len(seen) < expected: + for item in await workflow._read_stream_records(stream_id): + seen.append(item.record.body.data.decode()) + return seen + + +async def test_a_cached_workflow_consumes_across_sticky_tasks() -> None: + """The workflow cache stays on, which is what a real worker does. + + The server sends no replay slice for a sticky task, while the sticky + history still carries the previous task's consumed range, and the two + together have to agree on every task after the first consumed range. + """ + client = await _connect() + streams = StreamClient.connect(TARGET or "") + task_queue = "sticky-tq-" + uuid.uuid4().hex[:8] + workflow_id = "sticky-wf-" + uuid.uuid4().hex[:8] + stream_id = "sticky-src-" + uuid.uuid4().hex[:8] + + batches = [["a1", "a2"], ["b1"], ["c1", "c2", "c3"]] + expected = [tok for batch in batches for tok in batch] + + try: + await streams.create(stream_id) + async with Worker( + client, task_queue=task_queue, workflows=[ConsumeAcrossTasks] + ): + handle = await client.start_workflow( + ConsumeAcrossTasks.run, + args=[stream_id, len(expected)], + id=workflow_id, + task_queue=task_queue, + ) + + # Spaced out so the workflow drains, blocks and ends a task between + # them. Without the gap the appends coalesce into one task and the + # sticky path is never taken. + for batch in batches: + await streams.get(stream_id).append( + *[_record(t.encode()) for t in batch] + ) + await asyncio.sleep(0.4) + + assert await asyncio.wait_for(handle.result(), timeout=60) == expected + + counts = await _event_counts(client, workflow_id) + assert counts[EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED] >= 3, ( + "sticky path not exercised" + ) + finally: + await streams.close() + + +class _EncryptingCodec(PayloadCodec): + """A codec whose output never contains its input. + + The SDK's own test codec marks payloads as encrypted but leaves the bytes + as they are, so it cannot show that a stored body is not plaintext. This + one keeps its metadata convention and runs a repeating-key XOR over the + serialized payload, which is enough for the plaintext markers the tests + look for to be absent from what the server stores. + """ + + _KEY = b"ai198" + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [ + Payload( + metadata={"encoding": b"binary/encrypted"}, + data=self._xor(p.SerializeToString()), + ) + for p in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + out: list[Payload] = [] + for p in payloads: + if p.metadata.get("encoding", b"") != b"binary/encrypted": + out.append(p) + continue + out.append(Payload.FromString(self._xor(p.data))) + return out + + def _xor(self, data: bytes) -> bytes: + key = self._KEY + return bytes(b ^ key[i % len(key)] for i, b in enumerate(data)) + + +async def _drive_contract_loop(client: Client) -> WorkflowHistory: + """Run ``ContractLoop`` to completion on ``client`` and return its history.""" + task_queue = "codec-tq-" + uuid.uuid4().hex[:8] + workflow_id = "codec-wf-" + uuid.uuid4().hex[:8] + async with Worker(client, task_queue=task_queue, workflows=[ContractLoop]): + handle = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + await producer.finish() + trace = await asyncio.wait_for(handle.result(), 60) + # The workflow decoded what the outside producer appended. + assert trace == [ + {"kind": "decision", "n": 1}, + {"kind": "decision", "n": 2}, + {"kind": "finish", "producer": "model"}, + ] + # The outside read decodes what the workflow published. + decisions = [ + (r.kind, r.value) + async for r in stream.read(topic=DECISIONS, result_type=dict) + ] + assert decisions == [ + (RecordKind.DATA, {"decided": 1}), + (RecordKind.DATA, {"decided": 2}), + (RecordKind.FINISH, None), + ] + return await handle.fetch_history() + + +async def test_a_codec_encodes_records_on_both_halves() -> None: + """A payload codec on the client covers the workflow's records and an outside producer's. + + The worker's payload visitor runs the codec over the bodies a workflow + publishes and receives; the outside half applies the client's codec to each + body it sends and reads. Read raw, without the codec, every stored body is + ciphertext, and each side still reads the other's records in the clear. + """ + plain = await _connect() + config = plain.config() + config["data_converter"] = DataConverter(payload_codec=_EncryptingCodec()) + provider = NativeStreams() + config["plugins"] = [provider] + client = Client(**config) + raw = StreamClient.connect(TARGET or "") + try: + history = await _drive_contract_loop(client) + run_id = history.run_id + + # What the server holds, read through the stream service with no codec. + inputs = await raw.workflow_stream( + history.workflow_id, INPUTS, owner_run_id=run_id + ).poll(from_offset=0, wait=False) + decisions = await raw.workflow_stream( + history.workflow_id, DECISIONS, owner_run_id=run_id + ).poll(from_offset=0, wait=False) + + # The outside producer's two records, stored encoded. + stored_inputs = [e.record for e in inputs.entries if e.record.HasField("body")] + assert len(stored_inputs) == 2 + for record in stored_inputs: + assert record.body.metadata["encoding"] == b"binary/encrypted" + assert b'"n"' not in record.body.data + # The workflow's two decisions, stored encoded by the worker's visitor. + stored_decisions = [ + e.record for e in decisions.entries if e.record.HasField("body") + ] + assert len(stored_decisions) == 2 + for record in stored_decisions: + assert record.body.metadata["encoding"] == b"binary/encrypted" + assert b"decided" not in record.body.data + finally: + await raw.close() + await provider.close() + + +def _consumed_ranges(history: WorkflowHistory) -> list[StreamRange]: + return [ + consumed + for event in history.events + if event.HasField("workflow_task_completed_event_attributes") + for consumed in event.workflow_task_completed_event_attributes.consumed_stream_ranges + ] + + +def _copy_of( + history: WorkflowHistory, workflow_id: str | None = None +) -> WorkflowHistory: + events: list[HistoryEvent] = [] + for event in history.events: + copied = HistoryEvent() + copied.CopyFrom(event) + events.append(copied) + return WorkflowHistory(workflow_id or history.workflow_id, events) + + +async def test_a_replayer_with_a_client_replays_a_consuming_workflow() -> None: + """Rule 2 through the ``Replayer``: the records come back from the stream service. + + History holds the offsets each task consumed and never the records, so the + replayer is given a client to the server that still holds the streams and + fetches every recorded range before pushing the history. The same delivery + path as a live cache miss then hands each range to the task that consumed + it, and the reissued publishes match their events. + """ + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + # Something was consumed, or the replay would prove nothing. + assert any(r.to_offset > r.from_offset for r in _consumed_ranges(history)) + + replayer = Replayer( + workflows=[ContractLoop], plugins=[provider], stream_client=client + ) + result = await replayer.replay_workflow(history) + assert result.replay_failure is None + finally: + await provider.close() + + +async def test_a_replayer_fails_a_tampered_range_as_nondeterministic() -> None: + """A history whose recorded ranges were changed replays on other input and fails. + + Every recorded range is emptied. The task that read two records and + published two decisions is replayed with nothing to read, so it issues no + publish, and Core finds the recorded publish event with no command for it. + """ + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + tampered = _copy_of(history) + for consumed in _consumed_ranges(tampered): + consumed.to_offset = consumed.from_offset + + replayer = Replayer( + workflows=[ContractLoop], plugins=[provider], stream_client=client + ) + with pytest.raises(NondeterminismError): + await replayer.replay_workflow(tampered) + finally: + await provider.close() + + +async def test_a_replayer_fails_loudly_when_the_stream_is_gone() -> None: + """A range the stream service no longer serves is a ``StreamNotFoundError``, not a replay.""" + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + # The same history under a workflow id that owns no stream: the + # records it consumed are nowhere to be fetched from. + gone = _copy_of(history, workflow_id="gone-wf-" + uuid.uuid4().hex[:8]) + + replayer = Replayer( + workflows=[ContractLoop], plugins=[provider], stream_client=client + ) + with pytest.raises(StreamNotFoundError, match="cannot be replayed"): + await replayer.replay_workflow(gone) + finally: + await provider.close() + + +async def test_a_replayer_without_a_client_says_what_it_needs() -> None: + """Without a stream client a consuming workflow's history is refused, with the remedy.""" + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + replayer = Replayer(workflows=[ContractLoop], plugins=[provider]) + + with pytest.raises(RuntimeError, match="stream_client=") as raised: + await replayer.replay_workflow(history) + assert f"'{INPUTS}'" in str(raised.value) + + # The aggregating call reports it per run rather than aborting. + async def histories(): + yield history + + results = await replayer.replay_workflows( + histories(), raise_on_replay_failure=False + ) + assert isinstance(results.replay_failures[history.run_id], RuntimeError) + finally: + await provider.close() + + +async def test_a_replayer_fetches_a_standalone_stream_by_its_id() -> None: + """A subscribed name that is no stream of the workflow's is a standalone stream's id. + + The server resolves a subscription the same way, an owned stream by that + name first, so the replayer has to look in both places to hand the replay + the records the workflow actually read. + """ + client = await _connect() + streams = StreamClient.connect(TARGET or "") + task_queue = "replay-src-tq-" + uuid.uuid4().hex[:8] + workflow_id = "replay-src-wf-" + uuid.uuid4().hex[:8] + stream_id = "replay-src-" + uuid.uuid4().hex[:8] + tokens = ["a1", "a2", "b1"] + try: + await streams.create(stream_id) + async with Worker( + client, task_queue=task_queue, workflows=[ConsumeAcrossTasks] + ): + handle = await client.start_workflow( + ConsumeAcrossTasks.run, + args=[stream_id, len(tokens)], + id=workflow_id, + task_queue=task_queue, + ) + await streams.get(stream_id).append(*[_record(t.encode()) for t in tokens]) + assert await asyncio.wait_for(handle.result(), timeout=60) == tokens + history = await handle.fetch_history() + + result = await Replayer( + workflows=[ConsumeAcrossTasks], stream_client=client + ).replay_workflow(history) + assert result.replay_failure is None + finally: + await streams.close() + + +TWO_DECISIONS = [{"kind": "decision", "n": 1}, {"kind": "decision", "n": 2}] + + +async def _feed_two(client: Client, workflow_id: str) -> Any: + """Append two inputs and wait until both decisions are out, so their tasks have completed.""" + stream = client.get_stream_handle(workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="model", attempt=1) + await producer.append({"n": 1}, {"n": 2}) + await take(stream.read(topic=DECISIONS, result_type=dict), 2, timeout=60) + return producer + + +async def test_a_query_against_a_cold_worker_answers_from_replayed_state() -> None: + """A query task built through matching carries the recorded ranges. + + With the cache off every task is a full replay. A query dispatched to a + worker that holds nothing has to rebuild the run from History, and the + records the run read are not in it, so the server attaches them to the + query task the way it does to a task after a cache miss. Without them the + replay would have nothing to read and the query would fail. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "query-cold-tq-" + uuid.uuid4().hex[:8] + workflow_id = "query-cold-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[ContractLoop], + max_cached_workflows=0, + ): + handle = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + producer = await _feed_two(client, workflow_id) + assert await handle.query(ContractLoop.trace) == TWO_DECISIONS + await producer.finish() + await asyncio.wait_for(handle.result(), 60) + finally: + await provider.close() + + +async def test_a_sticky_query_after_an_eviction_still_answers() -> None: + """A query sent to the sticky queue of a run the worker evicted still answers. + + The sticky task carries the history since the last task and no records, + which a worker that lost the run cannot use. Core sees that the history it + fetched itself records a consumed range it was sent no records for, and + lets the query go unanswered rather than answer it from the wrong state. + The server's sticky attempt then times out, stickiness is reset, and the + query is dispatched again on the normal queue, which carries the records. + The sticky timeout is shortened so the test does not wait the default out. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "query-sticky-tq-" + uuid.uuid4().hex[:8] + first_id = "query-sticky-a-" + uuid.uuid4().hex[:8] + second_id = "query-sticky-b-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[ContractLoop], + max_cached_workflows=1, + sticky_queue_schedule_to_start_timeout=timedelta(seconds=2), + ): + first = await client.start_workflow( + ContractLoop.run, id=first_id, task_queue=task_queue + ) + first_producer = await _feed_two(client, first_id) + # A second run on a one-slot cache pushes the first out of it. + second = await client.start_workflow( + ContractLoop.run, id=second_id, task_queue=task_queue + ) + second_producer = await _feed_two(client, second_id) + + answer = await first.query( + ContractLoop.trace, rpc_timeout=timedelta(seconds=60) + ) + assert answer == TWO_DECISIONS + + await first_producer.finish() + await second_producer.finish() + await asyncio.wait_for(first.result(), 60) + await asyncio.wait_for(second.result(), 60) + finally: + await provider.close() + + +async def _collect( + client: Client, workflow_id: str, topic: str, run_id: str | None +) -> list[tuple[Any, str, int]]: + """Every record on ``topic`` as ``(value, run, offset)``, the run and offset from the cursor.""" + handle = client.get_stream_handle(workflow_id, run_id=run_id) + out: list[tuple[Any, str, int]] = [] + async for record in handle.read(topic=topic, result_type=dict): + # native:: + _, run, offset = record.cursor.token.split(":") + out.append((record.value, run, int(offset))) + return out + + +async def test_a_reset_run_is_followed_and_replayed() -> None: + """A run reset from a consuming one carries its subscriptions on, and readers follow. + + The reset re-runs the task named by the reset point, so the base run's + history is copied up to that task and the ranges recorded in the copy are + what the reset run replays, from the base run's streams. The inherited + ``inputs`` stream is the reset run's own from the inherited cursor on: the + server seeds it with the input the re-run task had consumed, at the offset + it held, so the re-run task reads it again and the next input follows. The + ``decisions`` stream, which the base run only published to, starts at zero. + A handle without a run id follows the base run into the reset run and starts + each stream at the floor it reports; a handle pinned to the base run ends + with it; the ``Replayer`` fetches each era of the reset run's history from + the run whose stream holds it. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "reset-tq-" + uuid.uuid4().hex[:8] + workflow_id = "reset-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker(client, task_queue=task_queue, workflows=[ContractLoop]): + base = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + base_run = base.result_run_id + assert base_run + producer = await _feed_two(client, workflow_id) + # A third input in a task of its own, which is the task the reset + # re-runs: the reset run is seeded with it and consumes it again. + await producer.append({"n": 3}) + base_stream = client.get_stream_handle(workflow_id, run_id=base_run) + await take( + base_stream.read(topic=DECISIONS, result_type=dict), 3, timeout=60 + ) + + # Reset to the completion of the last consuming task, while the base + # run waits for more input. + completion_id = 0 + async for event in client.get_workflow_handle( + workflow_id, run_id=base_run + ).fetch_history_events(): + if event.HasField("workflow_task_completed_event_attributes"): + completed = event.workflow_task_completed_event_attributes + if any( + r.to_offset > r.from_offset + for r in completed.consumed_stream_ranges + ): + completion_id = event.event_id + assert completion_id + reset = await client.workflow_service.reset_workflow_execution( + ResetWorkflowExecutionRequest( + namespace=client.namespace, + workflow_execution=WorkflowExecution( + workflow_id=workflow_id, run_id=base_run + ), + reason="re-run the last consuming task", + workflow_task_finish_event_id=completion_id, + request_id=uuid.uuid4().hex, + ) + ) + reset_run = reset.run_id + assert reset_run and reset_run != base_run + + # A fresh producer pins to the current run, the reset run, whose + # inherited inputs stream continues at the inherited offset. + continued = client.get_stream_handle(workflow_id).producer( + topic=INPUTS, producer_id="model2", attempt=1 + ) + await continued.append({"n": 4}) + await continued.finish() + trace = await asyncio.wait_for( + client.get_workflow_handle(workflow_id, run_id=reset_run).result(), + 60, + ) + # The first two decisions were replayed from the base run's stream; + # the third input came back with the task the reset re-ran. + assert trace == TWO_DECISIONS + [ + {"kind": "decision", "n": 3}, + {"kind": "decision", "n": 4}, + {"kind": "finish", "producer": "model2"}, + ] + + # The base run was terminated by the reset, and describe is the one + # place that names the run it was reset into. + described = await client.get_workflow_handle( + workflow_id, run_id=base_run + ).describe() + assert described.status == WorkflowExecutionStatus.TERMINATED + extended = described.raw_description.workflow_extended_info + assert extended.reset_run_id == reset_run + + # A reader pinned to the base run ends with the base run. + pinned = await asyncio.wait_for( + _collect(client, workflow_id, DECISIONS, base_run), 30 + ) + assert pinned == [ + ({"decided": 1}, base_run, 0), + ({"decided": 2}, base_run, 1), + ({"decided": 3}, base_run, 2), + ] + + # A chain-following reader crosses from the base run into the reset + # run on both topics, with no gap and no refusal: the published one + # restarts at zero, the inherited one continues at the floor. + decisions = await asyncio.wait_for( + _collect(client, workflow_id, DECISIONS, None), 30 + ) + assert decisions == [ + ({"decided": 1}, base_run, 0), + ({"decided": 2}, base_run, 1), + ({"decided": 3}, base_run, 2), + ({"decided": 3}, reset_run, 0), + ({"decided": 4}, reset_run, 1), + (None, reset_run, 2), + ] + # The seeded input sits at the offset it held in the base run. + inputs = await asyncio.wait_for( + _collect(client, workflow_id, INPUTS, None), 30 + ) + assert inputs == [ + ({"n": 1}, base_run, 0), + ({"n": 2}, base_run, 1), + ({"n": 3}, base_run, 2), + ({"n": 3}, reset_run, 2), + ({"n": 4}, reset_run, 3), + (None, reset_run, 4), + ] + latest = await client.get_stream_handle(workflow_id).latest(topic=INPUTS) + assert latest.token == f"native:{reset_run}:4" + + # Pinned to the reset run, BEGINNING is the floor its inherited + # stream starts at. Offset zero, which it never held, is refused. + from_floor = await asyncio.wait_for( + _collect(client, workflow_id, INPUTS, reset_run), 30 + ) + assert from_floor == [ + ({"n": 3}, reset_run, 2), + ({"n": 4}, reset_run, 3), + (None, reset_run, 4), + ] + + # The reset run's history: the base run's events, the reset marker + # naming both runs, then its own. The replayer fetches the first era + # from the base run's stream and the rest from the reset run's. + history = await client.get_workflow_handle( + workflow_id, run_id=reset_run + ).fetch_history() + reset_cause = WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_RESET_WORKFLOW + markers = [ + (failed.base_run_id, failed.new_run_id) + for event in history.events + if event.HasField("workflow_task_failed_event_attributes") + for failed in [event.workflow_task_failed_event_attributes] + if failed.cause == reset_cause + ] + assert markers == [(base_run, reset_run)] + result = await Replayer( + workflows=[ContractLoop], plugins=[provider], stream_client=client + ).replay_workflow(history) + assert result.replay_failure is None + + # Exported with its records, the reset run's history replays with no + # server: the slices come from both runs' streams. + bundle = await Replayer.fetch_stream_slices(client, history) + assert {s.run_id for s in bundle.stream_slices if s.records} == { + base_run, + reset_run, + } + restored = WorkflowHistory.from_json(workflow_id, bundle.to_json()) + offline = await Replayer( + workflows=[ContractLoop], plugins=[provider] + ).replay_workflow(restored) + assert offline.replay_failure is None + finally: + await provider.close() + + +async def test_an_exported_history_replays_offline_with_its_records() -> None: + """A history exported with its stream records is the whole replay input. + + ``fetch_stream_slices`` captures the records while the stream is retained, + ``to_json`` writes them beside the events, ``from_json`` reads them back, + and a replayer with no client replays the result. A plain export carries + none and is refused with both remedies named. + """ + provider = NativeStreams() + client = await _connect(provider) + try: + history = await _drive_contract_loop(client) + assert "streamSlices" not in history.to_json() + + bundle = await Replayer.fetch_stream_slices(client, history) + assert list(bundle.events) == list(history.events) + assert any(s.records for s in bundle.stream_slices) + text = bundle.to_json() + assert "streamSlices" in text + restored = WorkflowHistory.from_json(history.workflow_id, text) + assert list(restored.stream_slices) == list(bundle.stream_slices) + assert list(restored.events) == list(history.events) + + offline = Replayer(workflows=[ContractLoop], plugins=[provider]) + result = await offline.replay_workflow(restored) + assert result.replay_failure is None + + # With the records stripped it is a plain export again. + stripped = WorkflowHistory(restored.workflow_id, restored.events) + with pytest.raises(RuntimeError, match="stream_client=") as raised: + await offline.replay_workflow(stripped) + assert "fetch_stream_slices" in str(raised.value) + finally: + await provider.close() + + +async def test_a_chain_of_two_resets_is_followed_to_its_end() -> None: + """A run reset, and that run reset again: what a chain-following read walks. + + Resetting the same run twice is not a thing the server allows; the first + reset terminates it and the second is refused. So "reset more than once" + means a chain, base into A into B, and each run names only the one after + it. A chain-following read walks it one link at a time and stops at the + end, which is also the answer to whether it can loop: a reset always + makes a run that has never been reset itself. + """ + provider = NativeStreams() + client = await _connect(provider) + task_queue = "reset2-tq-" + uuid.uuid4().hex[:8] + workflow_id = "reset2-wf-" + uuid.uuid4().hex[:8] + + async def reset(run_id: str, reason: str) -> str: + # The last completed task of the run, whatever it did: a reset run + # picks up from there, and what matters here is the chain it makes. + completion_id = 0 + async for event in client.get_workflow_handle( + workflow_id, run_id=run_id + ).fetch_history_events(): + if event.HasField("workflow_task_completed_event_attributes"): + completion_id = event.event_id + assert completion_id + answer = await client.workflow_service.reset_workflow_execution( + ResetWorkflowExecutionRequest( + namespace=client.namespace, + workflow_execution=WorkflowExecution( + workflow_id=workflow_id, run_id=run_id + ), + reason=reason, + workflow_task_finish_event_id=completion_id, + request_id=uuid.uuid4().hex, + ) + ) + return answer.run_id + + async def reset_run_of(run_id: str) -> str: + described = await client.get_workflow_handle( + workflow_id, run_id=run_id + ).describe() + return described.raw_description.workflow_extended_info.reset_run_id + + try: + async with Worker(client, task_queue=task_queue, workflows=[ContractLoop]): + base = await client.start_workflow( + ContractLoop.run, id=workflow_id, task_queue=task_queue + ) + base_run = base.result_run_id + assert base_run + await _feed_two(client, workflow_id) + base_stream = client.get_stream_handle(workflow_id, run_id=base_run) + await take( + base_stream.read(topic=DECISIONS, result_type=dict), 2, timeout=60 + ) + + first = await reset(base_run, "the first reset") + second = await reset(first, "the second reset") + assert len({base_run, first, second}) == 3 + + # Each link names the one after it and nothing else. The last has + # not been reset, so the walk ends there rather than going round. + assert await reset_run_of(base_run) == first + assert await reset_run_of(first) == second + assert not await reset_run_of(second) + + # Let the end of the chain finish so a read can reach an end. + carry_on = client.get_stream_handle(workflow_id).producer( + topic=INPUTS, producer_id="model2", attempt=1 + ) + await carry_on.append({"n": 4}) + await carry_on.finish() + await asyncio.wait_for( + client.get_workflow_handle(workflow_id, run_id=second).result(), 60 + ) + + # The walk itself: one link at a time, ending at the run that was + # not reset. A middle run may have published nothing, so this is + # asked of the provider rather than read off the records. + handle = client.get_stream_handle(workflow_id) + walked = [base_run] + while True: + onward = await handle._successor(DECISIONS, walked[-1]) # type: ignore[attr-defined] + if onward is None: + break + assert onward[0] not in walked, "the chain must not double back" + walked.append(onward[0]) + assert walked == [base_run, first, second] + + # And a chain-following read ends, having crossed the same runs + # and repeated none of them. + decisions = await asyncio.wait_for( + _collect(client, workflow_id, DECISIONS, None), 60 + ) + seen: list[str] = [] + for _, run, _ in decisions: + if not seen or seen[-1] != run: + seen.append(run) + assert seen == sorted(set(seen), key=walked.index) + assert seen[0] == base_run and seen[-1] == second + finally: + await provider.close() + + +@workflow.defn +class StartsWhenTold: + """Subscribes to ``inputs`` at a start given by a signal and returns what it read.""" + + def __init__(self) -> None: + self._start: str | None = None + + @workflow.signal + def begin(self, start: str) -> None: + self._start = start + + @workflow.run + async def run(self) -> list[Any]: + await workflow.wait_condition(lambda: self._start is not None) + if self._start == "end": + reader = workflow.stream_reader(INPUTS, result_type=dict, after=END) + want = 1 + else: + reader = workflow.stream_reader(INPUTS, result_type=dict, last=2) + want = 2 + values: list[Any] = [] + async for value in reader.values(): + values.append(value["n"]) + if len(values) == want: + break + return values + + +async def _subscribed_offsets(client: Client, workflow_id: str) -> list[int]: + return [ + event.workflow_stream_subscribed_event_attributes.start_offset + async for event in client.get_workflow_handle( + workflow_id + ).fetch_history_events() + if event.event_type == EVENT_STREAM_SUBSCRIBED + ] + + +async def test_a_workflow_reader_starts_at_the_last_n_records_on_the_server() -> None: + provider = NativeStreams() + client = await _connect(provider) + task_queue = "last-tq-" + uuid.uuid4().hex[:8] + workflow_id = "last-wf-" + uuid.uuid4().hex[:8] + try: + # Cold, so every task replays and the recorded start is what places + # the reader, not a second resolution. + async with Worker( + client, + task_queue=task_queue, + workflows=[StartsWhenTold], + max_cached_workflows=0, + ): + handle = await client.start_workflow( + StartsWhenTold.run, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + await producer.append({"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}) + await handle.signal(StartsWhenTold.begin, "last") + assert await asyncio.wait_for(handle.result(), 60) == [3, 4] + assert await _subscribed_offsets(client, workflow_id) == [2] + finally: + await provider.close() + + +async def test_a_workflow_reader_at_end_skips_what_was_there_on_the_server() -> None: + provider = NativeStreams() + client = await _connect(provider) + task_queue = "end-tq-" + uuid.uuid4().hex[:8] + workflow_id = "end-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker( + client, + task_queue=task_queue, + workflows=[StartsWhenTold], + max_cached_workflows=0, + ): + handle = await client.start_workflow( + StartsWhenTold.run, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + await producer.append({"n": "old"}, {"n": "old"}) + await handle.signal(StartsWhenTold.begin, "end") + result = asyncio.ensure_future(handle.result()) + # The subscription registers when the signalled task completes, + # which the test does not observe, so appends keep coming. + for _ in range(150): + await producer.append({"n": "new"}) + done, _ = await asyncio.wait({result}, timeout=0.2) + if done: + break + assert await asyncio.wait_for(result, 60) == ["new"] + offsets = await _subscribed_offsets(client, workflow_id) + assert len(offsets) == 1 and offsets[0] >= 2 + finally: + await provider.close() diff --git a/tests/worker/test_workflow_stream_payloads_e2e.py b/tests/worker/test_workflow_stream_payloads_e2e.py new file mode 100644 index 0000000000..aa739fb1f0 --- /dev/null +++ b/tests/worker/test_workflow_stream_payloads_e2e.py @@ -0,0 +1,252 @@ +"""How a body's encoding meets the server: retry identity and external storage. + +A payload codec and an ``ExternalStorage`` driver both change the bytes a body +is stored as. The server must still recognize a retry, and a driver must be +applied on both halves of the native provider. Needs a Temporal server built +from the AI-198 branch: + + TEMPORAL_STREAM_TARGET=127.0.0.1:7333 uv run pytest tests/worker/test_workflow_stream_payloads_e2e.py +""" + +from __future__ import annotations + +import asyncio +import os +import uuid +from collections.abc import Sequence +from typing import Any + +import pytest + +from temporalio import workflow +from temporalio.api.common.v1 import Payload +from temporalio.api.sdk.v1.external_storage_pb2 import ExternalStorageReference +from temporalio.client import Client +from temporalio.client_stream import StreamClient +from temporalio.converter import ( + DataConverter, + ExternalStorage, + JSONProtoPayloadConverter, + PayloadCodec, +) +from temporalio.streams import CONTENT_HASH_KEY, RecordKind +from temporalio.streams.providers.native import NativeStreams +from temporalio.worker import Worker +from tests.streams.test_streams_conformance import take +from tests.test_extstore import InMemoryTestDriver + +TARGET = os.environ.get("TEMPORAL_STREAM_TARGET") + +pytestmark = pytest.mark.skipif( + not TARGET, + reason="set TEMPORAL_STREAM_TARGET to a server with the stream commands", +) + +INPUTS = "inputs" +DECISIONS = "decisions" + + +class _NonceCodec(PayloadCodec): + """Encodes to different bytes on every call, as a nonce-based cipher does. + + The plaintext is kept in the clear behind a counter so the test can read + the stored bytes back and see that two encodings of one value differ. + """ + + def __init__(self) -> None: + self.calls = 0 + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + self.calls += 1 + return [ + Payload( + metadata={"encoding": b"binary/nonce"}, + data=f"{self.calls}:".encode() + p.SerializeToString(), + ) + for p in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + out: list[Payload] = [] + for p in payloads: + if p.metadata.get("encoding", b"") != b"binary/nonce": + out.append(p) + continue + out.append(Payload.FromString(p.data.split(b":", 1)[1])) + return out + + +@workflow.defn +class Echo: + """Reads ``inputs`` and publishes each value on ``decisions``, until told to stop.""" + + def __init__(self) -> None: + self._stop = False + + @workflow.signal + def stop(self) -> None: + self._stop = True + + @workflow.run + async def run(self, count: int) -> list[Any]: + inputs = workflow.stream_reader(INPUTS, result_type=dict) + decisions = workflow.stream_writer(DECISIONS) + seen: list[Any] = [] + async for value in inputs.values(): + decisions.publish({"echo": value}) + seen.append(value) + if len(seen) >= count: + break + decisions.finish() + await workflow.wait_condition(lambda: self._stop) + return seen + + +async def _connect(converter: DataConverter, provider: NativeStreams) -> Client: + plain = await Client.connect(TARGET or "") + config = plain.config() + config["data_converter"] = converter + config["plugins"] = [provider] + return Client(**config) + + +async def test_a_retried_append_under_a_nonce_codec_is_deduplicated() -> None: + """Retry identity is the plaintext hash, not the encoded bytes. + + Two producers with the same identity append the same value at the same + sequence, as a retry after a lost response would. The codec encodes each + differently, so the server sees two different bodies and one hash, and + writes the record once. + """ + provider = NativeStreams() + client = await _connect(DataConverter(payload_codec=_NonceCodec()), provider) + raw = StreamClient.connect(TARGET or "") + task_queue = "nonce-tq-" + uuid.uuid4().hex[:8] + workflow_id = "nonce-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker(client, task_queue=task_queue, workflows=[Echo]): + handle = await client.start_workflow( + Echo.run, 1, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + first = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + retry = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + landed = await first.append({"n": 1}) + again = await retry.append({"n": 1}) + assert again == landed, "the repeat answers with the original position" + + echoed = await take(stream.read(topic=DECISIONS, result_type=dict), 1) + assert [r.value for r in echoed] == [{"echo": {"n": 1}}] + await handle.signal(Echo.stop) + assert await asyncio.wait_for(handle.result(), 60) == [{"n": 1}] + + run_id = (await handle.describe()).run_id + assert run_id is not None + page = await raw.workflow_stream(workflow_id, INPUTS, owner_run_id=run_id).poll( + from_offset=0, wait=False + ) + bodies = [e.record for e in page.entries if e.record.HasField("body")] + assert len(bodies) == 1, "written once" + assert bodies[0].body.metadata["encoding"] == b"binary/nonce" + assert bodies[0].metadata[CONTENT_HASH_KEY].data + finally: + await raw.close() + await provider.close() + + +async def test_a_workflow_publish_carries_the_plaintext_hash() -> None: + """The workflow's own records are stamped too, from the workflow thread.""" + provider = NativeStreams() + client = await _connect(DataConverter.default, provider) + raw = StreamClient.connect(TARGET or "") + task_queue = "stamp-tq-" + uuid.uuid4().hex[:8] + workflow_id = "stamp-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker(client, task_queue=task_queue, workflows=[Echo]): + handle = await client.start_workflow( + Echo.run, 1, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + await stream.producer(topic=INPUTS, producer_id="tool", attempt=1).append( + {"n": 7} + ) + await take(stream.read(topic=DECISIONS, result_type=dict), 1) + await handle.signal(Echo.stop) + await asyncio.wait_for(handle.result(), 60) + + run_id = (await handle.describe()).run_id + assert run_id is not None + page = await raw.workflow_stream( + workflow_id, DECISIONS, owner_run_id=run_id + ).poll(from_offset=0, wait=False) + kinds = {e.record.kind for e in page.entries} + assert len(page.entries) == 2, "one decision and the finish" + data = [e.record for e in page.entries if e.record.HasField("body")] + assert len(data) == 1 and data[0].metadata[CONTENT_HASH_KEY].data + finish = [e.record for e in page.entries if not e.record.HasField("body")] + assert CONTENT_HASH_KEY not in finish[0].metadata + assert RecordKind.FINISH in {RecordKind(k) for k in kinds} + finally: + await raw.close() + await provider.close() + + +async def test_external_storage_applies_on_both_halves() -> None: + """A ``StorageDriver`` on the client offloads stream bodies on both paths. + + Every body is over the threshold. The outside producer's append is stored + through the driver before it reaches the server; the workflow retrieves it + on its task, publishes, and the worker's payload pass offloads that too, off + the workflow thread, so the outside reader retrieves it. Read raw, the + server holds references on both topics and no plaintext. + """ + driver = InMemoryTestDriver() + converter = DataConverter( + external_storage=ExternalStorage(drivers=[driver], payload_size_threshold=0) + ) + provider = NativeStreams() + client = await _connect(converter, provider) + raw = StreamClient.connect(TARGET or "") + task_queue = "offload-tq-" + uuid.uuid4().hex[:8] + workflow_id = "offload-wf-" + uuid.uuid4().hex[:8] + try: + async with Worker(client, task_queue=task_queue, workflows=[Echo]): + handle = await client.start_workflow( + Echo.run, 1, id=workflow_id, task_queue=task_queue + ) + stream = client.get_stream_handle(workflow_id) + producer = stream.producer(topic=INPUTS, producer_id="tool", attempt=1) + await producer.append({"n": "plaintext-marker"}) + stores_after_append = driver._store_calls + assert stores_after_append >= 1, "the outside append offloaded" + + echoed = await take(stream.read(topic=DECISIONS, result_type=dict), 1) + assert [r.value for r in echoed] == [{"echo": {"n": "plaintext-marker"}}] + await handle.signal(Echo.stop) + assert await asyncio.wait_for(handle.result(), 60) == [ + {"n": "plaintext-marker"} + ] + + run_id = (await handle.describe()).run_id + assert run_id is not None + for topic in (INPUTS, DECISIONS): + page = await raw.workflow_stream( + workflow_id, topic, owner_run_id=run_id + ).poll(from_offset=0, wait=False) + bodies = [e.record.body for e in page.entries if e.record.HasField("body")] + assert len(bodies) == 1, topic + assert b"plaintext-marker" not in bodies[0].data, topic + reference = JSONProtoPayloadConverter().from_payload( + bodies[0], ExternalStorageReference + ) + assert reference.driver_name == driver.name(), topic + # The workflow's publish was offloaded by the worker, after the append. + assert driver._store_calls > stores_after_append + # Both halves retrieved: the worker on its task and the outside reader. + assert driver._retrieve_calls >= 2 + # The outside append was stored under the workflow that owns the stream. + targets = [ctx.target for ctx in driver._store_contexts if ctx.target] + assert any(t.id == workflow_id and t.run_id == run_id for t in targets) + finally: + await raw.close() + await provider.close() diff --git a/uv.lock b/uv.lock index cee51de68d..ea6e73b7dd 100644 --- a/uv.lock +++ b/uv.lock @@ -257,14 +257,14 @@ name = "anthropic" version = "0.117.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "python_full_version >= '3.11'" }, - { name = "distro", marker = "python_full_version >= '3.11'" }, - { name = "docstring-parser", marker = "python_full_version >= '3.11'" }, - { name = "httpx", marker = "python_full_version >= '3.11'" }, - { name = "jiter", marker = "python_full_version >= '3.11'" }, - { name = "pydantic", marker = "python_full_version >= '3.11'" }, - { name = "sniffio", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/41/0d/8f71d535edb0d438f023bd825fb65f67c14fa88a2bd6b75f292a58a63de4/anthropic-0.117.0.tar.gz", hash = "sha256:98107f2b76439641e0ae2a1754087534b8f178dbab99d6eb1bc4b7bc8c744496", size = 989933, upload-time = "2026-07-16T19:36:13.07Z" } wheels = [ @@ -942,12 +942,12 @@ name = "deepagents" version = "0.6.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain", marker = "python_full_version >= '3.11'" }, - { name = "langchain-anthropic", marker = "python_full_version >= '3.11'" }, - { name = "langchain-core", marker = "python_full_version >= '3.11'" }, - { name = "langchain-google-genai", marker = "python_full_version >= '3.11'" }, - { name = "langsmith", marker = "python_full_version >= '3.11'" }, - { name = "wcmatch", marker = "python_full_version >= '3.11'" }, + { name = "langchain" }, + { name = "langchain-anthropic" }, + { name = "langchain-core" }, + { name = "langchain-google-genai" }, + { name = "langsmith" }, + { name = "wcmatch" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/db/a6acdc72a9e90c3f07ed10de35c951734a02d4facb693bb59684ad368801/deepagents-0.6.12.tar.gz", hash = "sha256:1f281c0bc5a63132f62e2ee345c1dc593b23188da6e23016401f6879fbe54b5f", size = 211364, upload-time = "2026-06-25T17:26:52.775Z" } wheels = [ @@ -1022,7 +1022,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1972,9 +1972,9 @@ name = "langchain" version = "1.3.14" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core", marker = "python_full_version >= '3.11'" }, - { name = "langgraph", marker = "python_full_version >= '3.11'" }, - { name = "pydantic", marker = "python_full_version >= '3.11'" }, + { name = "langchain-core" }, + { name = "langgraph" }, + { name = "pydantic" }, ] sdist = { url = "https://files.pythonhosted.org/packages/29/68/a6dbad9c22df4087a0f9e79ddd46226c442b30128bfeee538d5889492a73/langchain-1.3.14.tar.gz", hash = "sha256:1b6696c72ba3bbbce54d745e0180742c9f6ece8bbc59ed5a46c3e20b9a435929", size = 645181, upload-time = "2026-07-16T13:28:18.29Z" } wheels = [ @@ -1986,9 +1986,9 @@ name = "langchain-anthropic" version = "1.4.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anthropic", marker = "python_full_version >= '3.11'" }, - { name = "langchain-core", marker = "python_full_version >= '3.11'" }, - { name = "pydantic", marker = "python_full_version >= '3.11'" }, + { name = "anthropic" }, + { name = "langchain-core" }, + { name = "pydantic" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/22/40ab129b08329ca295b391aa1d48267692b42594757084c6918e22b655ac/langchain_anthropic-1.4.8.tar.gz", hash = "sha256:c76891b2044d56105ff13c106ed12650637b53bd598a4bdf15b4796eefa2a4ec", size = 708524, upload-time = "2026-06-26T21:28:46.916Z" } wheels = [ @@ -2020,10 +2020,10 @@ name = "langchain-google-genai" version = "4.2.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filetype", marker = "python_full_version >= '3.11'" }, - { name = "google-genai", marker = "python_full_version >= '3.11'" }, - { name = "langchain-core", marker = "python_full_version >= '3.11'" }, - { name = "pydantic", marker = "python_full_version >= '3.11'" }, + { name = "filetype" }, + { name = "google-genai" }, + { name = "langchain-core" }, + { name = "pydantic" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/0c/bc60dabc362ca7c6ffe8c4bcc2f724c7e566b43eb230cee51419f88f784c/langchain_google_genai-4.2.7.tar.gz", hash = "sha256:03b1463ffe4d42435f43c7870467f2215f684bb46400d2543435d10157c80ac7", size = 281605, upload-time = "2026-07-06T13:51:58.724Z" } wheels = [ @@ -2839,7 +2839,7 @@ wheels = [ [package.optional-dependencies] litellm = [ - { name = "litellm", marker = "python_full_version < '3.14'" }, + { name = "litellm" }, ] [[package]] @@ -4026,6 +4026,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl", hash = "sha256:3385ed220117104a2bceb4a9dac8c5fdf6d1f96890d7ea2a9c7174fd5c84091f", size = 14134, upload-time = "2026-06-09T21:05:15.85Z" }, ] +[[package]] +name = "redis" +version = "8.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -4802,6 +4814,7 @@ dev = [ { name = "pytest-rerunfailures" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, + { name = "redis" }, { name = "ruff" }, { name = "setuptools" }, { name = "strands-agents" }, @@ -4886,6 +4899,7 @@ dev = [ { name = "pytest-rerunfailures", specifier = ">=16.1" }, { name = "pytest-timeout", specifier = "~=2.2" }, { name = "pytest-xdist", specifier = ">=3.6,<4" }, + { name = "redis", specifier = ">=5,<9" }, { name = "ruff", specifier = ">=0.15.12,<0.16" }, { name = "setuptools", specifier = "<82" }, { name = "strands-agents", specifier = ">=1.51.0" }, @@ -5379,7 +5393,7 @@ name = "wcmatch" version = "11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "bracex", marker = "python_full_version >= '3.11'" }, + { name = "bracex" }, ] sdist = { url = "https://files.pythonhosted.org/packages/16/25/1da725838132221e33568973da484ff43813662ccc06ebf7f6e3abddfcd5/wcmatch-11.0.tar.gz", hash = "sha256:55d95c2447789712774b198ceec72939e88b5618f1f8f0a9b605bf7740b63b96", size = 141360, upload-time = "2026-07-10T05:50:24.183Z" } wheels = [