From a229c1fccb149b165adeb2490ffc9fd6b80d14be Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 3 Oct 2026 01:10:17 -0700 Subject: [PATCH] Added the native stream protos. A Workflow appends to and subscribes to server-held streams with two commands. Workflow Tasks carry the records out of band, and History keeps only the offset ranges. The new values take the numbers after the channel's. --- openapi/openapiv2.json | 183 +++++++++++++++- openapi/openapiv3.yaml | 197 ++++++++++++++++++ temporal/api/command/v1/message.proto | 43 ++++ temporal/api/enums/v1/command_type.proto | 2 + temporal/api/enums/v1/event_type.proto | 8 + temporal/api/enums/v1/failed_cause.proto | 8 + temporal/api/history/v1/message.proto | 44 ++++ temporal/api/stream/v1/message.proto | 79 ++++++- .../workflowservice/v1/request_response.proto | 5 + 9 files changed, 562 insertions(+), 7 deletions(-) diff --git a/openapi/openapiv2.json b/openapi/openapiv2.json index 0db7a05ff..04b1fac03 100644 --- a/openapi/openapiv2.json +++ b/openapi/openapiv2.json @@ -18257,10 +18257,12 @@ "EVENT_TYPE_WORKFLOW_EXECUTION_UNPAUSED", "EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED", "EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED", - "EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED" + "EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED", + "EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED", + "EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED" ], "default": "EVENT_TYPE_UNSPECIFIED", - "description": "- EVENT_TYPE_UNSPECIFIED: Place holder and should never appear in a Workflow execution history\n - EVENT_TYPE_WORKFLOW_EXECUTION_STARTED: Workflow execution has been triggered/started\nIt contains Workflow execution inputs, as well as Workflow timeout configurations\n - EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED: Workflow execution has successfully completed and contains Workflow execution results\n - EVENT_TYPE_WORKFLOW_EXECUTION_FAILED: Workflow execution has unsuccessfully completed and contains the Workflow execution error\n - EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT: Workflow execution has timed out by the Temporal Server\nUsually due to the Workflow having not been completed within timeout settings\n - EVENT_TYPE_WORKFLOW_TASK_SCHEDULED: Workflow Task has been scheduled and the SDK client should now be able to process any new history events\n - EVENT_TYPE_WORKFLOW_TASK_STARTED: Workflow Task has started and the SDK client has picked up the Workflow Task and is processing new history events\n - EVENT_TYPE_WORKFLOW_TASK_COMPLETED: Workflow Task has completed\nThe SDK client picked up the Workflow Task and processed new history events\nSDK client may or may not ask the Temporal Server to do additional work, such as:\nEVENT_TYPE_ACTIVITY_TASK_SCHEDULED\nEVENT_TYPE_TIMER_STARTED\nEVENT_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES\nEVENT_TYPE_MARKER_RECORDED\nEVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED\nEVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED\nEVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED\nEVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED\nEVENT_TYPE_WORKFLOW_EXECUTION_FAILED\nEVENT_TYPE_WORKFLOW_EXECUTION_CANCELED\nEVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW\n - EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT: Workflow Task encountered a timeout\nEither an SDK client with a local cache was not available at the time, or it took too long for the SDK client to process the task\n - EVENT_TYPE_WORKFLOW_TASK_FAILED: Workflow Task encountered a failure\nUsually this means that the Workflow was non-deterministic\nHowever, the Workflow reset functionality also uses this event\n - EVENT_TYPE_ACTIVITY_TASK_SCHEDULED: Activity Task was scheduled\nThe SDK client should pick up this activity task and execute\nThis event type contains activity inputs, as well as activity timeout configurations\n - EVENT_TYPE_ACTIVITY_TASK_STARTED: Activity Task has started executing\nThe SDK client has picked up the Activity Task and is processing the Activity invocation\n - EVENT_TYPE_ACTIVITY_TASK_COMPLETED: Activity Task has finished successfully\nThe SDK client has picked up and successfully completed the Activity Task\nThis event type contains Activity execution results\n - EVENT_TYPE_ACTIVITY_TASK_FAILED: Activity Task has finished unsuccessfully\nThe SDK picked up the Activity Task but unsuccessfully completed it\nThis event type contains Activity execution errors\n - EVENT_TYPE_ACTIVITY_TASK_TIMED_OUT: Activity has timed out according to the Temporal Server\nActivity did not complete within the timeout settings\n - EVENT_TYPE_ACTIVITY_TASK_CANCEL_REQUESTED: A request to cancel the Activity has occurred\nThe SDK client will be able to confirm cancellation of an Activity during an Activity heartbeat\n - EVENT_TYPE_ACTIVITY_TASK_CANCELED: Activity has been cancelled\n - EVENT_TYPE_TIMER_STARTED: A timer has started\n - EVENT_TYPE_TIMER_FIRED: A timer has fired\n - EVENT_TYPE_TIMER_CANCELED: A time has been cancelled\n - EVENT_TYPE_WORKFLOW_EXECUTION_CANCEL_REQUESTED: A request has been made to cancel the Workflow execution\n - EVENT_TYPE_WORKFLOW_EXECUTION_CANCELED: SDK client has confirmed the cancellation request and the Workflow execution has been cancelled\n - EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED: Workflow has requested that the Temporal Server try to cancel another Workflow\n - EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_FAILED: Temporal Server could not cancel the targeted Workflow\nThis is usually because the target Workflow could not be found\n - EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_CANCEL_REQUESTED: Temporal Server has successfully requested the cancellation of the target Workflow\n - EVENT_TYPE_MARKER_RECORDED: A marker has been recorded.\nThis event type is transparent to the Temporal Server\nThe Server will only store it and will not try to understand it.\n - EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED: Workflow has received a Signal event\nThe event type contains the Signal name, as well as a Signal payload\n - EVENT_TYPE_WORKFLOW_EXECUTION_TERMINATED: Workflow execution has been forcefully terminated\nThis is usually because the terminate Workflow API was called\n - EVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW: Workflow has successfully completed and a new Workflow has been started within the same transaction\nContains last Workflow execution results as well as new Workflow execution inputs\n - EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED: Temporal Server will try to start a child Workflow\n - EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_FAILED: Child Workflow execution cannot be started/triggered\nUsually due to a child Workflow ID collision\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED: Child Workflow execution has successfully started/triggered\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED: Child Workflow execution has successfully completed\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_FAILED: Child Workflow execution has unsuccessfully completed\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_CANCELED: Child Workflow execution has been cancelled\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_TIMED_OUT: Child Workflow execution has timed out by the Temporal Server\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_TERMINATED: Child Workflow execution has been terminated\n - EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED: Temporal Server will try to Signal the targeted Workflow\nContains the Signal name, as well as a Signal payload\n - EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED: Temporal Server cannot Signal the targeted Workflow\nUsually because the Workflow could not be found\n - EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_SIGNALED: Temporal Server has successfully Signaled the targeted Workflow\n - EVENT_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES: Workflow search attributes should be updated and synchronized with the visibility store\n - EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED: An update was admitted. Note that not all admitted updates result in this\nevent. See UpdateAdmittedEventOrigin for situations in which this event\nis created.\n - EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ACCEPTED: An update was accepted (i.e. passed validation, perhaps because no validator was defined)\n - EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_REJECTED: This event is never written to history.\n - EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_COMPLETED: An update completed\n - EVENT_TYPE_WORKFLOW_PROPERTIES_MODIFIED_EXTERNALLY: Some property or properties of the workflow as a whole have changed by non-workflow code.\nThe distinction of external vs. command-based modification is important so the SDK can\nmaintain determinism when using the command-based approach.\n - EVENT_TYPE_ACTIVITY_PROPERTIES_MODIFIED_EXTERNALLY: Some property or properties of an already-scheduled activity have changed by non-workflow code.\nThe distinction of external vs. command-based modification is important so the SDK can\nmaintain determinism when using the command-based approach.\n - EVENT_TYPE_WORKFLOW_PROPERTIES_MODIFIED: Workflow properties modified by user workflow code\n - EVENT_TYPE_NEXUS_OPERATION_SCHEDULED: A Nexus operation was scheduled using a ScheduleNexusOperation command.\n - EVENT_TYPE_NEXUS_OPERATION_STARTED: An asynchronous Nexus operation was started by a Nexus handler.\n - EVENT_TYPE_NEXUS_OPERATION_COMPLETED: A Nexus operation completed successfully.\n - EVENT_TYPE_NEXUS_OPERATION_FAILED: A Nexus operation failed.\n - EVENT_TYPE_NEXUS_OPERATION_CANCELED: A Nexus operation completed as canceled.\n - EVENT_TYPE_NEXUS_OPERATION_TIMED_OUT: A Nexus operation timed out.\n - EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUESTED: A Nexus operation was requested to be canceled using a RequestCancelNexusOperation command.\n - EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED: Workflow execution options updated by user.\n - EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_COMPLETED: A cancellation request for a Nexus operation was successfully delivered to the Nexus handler.\n - EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_FAILED: A cancellation request for a Nexus operation resulted in an error.\n - EVENT_TYPE_WORKFLOW_EXECUTION_PAUSED: An event that indicates that the workflow execution has been paused.\n - EVENT_TYPE_WORKFLOW_EXECUTION_UNPAUSED: An event that indicates that the previously paused workflow execution has been unpaused.\n - EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED: An event that indicates time skipping advanced time or was disabled automatically after a bound was reached.\n - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED: A Workflow became a listener of a notification channel for its run.\nThe notifications themselves ride the WorkflowTaskScheduled event.\n - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED: A Workflow stopped listening on a notification channel for its run.\nRecorded for every UnsubscribeNotificationChannel command, including one\nnaming a channel the run was not subscribed to.", + "description": "- EVENT_TYPE_UNSPECIFIED: Place holder and should never appear in a Workflow execution history\n - EVENT_TYPE_WORKFLOW_EXECUTION_STARTED: Workflow execution has been triggered/started\nIt contains Workflow execution inputs, as well as Workflow timeout configurations\n - EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED: Workflow execution has successfully completed and contains Workflow execution results\n - EVENT_TYPE_WORKFLOW_EXECUTION_FAILED: Workflow execution has unsuccessfully completed and contains the Workflow execution error\n - EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT: Workflow execution has timed out by the Temporal Server\nUsually due to the Workflow having not been completed within timeout settings\n - EVENT_TYPE_WORKFLOW_TASK_SCHEDULED: Workflow Task has been scheduled and the SDK client should now be able to process any new history events\n - EVENT_TYPE_WORKFLOW_TASK_STARTED: Workflow Task has started and the SDK client has picked up the Workflow Task and is processing new history events\n - EVENT_TYPE_WORKFLOW_TASK_COMPLETED: Workflow Task has completed\nThe SDK client picked up the Workflow Task and processed new history events\nSDK client may or may not ask the Temporal Server to do additional work, such as:\nEVENT_TYPE_ACTIVITY_TASK_SCHEDULED\nEVENT_TYPE_TIMER_STARTED\nEVENT_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES\nEVENT_TYPE_MARKER_RECORDED\nEVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED\nEVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED\nEVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED\nEVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED\nEVENT_TYPE_WORKFLOW_EXECUTION_FAILED\nEVENT_TYPE_WORKFLOW_EXECUTION_CANCELED\nEVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW\n - EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT: Workflow Task encountered a timeout\nEither an SDK client with a local cache was not available at the time, or it took too long for the SDK client to process the task\n - EVENT_TYPE_WORKFLOW_TASK_FAILED: Workflow Task encountered a failure\nUsually this means that the Workflow was non-deterministic\nHowever, the Workflow reset functionality also uses this event\n - EVENT_TYPE_ACTIVITY_TASK_SCHEDULED: Activity Task was scheduled\nThe SDK client should pick up this activity task and execute\nThis event type contains activity inputs, as well as activity timeout configurations\n - EVENT_TYPE_ACTIVITY_TASK_STARTED: Activity Task has started executing\nThe SDK client has picked up the Activity Task and is processing the Activity invocation\n - EVENT_TYPE_ACTIVITY_TASK_COMPLETED: Activity Task has finished successfully\nThe SDK client has picked up and successfully completed the Activity Task\nThis event type contains Activity execution results\n - EVENT_TYPE_ACTIVITY_TASK_FAILED: Activity Task has finished unsuccessfully\nThe SDK picked up the Activity Task but unsuccessfully completed it\nThis event type contains Activity execution errors\n - EVENT_TYPE_ACTIVITY_TASK_TIMED_OUT: Activity has timed out according to the Temporal Server\nActivity did not complete within the timeout settings\n - EVENT_TYPE_ACTIVITY_TASK_CANCEL_REQUESTED: A request to cancel the Activity has occurred\nThe SDK client will be able to confirm cancellation of an Activity during an Activity heartbeat\n - EVENT_TYPE_ACTIVITY_TASK_CANCELED: Activity has been cancelled\n - EVENT_TYPE_TIMER_STARTED: A timer has started\n - EVENT_TYPE_TIMER_FIRED: A timer has fired\n - EVENT_TYPE_TIMER_CANCELED: A time has been cancelled\n - EVENT_TYPE_WORKFLOW_EXECUTION_CANCEL_REQUESTED: A request has been made to cancel the Workflow execution\n - EVENT_TYPE_WORKFLOW_EXECUTION_CANCELED: SDK client has confirmed the cancellation request and the Workflow execution has been cancelled\n - EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED: Workflow has requested that the Temporal Server try to cancel another Workflow\n - EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_FAILED: Temporal Server could not cancel the targeted Workflow\nThis is usually because the target Workflow could not be found\n - EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_CANCEL_REQUESTED: Temporal Server has successfully requested the cancellation of the target Workflow\n - EVENT_TYPE_MARKER_RECORDED: A marker has been recorded.\nThis event type is transparent to the Temporal Server\nThe Server will only store it and will not try to understand it.\n - EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED: Workflow has received a Signal event\nThe event type contains the Signal name, as well as a Signal payload\n - EVENT_TYPE_WORKFLOW_EXECUTION_TERMINATED: Workflow execution has been forcefully terminated\nThis is usually because the terminate Workflow API was called\n - EVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW: Workflow has successfully completed and a new Workflow has been started within the same transaction\nContains last Workflow execution results as well as new Workflow execution inputs\n - EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED: Temporal Server will try to start a child Workflow\n - EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_FAILED: Child Workflow execution cannot be started/triggered\nUsually due to a child Workflow ID collision\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED: Child Workflow execution has successfully started/triggered\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED: Child Workflow execution has successfully completed\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_FAILED: Child Workflow execution has unsuccessfully completed\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_CANCELED: Child Workflow execution has been cancelled\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_TIMED_OUT: Child Workflow execution has timed out by the Temporal Server\n - EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_TERMINATED: Child Workflow execution has been terminated\n - EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED: Temporal Server will try to Signal the targeted Workflow\nContains the Signal name, as well as a Signal payload\n - EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED: Temporal Server cannot Signal the targeted Workflow\nUsually because the Workflow could not be found\n - EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_SIGNALED: Temporal Server has successfully Signaled the targeted Workflow\n - EVENT_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES: Workflow search attributes should be updated and synchronized with the visibility store\n - EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED: An update was admitted. Note that not all admitted updates result in this\nevent. See UpdateAdmittedEventOrigin for situations in which this event\nis created.\n - EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ACCEPTED: An update was accepted (i.e. passed validation, perhaps because no validator was defined)\n - EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_REJECTED: This event is never written to history.\n - EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_COMPLETED: An update completed\n - EVENT_TYPE_WORKFLOW_PROPERTIES_MODIFIED_EXTERNALLY: Some property or properties of the workflow as a whole have changed by non-workflow code.\nThe distinction of external vs. command-based modification is important so the SDK can\nmaintain determinism when using the command-based approach.\n - EVENT_TYPE_ACTIVITY_PROPERTIES_MODIFIED_EXTERNALLY: Some property or properties of an already-scheduled activity have changed by non-workflow code.\nThe distinction of external vs. command-based modification is important so the SDK can\nmaintain determinism when using the command-based approach.\n - EVENT_TYPE_WORKFLOW_PROPERTIES_MODIFIED: Workflow properties modified by user workflow code\n - EVENT_TYPE_NEXUS_OPERATION_SCHEDULED: A Nexus operation was scheduled using a ScheduleNexusOperation command.\n - EVENT_TYPE_NEXUS_OPERATION_STARTED: An asynchronous Nexus operation was started by a Nexus handler.\n - EVENT_TYPE_NEXUS_OPERATION_COMPLETED: A Nexus operation completed successfully.\n - EVENT_TYPE_NEXUS_OPERATION_FAILED: A Nexus operation failed.\n - EVENT_TYPE_NEXUS_OPERATION_CANCELED: A Nexus operation completed as canceled.\n - EVENT_TYPE_NEXUS_OPERATION_TIMED_OUT: A Nexus operation timed out.\n - EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUESTED: A Nexus operation was requested to be canceled using a RequestCancelNexusOperation command.\n - EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED: Workflow execution options updated by user.\n - EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_COMPLETED: A cancellation request for a Nexus operation was successfully delivered to the Nexus handler.\n - EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_FAILED: A cancellation request for a Nexus operation resulted in an error.\n - EVENT_TYPE_WORKFLOW_EXECUTION_PAUSED: An event that indicates that the workflow execution has been paused.\n - EVENT_TYPE_WORKFLOW_EXECUTION_UNPAUSED: An event that indicates that the previously paused workflow execution has been unpaused.\n - EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED: An event that indicates time skipping advanced time or was disabled automatically after a bound was reached.\n - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED: A Workflow became a listener of a notification channel for its run.\nThe notifications themselves ride the WorkflowTaskScheduled event.\n - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED: A Workflow stopped listening on a notification channel for its run.\nRecorded for every UnsubscribeNotificationChannel command, including one\nnaming a channel the run was not subscribed to.\n - EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED: A Workflow subscribed to a stream. Recorded once per subscription, not\nper record: the offsets a task consumed ride WorkflowTaskCompleted and\nthe payloads never enter History at all.\n - EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED: A Workflow appended a batch of records to a stream. Recorded per\nbatch, and carrying only the offset range it landed at: the bodies go to\nthe stream's own log, never into History.", "title": "Whenever this list of events is changed do change the function shouldBufferEvent in mutableStateBuilder.go to make sure to do the correct event ordering" }, "v1Execution": { @@ -18933,6 +18935,12 @@ }, "workflowNotificationChannelUnsubscribedEventAttributes": { "$ref": "#/definitions/v1WorkflowNotificationChannelUnsubscribedEventAttributes" + }, + "workflowStreamSubscribedEventAttributes": { + "$ref": "#/definitions/v1WorkflowStreamSubscribedEventAttributes" + }, + "workflowStreamRecordsAppendedEventAttributes": { + "$ref": "#/definitions/v1WorkflowStreamRecordsAppendedEventAttributes" } }, "description": "History events are the method by which Temporal SDKs advance (or recreate) workflow state.\nSee the `EventType` enum for more info about what each event is for." @@ -20760,6 +20768,14 @@ "pollerGroupsInfo": { "$ref": "#/definitions/v1PollerGroupsInfo", "description": "The weighted, versioned list of poller groups IDs that client should use for future polls to\nthis task queue. Client should ignore this if it has already applied a snapshot with a\nversion greater than or equal to `poller_groups_info.version`. Client is expected to:\n 1. Maintain minimum number of pollers no less than the number of groups.\n 2. Try to assign the next poll to a group without any pending polls,\n 3. If every group has some pending polls, assign the next poll to a group randomly\n according to the weights." + }, + "streamSlices": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/v1StreamSlice" + }, + "description": "Stream data attached to this task. Delivered out of band so the payloads\nnever enter History; only the offset ranges are recorded there." } } }, @@ -22257,6 +22273,111 @@ } } }, + "v1StreamRange": { + "type": "object", + "properties": { + "streamId": { + "type": "string", + "description": "The stream, as the subscribing command addressed it: either the name of\na stream the consuming Workflow owns or the id of one in another\nexecution." + }, + "fromOffset": { + "type": "string", + "format": "int64", + "description": "Inclusive." + }, + "toOffset": { + "type": "string", + "format": "int64", + "description": "Exclusive." + } + }, + "description": "The offsets a Workflow Task consumed, without the payloads. Recorded on\nWorkflowTaskCompleted so History grows with Workflow Tasks rather than with\nrecords." + }, + "v1StreamRecord": { + "type": "object", + "properties": { + "body": { + "$ref": "#/definitions/v1Payload", + "description": "The value the producer published, stored as sent.\n\nA payload codec applies on the paths this API owns: the append command on\nRespondWorkflowTaskCompleted, and the slices on PollWorkflowTaskQueue.\nRecords a producer writes or reads through the stream service take a\ndifferent path, whose messages are not part of this API yet and so are\noutside what a codec-applying proxy walks." + }, + "metadata": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/v1Payload" + }, + "description": "Producer-supplied provenance, stored as sent." + }, + "topic": { + "type": "string", + "description": "Producer-supplied grouping label, stored as sent." + }, + "kind": { + "$ref": "#/definitions/v1StreamRecordKind", + "description": "How to read this record. Unspecified is read as DATA." + }, + "producerId": { + "type": "string", + "description": "Who wrote the record. Empty when the owning Workflow did." + }, + "attempt": { + "type": "string", + "format": "int64", + "description": "The producer's attempt. Readers treat a later attempt by the same\nproducer as superseding what the earlier one wrote." + }, + "sequence": { + "type": "string", + "format": "int64", + "description": "The producer's position within its attempt, zero when it does not number\nits records. Stored as sent; the server does not assign, validate or\norder by it, and the stream's own offsets are what order a read." + } + }, + "description": "One entry in a stream. The record is the wire format: stores keep it\nserialized as is and readers in every language decode the same bytes." + }, + "v1StreamRecordKind": { + "type": "string", + "enum": [ + "STREAM_RECORD_KIND_UNSPECIFIED", + "STREAM_RECORD_KIND_DATA", + "STREAM_RECORD_KIND_FINISH" + ], + "default": "STREAM_RECORD_KIND_UNSPECIFIED", + "description": "What a record means to a reader. Kept on the record itself so every store\nand every language reads it the same way without a private envelope.\n\n - STREAM_RECORD_KIND_UNSPECIFIED: Read as DATA.\n - STREAM_RECORD_KIND_DATA: A value the producer published; `body` carries it.\n - STREAM_RECORD_KIND_FINISH: The producer named by `producer_id` writes nothing more on `topic`.\nSays nothing about that producer's outcome and does not end the stream." + }, + "v1StreamSlice": { + "type": "object", + "properties": { + "streamId": { + "type": "string", + "description": "The stream, as the subscribing command addressed it: either the name of\na stream the consuming Workflow owns or the id of one in another\nexecution." + }, + "runId": { + "type": "string", + "description": "Run id of the execution that owns the stream. Set on both a slice for the\ntask being started and a re-supplied one." + }, + "fromOffset": { + "type": "string", + "format": "int64", + "description": "Inclusive." + }, + "toOffset": { + "type": "string", + "format": "int64", + "description": "Exclusive. Equal to from_offset when the subscription observed nothing,\nwhich is a fact replay has to reproduce rather than an absence of one." + }, + "records": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/v1StreamRecord" + } + }, + "workflowTaskCompletedEventId": { + "type": "string", + "format": "int64", + "description": "The WorkflowTaskCompleted event whose consumed_stream_ranges recorded\nthis range. Set only when the server is re-supplying a range for a task\nbeing replayed; a slice for the task now being started leaves it unset,\nbecause the event closing that task does not exist yet.\n\nReplay needs this because a Workflow Task response carries one slice set\nwhile a cache miss replays every prior task, so the ranges have to be\nmatched to the events that recorded them rather than to the response." + } + }, + "description": "A contiguous range of a stream delivered to a Workflow Task, along with the\noffsets it covers. The offsets are what History records; the records\nthemselves are never written to History." + }, "v1StructuredCalendarSpec": { "type": "object", "properties": { @@ -24682,6 +24803,49 @@ } } }, + "v1WorkflowStreamRecordsAppendedEventAttributes": { + "type": "object", + "properties": { + "workflowTaskCompletedEventId": { + "type": "string", + "format": "int64", + "description": "The WorkflowTaskCompleted event of the task whose command appended this\nbatch." + }, + "streamId": { + "type": "string", + "description": "Name of the stream the Workflow appended to." + }, + "fromOffset": { + "type": "string", + "format": "int64", + "description": "Inclusive. Same range vocabulary as StreamRange and StreamSlice, so a\nreader does not have to remember which of the three counts and which\nbounds." + }, + "toOffset": { + "type": "string", + "format": "int64", + "description": "Exclusive. With from_offset this names the range without carrying any of\nit, which is what keeps this event a fixed size no matter how large the\nbatch or its payloads are." + } + } + }, + "v1WorkflowStreamSubscribedEventAttributes": { + "type": "object", + "properties": { + "workflowTaskCompletedEventId": { + "type": "string", + "format": "int64", + "description": "The WorkflowTaskCompleted event of the task whose command created this\nsubscription." + }, + "streamId": { + "type": "string", + "description": "The stream the Workflow subscribed to, as the command addressed it:\neither the name of a stream this Workflow owns or the id of one in\nanother execution." + }, + "startOffset": { + "type": "string", + "format": "int64", + "description": "The offset the subscription actually starts from. Resolved by the server\nwhen the subscription is registered and recorded here, so replay reads\nthe resolved value rather than resolving it again against a stream that\nhas since moved." + } + } + }, "v1WorkflowTaskCompletedEventAttributes": { "type": "object", "properties": { @@ -24734,6 +24898,14 @@ "deploymentVersion": { "$ref": "#/definitions/v1WorkerDeploymentVersion", "description": "The Worker Deployment Version that completed this task. Must be set if `versioning_behavior`\nis set. This value updates workflow execution's `versioning_info.deployment_version`." + }, + "consumedStreamRanges": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/v1StreamRange" + }, + "description": "Offset ranges this Workflow Task consumed from streams it subscribes to.\nRecorded on every task where a subscription is active, including when it\nobserved nothing: an empty range is a fact replay must reproduce, and\nomitting it would let replay deliver records the Workflow did not have." } } }, @@ -24812,10 +24984,13 @@ "WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_PAUSE_REQUESTED_BEFORE_TASK_STARTED", "WORKFLOW_TASK_FAILED_CAUSE_REQUEST_TOO_LARGE", "WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES", - "WORKFLOW_TASK_FAILED_CAUSE_BAD_UNSUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES" + "WORKFLOW_TASK_FAILED_CAUSE_BAD_UNSUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES", + "WORKFLOW_TASK_FAILED_CAUSE_BAD_APPEND_STREAM_RECORDS_ATTRIBUTES", + "WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_STREAM_ATTRIBUTES", + "WORKFLOW_TASK_FAILED_CAUSE_STREAM_RANGE_UNAVAILABLE" ], "default": "WORKFLOW_TASK_FAILED_CAUSE_UNSPECIFIED", - "description": "Workflow tasks can fail for various reasons. Note that some of these reasons can only originate\nfrom the server, and some of them can only originate from the SDK/worker.\n\n - WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND: Between starting and completing the workflow task (with a workflow completion command), some\nnew command (like a signal) was processed into workflow history. The outstanding task will be\nfailed with this reason, and a worker must pick up a new task.\n - WORKFLOW_TASK_FAILED_CAUSE_RESET_STICKY_TASK_QUEUE: The worker wishes to fail the task and have the next one be generated on a normal, not sticky\nqueue. Generally workers should prefer to use the explicit `ResetStickyTaskQueue` RPC call.\n - WORKFLOW_TASK_FAILED_CAUSE_NON_DETERMINISTIC_ERROR: The worker encountered a mismatch while replaying history between what was expected, and\nwhat the workflow code actually did.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_CHILD_WORKFLOWS_LIMIT_EXCEEDED: We send the below error codes to users when their requests would violate a size constraint\nof their workflow. We do this to ensure that the state of their workflow does not become too\nlarge because that can cause severe performance degradation. You can modify the thresholds for\neach of these errors within your dynamic config.\n\nSpawning a new child workflow would cause this workflow to exceed its limit of pending child\nworkflows.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_ACTIVITIES_LIMIT_EXCEEDED: Starting a new activity would cause this workflow to exceed its limit of pending activities\nthat we track.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_SIGNALS_LIMIT_EXCEEDED: A workflow has a buffer of signals that have not yet reached their destination. We return this\nerror when sending a new signal would exceed the capacity of this buffer.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_REQUEST_CANCEL_LIMIT_EXCEEDED: Similarly, we have a buffer of pending requests to cancel other workflows. We return this error\nwhen our capacity for pending cancel requests is already reached.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_UPDATE_WORKFLOW_EXECUTION_MESSAGE: Workflow execution update message (update.Acceptance, update.Rejection, or update.Response)\nhas wrong format, or missing required fields.\n - WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_UPDATE: Similar to WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND, but for updates.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_SCHEDULE_NEXUS_OPERATION_ATTRIBUTES: A workflow task completed with an invalid ScheduleNexusOperation command.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_NEXUS_OPERATIONS_LIMIT_EXCEEDED: A workflow task completed requesting to schedule a Nexus Operation exceeding the server configured limit.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_REQUEST_CANCEL_NEXUS_OPERATION_ATTRIBUTES: A workflow task completed with an invalid RequestCancelNexusOperation command.\n - WORKFLOW_TASK_FAILED_CAUSE_FEATURE_DISABLED: A workflow task completed requesting a feature that's disabled on the server (either system wide or - typically -\nfor the workflow's namespace).\nCheck the workflow task failure message for more information.\n - WORKFLOW_TASK_FAILED_CAUSE_GRPC_MESSAGE_TOO_LARGE: A workflow task failed because a grpc message was too large.\n - WORKFLOW_TASK_FAILED_CAUSE_PAYLOADS_TOO_LARGE: A workflow task failed because payloads were too large.\n - WORKFLOW_TASK_FAILED_CAUSE_EXTERNAL_STORAGE_FAILURE: A workflow task failed because an external storage operation failed.\nCheck the workflow task failure message for more information.\n - WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_PAUSE_REQUESTED_BEFORE_TASK_STARTED: A workflow task is failed because the workflow is paused before the task is started.\n - WORKFLOW_TASK_FAILED_CAUSE_REQUEST_TOO_LARGE: A workflow task failed because the request exceeded a size limit.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES: A SubscribeNotificationChannel command named an empty or too-long channel, or hit a\nsubscription or listener limit.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_UNSUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES: An UnsubscribeNotificationChannel command named an empty or too-long channel." + "description": "Workflow tasks can fail for various reasons. Note that some of these reasons can only originate\nfrom the server, and some of them can only originate from the SDK/worker.\n\n - WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND: Between starting and completing the workflow task (with a workflow completion command), some\nnew command (like a signal) was processed into workflow history. The outstanding task will be\nfailed with this reason, and a worker must pick up a new task.\n - WORKFLOW_TASK_FAILED_CAUSE_RESET_STICKY_TASK_QUEUE: The worker wishes to fail the task and have the next one be generated on a normal, not sticky\nqueue. Generally workers should prefer to use the explicit `ResetStickyTaskQueue` RPC call.\n - WORKFLOW_TASK_FAILED_CAUSE_NON_DETERMINISTIC_ERROR: The worker encountered a mismatch while replaying history between what was expected, and\nwhat the workflow code actually did.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_CHILD_WORKFLOWS_LIMIT_EXCEEDED: We send the below error codes to users when their requests would violate a size constraint\nof their workflow. We do this to ensure that the state of their workflow does not become too\nlarge because that can cause severe performance degradation. You can modify the thresholds for\neach of these errors within your dynamic config.\n\nSpawning a new child workflow would cause this workflow to exceed its limit of pending child\nworkflows.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_ACTIVITIES_LIMIT_EXCEEDED: Starting a new activity would cause this workflow to exceed its limit of pending activities\nthat we track.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_SIGNALS_LIMIT_EXCEEDED: A workflow has a buffer of signals that have not yet reached their destination. We return this\nerror when sending a new signal would exceed the capacity of this buffer.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_REQUEST_CANCEL_LIMIT_EXCEEDED: Similarly, we have a buffer of pending requests to cancel other workflows. We return this error\nwhen our capacity for pending cancel requests is already reached.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_UPDATE_WORKFLOW_EXECUTION_MESSAGE: Workflow execution update message (update.Acceptance, update.Rejection, or update.Response)\nhas wrong format, or missing required fields.\n - WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_UPDATE: Similar to WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND, but for updates.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_SCHEDULE_NEXUS_OPERATION_ATTRIBUTES: A workflow task completed with an invalid ScheduleNexusOperation command.\n - WORKFLOW_TASK_FAILED_CAUSE_PENDING_NEXUS_OPERATIONS_LIMIT_EXCEEDED: A workflow task completed requesting to schedule a Nexus Operation exceeding the server configured limit.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_REQUEST_CANCEL_NEXUS_OPERATION_ATTRIBUTES: A workflow task completed with an invalid RequestCancelNexusOperation command.\n - WORKFLOW_TASK_FAILED_CAUSE_FEATURE_DISABLED: A workflow task completed requesting a feature that's disabled on the server (either system wide or - typically -\nfor the workflow's namespace).\nCheck the workflow task failure message for more information.\n - WORKFLOW_TASK_FAILED_CAUSE_GRPC_MESSAGE_TOO_LARGE: A workflow task failed because a grpc message was too large.\n - WORKFLOW_TASK_FAILED_CAUSE_PAYLOADS_TOO_LARGE: A workflow task failed because payloads were too large.\n - WORKFLOW_TASK_FAILED_CAUSE_EXTERNAL_STORAGE_FAILURE: A workflow task failed because an external storage operation failed.\nCheck the workflow task failure message for more information.\n - WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_PAUSE_REQUESTED_BEFORE_TASK_STARTED: A workflow task is failed because the workflow is paused before the task is started.\n - WORKFLOW_TASK_FAILED_CAUSE_REQUEST_TOO_LARGE: A workflow task failed because the request exceeded a size limit.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES: A SubscribeNotificationChannel command named an empty or too-long channel, or hit a\nsubscription or listener limit.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_UNSUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES: An UnsubscribeNotificationChannel command named an empty or too-long channel.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_APPEND_STREAM_RECORDS_ATTRIBUTES: A workflow task completed with an invalid AppendStreamRecords command.\n - WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_STREAM_ATTRIBUTES: A workflow task completed with an invalid SubscribeStream command.\n - WORKFLOW_TASK_FAILED_CAUSE_STREAM_RANGE_UNAVAILABLE: A workflow task could not be started because a stream range it consumed and recorded in\nHistory can no longer be served, for example after truncation or because it exceeds the\nreplay bound. Check the workflow task failure message for more information." }, "v1WorkflowTaskFailedEventAttributes": { "type": "object", diff --git a/openapi/openapiv3.yaml b/openapi/openapiv3.yaml index 3fd8ed8b1..257b5a02b 100644 --- a/openapi/openapiv3.yaml +++ b/openapi/openapiv3.yaml @@ -14687,6 +14687,8 @@ components: - EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED + - EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED + - EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED type: string format: enum version: @@ -14856,6 +14858,10 @@ components: $ref: '#/components/schemas/WorkflowNotificationChannelSubscribedEventAttributes' workflowNotificationChannelUnsubscribedEventAttributes: $ref: '#/components/schemas/WorkflowNotificationChannelUnsubscribedEventAttributes' + workflowStreamSubscribedEventAttributes: + $ref: '#/components/schemas/WorkflowStreamSubscribedEventAttributes' + workflowStreamRecordsAppendedEventAttributes: + $ref: '#/components/schemas/WorkflowStreamRecordsAppendedEventAttributes' description: |- History events are the method by which Temporal SDKs advance (or recreate) workflow state. See the `EventType` enum for more info about what each event is for. @@ -16750,6 +16756,13 @@ components: 2. Try to assign the next poll to a group without any pending polls, 3. If every group has some pending polls, assign the next poll to a group randomly according to the weights. + streamSlices: + type: array + items: + $ref: '#/components/schemas/StreamSlice' + description: |- + Stream data attached to this task. Delivered out of band so the payloads + never enter History; only the offset ranges are recorded there. PollerGroupInfo: type: object properties: @@ -17458,6 +17471,8 @@ components: - EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED + - EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED + - EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED type: string description: The event type of the history event generated by the request. format: enum @@ -19609,6 +19624,124 @@ components: type: type: string description: The type of the driver, required. + StreamRange: + type: object + properties: + streamId: + type: string + description: |- + The stream, as the subscribing command addressed it: either the name of + a stream the consuming Workflow owns or the id of one in another + execution. + fromOffset: + type: string + description: |- + Inclusive. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + toOffset: + type: string + description: |- + Exclusive. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + description: |- + The offsets a Workflow Task consumed, without the payloads. Recorded on + WorkflowTaskCompleted so History grows with Workflow Tasks rather than with + records. + StreamRecord: + type: object + properties: + body: + allOf: + - $ref: '#/components/schemas/Payload' + description: |- + The value the producer published, stored as sent. + + A payload codec applies on the paths this API owns: the append command on + RespondWorkflowTaskCompleted, and the slices on PollWorkflowTaskQueue. + Records a producer writes or reads through the stream service take a + different path, whose messages are not part of this API yet and so are + outside what a codec-applying proxy walks. + metadata: + type: object + additionalProperties: + $ref: '#/components/schemas/Payload' + description: Producer-supplied provenance, stored as sent. + topic: + type: string + description: Producer-supplied grouping label, stored as sent. + kind: + enum: + - STREAM_RECORD_KIND_UNSPECIFIED + - STREAM_RECORD_KIND_DATA + - STREAM_RECORD_KIND_FINISH + type: string + description: How to read this record. Unspecified is read as DATA. + format: enum + producerId: + type: string + description: Who wrote the record. Empty when the owning Workflow did. + attempt: + type: string + description: |- + The producer's attempt. Readers treat a later attempt by the same + producer as superseding what the earlier one wrote. + sequence: + type: string + description: |- + The producer's position within its attempt, zero when it does not number + its records. Stored as sent; the server does not assign, validate or + order by it, and the stream's own offsets are what order a read. + description: |- + One entry in a stream. The record is the wire format: stores keep it + serialized as is and readers in every language decode the same bytes. + StreamSlice: + type: object + properties: + streamId: + type: string + description: |- + The stream, as the subscribing command addressed it: either the name of + a stream the consuming Workflow owns or the id of one in another + execution. + runId: + type: string + description: |- + Run id of the execution that owns the stream. Set on both a slice for the + task being started and a re-supplied one. + fromOffset: + type: string + description: |- + Inclusive. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + toOffset: + type: string + description: |- + Exclusive. Equal to from_offset when the subscription observed nothing, + which is a fact replay has to reproduce rather than an absence of one. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + records: + type: array + items: + $ref: '#/components/schemas/StreamRecord' + workflowTaskCompletedEventId: + type: string + description: |- + The WorkflowTaskCompleted event whose consumed_stream_ranges recorded + this range. Set only when the server is re-supplying a range for a task + being replayed; a slice for the task now being started leaves it unset, + because the event closing that task does not exist yet. + + Replay needs this because a Workflow Task response carries one slice set + while a cache miss replays every prior task, so the ranges have to be + matched to the events that recorded them rather than to the response. + description: |- + A contiguous range of a stream delivered to a Workflow Task, along with the + offsets it covers. The offsets are what History records; the records + themselves are never written to History. StructuredCalendarSpec: type: object properties: @@ -21764,6 +21897,8 @@ components: - EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED + - EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED + - EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED type: string format: enum description: EventReference is a direct reference to a history event through the event ID. @@ -21837,6 +21972,8 @@ components: - EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_SUBSCRIBED - EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED + - EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED + - EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED type: string format: enum description: RequestIdReference is a indirect reference to a history event through the request ID. @@ -22998,6 +23135,54 @@ components: * BETWEEN ... AND STARTS_WITH description: Activity trigger will be triggered when an activity is about to start. + WorkflowStreamRecordsAppendedEventAttributes: + type: object + properties: + workflowTaskCompletedEventId: + type: string + description: |- + The WorkflowTaskCompleted event of the task whose command appended this + batch. + streamId: + type: string + description: Name of the stream the Workflow appended to. + fromOffset: + type: string + description: |- + Inclusive. Same range vocabulary as StreamRange and StreamSlice, so a + reader does not have to remember which of the three counts and which + bounds. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + toOffset: + type: string + description: |- + Exclusive. With from_offset this names the range without carrying any of + it, which is what keeps this event a fixed size no matter how large the + batch or its payloads are. + (-- api-linter: core::0140::prepositions=disabled + aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + WorkflowStreamSubscribedEventAttributes: + type: object + properties: + workflowTaskCompletedEventId: + type: string + description: |- + The WorkflowTaskCompleted event of the task whose command created this + subscription. + streamId: + type: string + description: |- + The stream the Workflow subscribed to, as the command addressed it: + either the name of a stream this Workflow owns or the id of one in + another execution. + startOffset: + type: string + description: |- + The offset the subscription actually starts from. Resolved by the server + when the subscription is registered and recorded here, so replay reads + the resolved value rather than resolving it again against a stream that + has since moved. WorkflowTaskCompletedEventAttributes: type: object properties: @@ -23069,6 +23254,15 @@ components: description: |- The Worker Deployment Version that completed this task. Must be set if `versioning_behavior` is set. This value updates workflow execution's `versioning_info.deployment_version`. + consumedStreamRanges: + type: array + items: + $ref: '#/components/schemas/StreamRange' + description: |- + Offset ranges this Workflow Task consumed from streams it subscribes to. + Recorded on every task where a subscription is active, including when it + observed nothing: an empty range is a fact replay must reproduce, and + omitting it would let replay deliver records the Workflow did not have. WorkflowTaskCompletedMetadata: type: object properties: @@ -23178,6 +23372,9 @@ components: - WORKFLOW_TASK_FAILED_CAUSE_REQUEST_TOO_LARGE - WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES - WORKFLOW_TASK_FAILED_CAUSE_BAD_UNSUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES + - WORKFLOW_TASK_FAILED_CAUSE_BAD_APPEND_STREAM_RECORDS_ATTRIBUTES + - WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_STREAM_ATTRIBUTES + - WORKFLOW_TASK_FAILED_CAUSE_STREAM_RANGE_UNAVAILABLE type: string format: enum failure: diff --git a/temporal/api/command/v1/message.proto b/temporal/api/command/v1/message.proto index 0af8ba569..aaec7e4b0 100644 --- a/temporal/api/command/v1/message.proto +++ b/temporal/api/command/v1/message.proto @@ -14,6 +14,7 @@ import "google/protobuf/duration.proto"; import "temporal/api/enums/v1/workflow.proto"; import "temporal/api/enums/v1/command_type.proto"; import "temporal/api/common/v1/message.proto"; +import "temporal/api/stream/v1/message.proto"; import "temporal/api/failure/v1/message.proto"; import "temporal/api/taskqueue/v1/message.proto"; import "temporal/api/workflow/v1/message.proto"; @@ -328,6 +329,8 @@ message Command { subscribe_notification_channel_command_attributes = 20; UnsubscribeNotificationChannelCommandAttributes unsubscribe_notification_channel_command_attributes = 21; + AppendStreamRecordsCommandAttributes append_stream_records_command_attributes = 22; + SubscribeStreamCommandAttributes subscribe_stream_command_attributes = 23; } } @@ -347,3 +350,43 @@ message UnsubscribeNotificationChannelCommandAttributes { // The channel to stop listening on, as the writers name it. string channel = 1; } + +// Appends records to a stream the Workflow owns. Applied inside the Workflow +// Task's own commit. Produces one `WorkflowStreamRecordsAppended` event +// carrying the offset range and none of the payload; it schedules no further +// work. +message AppendStreamRecordsCommandAttributes { + // Name of a stream this Workflow owns, scoped to the Workflow. Created on + // first use. Empty means the Workflow's default output stream. A Workflow + // cannot append to a stream in another execution, so this is never the id + // of a standalone stream. + string stream_name = 1; + // Stored in order. The server sets `producer_id` to empty on each record, + // because the owning Workflow is the producer here. + repeated temporal.api.stream.v1.StreamRecord records = 2; +} + +// Subscribe this Workflow to a stream, so later Workflow Tasks carry the ranges +// it has not consumed yet. +// +// The stream's addressing is resolved by the server rather than supplied here. +// A Workflow cannot look it up without doing I/O, and a value it carried would +// be a reading rather than a fact, so it could differ on replay. +message SubscribeStreamCommandAttributes { + // Stream to consume, named either way round: a stream this Workflow owns + // by the name it appends under, a stream in another execution by its id. + // The server tries them in that order, so a Workflow that owns a stream + // under this name cannot reach a standalone stream with the same id. When + // neither exists the Workflow gets a stream of its own by that name, which + // is how a reader subscribes before the first record is written. + string stream_name_or_id = 1; + // Where to start, as an absolute offset. Read only when `start_position` + // is unset. A negative value is refused: the head of the stream is asked + // for with `start_position.tail`. + int64 start_offset = 2; + // Where to start. The server resolves it once, when it registers the + // subscription, and records the resolved absolute offset on the subscribed + // event, so replay does not resolve it again. Setting it together with a + // non-zero `start_offset` fails the command. + temporal.api.stream.v1.StreamStartPosition start_position = 3; +} diff --git a/temporal/api/enums/v1/command_type.proto b/temporal/api/enums/v1/command_type.proto index c3b3d8e9f..a46064e48 100644 --- a/temporal/api/enums/v1/command_type.proto +++ b/temporal/api/enums/v1/command_type.proto @@ -31,4 +31,6 @@ enum CommandType { COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION = 18; COMMAND_TYPE_SUBSCRIBE_NOTIFICATION_CHANNEL = 19; COMMAND_TYPE_UNSUBSCRIBE_NOTIFICATION_CHANNEL = 20; + COMMAND_TYPE_APPEND_STREAM_RECORDS = 21; + COMMAND_TYPE_SUBSCRIBE_STREAM = 22; } diff --git a/temporal/api/enums/v1/event_type.proto b/temporal/api/enums/v1/event_type.proto index e7d7a31a2..6dd2f5a79 100644 --- a/temporal/api/enums/v1/event_type.proto +++ b/temporal/api/enums/v1/event_type.proto @@ -182,4 +182,12 @@ enum EventType { // Recorded for every UnsubscribeNotificationChannel command, including one // naming a channel the run was not subscribed to. EVENT_TYPE_WORKFLOW_NOTIFICATION_CHANNEL_UNSUBSCRIBED = 62; + // A Workflow subscribed to a stream. Recorded once per subscription, not + // per record: the offsets a task consumed ride WorkflowTaskCompleted and + // the payloads never enter History at all. + EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED = 63; + // A Workflow appended a batch of records to a stream. Recorded per + // batch, and carrying only the offset range it landed at: the bodies go to + // the stream's own log, never into History. + EVENT_TYPE_WORKFLOW_STREAM_RECORDS_APPENDED = 64; } diff --git a/temporal/api/enums/v1/failed_cause.proto b/temporal/api/enums/v1/failed_cause.proto index 96baff0f8..2ccfc6900 100644 --- a/temporal/api/enums/v1/failed_cause.proto +++ b/temporal/api/enums/v1/failed_cause.proto @@ -95,6 +95,14 @@ enum WorkflowTaskFailedCause { WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES = 41; // An UnsubscribeNotificationChannel command named an empty or too-long channel. WORKFLOW_TASK_FAILED_CAUSE_BAD_UNSUBSCRIBE_NOTIFICATION_CHANNEL_ATTRIBUTES = 42; + // A workflow task completed with an invalid AppendStreamRecords command. + WORKFLOW_TASK_FAILED_CAUSE_BAD_APPEND_STREAM_RECORDS_ATTRIBUTES = 43; + // A workflow task completed with an invalid SubscribeStream command. + WORKFLOW_TASK_FAILED_CAUSE_BAD_SUBSCRIBE_STREAM_ATTRIBUTES = 44; + // A workflow task could not be started because a stream range it consumed and recorded in + // History can no longer be served, for example after truncation or because it exceeds the + // replay bound. Check the workflow task failure message for more information. + WORKFLOW_TASK_FAILED_CAUSE_STREAM_RANGE_UNAVAILABLE = 45; } // Activity tasks can fail for various reasons. Note that some of these reasons can only originate diff --git a/temporal/api/history/v1/message.proto b/temporal/api/history/v1/message.proto index 40f2b0a11..85b071a23 100644 --- a/temporal/api/history/v1/message.proto +++ b/temporal/api/history/v1/message.proto @@ -17,6 +17,7 @@ import "temporal/api/enums/v1/failed_cause.proto"; import "temporal/api/enums/v1/update.proto"; import "temporal/api/enums/v1/workflow.proto"; import "temporal/api/common/v1/message.proto"; +import "temporal/api/stream/v1/message.proto"; import "temporal/api/notification/v1/message.proto"; import "temporal/api/deployment/v1/message.proto"; import "temporal/api/failure/v1/message.proto"; @@ -384,6 +385,12 @@ message WorkflowTaskCompletedEventAttributes { // The Worker Deployment Version that completed this task. Must be set if `versioning_behavior` // is set. This value updates workflow execution's `versioning_info.deployment_version`. temporal.api.deployment.v1.WorkerDeploymentVersion deployment_version = 11; + + // Offset ranges this Workflow Task consumed from streams it subscribes to. + // Recorded on every task where a subscription is active, including when it + // observed nothing: an empty range is a fact replay must reproduce, and + // omitting it would let replay deliver records the Workflow did not have. + repeated temporal.api.stream.v1.StreamRange consumed_stream_ranges = 14; } message WorkflowTaskTimedOutEventAttributes { @@ -980,6 +987,41 @@ message WorkflowNotificationChannelUnsubscribedEventAttributes { int64 subscribed_event_id = 3; } +message WorkflowStreamSubscribedEventAttributes { + // The WorkflowTaskCompleted event of the task whose command created this + // subscription. + int64 workflow_task_completed_event_id = 1; + // The stream the Workflow subscribed to, as the command addressed it: + // either the name of a stream this Workflow owns or the id of one in + // another execution. + string stream_id = 2; + // The offset the subscription actually starts from. Resolved by the server + // when the subscription is registered and recorded here, so replay reads + // the resolved value rather than resolving it again against a stream that + // has since moved. + int64 start_offset = 3; +} + +message WorkflowStreamRecordsAppendedEventAttributes { + // The WorkflowTaskCompleted event of the task whose command appended this + // batch. + int64 workflow_task_completed_event_id = 1; + // Name of the stream the Workflow appended to. + string stream_id = 2; + // Inclusive. Same range vocabulary as StreamRange and StreamSlice, so a + // reader does not have to remember which of the three counts and which + // bounds. + // (-- api-linter: core::0140::prepositions=disabled + // aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + int64 from_offset = 3; + // Exclusive. With from_offset this names the range without carrying any of + // it, which is what keeps this event a fixed size no matter how large the + // batch or its payloads are. + // (-- api-linter: core::0140::prepositions=disabled + // aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + int64 to_offset = 4; +} + message WorkflowExecutionUpdateAcceptedEventAttributes { // The instance ID of the update protocol that generated this event. string protocol_instance_id = 1; @@ -1307,6 +1349,8 @@ message HistoryEvent { workflow_notification_channel_subscribed_event_attributes = 66; WorkflowNotificationChannelUnsubscribedEventAttributes workflow_notification_channel_unsubscribed_event_attributes = 67; + WorkflowStreamSubscribedEventAttributes workflow_stream_subscribed_event_attributes = 68; + WorkflowStreamRecordsAppendedEventAttributes workflow_stream_records_appended_event_attributes = 69; } } diff --git a/temporal/api/stream/v1/message.proto b/temporal/api/stream/v1/message.proto index dd1905803..e6a8fab2d 100644 --- a/temporal/api/stream/v1/message.proto +++ b/temporal/api/stream/v1/message.proto @@ -14,9 +14,13 @@ import "temporal/api/common/v1/message.proto"; // One entry in a stream. The record is the wire format: stores keep it // serialized as is and readers in every language decode the same bytes. message StreamRecord { - // The value the producer published, stored as sent. A stream provider - // writes and reads it as part of the record, so applying a payload codec - // to it is the provider's job. + // The value the producer published, stored as sent. + // + // A payload codec applies on the paths this API owns: the append command on + // RespondWorkflowTaskCompleted, and the slices on PollWorkflowTaskQueue. + // Records a producer writes or reads through the stream service take a + // different path, whose messages are not part of this API yet and so are + // outside what a codec-applying proxy walks. temporal.api.common.v1.Payload body = 1; // Producer-supplied provenance, stored as sent. map metadata = 2; @@ -35,6 +39,75 @@ message StreamRecord { int64 sequence = 7; } +// A contiguous range of a stream delivered to a Workflow Task, along with the +// offsets it covers. The offsets are what History records; the records +// themselves are never written to History. +message StreamSlice { + // The stream, as the subscribing command addressed it: either the name of + // a stream the consuming Workflow owns or the id of one in another + // execution. + string stream_id = 1; + // Run id of the execution that owns the stream. Set on both a slice for the + // task being started and a re-supplied one. + string run_id = 2; + // Inclusive. + // (-- api-linter: core::0140::prepositions=disabled + // aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + int64 from_offset = 3; + // Exclusive. Equal to from_offset when the subscription observed nothing, + // which is a fact replay has to reproduce rather than an absence of one. + // (-- api-linter: core::0140::prepositions=disabled + // aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + int64 to_offset = 4; + repeated StreamRecord records = 5; + // The WorkflowTaskCompleted event whose consumed_stream_ranges recorded + // this range. Set only when the server is re-supplying a range for a task + // being replayed; a slice for the task now being started leaves it unset, + // because the event closing that task does not exist yet. + // + // Replay needs this because a Workflow Task response carries one slice set + // while a cache miss replays every prior task, so the ranges have to be + // matched to the events that recorded them rather than to the response. + int64 workflow_task_completed_event_id = 6; +} + +// The offsets a Workflow Task consumed, without the payloads. Recorded on +// WorkflowTaskCompleted so History grows with Workflow Tasks rather than with +// records. +message StreamRange { + // The stream, as the subscribing command addressed it: either the name of + // a stream the consuming Workflow owns or the id of one in another + // execution. + string stream_id = 1; + // Inclusive. + // (-- api-linter: core::0140::prepositions=disabled + // aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + int64 from_offset = 2; + // Exclusive. + // (-- api-linter: core::0140::prepositions=disabled + // aip.dev/not-precedent: "from" and "to" name a half-open offset range. --) + int64 to_offset = 3; +} + +// Where a new subscription or read begins. The server resolves it against the +// stream as it stands in the same transaction that registers the reader, so +// the result does not race with appends or truncation, and records the +// resolved absolute offset. +message StreamStartPosition { + oneof position { + // Absolute and inclusive. Refused when below the stream's floor. + int64 offset = 1; + // The last N records the stream holds, or all of them when it holds + // fewer. Counts records of every kind. Must be positive. + int64 last_n = 2; + // The oldest record the stream still holds. Must be true. + bool earliest = 3; + // Only records appended after registration: the stream's head offset. + // Must be true. + bool tail = 4; + } +} + // What a record means to a reader. Kept on the record itself so every store // and every language reads it the same way without a private envelope. enum StreamRecordKind { diff --git a/temporal/api/workflowservice/v1/request_response.proto b/temporal/api/workflowservice/v1/request_response.proto index 47f6ebe24..48485ac79 100644 --- a/temporal/api/workflowservice/v1/request_response.proto +++ b/temporal/api/workflowservice/v1/request_response.proto @@ -24,6 +24,7 @@ import "temporal/api/enums/v1/activity.proto"; import "temporal/api/enums/v1/nexus.proto"; import "temporal/api/activity/v1/message.proto"; import "temporal/api/common/v1/message.proto"; +import "temporal/api/stream/v1/message.proto"; import "temporal/api/notification/v1/message.proto"; import "temporal/api/history/v1/message.proto"; import "temporal/api/workflow/v1/message.proto"; @@ -385,6 +386,10 @@ message PollWorkflowTaskQueueResponse { // 3. If every group has some pending polls, assign the next poll to a group randomly // according to the weights. temporal.api.taskqueue.v1.PollerGroupsInfo poller_groups_info = 19; + + // Stream data attached to this task. Delivered out of band so the payloads + // never enter History; only the offset ranges are recorded there. + repeated temporal.api.stream.v1.StreamSlice stream_slices = 20; } message RespondWorkflowTaskCompletedRequest {