From bb4ae6ccb4e6abd6f1d0e69721394da3dcd6dff3 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Fri, 2 Oct 2026 10:06:57 -0500 Subject: [PATCH 01/22] Add eager start support for standalone activities --- chasm/lib/activity/config.go | 8 + chasm/lib/activity/frontend.go | 5 +- chasm/lib/activity/frontend_test.go | 48 ++++ chasm/lib/activity/handler.go | 32 ++- chasm/lib/activity/responses.go | 62 +++++ chasm/lib/activity/statemachine.go | 63 +++++- chasm/lib/activity/statemachine_test.go | 40 ++++ docs/architecture/saa-eager-start-plan.md | 263 ++++++++++++++++++++++ tests/activity_standalone_test.go | 54 +++++ 9 files changed, 562 insertions(+), 13 deletions(-) create mode 100644 docs/architecture/saa-eager-start-plan.md diff --git a/chasm/lib/activity/config.go b/chasm/lib/activity/config.go index a6551f49dd7..d883d58c3a5 100644 --- a/chasm/lib/activity/config.go +++ b/chasm/lib/activity/config.go @@ -35,6 +35,12 @@ var ( `Allows non-zero start_delay on StartActivityExecution requests.`, ) + EnableEagerStart = dynamicconfig.NewNamespaceBoolSetting( + "activity.enableEagerStart", + true, + `Allows the first standalone activity task to be returned directly by StartActivityExecution.`, + ) + EnableCallbacks = dynamicconfig.NewNamespaceBoolSetting( "activity.enableCallbacks", false, @@ -63,6 +69,7 @@ type Config struct { EnableCallbacks dynamicconfig.BoolPropertyFnWithNamespaceFilter EnabledCallbackKinds dynamicconfig.TypedPropertyFnWithNamespaceFilter[[]callbacks.Kind] Enabled dynamicconfig.BoolPropertyFnWithNamespaceFilter + EnableEagerStart dynamicconfig.BoolPropertyFnWithNamespaceFilter EnableStandaloneActivityOperatorCommands dynamicconfig.BoolPropertyFnWithNamespaceFilter LongPollBuffer dynamicconfig.DurationPropertyFnWithNamespaceFilter LongPollTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter @@ -85,6 +92,7 @@ func ConfigProvider(dc *dynamicconfig.Collection) *Config { EnableCallbacks: EnableCallbacks.Get(dc), EnabledCallbackKinds: EnabledCallbackKinds.Get(dc), Enabled: Enabled.Get(dc), + EnableEagerStart: EnableEagerStart.Get(dc), EnableStandaloneActivityOperatorCommands: EnableStandaloneActivityOperatorCommands.Get(dc), LongPollBuffer: LongPollBuffer.Get(dc), LongPollTimeout: LongPollTimeout.Get(dc), diff --git a/chasm/lib/activity/frontend.go b/chasm/lib/activity/frontend.go index 3bf32ec5964..9d11ab85ec5 100644 --- a/chasm/lib/activity/frontend.go +++ b/chasm/lib/activity/frontend.go @@ -386,7 +386,10 @@ func (h *frontendHandler) validateAndPopulateStartRequest( if req.GetStartDelay().AsDuration() > 0 && !h.config.StartDelayEnabled(req.GetNamespace()) { return nil, serviceerror.NewInvalidArgument("start_delay is not enabled for this namespace") } - // TODO(saa): when eager start is supported, deny it if start delay > 0 (same as workflow behavior). + if req.GetRequestEagerExecution() && + (req.GetStartDelay().AsDuration() > 0 || !h.config.EnableEagerStart(req.GetNamespace())) { + req.RequestEagerExecution = false + } opts := activityOptionsFromStartRequest(req) err := ValidateAndNormalizeStandaloneActivity( diff --git a/chasm/lib/activity/frontend_test.go b/chasm/lib/activity/frontend_test.go index 1e105364bf4..0ba70f4c09d 100644 --- a/chasm/lib/activity/frontend_test.go +++ b/chasm/lib/activity/frontend_test.go @@ -215,3 +215,51 @@ func TestRequestIdStableAcrossRetries(t *testing.T) { }) }) } + +func TestEagerStartFallback(t *testing.T) { + newHandler := func(eagerEnabled bool) *frontendHandler { + return &frontendHandler{ + config: &Config{ + BlobSizeLimitError: defaultBlobSizeLimitError, + BlobSizeLimitWarn: defaultBlobSizeLimitWarn, + DefaultActivityRetryPolicy: getDefaultRetrySettings, + EnableEagerStart: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(eagerEnabled), + MaxIDLengthLimit: func() int { return defaultMaxIDLengthLimit }, + MaxUserMetadataDetailsSize: defaultMaxUserMetadataDetailsSize, + MaxUserMetadataSummarySize: defaultMaxUserMetadataSummarySize, + StartDelayEnabled: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(true), + }, + linkValidator: newLinkValidator( + defaultMaxLinksPerRequest, + func(string) int { return 2000 }, + defaultLinkMaxSize, + ), + logger: log.NewNoopLogger(), + } + } + + newRequest := func() *workflowservice.StartActivityExecutionRequest { + return &workflowservice.StartActivityExecutionRequest{ + Namespace: "test-namespace", + RequestEagerExecution: true, + ActivityId: "test-activity", + ActivityType: &commonpb.ActivityType{Name: "test-type"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-queue"}, + StartToCloseTimeout: durationpb.New(time.Minute), + } + } + + t.Run("namespace disabled", func(t *testing.T) { + req, err := newHandler(false).validateAndPopulateStartRequest(context.Background(), newRequest(), "test-namespace-id") + require.NoError(t, err) + require.False(t, req.GetRequestEagerExecution()) + }) + + t.Run("start delay", func(t *testing.T) { + req := newRequest() + req.StartDelay = durationpb.New(time.Minute) + req, err := newHandler(true).validateAndPopulateStartRequest(context.Background(), req, "test-namespace-id") + require.NoError(t, err) + require.False(t, req.GetRequestEagerExecution()) + }) +} diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index a89d7934d2a..79555e937ae 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -99,7 +99,14 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St } } - err = TransitionScheduled.Apply(newActivity, mutableContext, nil) + if request.GetRequestEagerExecution() { + err = TransitionEagerStarted.Apply(newActivity, mutableContext, eagerStartEvent{ + requestID: request.GetRequestId(), + identity: request.GetIdentity(), + }) + } else { + err = TransitionScheduled.Apply(newActivity, mutableContext, nil) + } if err != nil { return nil, err } @@ -129,6 +136,23 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St ) } + var eagerTask *workflowservice.PollActivityTaskQueueResponse + if result.Created && frontendReq.GetRequestEagerExecution() { + eagerTask, err = chasm.ReadComponent( + ctx, + result.ExecutionRef, + (*Activity).buildEagerActivityTask, + eagerActivityTaskRequest{ + namespaceID: req.GetNamespaceId(), + namespace: frontendReq.GetNamespace(), + requestID: frontendReq.GetRequestId(), + }, + ) + if err != nil { + return nil, err + } + } + // Apply on_conflict_options to an existing activity. // TODO: Use chasm.UpdateWithStartExecution to avoid a second transaction once the engine supports BusinessIDConflictPolicyFail in the updateFn path. cbs := frontendReq.GetCompletionCallbacks() @@ -165,8 +189,9 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St return &activitypb.StartActivityExecutionResponse{ FrontendResponse: &workflowservice.StartActivityExecutionResponse{ - RunId: result.ExecutionKey.RunID, - Started: result.Created, + RunId: result.ExecutionKey.RunID, + Started: result.Created, + EagerActivityTask: eagerTask, Link: &commonpb.Link{ Variant: &commonpb.Link_Activity_{ Activity: &commonpb.Link_Activity{ @@ -176,7 +201,6 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St }, }, }, - // EagerTask: TODO when supported, need to call the same code that would handle the HandleStarted API }, }, nil } diff --git a/chasm/lib/activity/responses.go b/chasm/lib/activity/responses.go index 340b21ca11c..4c53ef451f2 100644 --- a/chasm/lib/activity/responses.go +++ b/chasm/lib/activity/responses.go @@ -10,10 +10,72 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common/tasktoken" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) +type eagerActivityTaskRequest struct { + namespaceID string + namespace string + requestID string +} + +func (a *Activity) buildEagerActivityTask( + ctx chasm.Context, + request eagerActivityTaskRequest, +) (*workflowservice.PollActivityTaskQueueResponse, error) { + attempt := a.LastAttempt.Get(ctx) + if !a.hasAttemptInProgress() || attempt.GetCount() != 1 || attempt.GetStartRequestId() != request.requestID { + return nil, nil + } + + componentRef, err := ctx.Ref(a) + if err != nil { + return nil, err + } + key := ctx.ExecutionKey() + token, err := tasktoken.NewSerializer().Serialize(tasktoken.NewActivityTaskToken( + request.namespaceID, + "", + key.RunID, + 0, + key.BusinessID, + a.GetActivityType().GetName(), + attempt.GetCount(), + nil, + 0, + 0, + componentRef, + attempt.GetStartedStamp(), + )) + if err != nil { + return nil, err + } + + requestData := a.RequestData.Get(ctx) + lastHeartbeat, _ := a.LastHeartbeat.TryGet(ctx) + return &workflowservice.PollActivityTaskQueueResponse{ + TaskToken: token, + WorkflowNamespace: request.namespace, + ActivityType: a.GetActivityType(), + ActivityId: key.BusinessID, + Header: requestData.GetHeader(), + Input: requestData.GetInput(), + HeartbeatDetails: lastHeartbeat.GetDetails(), + ScheduledTime: a.GetScheduleTime(), + CurrentAttemptScheduledTime: a.dispatchTimeForAttempt(attempt), + StartedTime: attempt.GetStartedTime(), + Attempt: attempt.GetCount(), + ScheduleToCloseTimeout: a.GetScheduleToCloseTimeout(), + StartToCloseTimeout: a.GetStartToCloseTimeout(), + HeartbeatTimeout: a.GetHeartbeatTimeout(), + RetryPolicy: a.GetRetryPolicy(), + Priority: a.GetPriority(), + ActivityRunId: key.RunID, + }, nil +} + // Projection of activity state onto the API response protos. // InternalStatusToAPIStatus converts internal activity execution status to API status. diff --git a/chasm/lib/activity/statemachine.go b/chasm/lib/activity/statemachine.go index a9cd9be2d07..2fe2fc7c053 100644 --- a/chasm/lib/activity/statemachine.go +++ b/chasm/lib/activity/statemachine.go @@ -35,6 +35,11 @@ func (a *Activity) SetStateMachineState(state activitypb.ActivityExecutionStatus } type ( + eagerStartEvent struct { + requestID string + identity string + } + rescheduleEvent struct { retryInterval time.Duration retryIntervalSource activitypb.ActivityRetryIntervalSource @@ -98,7 +103,22 @@ var TransitionScheduled = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, func(a *Activity, ctx chasm.MutableContext, _ any) error { - return a.applyScheduled(ctx) + return a.applyScheduled(ctx, true) + }, +) + +// TransitionEagerStarted atomically schedules and starts the first attempt without creating a +// Matching dispatch task. The task itself is returned by StartActivityExecution. +var TransitionEagerStarted = chasm.NewTransition( + []activitypb.ActivityExecutionStatus{ + activitypb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED, + }, + activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, + func(a *Activity, ctx chasm.MutableContext, event eagerStartEvent) error { + if err := a.applyScheduled(ctx, false); err != nil { + return err + } + return a.applyEagerStarted(ctx, event) }, ) @@ -346,7 +366,7 @@ var TransitionResetAttemptFailedToScheduled = chasm.NewTransition( // The methods below are transition bodies, invoked from the transitions above once the // source state has been validated. Do not call them from outside this file. -func (a *Activity) applyScheduled(ctx chasm.MutableContext) error { +func (a *Activity) applyScheduled(ctx chasm.MutableContext, dispatch bool) error { attempt := a.LastAttempt.Get(ctx) attempt.Count++ @@ -357,7 +377,8 @@ func (a *Activity) applyScheduled(ctx chasm.MutableContext) error { dispatchTime := a.firstDispatchTime() attempt.DispatchTime = timestamppb.New(dispatchTime) - if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { + if dispatch && a.GetScheduleToStartTimeout().AsDuration() > 0 { + timeout := a.GetScheduleToStartTimeout().AsDuration() ctx.AddTask( a, chasm.TaskAttributes{ @@ -378,15 +399,41 @@ func (a *Activity) applyScheduled(ctx chasm.MutableContext) error { &activitypb.ScheduleToCloseTimeoutTask{Stamp: a.GetScheduleToCloseStamp()}) } - dispatchAttrs := chasm.TaskAttributes{} - if dispatchTime.After(a.ScheduleTime.AsTime()) { - dispatchAttrs.ScheduledTime = dispatchTime + if dispatch { + dispatchAttrs := chasm.TaskAttributes{} + if dispatchTime.After(a.ScheduleTime.AsTime()) { + dispatchAttrs.ScheduledTime = dispatchTime + } + ctx.AddTask( + a, + dispatchAttrs, + a.newActivityDispatchTask(ctx)) + } + + return nil +} + +func (a *Activity) applyEagerStarted(ctx chasm.MutableContext, event eagerStartEvent) error { + attempt := a.LastAttempt.Get(ctx) + attempt.StartedTime = timestamppb.New(ctx.Now(a)) + attempt.StartedStamp = attempt.GetStamp() + if a.FirstAttemptStartedTime == nil { + a.FirstAttemptStartedTime = attempt.GetStartedTime() } + attempt.StartRequestId = event.requestID + attempt.LastWorkerIdentity = event.identity + startTime := attempt.GetStartedTime().AsTime() ctx.AddTask( a, - dispatchAttrs, - a.newActivityDispatchTask(ctx)) + chasm.TaskAttributes{ScheduledTime: startTime.Add(a.GetStartToCloseTimeout().AsDuration())}, + &activitypb.StartToCloseTimeoutTask{Stamp: attempt.GetStamp()}) + if heartbeatTimeout := a.GetHeartbeatTimeout().AsDuration(); heartbeatTimeout > 0 { + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: startTime.Add(heartbeatTimeout)}, + &activitypb.HeartbeatTimeoutTask{Stamp: attempt.GetStamp()}) + } return nil } diff --git a/chasm/lib/activity/statemachine_test.go b/chasm/lib/activity/statemachine_test.go index 1583c7b7ff5..a940746de7a 100644 --- a/chasm/lib/activity/statemachine_test.go +++ b/chasm/lib/activity/statemachine_test.go @@ -196,6 +196,46 @@ func TestTransitionScheduled(t *testing.T) { } } +func TestTransitionEagerStarted(t *testing.T) { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return defaultTime }, + }, + } + attempt := &activitypb.ActivityAttemptState{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + ScheduleTime: timestamppb.New(defaultTime), + ScheduleToCloseTimeout: durationpb.New(defaultScheduleToCloseTimeout), + ScheduleToStartTimeout: durationpb.New(defaultScheduleToStartTimeout), + StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), + HeartbeatTimeout: durationpb.New(time.Minute), + Status: activitypb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + }, + LastAttempt: chasm.NewDataField(ctx, attempt), + Outcome: chasm.NewDataField(ctx, &activitypb.ActivityOutcome{}), + } + + err := TransitionEagerStarted.Apply(activity, ctx, eagerStartEvent{ + requestID: "start-request-id", + identity: "starter", + }) + require.NoError(t, err) + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, activity.GetStatus()) + require.EqualValues(t, 1, attempt.GetCount()) + require.EqualValues(t, 1, attempt.GetStamp()) + require.EqualValues(t, 1, attempt.GetStartedStamp()) + require.Equal(t, "start-request-id", attempt.GetStartRequestId()) + require.Equal(t, "starter", attempt.GetLastWorkerIdentity()) + + require.Len(t, ctx.Tasks, 3) + require.IsType(t, &activitypb.ScheduleToCloseTimeoutTask{}, ctx.Tasks[0].Payload) + require.IsType(t, &activitypb.StartToCloseTimeoutTask{}, ctx.Tasks[1].Payload) + require.IsType(t, &activitypb.HeartbeatTimeoutTask{}, ctx.Tasks[2].Payload) +} + func TestTransitionRescheduled(t *testing.T) { testCases := []struct { name string diff --git a/docs/architecture/saa-eager-start-plan.md b/docs/architecture/saa-eager-start-plan.md new file mode 100644 index 00000000000..5792f3f9400 --- /dev/null +++ b/docs/architecture/saa-eager-start-plan.md @@ -0,0 +1,263 @@ +# Standalone Activity Eager Start Plan + +## Goal + +Allow a caller of `StartActivityExecution` to request an eager first activity +attempt. When accepted, the server returns a worker-ready activity task directly +in the start response, avoiding the initial Chasm dispatch task and Matching +round trip. + +The eager path must leave the standalone activity (SAA) in the same persisted +state that normal Matching dispatch would create after a worker accepts its +first task. + +## Scope + +This change is server-only. It covers the API contract and Temporal server +behavior needed to accept an eager-start request and return an eager activity +task. SDK support—including deciding when a local worker has capacity, +requesting eager start, handing the returned task to that worker, and samples— +is explicitly out of scope and will be handled as follow-up work by the SDK +team. + +## Current lifecycle + +```text +StartActivityExecution + -> Chasm Activity created + -> SCHEDULED + durable dispatch/timer tasks + -> ActivityDispatchTask -> Matching.AddActivityTask + -> worker poll -> RecordActivityTaskStarted + -> STARTED -> worker completes, fails, or heartbeats + -> retry: SCHEDULED; terminal: COMPLETED, FAILED, CANCELED, TERMINATED, or TIMED_OUT +``` + +The relevant implementation points are: + +- `chasm/lib/activity/handler.go`: creates the execution and applies the initial + `TransitionScheduled` transition. +- `chasm/lib/activity/statemachine.go`: owns state transitions and durable task + creation. +- `chasm/lib/activity/tasks.go`: dispatches a valid scheduled attempt to Matching. +- `service/matching/matching_engine.go`: records the worker pickup and builds + `PollActivityTaskQueueResponse`. +- `service/history/handler.go`: routes a standalone `RecordActivityTaskStarted` + request to `Activity.HandleStarted` through Chasm. + +## Proposed eager lifecycle + +```text +StartActivityExecution(request_eager_execution=true) + -> validate eligibility + -> Chasm Activity created + -> initialize attempt 1 and schedule durable lifetime timers + -> atomically transition SCHEDULED -> STARTED + -> return EagerActivityTask with a normal activity task token + -> caller executes task directly + +Subsequent activity attempts use normal SCHEDULED -> Matching dispatch. A +retry of the *start request* returns the existing execution without +re-delivering its eager task. +``` + +The worker-facing eager task must be equivalent to the response normally built +by Matching: same input, header, activity/run IDs, timeouts, attempt, component +reference, and attempt stamp in its task token. + +### Idempotent eager-start retry lifecycle + +The initial eager response is created only after `StartExecution` succeeds, +using the persisted execution in `result.ExecutionRef`. It is returned only +when `result.Created` is true. A same-request-ID retry returns the existing +execution without another eager task, even if the original first attempt is +still current. Unlike a workflow task, an activity can have external side +effects, so re-delivering an eager activity task could run user code twice. + +```text +StartActivityExecution(request ID = R, eager = true) + -> StartExecution succeeds with Created = true + -> create and persist eager attempt 1 as STARTED + -> return EagerActivityTask + +Retry StartActivityExecution(request ID = R, eager = true) + -> StartExecution returns the existing execution + -> return the idempotent start response without EagerActivityTask + +StartActivityExecution(request ID != R, conflict policy = USE_EXISTING) + -> result.Created = false + -> return the existing execution without EagerActivityTask +``` + +If the first eager response is unavailable, the started attempt is recovered by +the normal timeout and retry path; subsequent attempts are dispatched through +Matching. The initial response token represents the first logical attempt and +must pass the normal activity-token validation path. + +## Implementation steps + +1. Define and regenerate the API contract. + + - Add `request_eager_execution` to `StartActivityExecutionRequest`. + - Add `eager_activity_task` to `StartActivityExecutionResponse`. + - Update the pinned Temporal API dependency and regenerate code as required. + - Confirm the final field names and numbers against the API repository before + implementation. + +2. Add eligibility and fallback policy. + + - Add a namespace-scoped dynamic configuration setting for standalone eager + start. + - Call `StartExecution` before constructing an eager task. Use + `result.ExecutionRef`, rather than a precomputed reference, so its token + identifies the execution actually persisted. + - Return an eager task only for a newly created execution, when + `result.Created == true`. + - A distinct request that resolves through `USE_EXISTING` must return no + eager task. + - Do not eagerly start an activity with non-zero `start_delay`. + - For ineligible requests, clear the effective eager request and use the + existing scheduling path. Return a normal start response with no eager task. + - Add accepted and denied metrics, including denial reasons. + +3. Preserve the state-machine invariant atomically. + + - Generalize the initial scheduling event/body so eager initialization creates + attempt 1, its stamp, dispatch time, and schedule-to-close timer without + emitting an `ActivityDispatchTask` or schedule-to-start timer. + - In the same Chasm creation transaction, apply the existing + `SCHEDULED -> STARTED` behavior. + - Persist start-to-close and heartbeat timers exactly as a Matching-started + task does. + - Do not rely on a later stale-task validation check as the primary mechanism; + avoid creating the initial Matching dispatch task in the eager case. + +4. Share start and task-response construction. + + - Extract the common state update and task-response data preparation currently + split between `Activity.HandleStarted` and Matching's + `createPollActivityTaskQueueResponse`. + - Keep `HandleStarted` as the normal Matching entry point, but make eager + start call the same transition and response-building helpers. + - Inject the token serializer into the SAA history handler or another local + dependency that owns the shared response builder. + - Include the serialized Chasm component reference and current attempt stamp + in the eager activity task token. + - Make the response builder usable by the request-ID deduplication path: it + must load the activity referenced by `result.ExecutionRef` and rebuild a + response only for the still-current original eager attempt. + +5. Keep post-start behavior unchanged. + + - Completion, failure, cancellation, heartbeat, pause, reset, and timeout + paths continue to validate the existing activity task token. + - Retryable worker failures and start-to-close/heartbeat timeouts transition + back to `SCHEDULED` and dispatch through Matching. + - Schedule-to-close remains an activity-lifetime timeout; it must still be + installed for an eager attempt. + +6. Implement idempotency and conflict behavior. + + - Let `StartExecution` perform request-ID deduplication and conflict/reuse + resolution before any eager task is built. + - When `result.Created == true`, construct the initial eager activity token + from `result.ExecutionRef`. + - When `result.Created == false`, including a request-ID retry or a + `USE_EXISTING` conflict, return no eager task. Do not create an attempt, + emit a dispatch task, or mutate activity state on this path. + - Keep token validation unchanged. The initial response token must carry the + persisted component reference and attempt stamp. + +## Test plan + +### State-machine tests + +- Eager initialization enters `STARTED`, creates attempt 1, and records the + expected started metadata. +- It creates start-to-close and heartbeat timers when configured. +- It does not create initial `ActivityDispatchTask` or schedule-to-start timer. +- A normal first attempt remains unchanged. + +### Handler tests + +- Accepted eager start returns an eager `PollActivityTaskQueueResponse`. +- The response token completes and heartbeats the SAA successfully. +- The token contains the activity's component reference and current attempt stamp. +- Disabled dynamic config and non-zero start delay fall back without an eager + task. +- An immediate retry with the same request ID returns the existing execution + without an eager task and without creating another attempt or invoking + Matching. +- A distinct `USE_EXISTING` request returns the existing execution without an + eager task. + +### End-to-end tests + +- Eager first attempt does not call `Matching.AddActivityTask`. +- Retrying an eager attempt sends the subsequent attempt through Matching. +- Completion, cancellation, start-to-close timeout, heartbeat timeout, and + schedule-to-close timeout retain current behavior. +- A stale normal dispatch task cannot start or duplicate an eagerly started + attempt. + +## Failure modes and trade-offs + +- **Lost start response:** a same-request-ID retry does not re-deliver the + original eager task. Normal timeout and retry behavior recover delivery + through Matching, without risking duplicate execution of the same activity + attempt. +- **Concurrent dispatch:** atomically omitting the initial dispatch task avoids + duplicate delivery and removes unnecessary queue work. +- **Worker crash after receipt:** normal start-to-close and heartbeat timers + recover the attempt through existing retry behavior. +- **Load increase:** eager start moves first-attempt work from Matching to the + synchronous start RPC. It reduces queue latency but increases start-response + size and History/Chasm transaction work; configuration gating provides a safe + rollout control. +- **Security:** reuse the normal task token format and validation path. Never + introduce a special eager-only token that bypasses component-reference, + namespace, attempt, or stamp checks. + +## Verification + +Run focused unit tests with `-tags test_dep`, then run the relevant integration +coverage if added. + +After the server/API changes are available, perform a local latency comparison +with the standalone-activity sample at +`/Users/wenlonggu/Desktop/workspaces/samples-go`: + +1. Build and run the local server from this checkout, + `/Users/wenlonggu/Desktop/workspaces/temporal`, then configure the sample + to use that local endpoint. The sample is an SDK client and does not import + the server module directly. +2. Run the same SAA workload without eager start and record end-to-end + start-to-running latency from the `StartActivityExecution` call until the + activity begins running. +3. Enable eager start, run the identical workload against a local worker on the + same task queue, and record the same latency measurement. +4. Compare the before/after results (at least p50 and p95) and verify that the + eager first attempt has no Matching dispatch or poll. Keep worker capacity, + task queue, payload, and repetition count equivalent between the two runs. + +Finish with: + +```text +make lint-code-fast +``` + +### Local direct-RPC result + +On 2026-10-01, 200 paired local runs against the modified server measured the +time from `StartActivityExecution` until a worker-ready task was available. +Normal runs used a waiting `PollActivityTaskQueue`; eager runs read the task +from the start response. Each task was immediately completed after measurement. + +| Path | p50 | p95 | p99 | +| --- | ---: | ---: | ---: | +| Normal | 2.372 ms | 58.654 ms | 88.855 ms | +| Eager | 1.588 ms | 5.028 ms | 7.362 ms | + +This local result shows lower task-delivery latency and much tighter tail +latency for eager start. It is not a production end-to-end worker benchmark: +it excludes SDK activity-function execution and measures only server-side task +delivery. diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index ded66e633ac..3cfe9c41572 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1207,6 +1207,60 @@ func (s *standaloneActivityTestSuite) TestStart() { }) } +func (s *standaloneActivityTestSuite) TestEagerStart() { + env := s.newTestEnv() + t := s.T() + ctx := s.Context() + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + requestID := testcore.RandomizeStr(t.Name()) + request := &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + RequestEagerExecution: true, + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + Identity: defaultIdentity, + Input: defaultInput, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), + RequestId: requestID, + } + + first, err := env.FrontendClient().StartActivityExecution(ctx, request) + require.NoError(t, err) + require.True(t, first.GetStarted()) + require.NotNil(t, first.GetEagerActivityTask()) + require.Equal(t, activityID, first.GetEagerActivityTask().GetActivityId()) + require.Equal(t, first.GetRunId(), first.GetEagerActivityTask().GetActivityRunId()) + require.EqualValues(t, 1, first.GetEagerActivityTask().GetAttempt()) + + retry, err := env.FrontendClient().StartActivityExecution(ctx, request) + require.NoError(t, err) + require.False(t, retry.GetStarted()) + require.Nil(t, retry.GetEagerActivityTask()) + + useExisting := common.CloneProto(request) + useExisting.RequestId = testcore.RandomizeStr(t.Name()) + useExisting.IdConflictPolicy = enumspb.ACTIVITY_ID_CONFLICT_POLICY_USE_EXISTING + conflict, err := env.FrontendClient().StartActivityExecution(ctx, useExisting) + require.NoError(t, err) + require.False(t, conflict.GetStarted()) + require.Nil(t, conflict.GetEagerActivityTask()) + + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: first.GetEagerActivityTask().GetTaskToken(), + Result: defaultResult, + Identity: defaultIdentity, + }) + require.NoError(t, err) + + afterCompletion, err := env.FrontendClient().StartActivityExecution(ctx, request) + require.NoError(t, err) + require.False(t, afterCompletion.GetStarted()) + require.Nil(t, afterCompletion.GetEagerActivityTask()) +} + func (s *standaloneActivityTestSuite) TestComplete() { env := s.newTestEnv() t := s.T() From 7e839054199075d998533971a7d2f140d8838fb5 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Fri, 2 Oct 2026 11:59:40 -0500 Subject: [PATCH 02/22] Add eager activity parity coverage --- chasm/lib/activity/model/model.go | 4 ++ chasm/lib/activity/model/model_test.go | 26 +++++++- chasm/lib/activity/model/vocabulary.go | 12 ++-- tests/activity_driver.go | 14 +++-- tests/activity_parity_test.go | 27 ++++++++ tests/activity_standalone_driver.go | 10 ++- tests/activity_standalone_test.go | 14 +---- tests/activity_workflow_driver.go | 87 ++++++++++++++++++++++++++ 8 files changed, 168 insertions(+), 26 deletions(-) diff --git a/chasm/lib/activity/model/model.go b/chasm/lib/activity/model/model.go index a104b18a1c8..c27d0b4af45 100644 --- a/chasm/lib/activity/model/model.go +++ b/chasm/lib/activity/model/model.go @@ -33,6 +33,10 @@ type Outcome struct { // Initial is the state of a newly created Activity. func Initial(cfg Config) AbstractState { s := AbstractState{Status: Scheduled, AttemptCount: 1} + if cfg.InitialAttemptStarted { + s.Status = Started + return s + } if cfg.HasStartDelay { s.Dispatchability = StartDelayPending } diff --git a/chasm/lib/activity/model/model_test.go b/chasm/lib/activity/model/model_test.go index 39187e104d5..54c82e1a407 100644 --- a/chasm/lib/activity/model/model_test.go +++ b/chasm/lib/activity/model/model_test.go @@ -12,6 +12,27 @@ func TestInitial(t *testing.T) { require.Equal(t, AbstractState{Status: Scheduled, AttemptCount: 1}, Initial(Config{HasScheduleToClose: true})) } +func TestEagerInitialAttempt(t *testing.T) { + cfg := Config{ + InitialAttemptStarted: true, + HasScheduleToClose: true, + HasScheduleToStart: true, + HasHeartbeat: true, + } + initial := Initial(cfg) + require.Equal(t, AbstractState{Status: Started, AttemptCount: 1}, initial) + require.False(t, FindsTask(initial), "an eager task is already started and must not be dispatched again") + require.False(t, Possible(cfg, initial, ScheduleToStartElapsesType), "an eager first attempt has no schedule-to-start window") + require.True(t, Possible(cfg, initial, ScheduleToCloseElapsesType), "schedule-to-close still protects an eager first attempt") + require.True(t, Possible(cfg, initial, StartToCloseElapsesType), "start-to-close still protects an eager first attempt") + require.True(t, Possible(cfg, initial, HeartbeatElapsesType), "heartbeat still protects an eager first attempt") + + retry := Transition(cfg, initial, FailRetryably).Next + require.Equal(t, AbstractState{Status: Scheduled, AttemptCount: 2, Dispatchability: BackoffPending}, retry) + retry = Transition(cfg, retry, BackoffElapses).Next + require.True(t, FindsTask(retry), "an eager first attempt's retry must return to normal dispatch") +} + func TestPollFromScheduledStarts(t *testing.T) { out := Transition(Config{}, Initial(Config{}), Event{Type: PollType}) require.Equal(t, NoError, out.Reject) @@ -40,7 +61,10 @@ func TestPauseWhileStartedIsPauseRequested(t *testing.T) { // backedOffRetry returns a Scheduled state with a pending retry backoff (attempt 2), reached the way // a worker would: poll the first attempt, then fail it retryably. func backedOffRetry(t require.TestingT, cfg Config) AbstractState { - started := Transition(cfg, Initial(cfg), Event{Type: PollType}).Next + started := Initial(cfg) + if started.Status != Started { + started = Transition(cfg, started, Event{Type: PollType}).Next + } s := Transition(cfg, started, Event{Type: RespondFailedType, Failure: &Failure{}}).Next require.Equal(t, Scheduled, s.Status, "a retryable failure must schedule a retry") require.Equal(t, BackoffPending, s.Dispatchability, "a retry must wait for its backoff") diff --git a/chasm/lib/activity/model/vocabulary.go b/chasm/lib/activity/model/vocabulary.go index 02e646a090e..e00c19ca20e 100644 --- a/chasm/lib/activity/model/vocabulary.go +++ b/chasm/lib/activity/model/vocabulary.go @@ -43,11 +43,13 @@ type AbstractState struct { // Config is what the model needs to know about an activity's configuration: which options are set, // and what their durations imply regarding retries. type Config struct { - HasScheduleToClose bool - HasScheduleToStart bool - HasHeartbeat bool - HasStartDelay bool - MaxAttempts int32 // 0 = unlimited + // InitialAttemptStarted means the first attempt was delivered eagerly and is already running. + InitialAttemptStarted bool + HasScheduleToClose bool + HasScheduleToStart bool + HasHeartbeat bool + HasStartDelay bool + MaxAttempts int32 // 0 = unlimited // NonRetryableTimeouts are the timeout elapses whose failure the retry policy refuses to retry, // so that the timeout closes the activity instead of scheduling another attempt. diff --git a/tests/activity_driver.go b/tests/activity_driver.go index 360c42f23dc..a563bcd30ee 100644 --- a/tests/activity_driver.go +++ b/tests/activity_driver.go @@ -41,6 +41,7 @@ import ( // always send start-to-close, defaulted long enough not to fire. The other timeouts are simply // absent when unset. type activityConfig struct { + EagerStart bool MaxAttempts int32 // RetryPolicy MaximumAttempts; 0 = unlimited RetryInterval time.Duration // RetryPolicy InitialInterval; 0 => activityShortRetryInterval BackoffCoefficient float64 // RetryPolicy BackoffCoefficient; 0 => 1.0 (constant interval) @@ -157,12 +158,13 @@ type activityTerminalOutcome struct { // ways the options alone settle that no retry can follow. Deriving it means the two cannot disagree. func (c activityConfig) modelConfig() model.Config { return model.Config{ - MaxAttempts: c.MaxAttempts, - HasStartDelay: c.StartDelay > 0, - HasScheduleToClose: c.ScheduleToClose > 0, - HasScheduleToStart: c.ScheduleToStart > 0, - HasHeartbeat: c.HeartbeatTimeout > 0, - NonRetryableTimeouts: c.nonRetryableTimeouts(), + InitialAttemptStarted: c.EagerStart, + MaxAttempts: c.MaxAttempts, + HasStartDelay: c.StartDelay > 0, + HasScheduleToClose: c.ScheduleToClose > 0, + HasScheduleToStart: c.ScheduleToStart > 0, + HasHeartbeat: c.HeartbeatTimeout > 0, + NonRetryableTimeouts: c.nonRetryableTimeouts(), // In reality, the server decides this based on how long the attempt took, and any // NextRetryDelay sent by the worker. The model cannot express that. Instead it supports a // fixed retry delay and the calculation here pretends that the attempt duration is zero. diff --git a/tests/activity_parity_test.go b/tests/activity_parity_test.go index bae153f3ec8..e3c0e967c32 100644 --- a/tests/activity_parity_test.go +++ b/tests/activity_parity_test.go @@ -41,10 +41,37 @@ func newActivityParityEnv(t *testing.T) *testcore.TestEnv { cluster := env.GetTestCluster() cluster.OverrideDynamicConfig(t, dynamicconfig.EnableChasm, nsValues(true)) cluster.OverrideDynamicConfig(t, activity.Enabled, nsValues(true)) + cluster.OverrideDynamicConfig(t, dynamicconfig.EnableActivityEagerExecution, nsValues(true)) cluster.OverrideDynamicConfig(t, activity.EnableStandaloneActivityOperatorCommands, nsValues(true)) return env } +func (s *activityParityTestSuite) TestEagerActivityStartParity() { + env := newActivityParityEnv(s.T()) + cfg := activityConfig{ + EagerStart: true, + MaxAttempts: 2, + RetryInterval: activityShortDispatchDelay, + } + + // Both implementations return the first task already started. A retry of that task must then + // return to the ordinary Matching dispatch path before its second attempt starts. + trace := []model.Event{ + model.Heartbeat, + model.FailRetryably, + model.BackoffElapses, + model.Poll, + model.Complete, + } + + s.Run("WorkflowActivity", func(s *activityParityTestSuite) { + newWFADriver(s.T(), env, cfg).driveTrace(s.T(), trace) + }) + s.Run("StandaloneActivity", func(s *activityParityTestSuite) { + newSAADriver(s.T(), env, cfg).driveTrace(s.T(), trace) + }) +} + func assertActivityTaskNotCancelRequested(t *testing.T, err error) { var invalidArgumentErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgumentErr) diff --git a/tests/activity_standalone_driver.go b/tests/activity_standalone_driver.go index d9ee39457a7..6807b4ffa0f 100644 --- a/tests/activity_standalone_driver.go +++ b/tests/activity_standalone_driver.go @@ -118,8 +118,15 @@ func (d *saaDriver) start(t require.TestingT, cfg activityConfig) *saaHandle { id := fmt.Sprintf("%s-%d", d.activityIDPrefix, d.numStarted) resp, err := d.env.FrontendClient().StartActivityExecution(d.testContext(), d.startRequest(cfg, id, id)) require.NoError(t, err) + state := activityDriverState{cfg: cfg} + if cfg.EagerStart { + task := resp.GetEagerActivityTask() + require.NotNil(t, task, "eager standalone activity start did not return a task") + state.token = task.GetTaskToken() + state.startedAttempt = task.GetAttempt() + } return &saaHandle{ - activityDriverState: activityDriverState{cfg: cfg}, + activityDriverState: state, model: newActivityModel(cfg), d: d, activityID: id, @@ -137,6 +144,7 @@ func (d *saaDriver) startRequest(c activityConfig, activityID, taskQueue string) } return &workflowservice.StartActivityExecutionRequest{ Namespace: d.env.Namespace().String(), + RequestEagerExecution: c.EagerStart, ActivityId: activityID, ActivityType: d.env.Tv().ActivityType(), Identity: d.env.Tv().ClientIdentity(), diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 3cfe9c41572..990c4767873 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1207,7 +1207,7 @@ func (s *standaloneActivityTestSuite) TestStart() { }) } -func (s *standaloneActivityTestSuite) TestEagerStart() { +func (s *standaloneActivityTestSuite) TestEagerStartResponseAndNoRedelivery() { env := s.newTestEnv() t := s.T() ctx := s.Context() @@ -1247,18 +1247,6 @@ func (s *standaloneActivityTestSuite) TestEagerStart() { require.False(t, conflict.GetStarted()) require.Nil(t, conflict.GetEagerActivityTask()) - _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ - Namespace: env.Namespace().String(), - TaskToken: first.GetEagerActivityTask().GetTaskToken(), - Result: defaultResult, - Identity: defaultIdentity, - }) - require.NoError(t, err) - - afterCompletion, err := env.FrontendClient().StartActivityExecution(ctx, request) - require.NoError(t, err) - require.False(t, afterCompletion.GetStarted()) - require.Nil(t, afterCompletion.GetEagerActivityTask()) } func (s *standaloneActivityTestSuite) TestComplete() { diff --git a/tests/activity_workflow_driver.go b/tests/activity_workflow_driver.go index 73917e5a2cd..0e088b6f8e2 100644 --- a/tests/activity_workflow_driver.go +++ b/tests/activity_workflow_driver.go @@ -16,6 +16,8 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" activitypb "go.temporal.io/api/activity/v1" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" workflowpb "go.temporal.io/api/workflow/v1" @@ -122,6 +124,9 @@ func (a *wfaHandle) awaitDispatchDelay(t testing.TB, e model.Event) { } func (d *wfaDriver) start(t *testing.T, cfg activityConfig) *wfaHandle { + if cfg.EagerStart { + return d.startEager(t, cfg) + } cfg.StartDelay = 0 // WFA does not support start delay, but SAA/WFA tests often share config wfTQ := testcore.RandomizeStr("wfa-wf") actTQ := testcore.RandomizeStr("wfa-act") @@ -158,6 +163,88 @@ func (d *wfaDriver) start(t *testing.T, cfg activityConfig) *wfaHandle { return a } +func (d *wfaDriver) startEager(t *testing.T, cfg activityConfig) *wfaHandle { + wfTQ := testcore.RandomizeStr("wfa-eager-wf") + actTQ := testcore.RandomizeStr("wfa-eager-act") + const actID = "act" + wfID := testcore.RandomizeStr("wfa-eager-run") + fc := d.env.FrontendClient() + + started, err := fc.StartWorkflowExecution(d.testContext(), &workflowservice.StartWorkflowExecutionRequest{ + Namespace: d.env.Namespace().String(), + WorkflowId: wfID, + WorkflowType: &commonpb.WorkflowType{Name: "eagerActivityDriver"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: wfTQ}, + RequestId: uuid.NewString(), + }) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(d.testContext(), activityDriverTimeout) + defer cancel() + workflowTask, err := fc.PollWorkflowTaskQueue(ctx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: d.env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{Name: wfTQ}, + Identity: d.env.Tv().WorkerIdentity(), + }) + require.NoError(t, err) + require.NotEmpty(t, workflowTask.GetTaskToken(), "workflow start did not produce a workflow task") + + activityTask, err := fc.RespondWorkflowTaskCompleted(d.testContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: d.env.Namespace().String(), + Identity: d.env.Tv().WorkerIdentity(), + TaskToken: workflowTask.GetTaskToken(), + Commands: []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: actID, + ActivityType: d.env.Tv().ActivityType(), + TaskQueue: &taskqueuepb.TaskQueue{Name: actTQ}, + Input: payloads.EncodeString(activityInput), + StartToCloseTimeout: durationpb.New(cfg.startToClose()), + ScheduleToCloseTimeout: optionalDuration(cfg.ScheduleToClose), + ScheduleToStartTimeout: optionalDuration(cfg.ScheduleToStart), + HeartbeatTimeout: optionalDuration(cfg.HeartbeatTimeout), + RetryPolicy: &commonpb.RetryPolicy{ + InitialInterval: durationpb.New(cfg.retryInterval()), + BackoffCoefficient: cmp.Or(cfg.BackoffCoefficient, 1.0), + MaximumInterval: durationpb.New(cmp.Or(cfg.MaxRetryInterval, cfg.retryInterval())), + MaximumAttempts: cfg.MaxAttempts, + NonRetryableErrorTypes: cfg.NonRetryableErrorTypes, + }, + RequestEagerExecution: true, + }, + }, + }}, + }) + require.NoError(t, err) + require.Len(t, activityTask.GetActivityTasks(), 1, "eager workflow activity start did not return a task") + task := activityTask.GetActivityTasks()[0] + + t.Cleanup(func() { + if err := d.env.SdkClient().TerminateWorkflow(context.Background(), wfID, started.GetRunId(), "eager activity driver cleanup"); err != nil { + t.Logf("eager activity driver cleanup: %v", err) + } + }) + return &wfaHandle{ + activityDriverState: activityDriverState{cfg: cfg, token: task.GetTaskToken(), startedAttempt: task.GetAttempt()}, + model: newActivityModel(cfg), + d: d, + run: d.env.SdkClient().GetWorkflow(d.testContext(), wfID, started.GetRunId()), + workflowID: wfID, + runID: started.GetRunId(), + activityID: actID, + taskQueue: actTQ, + } +} + +func optionalDuration(value time.Duration) *durationpb.Duration { + if value == 0 { + return nil + } + return durationpb.New(value) +} + // wfaActivityParams is what the helper workflow needs to schedule the activity: the activity the // test described, and where to put it. type wfaActivityParams struct { From 76f9b8c34373da9c9e328db09e3cae93508c422e Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 5 Oct 2026 15:16:20 -0500 Subject: [PATCH 03/22] Use eager activity API dependency --- go.mod | 4 +++- go.sum | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 26dc89de4c7..724b8c47252 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.44.0 go.opentelemetry.io/otel/trace v1.44.0 go.opentelemetry.io/proto/otlp v1.10.0 - go.temporal.io/api v1.63.6-0.20260909222256-20151aa90480 + go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed go.temporal.io/auto-scaled-workers v0.0.0-20260928183627-719679297cbe go.temporal.io/sdk v1.48.0 go.uber.org/fx v1.24.0 @@ -92,6 +92,8 @@ require ( modernc.org/sqlite v1.51.0 ) +replace go.temporal.io/api => github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed + require ( cel.dev/expr v0.25.2 // indirect cloud.google.com/go v0.123.0 // indirect; indirect e diff --git a/go.sum b/go.sum index 7c71b418ed4..133fcbe4126 100644 --- a/go.sum +++ b/go.sum @@ -425,6 +425,8 @@ github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ= github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= +github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed h1:Jr0wN0pOdf8USIjmOL/v7t8P1dH1oBnnrMsZhaM3klg= +github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= From d9e4ced341a42db5460434cb908e2ab0f6d0259c Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 5 Oct 2026 15:46:41 -0500 Subject: [PATCH 04/22] Address eager standalone activity review comments --- chasm/lib/activity/frontend.go | 25 +- chasm/lib/activity/frontend_test.go | 41 +++- chasm/lib/activity/responses.go | 1 + chasm/lib/activity/statemachine.go | 2 + chasm/lib/activity/statemachine_test.go | 3 + common/metrics/metric_defs.go | 2 + docs/architecture/saa-eager-start-plan.md | 263 ---------------------- tests/activity_parity_test.go | 1 + tests/activity_standalone_test.go | 17 +- 9 files changed, 84 insertions(+), 271 deletions(-) delete mode 100644 docs/architecture/saa-eager-start-plan.md diff --git a/chasm/lib/activity/frontend.go b/chasm/lib/activity/frontend.go index 9d11ab85ec5..314701fea24 100644 --- a/chasm/lib/activity/frontend.go +++ b/chasm/lib/activity/frontend.go @@ -42,6 +42,11 @@ var ErrStandaloneActivityDisabled = serviceerror.NewUnimplemented("Standalone ac var ErrStandaloneActivityOperatorCommandsDisabled = serviceerror.NewUnimplemented("Standalone activity operator commands are disabled") +const ( + eagerStartDeniedReasonDynamicConfigDisabled metrics.ReasonString = "dynamic_config_disabled" + eagerStartDeniedReasonStartDelay metrics.ReasonString = "start_delay" +) + type frontendHandler struct { FrontendHandler callbackValidator callbacks.Validator @@ -386,9 +391,23 @@ func (h *frontendHandler) validateAndPopulateStartRequest( if req.GetStartDelay().AsDuration() > 0 && !h.config.StartDelayEnabled(req.GetNamespace()) { return nil, serviceerror.NewInvalidArgument("start_delay is not enabled for this namespace") } - if req.GetRequestEagerExecution() && - (req.GetStartDelay().AsDuration() > 0 || !h.config.EnableEagerStart(req.GetNamespace())) { - req.RequestEagerExecution = false + if req.GetRequestEagerExecution() { + metricsHandler := h.metricsHandler.WithTags( + metrics.NamespaceTag(req.GetNamespace()), + metrics.OperationTag("StartActivityExecution"), + ) + switch { + case !h.config.EnableEagerStart(req.GetNamespace()): + metrics.StandaloneActivityEagerStartDeniedCounter.With(metricsHandler). + Record(1, metrics.ReasonTag(eagerStartDeniedReasonDynamicConfigDisabled)) + req.RequestEagerExecution = false + case req.GetStartDelay().AsDuration() > 0: + metrics.StandaloneActivityEagerStartDeniedCounter.With(metricsHandler). + Record(1, metrics.ReasonTag(eagerStartDeniedReasonStartDelay)) + req.RequestEagerExecution = false + default: + metrics.StandaloneActivityEagerStartAcceptedCounter.With(metricsHandler).Record(1) + } } opts := activityOptionsFromStartRequest(req) diff --git a/chasm/lib/activity/frontend_test.go b/chasm/lib/activity/frontend_test.go index 0ba70f4c09d..ab20e9bf504 100644 --- a/chasm/lib/activity/frontend_test.go +++ b/chasm/lib/activity/frontend_test.go @@ -11,6 +11,8 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "google.golang.org/protobuf/types/known/durationpb" ) @@ -217,7 +219,7 @@ func TestRequestIdStableAcrossRetries(t *testing.T) { } func TestEagerStartFallback(t *testing.T) { - newHandler := func(eagerEnabled bool) *frontendHandler { + newHandler := func(eagerEnabled bool, metricsHandler metrics.Handler) *frontendHandler { return &frontendHandler{ config: &Config{ BlobSizeLimitError: defaultBlobSizeLimitError, @@ -234,7 +236,8 @@ func TestEagerStartFallback(t *testing.T) { func(string) int { return 2000 }, defaultLinkMaxSize, ), - logger: log.NewNoopLogger(), + logger: log.NewNoopLogger(), + metricsHandler: metricsHandler, } } @@ -249,17 +252,47 @@ func TestEagerStartFallback(t *testing.T) { } } + assertMetric := func(t *testing.T, capture *metricstest.Capture, metricName string, reason string) { + t.Helper() + recordings := capture.SnapshotMetric(metricName) + require.Len(t, recordings, 1) + if reason != "" { + require.Equal(t, reason, recordings[0].Tags["reason"]) + } + } + + t.Run("enabled", func(t *testing.T) { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + + req, err := newHandler(true, metricsHandler).validateAndPopulateStartRequest(context.Background(), newRequest(), "test-namespace-id") + require.NoError(t, err) + require.True(t, req.GetRequestEagerExecution()) + assertMetric(t, capture, metrics.StandaloneActivityEagerStartAcceptedCounter.Name(), "") + }) + t.Run("namespace disabled", func(t *testing.T) { - req, err := newHandler(false).validateAndPopulateStartRequest(context.Background(), newRequest(), "test-namespace-id") + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + + req, err := newHandler(false, metricsHandler).validateAndPopulateStartRequest(context.Background(), newRequest(), "test-namespace-id") require.NoError(t, err) require.False(t, req.GetRequestEagerExecution()) + assertMetric(t, capture, metrics.StandaloneActivityEagerStartDeniedCounter.Name(), "dynamic_config_disabled") }) t.Run("start delay", func(t *testing.T) { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + req := newRequest() req.StartDelay = durationpb.New(time.Minute) - req, err := newHandler(true).validateAndPopulateStartRequest(context.Background(), req, "test-namespace-id") + req, err := newHandler(true, metricsHandler).validateAndPopulateStartRequest(context.Background(), req, "test-namespace-id") require.NoError(t, err) require.False(t, req.GetRequestEagerExecution()) + assertMetric(t, capture, metrics.StandaloneActivityEagerStartDeniedCounter.Name(), "start_delay") }) } diff --git a/chasm/lib/activity/responses.go b/chasm/lib/activity/responses.go index c82fc21ba37..4e3ed739dae 100644 --- a/chasm/lib/activity/responses.go +++ b/chasm/lib/activity/responses.go @@ -59,6 +59,7 @@ func (a *Activity) buildEagerActivityTask( return &workflowservice.PollActivityTaskQueueResponse{ TaskToken: token, WorkflowNamespace: request.namespace, + WorkflowExecution: &commonpb.WorkflowExecution{RunId: key.RunID}, ActivityType: a.GetActivityType(), ActivityId: key.BusinessID, Header: requestData.GetHeader(), diff --git a/chasm/lib/activity/statemachine.go b/chasm/lib/activity/statemachine.go index 2fe2fc7c053..5b5e080e818 100644 --- a/chasm/lib/activity/statemachine.go +++ b/chasm/lib/activity/statemachine.go @@ -422,6 +422,8 @@ func (a *Activity) applyEagerStarted(ctx chasm.MutableContext, event eagerStartE } attempt.StartRequestId = event.requestID attempt.LastWorkerIdentity = event.identity + attempt.SdkName = ctx.RequestHeader(headers.ClientNameHeaderName) + attempt.SdkVersion = ctx.RequestHeader(headers.ClientVersionHeaderName) startTime := attempt.GetStartedTime().AsTime() ctx.AddTask( a, diff --git a/chasm/lib/activity/statemachine_test.go b/chasm/lib/activity/statemachine_test.go index a940746de7a..68698c9de4f 100644 --- a/chasm/lib/activity/statemachine_test.go +++ b/chasm/lib/activity/statemachine_test.go @@ -200,6 +200,7 @@ func TestTransitionEagerStarted(t *testing.T) { ctx := &chasm.MockMutableContext{ MockContext: chasm.MockContext{ HandleNow: func(chasm.Component) time.Time { return defaultTime }, + GoCtx: headers.SetVersionsForTests(context.Background(), temporal.SDKVersion, headers.ClientNameGoSDK, "", ""), }, } attempt := &activitypb.ActivityAttemptState{} @@ -229,6 +230,8 @@ func TestTransitionEagerStarted(t *testing.T) { require.EqualValues(t, 1, attempt.GetStartedStamp()) require.Equal(t, "start-request-id", attempt.GetStartRequestId()) require.Equal(t, "starter", attempt.GetLastWorkerIdentity()) + require.Equal(t, headers.ClientNameGoSDK, attempt.GetSdkName()) + require.Equal(t, temporal.SDKVersion, attempt.GetSdkVersion()) require.Len(t, ctx.Tasks, 3) require.IsType(t, &activitypb.ScheduleToCloseTimeoutTask{}, ctx.Tasks[0].Payload) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index c425bf729c5..f985cd77108 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1025,6 +1025,8 @@ var ( SpeculativeWorkflowTaskRollbacks = NewCounterDef("speculative_workflow_task_rollbacks") ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") + StandaloneActivityEagerStartAcceptedCounter = NewCounterDef("standalone_activity_eager_start_accepted") + StandaloneActivityEagerStartDeniedCounter = NewCounterDef("standalone_activity_eager_start_denied") // WorkflowEagerExecutionCounter is emitted any time eager workflow start is requested. WorkflowEagerExecutionCounter = NewCounterDef("workflow_eager_execution") // WorkflowEagerExecutionDeniedCounter is emitted any time eager workflow start is requested and the serer fell back diff --git a/docs/architecture/saa-eager-start-plan.md b/docs/architecture/saa-eager-start-plan.md deleted file mode 100644 index 5792f3f9400..00000000000 --- a/docs/architecture/saa-eager-start-plan.md +++ /dev/null @@ -1,263 +0,0 @@ -# Standalone Activity Eager Start Plan - -## Goal - -Allow a caller of `StartActivityExecution` to request an eager first activity -attempt. When accepted, the server returns a worker-ready activity task directly -in the start response, avoiding the initial Chasm dispatch task and Matching -round trip. - -The eager path must leave the standalone activity (SAA) in the same persisted -state that normal Matching dispatch would create after a worker accepts its -first task. - -## Scope - -This change is server-only. It covers the API contract and Temporal server -behavior needed to accept an eager-start request and return an eager activity -task. SDK support—including deciding when a local worker has capacity, -requesting eager start, handing the returned task to that worker, and samples— -is explicitly out of scope and will be handled as follow-up work by the SDK -team. - -## Current lifecycle - -```text -StartActivityExecution - -> Chasm Activity created - -> SCHEDULED + durable dispatch/timer tasks - -> ActivityDispatchTask -> Matching.AddActivityTask - -> worker poll -> RecordActivityTaskStarted - -> STARTED -> worker completes, fails, or heartbeats - -> retry: SCHEDULED; terminal: COMPLETED, FAILED, CANCELED, TERMINATED, or TIMED_OUT -``` - -The relevant implementation points are: - -- `chasm/lib/activity/handler.go`: creates the execution and applies the initial - `TransitionScheduled` transition. -- `chasm/lib/activity/statemachine.go`: owns state transitions and durable task - creation. -- `chasm/lib/activity/tasks.go`: dispatches a valid scheduled attempt to Matching. -- `service/matching/matching_engine.go`: records the worker pickup and builds - `PollActivityTaskQueueResponse`. -- `service/history/handler.go`: routes a standalone `RecordActivityTaskStarted` - request to `Activity.HandleStarted` through Chasm. - -## Proposed eager lifecycle - -```text -StartActivityExecution(request_eager_execution=true) - -> validate eligibility - -> Chasm Activity created - -> initialize attempt 1 and schedule durable lifetime timers - -> atomically transition SCHEDULED -> STARTED - -> return EagerActivityTask with a normal activity task token - -> caller executes task directly - -Subsequent activity attempts use normal SCHEDULED -> Matching dispatch. A -retry of the *start request* returns the existing execution without -re-delivering its eager task. -``` - -The worker-facing eager task must be equivalent to the response normally built -by Matching: same input, header, activity/run IDs, timeouts, attempt, component -reference, and attempt stamp in its task token. - -### Idempotent eager-start retry lifecycle - -The initial eager response is created only after `StartExecution` succeeds, -using the persisted execution in `result.ExecutionRef`. It is returned only -when `result.Created` is true. A same-request-ID retry returns the existing -execution without another eager task, even if the original first attempt is -still current. Unlike a workflow task, an activity can have external side -effects, so re-delivering an eager activity task could run user code twice. - -```text -StartActivityExecution(request ID = R, eager = true) - -> StartExecution succeeds with Created = true - -> create and persist eager attempt 1 as STARTED - -> return EagerActivityTask - -Retry StartActivityExecution(request ID = R, eager = true) - -> StartExecution returns the existing execution - -> return the idempotent start response without EagerActivityTask - -StartActivityExecution(request ID != R, conflict policy = USE_EXISTING) - -> result.Created = false - -> return the existing execution without EagerActivityTask -``` - -If the first eager response is unavailable, the started attempt is recovered by -the normal timeout and retry path; subsequent attempts are dispatched through -Matching. The initial response token represents the first logical attempt and -must pass the normal activity-token validation path. - -## Implementation steps - -1. Define and regenerate the API contract. - - - Add `request_eager_execution` to `StartActivityExecutionRequest`. - - Add `eager_activity_task` to `StartActivityExecutionResponse`. - - Update the pinned Temporal API dependency and regenerate code as required. - - Confirm the final field names and numbers against the API repository before - implementation. - -2. Add eligibility and fallback policy. - - - Add a namespace-scoped dynamic configuration setting for standalone eager - start. - - Call `StartExecution` before constructing an eager task. Use - `result.ExecutionRef`, rather than a precomputed reference, so its token - identifies the execution actually persisted. - - Return an eager task only for a newly created execution, when - `result.Created == true`. - - A distinct request that resolves through `USE_EXISTING` must return no - eager task. - - Do not eagerly start an activity with non-zero `start_delay`. - - For ineligible requests, clear the effective eager request and use the - existing scheduling path. Return a normal start response with no eager task. - - Add accepted and denied metrics, including denial reasons. - -3. Preserve the state-machine invariant atomically. - - - Generalize the initial scheduling event/body so eager initialization creates - attempt 1, its stamp, dispatch time, and schedule-to-close timer without - emitting an `ActivityDispatchTask` or schedule-to-start timer. - - In the same Chasm creation transaction, apply the existing - `SCHEDULED -> STARTED` behavior. - - Persist start-to-close and heartbeat timers exactly as a Matching-started - task does. - - Do not rely on a later stale-task validation check as the primary mechanism; - avoid creating the initial Matching dispatch task in the eager case. - -4. Share start and task-response construction. - - - Extract the common state update and task-response data preparation currently - split between `Activity.HandleStarted` and Matching's - `createPollActivityTaskQueueResponse`. - - Keep `HandleStarted` as the normal Matching entry point, but make eager - start call the same transition and response-building helpers. - - Inject the token serializer into the SAA history handler or another local - dependency that owns the shared response builder. - - Include the serialized Chasm component reference and current attempt stamp - in the eager activity task token. - - Make the response builder usable by the request-ID deduplication path: it - must load the activity referenced by `result.ExecutionRef` and rebuild a - response only for the still-current original eager attempt. - -5. Keep post-start behavior unchanged. - - - Completion, failure, cancellation, heartbeat, pause, reset, and timeout - paths continue to validate the existing activity task token. - - Retryable worker failures and start-to-close/heartbeat timeouts transition - back to `SCHEDULED` and dispatch through Matching. - - Schedule-to-close remains an activity-lifetime timeout; it must still be - installed for an eager attempt. - -6. Implement idempotency and conflict behavior. - - - Let `StartExecution` perform request-ID deduplication and conflict/reuse - resolution before any eager task is built. - - When `result.Created == true`, construct the initial eager activity token - from `result.ExecutionRef`. - - When `result.Created == false`, including a request-ID retry or a - `USE_EXISTING` conflict, return no eager task. Do not create an attempt, - emit a dispatch task, or mutate activity state on this path. - - Keep token validation unchanged. The initial response token must carry the - persisted component reference and attempt stamp. - -## Test plan - -### State-machine tests - -- Eager initialization enters `STARTED`, creates attempt 1, and records the - expected started metadata. -- It creates start-to-close and heartbeat timers when configured. -- It does not create initial `ActivityDispatchTask` or schedule-to-start timer. -- A normal first attempt remains unchanged. - -### Handler tests - -- Accepted eager start returns an eager `PollActivityTaskQueueResponse`. -- The response token completes and heartbeats the SAA successfully. -- The token contains the activity's component reference and current attempt stamp. -- Disabled dynamic config and non-zero start delay fall back without an eager - task. -- An immediate retry with the same request ID returns the existing execution - without an eager task and without creating another attempt or invoking - Matching. -- A distinct `USE_EXISTING` request returns the existing execution without an - eager task. - -### End-to-end tests - -- Eager first attempt does not call `Matching.AddActivityTask`. -- Retrying an eager attempt sends the subsequent attempt through Matching. -- Completion, cancellation, start-to-close timeout, heartbeat timeout, and - schedule-to-close timeout retain current behavior. -- A stale normal dispatch task cannot start or duplicate an eagerly started - attempt. - -## Failure modes and trade-offs - -- **Lost start response:** a same-request-ID retry does not re-deliver the - original eager task. Normal timeout and retry behavior recover delivery - through Matching, without risking duplicate execution of the same activity - attempt. -- **Concurrent dispatch:** atomically omitting the initial dispatch task avoids - duplicate delivery and removes unnecessary queue work. -- **Worker crash after receipt:** normal start-to-close and heartbeat timers - recover the attempt through existing retry behavior. -- **Load increase:** eager start moves first-attempt work from Matching to the - synchronous start RPC. It reduces queue latency but increases start-response - size and History/Chasm transaction work; configuration gating provides a safe - rollout control. -- **Security:** reuse the normal task token format and validation path. Never - introduce a special eager-only token that bypasses component-reference, - namespace, attempt, or stamp checks. - -## Verification - -Run focused unit tests with `-tags test_dep`, then run the relevant integration -coverage if added. - -After the server/API changes are available, perform a local latency comparison -with the standalone-activity sample at -`/Users/wenlonggu/Desktop/workspaces/samples-go`: - -1. Build and run the local server from this checkout, - `/Users/wenlonggu/Desktop/workspaces/temporal`, then configure the sample - to use that local endpoint. The sample is an SDK client and does not import - the server module directly. -2. Run the same SAA workload without eager start and record end-to-end - start-to-running latency from the `StartActivityExecution` call until the - activity begins running. -3. Enable eager start, run the identical workload against a local worker on the - same task queue, and record the same latency measurement. -4. Compare the before/after results (at least p50 and p95) and verify that the - eager first attempt has no Matching dispatch or poll. Keep worker capacity, - task queue, payload, and repetition count equivalent between the two runs. - -Finish with: - -```text -make lint-code-fast -``` - -### Local direct-RPC result - -On 2026-10-01, 200 paired local runs against the modified server measured the -time from `StartActivityExecution` until a worker-ready task was available. -Normal runs used a waiting `PollActivityTaskQueue`; eager runs read the task -from the start response. Each task was immediately completed after measurement. - -| Path | p50 | p95 | p99 | -| --- | ---: | ---: | ---: | -| Normal | 2.372 ms | 58.654 ms | 88.855 ms | -| Eager | 1.588 ms | 5.028 ms | 7.362 ms | - -This local result shows lower task-delivery latency and much tighter tail -latency for eager start. It is not a production end-to-end worker benchmark: -it excludes SDK activity-function execution and measures only server-side task -delivery. diff --git a/tests/activity_parity_test.go b/tests/activity_parity_test.go index e3c0e967c32..9218ac2f11d 100644 --- a/tests/activity_parity_test.go +++ b/tests/activity_parity_test.go @@ -42,6 +42,7 @@ func newActivityParityEnv(t *testing.T) *testcore.TestEnv { cluster.OverrideDynamicConfig(t, dynamicconfig.EnableChasm, nsValues(true)) cluster.OverrideDynamicConfig(t, activity.Enabled, nsValues(true)) cluster.OverrideDynamicConfig(t, dynamicconfig.EnableActivityEagerExecution, nsValues(true)) + cluster.OverrideDynamicConfig(t, activity.EnableEagerStart, nsValues(true)) cluster.OverrideDynamicConfig(t, activity.EnableStandaloneActivityOperatorCommands, nsValues(true)) return env } diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 990c4767873..4e5827a8874 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1210,7 +1210,10 @@ func (s *standaloneActivityTestSuite) TestStart() { func (s *standaloneActivityTestSuite) TestEagerStartResponseAndNoRedelivery() { env := s.newTestEnv() t := s.T() - ctx := s.Context() + ctx := metadata.NewOutgoingContext(s.Context(), metadata.Pairs( + headers.ClientNameHeaderName, headers.ClientNameGoSDK, + headers.ClientVersionHeaderName, temporal.SDKVersion, + )) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) requestID := testcore.RandomizeStr(t.Name()) @@ -1232,8 +1235,20 @@ func (s *standaloneActivityTestSuite) TestEagerStartResponseAndNoRedelivery() { require.NotNil(t, first.GetEagerActivityTask()) require.Equal(t, activityID, first.GetEagerActivityTask().GetActivityId()) require.Equal(t, first.GetRunId(), first.GetEagerActivityTask().GetActivityRunId()) + require.NotNil(t, first.GetEagerActivityTask().GetWorkflowExecution()) + require.Empty(t, first.GetEagerActivityTask().GetWorkflowExecution().GetWorkflowId()) + require.Equal(t, first.GetRunId(), first.GetEagerActivityTask().GetWorkflowExecution().GetRunId()) require.EqualValues(t, 1, first.GetEagerActivityTask().GetAttempt()) + describe, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + RunId: first.GetRunId(), + }) + require.NoError(t, err) + require.Equal(t, headers.ClientNameGoSDK, describe.GetInfo().GetSdkName()) + require.Equal(t, temporal.SDKVersion, describe.GetInfo().GetSdkVersion()) + retry, err := env.FrontendClient().StartActivityExecution(ctx, request) require.NoError(t, err) require.False(t, retry.GetStarted()) From a4e52c321e3e6f8f29c907a2f07c1456d9497fe8 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 5 Oct 2026 16:09:52 -0500 Subject: [PATCH 05/22] Fix mixed brain module dependencies --- go.mod | 4 +- go.sum | 2 - tests/mixedbrain/go.mod | 4 +- tests/mixedbrain/go.sum | 219 +--------------------------------------- 4 files changed, 7 insertions(+), 222 deletions(-) diff --git a/go.mod b/go.mod index 724b8c47252..299d7067cb6 100644 --- a/go.mod +++ b/go.mod @@ -92,8 +92,6 @@ require ( modernc.org/sqlite v1.51.0 ) -replace go.temporal.io/api => github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed - require ( cel.dev/expr v0.25.2 // indirect cloud.google.com/go v0.123.0 // indirect; indirect e @@ -225,4 +223,6 @@ require ( sigs.k8s.io/yaml v1.6.0 // indirect ) +replace go.temporal.io/api => github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed + tool golang.org/x/perf/cmd/benchstat diff --git a/go.sum b/go.sum index 133fcbe4126..dc276bb794a 100644 --- a/go.sum +++ b/go.sum @@ -481,8 +481,6 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.63.6-0.20260909222256-20151aa90480 h1:1w2xEZFpL7R9NagEQeWTh7oZmZU3YUJGogusn9/LfCg= -go.temporal.io/api v1.63.6-0.20260909222256-20151aa90480/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= go.temporal.io/auto-scaled-workers v0.0.0-20260928183627-719679297cbe h1:BlshPqVI1uh/AsdfGPPjfKc37HEZdGfcmvx+uW+rbe8= go.temporal.io/auto-scaled-workers v0.0.0-20260928183627-719679297cbe/go.mod h1:6Q4guHwlbJldBOZwvc201E0+Jz9Tly1L1Db6C7RJuLw= go.temporal.io/sdk v1.48.0 h1:WDctKDVuh0Z8Nf7euAyqs/EwcPg1JTIIq1Fut8Tq118= diff --git a/tests/mixedbrain/go.mod b/tests/mixedbrain/go.mod index 533f6e6183f..4e44d0a6674 100644 --- a/tests/mixedbrain/go.mod +++ b/tests/mixedbrain/go.mod @@ -7,7 +7,7 @@ require ( github.com/siderolabs/grpc-proxy v0.5.2 github.com/stretchr/testify v1.11.1 github.com/temporalio/omes v0.0.0-20260915193711-9667582a3051 - go.temporal.io/api v1.63.6-0.20260909222256-20151aa90480 + go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed go.temporal.io/server v0.0.0-00010101000000-000000000000 google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.11 @@ -31,6 +31,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) +replace go.temporal.io/api => github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed + replace go.temporal.io/server => ../.. replace go.temporal.io/sdk => go.temporal.io/sdk v1.42.0 diff --git a/tests/mixedbrain/go.sum b/tests/mixedbrain/go.sum index b2e0bb45bdb..881af88d9ed 100644 --- a/tests/mixedbrain/go.sum +++ b/tests/mixedbrain/go.sum @@ -1,230 +1,54 @@ -cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= -cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= -cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.9.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= -cloud.google.com/go/longrunning v0.11.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= -cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= -cloud.google.com/go/run v1.19.0/go.mod h1:Z5wHbyFirI8XU48EPs5XJf/qmVm1SXZEhuS8EvZOuQU= -cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= -dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= -filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0/go.mod h1:hEpiGU18xf70qb3jbTcIggWAiEfX/cOIVc2OTe4OegA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0/go.mod h1:6ZZMQhZKDvUvkJw2rc+oDP90tMMzuU/J+5HG1ZmPOmE= -github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= -github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= -github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9/go.mod h1:uOYhgfgThm/ZyAuJGNQ5YgNyOlYfqnGpTHXvk3cpykg= -github.com/aws/aws-sdk-go-v2/config v1.32.16/go.mod h1:duCCnJEFqpt2RC6no1iK6q+8HpwOAkiUua0pY507dQc= -github.com/aws/aws-sdk-go-v2/credentials v1.19.15/go.mod h1:gJiYyMOjNg8OEdRWOf3CrFQxM2a98qmrtjx1zuiQfB8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22/go.mod h1:b+hYdbU+jGKfXE8kKM6g1+h+L/Go3vMvzlxBsiuGsxg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22/go.mod h1:6sW9iWm9DK9YRpRGga/qzrzNLgKpT2cIxb7Vo2eNOp0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22/go.mod h1:KIpEUx0JuRZLO7U6cbV204cWAEco2iC3l061IxlwLtI= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23/go.mod h1:7J8iGMdRKk6lw2C+cMIphgAnT8uTwBwNOsGkyOCm80U= -github.com/aws/aws-sdk-go-v2/service/ecs v1.78.1/go.mod h1:1DlTqkp+8uc5At3UXyJAvJXFaWoMmxSHcp2Zdor0qGw= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8/go.mod h1:VsK9abqQeGlzPgUr+isNWzPlK2vKe9INMLWnY65f5Xs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14/go.mod h1:yS5rNogD8e0Wu9+l3MUwr6eENBzEeGejvINpN5PAYfY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22/go.mod h1:nO6egFBoAaoXze24a2C0NjQCvdpk8OueRoYimvEB9jo= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22/go.mod h1:ES3ynECd7fYeJIL6+oax+uIEljmfps0S70BaQzbMd/o= -github.com/aws/aws-sdk-go-v2/service/lambda v1.89.1/go.mod h1:7qoh/MlWG5QCnZwq9bvdXomEAkmumayXcjEjIemIV7U= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1/go.mod h1:Fw9aqhJicIVee1VytBBjH+l+5ov6/PhbtIK/u3rt/ls= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.10/go.mod h1:p6+MXNxW7IA6dMgHfTAzljuwSKD0NCm/4lbS4t6+7vI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.16/go.mod h1:CudnEVKRtLn0+3uMV0yEXZ+YZOKnAtUJ5DmDhilVnIw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20/go.mod h1:JHs8/y1f3zY7U5WcuzoJ/yAYGYtNIVPKLIbp61euvmg= -github.com/aws/aws-sdk-go-v2/service/sts v1.42.0/go.mod h1:pFw33T0WLvXU3rw1WBkpMlkgIn54eCB5FYLhjDc9Foo= -github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= -github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= -github.com/cactus/go-statsd-client/v5 v5.1.0/go.mod h1:COEvJ1E+/E2L4q6QE5CkjWPi4eeDw9maJBMIuMPBZbY= -github.com/caio/go-tdigest/v5 v5.0.0/go.mod h1:wI618wZoAYzIDZlpX2CfyTQdrdGtwEZOJuXdrI3zk/Y= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= -github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebitengine/purego v0.9.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= -github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= -github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= -github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fullstorydev/grpchan v1.1.1 h1:heQqIJlAv5Cnks9a70GRL2EJke6QQoUB25VGR6TZQas= github.com/fullstorydev/grpchan v1.1.1/go.mod h1:f4HpiV8V6htfY/K44GWV1ESQzHBTq7DinhzqQ95lpgc= -github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-faker/faker/v4 v4.7.0/go.mod h1:u1dIRP5neLB6kTzgyVjdBOV5R1uP7BdxkcWk7tiKQXk= -github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.23.1/go.mod h1:iWRmZTrGn7XwYhtPt/fvdSFj1OfNBngqRT2UG3BxSqY= -github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw= -github.com/go-openapi/swag v0.26.0/go.mod h1:82g3193sZJRbocs7bNCqGfIgq8pkuwVwCfhKIRlEQF0= -github.com/go-openapi/swag/cmdutils v0.26.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= -github.com/go-openapi/swag/conv v0.26.0/go.mod h1:tpAmIL7X58VPnHHiSO4uE3jBeRamGsFsfdDeDtb5ECE= -github.com/go-openapi/swag/fileutils v0.26.0/go.mod h1:0WDJ7lp67eNjPMO50wAWYlKvhOb6CQ37rzR7wrgI8Tc= -github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTeIz0ARZxHkAPktJK2M= -github.com/go-openapi/swag/jsonutils v0.26.0/go.mod h1:2VmA0CJlyFqgawOaPI9psnjFDqzyivIqLYN34t9p91E= -github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= -github.com/go-openapi/swag/mangling v0.26.0/go.mod h1:jifS7W9vbg+pw63bT+GI53otluMQL3CeemuyCHKwVx0= -github.com/go-openapi/swag/netutils v0.26.0/go.mod h1:5iK+Ok3ZohWWex1C50BFTPexi03UaPwjW4Oj8kgrpwo= -github.com/go-openapi/swag/stringutils v0.26.0/go.mod h1:sWn5uY+QIIspwPhvgnqJsH8xqFT2ZbYcvbcFanRyhFE= -github.com/go-openapi/swag/typeutils v0.26.0/go.mod h1:oovDuIUvTrEHVMqWilQzKzV4YlSKgyZmFh7AlfABNVE= -github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU= -github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= -github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jhump/grpctunnel v0.3.0 h1:itddWDKl7J4CeW4nzY3S/a1s7mPZUb8UtUzEhc/R8mg= github.com/jhump/grpctunnel v0.3.0/go.mod h1:dn5zls1F+1ftPMkbh4kVTVgGuY5t/v3ZgdjtnSMC3f4= -github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/mailru/easyjson v0.9.2/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= -github.com/maruel/panicparse/v2 v2.5.0/go.mod h1:DA2fDiBk63bKfBf4CVZP9gb4fuvzdPbLDsSI873hweQ= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nexus-rpc/nexus-proto-annotations v0.1.0 h1:2fELd+9sqUtNu6Fg//pw8YFsxOvp8vZ8hfP0nHhNI80= github.com/nexus-rpc/nexus-proto-annotations v0.1.0/go.mod h1:n3UjF1bPCW8llR8tHvbxJ+27yPWrhpo8w/Yg1IOuY0Y= -github.com/nexus-rpc/sdk-go v0.7.0/go.mod h1:FHdPfVQwRuJFZFTF0Y2GOAxCrbIBNrcPna9slkGKPYk= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY= -github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shirou/gopsutil/v4 v4.25.10/go.mod h1:+kSwyC8DRUD9XXEHCAFjK+0nuArFJM0lva+StQAcskM= -github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/siderolabs/grpc-proxy v0.5.2 h1:o34tS02IxbX3qeXe0MM99zE2XhfWHuvdBtSWWRD9HNI= github.com/siderolabs/grpc-proxy v0.5.2/go.mod h1:ygf+gqFxWdymAXuP4qMHTa+j6SvasdcFg3XkhpvUvDw= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= -github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/temporalio/features v0.0.0-20260908141410-6f64678c5b39/go.mod h1:Lo/kKS4Xz8QhK1DFa3B/lmaBiQCeFbGdob4i8ewuZQE= github.com/temporalio/omes v0.0.0-20260915193711-9667582a3051 h1:lJ/bCaq7PVGpSwON+MgqAK8/0oih71Jrq9OP3Gxajns= github.com/temporalio/omes v0.0.0-20260915193711-9667582a3051/go.mod h1:ECa3gdOOjrNXSR1OXjI3s+iO1DhoxDIqcyS4fxK2j0Y= -github.com/temporalio/omes/workers/go/harness/api v0.0.0-20260601200529-2419bd37e739/go.mod h1:Aa80OftxTrGB+4Vsi4MRbLA117rnrqfqC6KWVeGvFQg= -github.com/temporalio/ringpop-go v0.1.0/go.mod h1:RE+CHmY+kOZQk47AQaVzwrGmxpflnLgTd6EOK0853j4= -github.com/temporalio/sqlparser v0.1.0/go.mod h1:143qKdh3G45IgV9p+gbAwp3ikRDI8mxsijFiXDfuxsw= -github.com/temporalio/tchannel-go v1.22.1/go.mod h1:ezRQRwu9KQXy8Wuuv1aaFFxoCNz5CeNbVOOkh3xctbY= -github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A= -github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= -github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= -github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/uber-common/bark v1.3.0/go.mod h1:5fDe/YcIVP55XhFF9hUihX2lDsDcpFrTZEAwAVwtPDw= -github.com/uber-go/tally/v4 v4.1.17/go.mod h1:ZdpiHRGSa3z4NIAc1VlEH4SiknR885fOIF08xmS0gaU= -github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo= -github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed h1:Jr0wN0pOdf8USIjmOL/v7t8P1dH1oBnnrMsZhaM3klg= +github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/collector/featuregate v1.56.0/go.mod h1:4ga1QBMPEejXXmpyJS8lmaRpknJ3Lb9Bvk6e420bUFU= -go.opentelemetry.io/collector/pdata v1.56.0/go.mod h1:usR9utboXufbD1rp1oJy+3smQXXpZ+CsI3WN7QsiOs0= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0/go.mod h1:2lmweYCiHYpEjQ/lSJBYhj9jP1zvCvQW4BqL9dnT7FQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0/go.mod h1:AGmbycVGEsRx9mXMZ75CsOyhSP6MFIcj/6dnG+vhVjk= -go.opentelemetry.io/otel/exporters/prometheus v0.56.0/go.mod h1:JQcVZtbIIPM+7SWBB+T6FK+xunlyidwLp++fN0sUaOk= -go.opentelemetry.io/otel/log v0.20.0/go.mod h1:wOcMcjsZpG8x7Bak7IhSi/lg8wscV2C1VdrKCLPlt0E= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= @@ -233,42 +57,20 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= -go.temporal.io/api v1.63.6-0.20260909222256-20151aa90480 h1:1w2xEZFpL7R9NagEQeWTh7oZmZU3YUJGogusn9/LfCg= -go.temporal.io/api v1.63.6-0.20260909222256-20151aa90480/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= -go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab/go.mod h1:hhHijO9XRPIkAflLJJHix61M9FzbRPqk8fSydkcLkqw= -go.temporal.io/sdk v1.42.0/go.mod h1:Xp4TMHsie6kdw0lc0Ae4o8vktze5HZXBynF2DkiXcrQ= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= -go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= -golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/perf v0.0.0-20260709024250-82a0b07e230d/go.mod h1:N6rE9oqNql+XyUUA7hIpm+CorIvF5RWMdE7+p0BiB5s= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.276.0/go.mod h1:Fnag/EWUPIcJXuIkP1pjoTgS5vdxlk3eeemL7Do6bvw= -google.golang.org/genproto v0.0.0-20260420184626-e10c466a9529/go.mod h1:EjLmDZ8liSLBrCTK5vP+bGIxRQHE3ovGvOI0CzGk1PI= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= @@ -280,22 +82,5 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/validator.v2 v2.0.1/go.mod h1:lIUZBlB3Im4s/eYp39Ry/wkR02yOPhZ9IwIRBjuPuG8= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.35.4/go.mod h1:yl4lqySWOgYJJf9RERXKUwE9g2y+CkuwG+xmcOK8wXU= -k8s.io/apimachinery v0.35.4/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= -k8s.io/client-go v0.35.4/go.mod h1:2Pg9WpsS4NeOpoYTfHHfMxBG8zFMSAUi4O/qoiJC3nY= -k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -k8s.io/kube-openapi v0.0.0-20260414162039-ec9c827d403f/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= -k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= -modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/sqlite v1.51.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= -sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= From 7bb39d1fb96f13aea170ace4379c6bca4f1abd87 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 5 Oct 2026 16:40:50 -0500 Subject: [PATCH 06/22] Support temporary API dependency replacements --- Makefile | 14 ++++ cmd/tools/check-dependencies/main.go | 85 +++++++++++++++++++++++ cmd/tools/check-dependencies/main_test.go | 55 +++++++++++++++ 3 files changed, 154 insertions(+) diff --git a/Makefile b/Makefile index 684900b1a1d..ad25d70ccb7 100644 --- a/Makefile +++ b/Makefile @@ -204,9 +204,17 @@ $(BUF): | $(LOCALBIN) GO_API_VER = $(shell go list -m -f '{{.Version}}' go.temporal.io/api \ || (echo "failed to fetch version for go.temporal.io/api" >&2)) +GO_API_REPLACED = $(shell go list -m -f '{{if .Replace}}true{{end}}' go.temporal.io/api \ + || (echo "failed to resolve replacement for go.temporal.io/api" >&2)) PROTOGEN := $(LOCALBIN)/protogen-$(GO_API_VER) +ifeq ($(GO_API_REPLACED),true) +$(PROTOGEN): | $(LOCALBIN) + @printf $(COLOR) "Building protogen through the replaced go.temporal.io/api module..." + @go build -o $@ go.temporal.io/api/cmd/protogen +else $(PROTOGEN): | $(LOCALBIN) $(call go-install-tool,$(PROTOGEN),go.temporal.io/api/cmd/protogen,$(GO_API_VER)) +endif ACTIONLINT_VER := v1.7.7 ACTIONLINT := $(LOCALBIN)/actionlint-$(ACTIONLINT_VER) @@ -285,8 +293,14 @@ $(STAMPDIR)/protoc-gen-go-grpc-$(PROTOC_GEN_GO_GRPC_VER): | $(STAMPDIR) $(LOCALB $(PROTOC_GEN_GO_GRPC): $(STAMPDIR)/protoc-gen-go-grpc-$(PROTOC_GEN_GO_GRPC_VER) PROTOC_GEN_GO_HELPERS := $(LOCALBIN)/protoc-gen-go-helpers-$(GO_API_VER) +ifeq ($(GO_API_REPLACED),true) +$(STAMPDIR)/protoc-gen-go-helpers-$(GO_API_VER): | $(STAMPDIR) $(LOCALBIN) + @printf $(COLOR) "Building protoc-gen-go-helpers through the replaced go.temporal.io/api module..." + @go build -o $(PROTOC_GEN_GO_HELPERS) go.temporal.io/api/cmd/protoc-gen-go-helpers +else $(STAMPDIR)/protoc-gen-go-helpers-$(GO_API_VER): | $(STAMPDIR) $(LOCALBIN) $(call go-install-tool,$(PROTOC_GEN_GO_HELPERS),go.temporal.io/api/cmd/protoc-gen-go-helpers,$(GO_API_VER)) +endif @touch $@ $(PROTOC_GEN_GO_HELPERS): $(STAMPDIR)/protoc-gen-go-helpers-$(GO_API_VER) diff --git a/cmd/tools/check-dependencies/main.go b/cmd/tools/check-dependencies/main.go index 2410ba4eb40..e10f4d45c02 100644 --- a/cmd/tools/check-dependencies/main.go +++ b/cmd/tools/check-dependencies/main.go @@ -43,6 +43,8 @@ type moduleSpec struct { // strict rule. const apiModulePath = "go.temporal.io/api" +var replacementContainsCommit = repositoryContainsCommit + var knownModules = []moduleSpec{ { modulePath: apiModulePath, @@ -325,6 +327,12 @@ func validateMainModule( version := modVersion.Version fmt.Printf("Found %s version: %s\n", mod.modulePath, version) + if replacement, ok := findReplacement(modFile, modVersion); ok { + if mod.modulePath != apiModulePath { + return fmt.Errorf("%s@%s: replacements are not allowed", mod.modulePath, version) + } + return validateTemporaryAPIReplacement(ctx, modVersion, replacement) + } if !module.IsPseudoVersion(version) { if !semver.IsValid(version) { @@ -362,6 +370,83 @@ func findRequiredModuleVersion(modFile *modfile.File, modulePath string) (module return module.Version{}, false } +func findReplacement(modFile *modfile.File, required module.Version) (module.Version, bool) { + for _, replacement := range modFile.Replace { + if replacement.Old.Path == required.Path && + (replacement.Old.Version == "" || replacement.Old.Version == required.Version) { + return replacement.New, true + } + } + return module.Version{}, false +} + +func validateTemporaryAPIReplacement( + ctx context.Context, + required module.Version, + replacement module.Version, +) error { + if !module.IsPseudoVersion(required.Version) || !module.IsPseudoVersion(replacement.Version) { + return fmt.Errorf("%s@%s: temporary replacement %s@%s must use matching pseudo-versions", + required.Path, required.Version, replacement.Path, replacement.Version) + } + + requiredRevision, err := module.PseudoVersionRev(required.Version) + if err != nil { + return fmt.Errorf("%s@%s: failed to parse pseudo-version revision: %w", required.Path, required.Version, err) + } + replacementRevision, err := module.PseudoVersionRev(replacement.Version) + if err != nil { + return fmt.Errorf("%s@%s: failed to parse temporary replacement revision: %w", replacement.Path, replacement.Version, err) + } + if requiredRevision != replacementRevision { + return fmt.Errorf("%s@%s: temporary replacement %s@%s names a different commit", + required.Path, required.Version, replacement.Path, replacement.Version) + } + + parts := strings.Split(replacement.Path, "/") + if len(parts) != 3 || parts[0] != "github.com" || parts[2] != "api-go" { + return fmt.Errorf("%s@%s: temporary replacement must point to a GitHub api-go repository, got %s", + required.Path, required.Version, replacement.Path) + } + + repoURL := "https://" + replacement.Path + ".git" + available, err := replacementContainsCommit(ctx, repoURL, replacementRevision) + if err != nil { + return fmt.Errorf("%s@%s: failed to resolve temporary replacement: %w", required.Path, required.Version, err) + } + if !available { + return fmt.Errorf("%s@%s: temporary replacement commit %s is not available from %s", + required.Path, required.Version, replacementRevision, repoURL) + } + + fmt.Printf(" - %s@%s temporarily replaced by %s@%s (ok)\n", + required.Path, required.Version, replacement.Path, replacement.Version) + return nil +} + +func repositoryContainsCommit(ctx context.Context, repoURL string, shortHash string) (bool, error) { + tmpRepo, err := os.MkdirTemp("", "check-dependencies-*") + if err != nil { + return false, fmt.Errorf("failed to create temp repo dir: %w", err) + } + defer func() { _ = os.RemoveAll(tmpRepo) }() + + out, err := exec.CommandContext(ctx, "git", "clone", "--bare", "--filter=blob:none", repoURL, tmpRepo).CombinedOutput() + if err != nil { + return false, fmt.Errorf("git clone failed: %w: %s", err, strings.TrimSpace(string(out))) + } + + out, err = exec.CommandContext(ctx, "git", "-C", tmpRepo, "cat-file", "-e", shortHash+"^{commit}").CombinedOutput() + if err == nil { + return true, nil + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return false, nil + } + return false, fmt.Errorf("git cat-file failed: %w: %s", err, strings.TrimSpace(string(out))) +} + // resolveModuleOriginForSpec reports whether shortHash is reachable from the // default branch of mod's repository. // diff --git a/cmd/tools/check-dependencies/main_test.go b/cmd/tools/check-dependencies/main_test.go index 2778a2dd2d2..26189647ccd 100644 --- a/cmd/tools/check-dependencies/main_test.go +++ b/cmd/tools/check-dependencies/main_test.go @@ -48,6 +48,61 @@ func TestFindRequiredModuleVersion(t *testing.T) { }) } +func TestFindReplacement(t *testing.T) { + f := parseGoMod(t, `module test + +go 1.21 + +require go.temporal.io/api v1.2.4-0.20240101000000-abcdef012345 + +replace go.temporal.io/api => github.com/example/api-go v1.2.4-0.20240101000000-abcdef012345 +`) + + replacement, ok := findReplacement(f, module.Version{ + Path: "go.temporal.io/api", + Version: "v1.2.4-0.20240101000000-abcdef012345", + }) + require.True(t, ok) + require.Equal(t, "github.com/example/api-go", replacement.Path) + require.Equal(t, "v1.2.4-0.20240101000000-abcdef012345", replacement.Version) +} + +func TestValidateTemporaryAPIReplacement(t *testing.T) { + const version = "v1.2.4-0.20240101000000-abcdef012345" + originalResolver := replacementContainsCommit + t.Cleanup(func() { replacementContainsCommit = originalResolver }) + + var gotRepoURL string + var gotRevision string + replacementContainsCommit = func(_ context.Context, repoURL, revision string) (bool, error) { + gotRepoURL = repoURL + gotRevision = revision + return true, nil + } + + required := module.Version{Path: apiModulePath, Version: version} + replacement := module.Version{Path: "github.com/example/api-go", Version: version} + require.NoError(t, validateTemporaryAPIReplacement(context.Background(), required, replacement)) + require.Equal(t, "https://github.com/example/api-go.git", gotRepoURL) + require.Equal(t, "abcdef012345", gotRevision) + + t.Run("rejects a different revision", func(t *testing.T) { + err := validateTemporaryAPIReplacement(context.Background(), required, module.Version{ + Path: replacement.Path, + Version: "v1.2.4-0.20240101000000-fedcba987654", + }) + require.ErrorContains(t, err, "names a different commit") + }) + + t.Run("rejects a non api-go repository", func(t *testing.T) { + err := validateTemporaryAPIReplacement(context.Background(), required, module.Version{ + Path: "github.com/example/not-api-go", + Version: version, + }) + require.ErrorContains(t, err, "must point to a GitHub api-go repository") + }) +} + func TestValidateReleaseBranch(t *testing.T) { tests := []struct { name string From d69cb3176172cf607b82a92d4659364098be6c81 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 5 Oct 2026 17:06:55 -0500 Subject: [PATCH 07/22] Format eager activity metrics --- common/metrics/metric_defs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index f985cd77108..7bb948f0e45 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1024,7 +1024,7 @@ var ( SpeculativeWorkflowTaskCommits = NewCounterDef("speculative_workflow_task_commits") SpeculativeWorkflowTaskRollbacks = NewCounterDef("speculative_workflow_task_rollbacks") - ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") + ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") StandaloneActivityEagerStartAcceptedCounter = NewCounterDef("standalone_activity_eager_start_accepted") StandaloneActivityEagerStartDeniedCounter = NewCounterDef("standalone_activity_eager_start_denied") // WorkflowEagerExecutionCounter is emitted any time eager workflow start is requested. From b7bd052171d78a280f95287630f2889a0cc7a8e7 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Wed, 7 Oct 2026 09:23:01 -0500 Subject: [PATCH 08/22] Build eager activity task response transactionally --- chasm/lib/activity/handler.go | 24 +++---- chasm/lib/activity/responses.go | 101 +++++++++++++++++++++--------- tests/activity_standalone_test.go | 13 +++- 3 files changed, 93 insertions(+), 45 deletions(-) diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index 79555e937ae..133ea9381df 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -63,6 +63,7 @@ func newHandler( // matching being delivered to a worker poll request. func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.StartActivityExecutionRequest) (*activitypb.StartActivityExecutionResponse, error) { frontendReq := req.GetFrontendRequest() + var eagerTaskData *eagerActivityTaskData reusePolicy, ok := businessIDReusePolicyMap[frontendReq.GetIdReusePolicy()] if !ok { @@ -110,6 +111,16 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St if err != nil { return nil, err } + if request.GetRequestEagerExecution() { + eagerTaskData, err = newActivity.eagerActivityTaskData(mutableContext, eagerActivityTaskRequest{ + namespaceID: req.GetNamespaceId(), + namespace: request.GetNamespace(), + requestID: request.GetRequestId(), + }) + if err != nil { + return nil, err + } + } return newActivity, nil }, @@ -137,17 +148,8 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St } var eagerTask *workflowservice.PollActivityTaskQueueResponse - if result.Created && frontendReq.GetRequestEagerExecution() { - eagerTask, err = chasm.ReadComponent( - ctx, - result.ExecutionRef, - (*Activity).buildEagerActivityTask, - eagerActivityTaskRequest{ - namespaceID: req.GetNamespaceId(), - namespace: frontendReq.GetNamespace(), - requestID: frontendReq.GetRequestId(), - }, - ) + if result.Created && eagerTaskData != nil { + eagerTask, err = eagerTaskData.response(result.ExecutionRef) if err != nil { return nil, err } diff --git a/chasm/lib/activity/responses.go b/chasm/lib/activity/responses.go index 4e3ed739dae..f817790f8ea 100644 --- a/chasm/lib/activity/responses.go +++ b/chasm/lib/activity/responses.go @@ -22,59 +22,98 @@ type eagerActivityTaskRequest struct { requestID string } -func (a *Activity) buildEagerActivityTask( +type eagerActivityTaskData struct { + namespaceID string + namespace string + runID string + activityType *commonpb.ActivityType + activityID string + header *commonpb.Header + input *commonpb.Payloads + heartbeatDetails *commonpb.Payloads + scheduledTime *timestamppb.Timestamp + currentAttemptScheduledTime *timestamppb.Timestamp + startedTime *timestamppb.Timestamp + attempt int32 + scheduleToCloseTimeout *durationpb.Duration + startToCloseTimeout *durationpb.Duration + heartbeatTimeout *durationpb.Duration + retryPolicy *commonpb.RetryPolicy + priority *commonpb.Priority + activityAttemptStartedStamp int32 +} + +func (a *Activity) eagerActivityTaskData( ctx chasm.Context, request eagerActivityTaskRequest, -) (*workflowservice.PollActivityTaskQueueResponse, error) { +) (*eagerActivityTaskData, error) { attempt := a.LastAttempt.Get(ctx) if !a.hasAttemptInProgress() || attempt.GetCount() != 1 || attempt.GetStartRequestId() != request.requestID { return nil, nil } - componentRef, err := ctx.Ref(a) - if err != nil { - return nil, err - } key := ctx.ExecutionKey() + requestData := a.RequestData.Get(ctx) + lastHeartbeat, _ := a.LastHeartbeat.TryGet(ctx) + return &eagerActivityTaskData{ + namespaceID: request.namespaceID, + namespace: request.namespace, + runID: key.RunID, + activityType: a.GetActivityType(), + activityID: key.BusinessID, + header: requestData.GetHeader(), + input: requestData.GetInput(), + heartbeatDetails: lastHeartbeat.GetDetails(), + scheduledTime: a.GetScheduleTime(), + currentAttemptScheduledTime: a.dispatchTimeForAttempt(attempt), + startedTime: attempt.GetStartedTime(), + attempt: attempt.GetCount(), + scheduleToCloseTimeout: a.GetScheduleToCloseTimeout(), + startToCloseTimeout: a.GetStartToCloseTimeout(), + heartbeatTimeout: a.GetHeartbeatTimeout(), + retryPolicy: a.GetRetryPolicy(), + priority: a.GetPriority(), + activityAttemptStartedStamp: attempt.GetStartedStamp(), + }, nil +} + +func (d *eagerActivityTaskData) response(componentRef []byte) (*workflowservice.PollActivityTaskQueueResponse, error) { token, err := tasktoken.NewSerializer().Serialize(tasktoken.NewActivityTaskToken( - request.namespaceID, + d.namespaceID, "", - key.RunID, + d.runID, 0, - key.BusinessID, - a.GetActivityType().GetName(), - attempt.GetCount(), + d.activityID, + d.activityType.GetName(), + d.attempt, nil, 0, 0, componentRef, - attempt.GetStartedStamp(), + d.activityAttemptStartedStamp, )) if err != nil { return nil, err } - requestData := a.RequestData.Get(ctx) - lastHeartbeat, _ := a.LastHeartbeat.TryGet(ctx) return &workflowservice.PollActivityTaskQueueResponse{ TaskToken: token, - WorkflowNamespace: request.namespace, - WorkflowExecution: &commonpb.WorkflowExecution{RunId: key.RunID}, - ActivityType: a.GetActivityType(), - ActivityId: key.BusinessID, - Header: requestData.GetHeader(), - Input: requestData.GetInput(), - HeartbeatDetails: lastHeartbeat.GetDetails(), - ScheduledTime: a.GetScheduleTime(), - CurrentAttemptScheduledTime: a.dispatchTimeForAttempt(attempt), - StartedTime: attempt.GetStartedTime(), - Attempt: attempt.GetCount(), - ScheduleToCloseTimeout: a.GetScheduleToCloseTimeout(), - StartToCloseTimeout: a.GetStartToCloseTimeout(), - HeartbeatTimeout: a.GetHeartbeatTimeout(), - RetryPolicy: a.GetRetryPolicy(), - Priority: a.GetPriority(), - ActivityRunId: key.RunID, + WorkflowNamespace: d.namespace, + ActivityType: d.activityType, + ActivityId: d.activityID, + Header: d.header, + Input: d.input, + HeartbeatDetails: d.heartbeatDetails, + ScheduledTime: d.scheduledTime, + CurrentAttemptScheduledTime: d.currentAttemptScheduledTime, + StartedTime: d.startedTime, + Attempt: d.attempt, + ScheduleToCloseTimeout: d.scheduleToCloseTimeout, + StartToCloseTimeout: d.startToCloseTimeout, + HeartbeatTimeout: d.heartbeatTimeout, + RetryPolicy: d.retryPolicy, + Priority: d.priority, + ActivityRunId: d.runID, }, nil } diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 4e5827a8874..6304e689d7d 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1235,9 +1235,7 @@ func (s *standaloneActivityTestSuite) TestEagerStartResponseAndNoRedelivery() { require.NotNil(t, first.GetEagerActivityTask()) require.Equal(t, activityID, first.GetEagerActivityTask().GetActivityId()) require.Equal(t, first.GetRunId(), first.GetEagerActivityTask().GetActivityRunId()) - require.NotNil(t, first.GetEagerActivityTask().GetWorkflowExecution()) - require.Empty(t, first.GetEagerActivityTask().GetWorkflowExecution().GetWorkflowId()) - require.Equal(t, first.GetRunId(), first.GetEagerActivityTask().GetWorkflowExecution().GetRunId()) + require.Nil(t, first.GetEagerActivityTask().GetWorkflowExecution()) require.EqualValues(t, 1, first.GetEagerActivityTask().GetAttempt()) describe, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ @@ -1262,6 +1260,15 @@ func (s *standaloneActivityTestSuite) TestEagerStartResponseAndNoRedelivery() { require.False(t, conflict.GetStarted()) require.Nil(t, conflict.GetEagerActivityTask()) + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: first.GetEagerActivityTask().GetTaskToken(), + Result: defaultResult, + Identity: defaultIdentity, + }) + require.NoError(t, err) + env.validateCompletion(ctx, t, activityID, first.GetRunId(), defaultIdentity) + } func (s *standaloneActivityTestSuite) TestComplete() { From ef78df7f3a608d79a1b95f5d92e3e63423eb6dcd Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Wed, 7 Oct 2026 10:10:01 -0500 Subject: [PATCH 09/22] Align eager activity execution metrics --- chasm/lib/activity/frontend.go | 15 ----- chasm/lib/activity/frontend_test.go | 35 ++-------- chasm/lib/activity/handler.go | 1 + chasm/lib/activity/metrics.go | 17 +++++ common/metrics/activity.go | 17 +++++ common/metrics/metric_defs.go | 4 +- .../workflow_task_completed_handler.go | 12 +++- tests/activity_parity_test.go | 67 +++++++++++++++++++ 8 files changed, 119 insertions(+), 49 deletions(-) diff --git a/chasm/lib/activity/frontend.go b/chasm/lib/activity/frontend.go index 314701fea24..e97baaa4521 100644 --- a/chasm/lib/activity/frontend.go +++ b/chasm/lib/activity/frontend.go @@ -42,11 +42,6 @@ var ErrStandaloneActivityDisabled = serviceerror.NewUnimplemented("Standalone ac var ErrStandaloneActivityOperatorCommandsDisabled = serviceerror.NewUnimplemented("Standalone activity operator commands are disabled") -const ( - eagerStartDeniedReasonDynamicConfigDisabled metrics.ReasonString = "dynamic_config_disabled" - eagerStartDeniedReasonStartDelay metrics.ReasonString = "start_delay" -) - type frontendHandler struct { FrontendHandler callbackValidator callbacks.Validator @@ -392,21 +387,11 @@ func (h *frontendHandler) validateAndPopulateStartRequest( return nil, serviceerror.NewInvalidArgument("start_delay is not enabled for this namespace") } if req.GetRequestEagerExecution() { - metricsHandler := h.metricsHandler.WithTags( - metrics.NamespaceTag(req.GetNamespace()), - metrics.OperationTag("StartActivityExecution"), - ) switch { case !h.config.EnableEagerStart(req.GetNamespace()): - metrics.StandaloneActivityEagerStartDeniedCounter.With(metricsHandler). - Record(1, metrics.ReasonTag(eagerStartDeniedReasonDynamicConfigDisabled)) req.RequestEagerExecution = false case req.GetStartDelay().AsDuration() > 0: - metrics.StandaloneActivityEagerStartDeniedCounter.With(metricsHandler). - Record(1, metrics.ReasonTag(eagerStartDeniedReasonStartDelay)) req.RequestEagerExecution = false - default: - metrics.StandaloneActivityEagerStartAcceptedCounter.With(metricsHandler).Record(1) } } diff --git a/chasm/lib/activity/frontend_test.go b/chasm/lib/activity/frontend_test.go index ab20e9bf504..bbe3abe6764 100644 --- a/chasm/lib/activity/frontend_test.go +++ b/chasm/lib/activity/frontend_test.go @@ -12,7 +12,6 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" - "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "google.golang.org/protobuf/types/known/durationpb" ) @@ -219,7 +218,7 @@ func TestRequestIdStableAcrossRetries(t *testing.T) { } func TestEagerStartFallback(t *testing.T) { - newHandler := func(eagerEnabled bool, metricsHandler metrics.Handler) *frontendHandler { + newHandler := func(eagerEnabled bool) *frontendHandler { return &frontendHandler{ config: &Config{ BlobSizeLimitError: defaultBlobSizeLimitError, @@ -237,7 +236,7 @@ func TestEagerStartFallback(t *testing.T) { defaultLinkMaxSize, ), logger: log.NewNoopLogger(), - metricsHandler: metricsHandler, + metricsHandler: metrics.NoopMetricsHandler, } } @@ -252,47 +251,23 @@ func TestEagerStartFallback(t *testing.T) { } } - assertMetric := func(t *testing.T, capture *metricstest.Capture, metricName string, reason string) { - t.Helper() - recordings := capture.SnapshotMetric(metricName) - require.Len(t, recordings, 1) - if reason != "" { - require.Equal(t, reason, recordings[0].Tags["reason"]) - } - } - t.Run("enabled", func(t *testing.T) { - metricsHandler := metricstest.NewCaptureHandler() - capture := metricsHandler.StartCapture() - defer metricsHandler.StopCapture(capture) - - req, err := newHandler(true, metricsHandler).validateAndPopulateStartRequest(context.Background(), newRequest(), "test-namespace-id") + req, err := newHandler(true).validateAndPopulateStartRequest(context.Background(), newRequest(), "test-namespace-id") require.NoError(t, err) require.True(t, req.GetRequestEagerExecution()) - assertMetric(t, capture, metrics.StandaloneActivityEagerStartAcceptedCounter.Name(), "") }) t.Run("namespace disabled", func(t *testing.T) { - metricsHandler := metricstest.NewCaptureHandler() - capture := metricsHandler.StartCapture() - defer metricsHandler.StopCapture(capture) - - req, err := newHandler(false, metricsHandler).validateAndPopulateStartRequest(context.Background(), newRequest(), "test-namespace-id") + req, err := newHandler(false).validateAndPopulateStartRequest(context.Background(), newRequest(), "test-namespace-id") require.NoError(t, err) require.False(t, req.GetRequestEagerExecution()) - assertMetric(t, capture, metrics.StandaloneActivityEagerStartDeniedCounter.Name(), "dynamic_config_disabled") }) t.Run("start delay", func(t *testing.T) { - metricsHandler := metricstest.NewCaptureHandler() - capture := metricsHandler.StartCapture() - defer metricsHandler.StopCapture(capture) - req := newRequest() req.StartDelay = durationpb.New(time.Minute) - req, err := newHandler(true, metricsHandler).validateAndPopulateStartRequest(context.Background(), req, "test-namespace-id") + req, err := newHandler(true).validateAndPopulateStartRequest(context.Background(), req, "test-namespace-id") require.NoError(t, err) require.False(t, req.GetRequestEagerExecution()) - assertMetric(t, capture, metrics.StandaloneActivityEagerStartDeniedCounter.Name(), "start_delay") }) } diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index 133ea9381df..a61d522af3b 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -153,6 +153,7 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St if err != nil { return nil, err } + metrics.ActivityEagerExecutionCounter.With(eagerActivityMetricsHandler(h.metricsHandler, h.config, frontendReq)).Record(1) } // Apply on_conflict_options to an existing activity. diff --git a/chasm/lib/activity/metrics.go b/chasm/lib/activity/metrics.go index 6e680062a9a..20213d28147 100644 --- a/chasm/lib/activity/metrics.go +++ b/chasm/lib/activity/metrics.go @@ -7,6 +7,7 @@ import ( commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" + "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" "go.temporal.io/server/common/metrics" @@ -28,6 +29,22 @@ func (a *Activity) taskScheduleToStartMetricsHandler(ctx chasm.Context) metrics. )) } +func eagerActivityMetricsHandler( + handler metrics.Handler, + config *Config, + request *workflowservice.StartActivityExecutionRequest, +) metrics.Handler { + namespaceName := request.GetNamespace() + taskQueueName := request.GetTaskQueue().GetName() + return metrics.GetPerActivityEagerExecutionScope( + handler, + namespaceName, + taskQueueName, + config.BreakdownMetricsByTaskQueue(namespaceName, taskQueueName, enumspb.TASK_QUEUE_TYPE_ACTIVITY), + "StartActivityExecution", + ) +} + // baseMetricsHandler adds only the operation tag. func (a *Activity) baseMetricsHandler(ctx chasm.Context, operation string) metrics.Handler { return ctx.MetricsHandler().WithTags(metrics.OperationTag(operation)) diff --git a/common/metrics/activity.go b/common/metrics/activity.go index ca1b377828c..9f9f571b657 100644 --- a/common/metrics/activity.go +++ b/common/metrics/activity.go @@ -27,3 +27,20 @@ func GetPerActivityScope( WorkflowTypeTag(workflowType), ) } + +// GetPerActivityEagerExecutionScope returns the tags shared by eager activity execution metrics. +func GetPerActivityEagerExecutionScope( + handler Handler, + namespaceName string, + taskQueueName string, + taskQueueBreakdown bool, + operation string, +) Handler { + return GetPerTaskQueueFamilyScope( + handler, + namespaceName, + tqid.UnsafeTaskQueueFamily(namespaceName, taskQueueName), + taskQueueBreakdown, + OperationTag(operation), + ) +} diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 7bb948f0e45..c425bf729c5 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1024,9 +1024,7 @@ var ( SpeculativeWorkflowTaskCommits = NewCounterDef("speculative_workflow_task_commits") SpeculativeWorkflowTaskRollbacks = NewCounterDef("speculative_workflow_task_rollbacks") - ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") - StandaloneActivityEagerStartAcceptedCounter = NewCounterDef("standalone_activity_eager_start_accepted") - StandaloneActivityEagerStartDeniedCounter = NewCounterDef("standalone_activity_eager_start_denied") + ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") // WorkflowEagerExecutionCounter is emitted any time eager workflow start is requested. WorkflowEagerExecutionCounter = NewCounterDef("workflow_eager_execution") // WorkflowEagerExecutionDeniedCounter is emitted any time eager workflow start is requested and the serer fell back diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index 2e4abb0d9c6..2574fefd194 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -712,7 +712,17 @@ func (handler *workflowTaskCompletedHandler) handlePostCommandEagerExecuteActivi }, } metrics.ActivityEagerExecutionCounter.With( - workflow.GetPerTaskQueueFamilyScope(handler.metricsHandler, handler.mutableState.GetNamespaceEntry().Name(), ai.TaskQueue, handler.config), + metrics.GetPerActivityEagerExecutionScope( + handler.metricsHandler, + handler.mutableState.GetNamespaceEntry().Name().String(), + ai.TaskQueue, + handler.config.BreakdownMetricsByTaskQueue( + handler.mutableState.GetNamespaceEntry().Name().String(), + ai.TaskQueue, + enumspb.TASK_QUEUE_TYPE_WORKFLOW, + ), + metrics.HistoryRespondWorkflowTaskCompletedScope, + ), ).Record(1) return func(resp *historyservice.RespondWorkflowTaskCompletedResponse) error { diff --git a/tests/activity_parity_test.go b/tests/activity_parity_test.go index 9218ac2f11d..3245e2a0504 100644 --- a/tests/activity_parity_test.go +++ b/tests/activity_parity_test.go @@ -4,6 +4,7 @@ package tests import ( "errors" + "slices" "strings" "testing" "time" @@ -17,6 +18,8 @@ import ( "go.temporal.io/server/chasm/lib/activity/model" "go.temporal.io/server/common" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/retrypolicy" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/service/history/consts" @@ -73,6 +76,70 @@ func (s *activityParityTestSuite) TestEagerActivityStartParity() { }) } +func (s *activityParityTestSuite) TestEagerActivityMetricParity() { + type captureResult struct { + recordings []*metricstest.CapturedRecording + taskQueue string + } + + capture := func(t *testing.T, standalone, breakdownByTaskQueue bool) captureResult { + env := newActivityParityEnv(t) + env.GetTestCluster().OverrideDynamicConfig(t, dynamicconfig.MetricsBreakdownByTaskQueue, + []dynamicconfig.ConstrainedValue{{ + Constraints: dynamicconfig.Constraints{Namespace: env.Namespace().String()}, + Value: breakdownByTaskQueue, + }}) + metricCapture := env.StartNamespaceMetricCapture() + + cfg := activityConfig{EagerStart: true} + var taskQueue string + if standalone { + taskQueue = newSAADriver(t, env, cfg).start(t, cfg).taskQueue + } else { + taskQueue = newWFADriver(t, env, cfg).start(t, cfg).taskQueue + } + + return captureResult{ + recordings: metricCapture.Metric(metrics.ActivityEagerExecutionCounter.Name()), + taskQueue: taskQueue, + } + } + + tagKeys := func(recording *metricstest.CapturedRecording) []string { + keys := make([]string, 0, len(recording.Tags)) + for key := range recording.Tags { + keys = append(keys, key) + } + slices.Sort(keys) + return keys + } + + for _, breakdownByTaskQueue := range []bool{true, false} { + name := "TaskQueueBreakdownEnabled" + if !breakdownByTaskQueue { + name = "TaskQueueBreakdownDisabled" + } + s.Run(name, func(s *activityParityTestSuite) { + t := s.T() + wfa := capture(t, false, breakdownByTaskQueue) + saa := capture(t, true, breakdownByTaskQueue) + require.Len(t, wfa.recordings, 1) + require.Len(t, saa.recordings, 1) + require.Equal(t, tagKeys(wfa.recordings[0]), tagKeys(saa.recordings[0]), + "WFA and SAA eager activity metrics must use the same label keys") + + expectedTaskQueue := "__omitted__" + if breakdownByTaskQueue { + require.Equal(t, wfa.taskQueue, wfa.recordings[0].Tags["taskqueue"]) + require.Equal(t, saa.taskQueue, saa.recordings[0].Tags["taskqueue"]) + } else { + require.Equal(t, expectedTaskQueue, wfa.recordings[0].Tags["taskqueue"]) + require.Equal(t, expectedTaskQueue, saa.recordings[0].Tags["taskqueue"]) + } + }) + } +} + func assertActivityTaskNotCancelRequested(t *testing.T, err error) { var invalidArgumentErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgumentErr) From bb028b0fdad59c35ee642ad13251248b06e4deca Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Wed, 7 Oct 2026 11:31:42 -0500 Subject: [PATCH 10/22] Honor Matching grants for eager activities --- chasm/lib/activity/config.go | 2 + chasm/lib/activity/handler.go | 44 ++++++++++++++ chasm/lib/activity/handler_test.go | 96 ++++++++++++++++++++++++++++++ tests/activity_standalone_test.go | 64 +++++++++++++++++++- 4 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 chasm/lib/activity/handler_test.go diff --git a/chasm/lib/activity/config.go b/chasm/lib/activity/config.go index d883d58c3a5..8e708980c7e 100644 --- a/chasm/lib/activity/config.go +++ b/chasm/lib/activity/config.go @@ -67,6 +67,7 @@ type Config struct { BlobSizeLimitWarn dynamicconfig.IntPropertyFnWithNamespaceFilter BreakdownMetricsByTaskQueue dynamicconfig.TypedPropertyFnWithTaskQueueFilter[bool] EnableCallbacks dynamicconfig.BoolPropertyFnWithNamespaceFilter + EnableActivityEagerDispatchCheck dynamicconfig.BoolPropertyFnWithNamespaceFilter EnabledCallbackKinds dynamicconfig.TypedPropertyFnWithNamespaceFilter[[]callbacks.Kind] Enabled dynamicconfig.BoolPropertyFnWithNamespaceFilter EnableEagerStart dynamicconfig.BoolPropertyFnWithNamespaceFilter @@ -89,6 +90,7 @@ func ConfigProvider(dc *dynamicconfig.Collection) *Config { BlobSizeLimitWarn: dynamicconfig.BlobSizeLimitWarn.Get(dc), BreakdownMetricsByTaskQueue: dynamicconfig.MetricsBreakdownByTaskQueue.Get(dc), DefaultActivityRetryPolicy: dynamicconfig.DefaultActivityRetryPolicy.Get(dc), + EnableActivityEagerDispatchCheck: dynamicconfig.EnableActivityEagerDispatchCheck.Get(dc), EnableCallbacks: EnableCallbacks.Get(dc), EnabledCallbackKinds: EnabledCallbackKinds.Get(dc), Enabled: Enabled.Get(dc), diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index a61d522af3b..da00cb07fc2 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -9,12 +9,15 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/matchingservice/v1" + taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" "go.temporal.io/server/common/contextutil" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/resource" ) var ( @@ -37,6 +40,7 @@ type handler struct { linkValidator *linkValidator logger log.Logger metricsHandler metrics.Handler + matchingClient resource.MatchingClient namespaceRegistry namespace.Registry } @@ -46,6 +50,7 @@ func newHandler( linkValidator *linkValidator, metricsHandler metrics.Handler, logger log.Logger, + matchingClient resource.MatchingClient, namespaceRegistry namespace.Registry, ) *handler { return &handler{ @@ -54,6 +59,7 @@ func newHandler( linkValidator: linkValidator, logger: logger, metricsHandler: metricsHandler, + matchingClient: matchingClient, namespaceRegistry: namespaceRegistry, } } @@ -76,6 +82,12 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St } maxCallbacks := h.config.MaxCallbacksPerExecution(frontendReq.GetNamespace()) + if frontendReq.GetRequestEagerExecution() && + !h.eagerActivityDispatchAllowed(ctx, req.GetNamespaceId(), frontendReq) { + // Matching grants are best-effort. On a denial or any Matching failure, schedule the + // activity normally instead of failing the start request. + frontendReq.RequestEagerExecution = false + } result, err := chasm.StartExecution( ctx, @@ -208,6 +220,38 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St }, nil } +func (h *handler) eagerActivityDispatchAllowed( + ctx context.Context, + namespaceID string, + request *workflowservice.StartActivityExecutionRequest, +) bool { + return !h.config.EnableActivityEagerDispatchCheck(request.GetNamespace()) || + h.grantEagerActivityDispatch(ctx, namespaceID, request) +} + +func (h *handler) grantEagerActivityDispatch( + ctx context.Context, + namespaceID string, + request *workflowservice.StartActivityExecutionRequest, +) bool { + if h.matchingClient == nil { + return false + } + + response, err := h.matchingClient.GrantEagerDispatch(ctx, &matchingservice.GrantEagerDispatchRequest{ + NamespaceId: namespaceID, + TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ + TaskQueue: request.GetTaskQueue().GetName(), + TaskQueueType: enumspb.TASK_QUEUE_TYPE_ACTIVITY, + }, + Items: []*matchingservice.GrantEagerDispatchRequest_Item{{ + Count: 1, + Priority: request.GetPriority(), + }}, + }) + return err == nil && len(response.GetItems()) == 1 && response.GetItems()[0].GetGrantedCount() == 1 +} + // DescribeActivityExecution queries current activity state, optionally as a long-poll that waits // for any state change. When used to long-poll, it returns an empty non-error response on context // deadline expiry, to indicate that the state being waited for was not reached. Callers should diff --git a/chasm/lib/activity/handler_test.go b/chasm/lib/activity/handler_test.go new file mode 100644 index 00000000000..5ab1ab317ef --- /dev/null +++ b/chasm/lib/activity/handler_test.go @@ -0,0 +1,96 @@ +package activity + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/api/matchingservicemock/v1" + "go.temporal.io/server/common/dynamicconfig" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +func TestGrantEagerActivityDispatch(t *testing.T) { + priority := &commonpb.Priority{PriorityKey: 2, FairnessKey: "fairness-key"} + request := &workflowservice.StartActivityExecutionRequest{ + Namespace: "namespace", + TaskQueue: &taskqueuepb.TaskQueue{Name: "activity-task-queue"}, + Priority: priority, + } + + testCases := []struct { + name string + response *matchingservice.GrantEagerDispatchResponse + err error + granted bool + }{ + { + name: "granted", + response: &matchingservice.GrantEagerDispatchResponse{Items: []*matchingservice.GrantEagerDispatchResponse_Item{ + {GrantedCount: 1}, + }}, + granted: true, + }, + { + name: "denied", + response: &matchingservice.GrantEagerDispatchResponse{Items: []*matchingservice.GrantEagerDispatchResponse_Item{ + {}, + }}, + }, + { + name: "matching error", + err: errors.New("matching unavailable"), + }, + { + name: "missing response item", + response: &matchingservice.GrantEagerDispatchResponse{}, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + controller := gomock.NewController(t) + matchingClient := matchingservicemock.NewMockMatchingServiceClient(controller) + matchingClient.EXPECT().GrantEagerDispatch(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, actual *matchingservice.GrantEagerDispatchRequest, _ ...grpc.CallOption) (*matchingservice.GrantEagerDispatchResponse, error) { + require.Equal(t, "namespace-id", actual.GetNamespaceId()) + require.Equal(t, "activity-task-queue", actual.GetTaskQueuePartition().GetTaskQueue()) + require.Equal(t, enumspb.TASK_QUEUE_TYPE_ACTIVITY, actual.GetTaskQueuePartition().GetTaskQueueType()) + require.Len(t, actual.GetItems(), 1) + require.EqualValues(t, 1, actual.GetItems()[0].GetCount()) + require.True(t, proto.Equal(priority, actual.GetItems()[0].GetPriority())) + return test.response, test.err + }, + ) + + handler := &handler{matchingClient: matchingClient} + require.Equal(t, test.granted, handler.grantEagerActivityDispatch(context.Background(), "namespace-id", request)) + }) + } +} + +func TestEagerActivityDispatchCheck(t *testing.T) { + request := &workflowservice.StartActivityExecutionRequest{Namespace: "namespace"} + + t.Run("disabled skips Matching", func(t *testing.T) { + handler := &handler{config: &Config{ + EnableActivityEagerDispatchCheck: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(false), + }} + require.True(t, handler.eagerActivityDispatchAllowed(context.Background(), "namespace-id", request)) + }) + + t.Run("enabled falls back when Matching is unavailable", func(t *testing.T) { + handler := &handler{config: &Config{ + EnableActivityEagerDispatchCheck: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(true), + }} + require.False(t, handler.eagerActivityDispatchAllowed(context.Background(), "namespace-id", request)) + }) +} diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 6304e689d7d..b3c787c9626 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1208,7 +1208,7 @@ func (s *standaloneActivityTestSuite) TestStart() { } func (s *standaloneActivityTestSuite) TestEagerStartResponseAndNoRedelivery() { - env := s.newTestEnv() + env := s.newTestEnv(testcore.WithDynamicConfig(dynamicconfig.EnableActivityEagerDispatchCheck, true)) t := s.T() ctx := metadata.NewOutgoingContext(s.Context(), metadata.Pairs( headers.ClientNameHeaderName, headers.ClientNameGoSDK, @@ -1271,6 +1271,68 @@ func (s *standaloneActivityTestSuite) TestEagerStartResponseAndNoRedelivery() { } +func (s *standaloneActivityTestSuite) TestEagerStartFallsBackWhenMatchingDenies() { + env := s.newTestEnv( + testcore.WithDynamicConfig(dynamicconfig.EnableActivityEagerDispatchCheck, true), + testcore.WithDynamicConfig(dynamicconfig.MatchingBacklogNegligibleAge, time.Duration(0)), + ) + t := s.T() + ctx := s.Context() + taskQueue := testcore.RandomizeStr(t.Name()) + backloggedActivityID := testcore.RandomizeStr(t.Name() + "-backlogged") + eagerActivityID := testcore.RandomizeStr(t.Name() + "-eager") + + start := func(activityID string, eager bool, priority int32) *workflowservice.StartActivityExecutionResponse { + response, err := env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + RequestEagerExecution: eager, + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + Identity: defaultIdentity, + Input: defaultInput, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + Priority: &commonpb.Priority{PriorityKey: priority}, + StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), + RequestId: testcore.RandomizeStr(t.Name()), + }) + require.NoError(t, err) + require.True(t, response.GetStarted()) + return response + } + + backlogged := start(backloggedActivityID, false, 1) + await.RequireTruef(t, func() bool { + response, err := env.FrontendClient().DescribeTaskQueue(ctx, &workflowservice.DescribeTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + TaskQueueType: enumspb.TASK_QUEUE_TYPE_ACTIVITY, + ReportStats: true, + }) + return err == nil && response.GetStats().GetApproximateBacklogCount() >= 1 + }, 10*time.Second, 100*time.Millisecond, "backlogged standalone activity never reached Matching") + + eager := start(eagerActivityID, true, 5) + require.Nil(t, eager.GetEagerActivityTask(), "Matching denial must fall back to normal dispatch") + + for _, activity := range []struct { + id string + runID string + }{ + {backloggedActivityID, backlogged.GetRunId()}, + {eagerActivityID, eager.GetRunId()}, + } { + task := env.pollActivityTaskAndValidate(ctx, t, activity.id, taskQueue, activity.runID) + _, err := env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: task.GetTaskToken(), + Result: defaultResult, + Identity: defaultIdentity, + }) + require.NoError(t, err) + env.validateCompletion(ctx, t, activity.id, activity.runID, defaultIdentity) + } +} + func (s *standaloneActivityTestSuite) TestComplete() { env := s.newTestEnv() t := s.T() From 969beae34047352e04ab9af16bb5a32b3712a47d Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Wed, 7 Oct 2026 12:50:24 -0500 Subject: [PATCH 11/22] Test eager activity response loss recovery --- tests/activity_standalone_test.go | 113 ++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index b3c787c9626..095b4f3dfed 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1333,6 +1333,119 @@ func (s *standaloneActivityTestSuite) TestEagerStartFallsBackWhenMatchingDenies( } } +func (s *standaloneActivityTestSuite) TestLostEagerActivityTaskResponseRecovery() { + env := s.newTestEnv(testcore.WithDynamicConfig(dynamicconfig.EnableActivityEagerDispatchCheck, false)) + t := s.T() + + startEagerActivity := func( + ctx context.Context, + t *testing.T, + activityID string, + taskQueue string, + startToCloseTimeout time.Duration, + heartbeatTimeout time.Duration, + scheduleToCloseTimeout time.Duration, + ) *workflowservice.StartActivityExecutionResponse { + response, err := env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + RequestEagerExecution: true, + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + Identity: defaultIdentity, + Input: defaultInput, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + ScheduleToCloseTimeout: durationpb.New(scheduleToCloseTimeout), + StartToCloseTimeout: durationpb.New(startToCloseTimeout), + HeartbeatTimeout: durationpb.New(heartbeatTimeout), + RetryPolicy: &commonpb.RetryPolicy{ + InitialInterval: durationpb.New(time.Second), + BackoffCoefficient: 1, + MaximumAttempts: 2, + }, + RequestId: testcore.RandomizeStr(t.Name()), + }) + require.NoError(t, err) + require.True(t, response.GetStarted()) + require.NotNil(t, response.GetEagerActivityTask()) + return response + } + + completeRetriedActivity := func( + ctx context.Context, + t *testing.T, + activityID string, + taskQueue string, + runID string, + ) { + task, err := env.pollActivityTaskQueue(ctx, taskQueue) + require.NoError(t, err) + require.Equal(t, activityID, task.GetActivityId()) + require.EqualValues(t, 2, task.GetAttempt(), "retry must use the normal task-queue poll path") + + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: task.GetTaskToken(), + Result: defaultResult, + Identity: defaultIdentity, + }) + require.NoError(t, err) + + describe, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + RunId: runID, + IncludeOutcome: true, + }) + require.NoError(t, err) + require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED, describe.GetInfo().GetStatus()) + require.EqualValues(t, 2, describe.GetInfo().GetAttempt()) + protorequire.ProtoEqual(t, defaultResult, describe.GetOutcome().GetResult()) + } + + t.Run("StartToCloseTimeout", func(t *testing.T) { + ctx, cancel := context.WithTimeout(s.Context(), 20*time.Second) + defer cancel() + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + + response := startEagerActivity(ctx, t, activityID, taskQueue, time.Second, 0, time.Minute) + // Simulate a client that lost the eager response: it never responds using this token. + require.NotEmpty(t, response.GetEagerActivityTask().GetTaskToken()) + completeRetriedActivity(ctx, t, activityID, taskQueue, response.GetRunId()) + }) + + t.Run("HeartbeatTimeout", func(t *testing.T) { + ctx, cancel := context.WithTimeout(s.Context(), 20*time.Second) + defer cancel() + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + + response := startEagerActivity(ctx, t, activityID, taskQueue, time.Minute, time.Second, time.Minute) + // Simulate a client that lost the eager response: it never heartbeats or completes. + require.NotEmpty(t, response.GetEagerActivityTask().GetTaskToken()) + completeRetriedActivity(ctx, t, activityID, taskQueue, response.GetRunId()) + }) + + t.Run("ScheduleToCloseTimeout", func(t *testing.T) { + ctx, cancel := context.WithTimeout(s.Context(), 20*time.Second) + defer cancel() + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + + response := startEagerActivity(ctx, t, activityID, taskQueue, time.Minute, 0, time.Second) + // Schedule-to-close is the activity's terminal deadline, so a lost eager task must not retry. + require.NotEmpty(t, response.GetEagerActivityTask().GetTaskToken()) + + outcome, err := env.FrontendClient().PollActivityExecution(ctx, &workflowservice.PollActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + RunId: response.GetRunId(), + }) + require.NoError(t, err) + require.Equal(t, enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, outcome.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) + }) +} + func (s *standaloneActivityTestSuite) TestComplete() { env := s.newTestEnv() t := s.T() From aa3ef43ee8c07ed2dfff84b13a434394729d61a5 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Wed, 7 Oct 2026 14:21:45 -0500 Subject: [PATCH 12/22] Clean up standalone activity state transitions --- chasm/lib/activity/statemachine.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chasm/lib/activity/statemachine.go b/chasm/lib/activity/statemachine.go index 5b5e080e818..6aa202a1096 100644 --- a/chasm/lib/activity/statemachine.go +++ b/chasm/lib/activity/statemachine.go @@ -377,8 +377,7 @@ func (a *Activity) applyScheduled(ctx chasm.MutableContext, dispatch bool) error dispatchTime := a.firstDispatchTime() attempt.DispatchTime = timestamppb.New(dispatchTime) - if dispatch && a.GetScheduleToStartTimeout().AsDuration() > 0 { - timeout := a.GetScheduleToStartTimeout().AsDuration() + if timeout := a.GetScheduleToStartTimeout().AsDuration(); dispatch && timeout > 0 { ctx.AddTask( a, chasm.TaskAttributes{ @@ -424,6 +423,7 @@ func (a *Activity) applyEagerStarted(ctx chasm.MutableContext, event eagerStartE attempt.LastWorkerIdentity = event.identity attempt.SdkName = ctx.RequestHeader(headers.ClientNameHeaderName) attempt.SdkVersion = ctx.RequestHeader(headers.ClientVersionHeaderName) + // TODO: Populate LastDeploymentVersion when standalone eager start supports worker versioning. startTime := attempt.GetStartedTime().AsTime() ctx.AddTask( a, From dbc41ead6be2f514e6cb90f81006acadccbc2397 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Wed, 7 Oct 2026 14:21:53 -0500 Subject: [PATCH 13/22] Resolve eager activity lint findings --- chasm/lib/activity/frontend.go | 10 +- chasm/lib/activity/handler.go | 220 ++++++++++++++++----------- chasm/lib/activity/validator_test.go | 14 +- tests/activity_standalone_test.go | 8 +- 4 files changed, 149 insertions(+), 103 deletions(-) diff --git a/chasm/lib/activity/frontend.go b/chasm/lib/activity/frontend.go index e97baaa4521..9d8dacea751 100644 --- a/chasm/lib/activity/frontend.go +++ b/chasm/lib/activity/frontend.go @@ -386,13 +386,9 @@ func (h *frontendHandler) validateAndPopulateStartRequest( if req.GetStartDelay().AsDuration() > 0 && !h.config.StartDelayEnabled(req.GetNamespace()) { return nil, serviceerror.NewInvalidArgument("start_delay is not enabled for this namespace") } - if req.GetRequestEagerExecution() { - switch { - case !h.config.EnableEagerStart(req.GetNamespace()): - req.RequestEagerExecution = false - case req.GetStartDelay().AsDuration() > 0: - req.RequestEagerExecution = false - } + if req.GetRequestEagerExecution() && + (!h.config.EnableEagerStart(req.GetNamespace()) || req.GetStartDelay().AsDuration() > 0) { + req.RequestEagerExecution = false } opts := activityOptionsFromStartRequest(req) diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index da00cb07fc2..62b501e1b51 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -71,23 +71,13 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St frontendReq := req.GetFrontendRequest() var eagerTaskData *eagerActivityTaskData - reusePolicy, ok := businessIDReusePolicyMap[frontendReq.GetIdReusePolicy()] - if !ok { - return nil, serviceerror.NewInvalidArgumentf("unsupported ID reuse policy: %v", frontendReq.GetIdReusePolicy()) - } - - conflictPolicy, ok := businessIDConflictPolicyMap[frontendReq.GetIdConflictPolicy()] - if !ok { - return nil, serviceerror.NewInvalidArgumentf("unsupported ID conflict policy: %v", frontendReq.GetIdConflictPolicy()) + reusePolicy, conflictPolicy, err := businessIDPolicies(frontendReq) + if err != nil { + return nil, err } maxCallbacks := h.config.MaxCallbacksPerExecution(frontendReq.GetNamespace()) - if frontendReq.GetRequestEagerExecution() && - !h.eagerActivityDispatchAllowed(ctx, req.GetNamespaceId(), frontendReq) { - // Matching grants are best-effort. On a denial or any Matching failure, schedule the - // activity normally instead of failing the start request. - frontendReq.RequestEagerExecution = false - } + h.maybeDisableEagerActivityDispatch(ctx, req.GetNamespaceId(), frontendReq) result, err := chasm.StartExecution( ctx, @@ -96,44 +86,11 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St BusinessID: frontendReq.GetActivityId(), }, func(mutableContext chasm.MutableContext, request *workflowservice.StartActivityExecutionRequest) (*Activity, error) { - newActivity, err := NewStandaloneActivity(mutableContext, request) + newActivity, newEagerTaskData, err := h.newActivityForStartExecution(mutableContext, request, req.GetNamespaceId(), maxCallbacks) if err != nil { return nil, err } - - if cbs := request.GetCompletionCallbacks(); len(cbs) > 0 { - if err := newActivity.addCompletionCallbacks(mutableContext, request.GetRequestId(), cbs, maxCallbacks); err != nil { - return nil, err - } - } - if len(request.GetLinks()) > 0 { - if err := newActivity.attachLinks(mutableContext, request.GetLinks(), request.GetRequestId(), h.linkValidator, frontendReq.GetNamespace()); err != nil { - return nil, err - } - } - - if request.GetRequestEagerExecution() { - err = TransitionEagerStarted.Apply(newActivity, mutableContext, eagerStartEvent{ - requestID: request.GetRequestId(), - identity: request.GetIdentity(), - }) - } else { - err = TransitionScheduled.Apply(newActivity, mutableContext, nil) - } - if err != nil { - return nil, err - } - if request.GetRequestEagerExecution() { - eagerTaskData, err = newActivity.eagerActivityTaskData(mutableContext, eagerActivityTaskRequest{ - namespaceID: req.GetNamespaceId(), - namespace: request.GetNamespace(), - requestID: request.GetRequestId(), - }) - if err != nil { - return nil, err - } - } - + eagerTaskData = newEagerTaskData return newActivity, nil }, frontendReq, @@ -142,11 +99,7 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St ) if err != nil { - if alreadyStartedErr, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { - return nil, serviceerror.NewActivityExecutionAlreadyStarted("activity execution already started", alreadyStartedErr.CurrentRequestID, alreadyStartedErr.CurrentRunID) - } - - return nil, err + return nil, startActivityExecutionError(err) } if result.Created { @@ -168,38 +121,8 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St metrics.ActivityEagerExecutionCounter.With(eagerActivityMetricsHandler(h.metricsHandler, h.config, frontendReq)).Record(1) } - // Apply on_conflict_options to an existing activity. - // TODO: Use chasm.UpdateWithStartExecution to avoid a second transaction once the engine supports BusinessIDConflictPolicyFail in the updateFn path. - cbs := frontendReq.GetCompletionCallbacks() - links := frontendReq.GetLinks() - onConflict := frontendReq.GetOnConflictOptions() - attachCallbacks := onConflict.GetAttachCompletionCallbacks() && len(cbs) > 0 - attachLinks := onConflict.GetAttachLinks() && len(links) > 0 - if !result.Created && (attachCallbacks || attachLinks) { - requestID := frontendReq.GetRequestId() - ref := chasm.NewComponentRef[*Activity](result.ExecutionKey) - _, _, err := chasm.UpdateComponent( - ctx, - ref, - func(a *Activity, ctx chasm.MutableContext, _ any) (any, error) { - if attachCallbacks { - if err := a.addCompletionCallbacks(ctx, requestID, cbs, maxCallbacks); err != nil { - return nil, err - } - } - if attachLinks { - if err := a.attachLinks(ctx, links, requestID, h.linkValidator, frontendReq.GetNamespace()); err != nil { - return nil, err - } - } - return nil, nil - }, - nil, - chasm.WithRequestID(requestID), - ) - if err != nil && !errors.Is(err, chasm.ErrRequestIDAlreadyUsed) { - return nil, err - } + if err := h.applyOnConflictOptions(ctx, frontendReq, result.Created, result.ExecutionKey, maxCallbacks); err != nil { + return nil, err } return &activitypb.StartActivityExecutionResponse{ @@ -220,6 +143,131 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St }, nil } +func (h *handler) newActivityForStartExecution( + mutableContext chasm.MutableContext, + request *workflowservice.StartActivityExecutionRequest, + namespaceID string, + maxCallbacks int, +) (*Activity, *eagerActivityTaskData, error) { + newActivity, err := NewStandaloneActivity(mutableContext, request) + if err != nil { + return nil, nil, err + } + + if cbs := request.GetCompletionCallbacks(); len(cbs) > 0 { + if err := newActivity.addCompletionCallbacks(mutableContext, request.GetRequestId(), cbs, maxCallbacks); err != nil { + return nil, nil, err + } + } + if len(request.GetLinks()) > 0 { + if err := newActivity.attachLinks(mutableContext, request.GetLinks(), request.GetRequestId(), h.linkValidator, request.GetNamespace()); err != nil { + return nil, nil, err + } + } + + if request.GetRequestEagerExecution() { + err = TransitionEagerStarted.Apply(newActivity, mutableContext, eagerStartEvent{ + requestID: request.GetRequestId(), + identity: request.GetIdentity(), + }) + } else { + err = TransitionScheduled.Apply(newActivity, mutableContext, nil) + } + if err != nil { + return nil, nil, err + } + if !request.GetRequestEagerExecution() { + return newActivity, nil, nil + } + + eagerTaskData, err := newActivity.eagerActivityTaskData(mutableContext, eagerActivityTaskRequest{ + namespaceID: namespaceID, + namespace: request.GetNamespace(), + requestID: request.GetRequestId(), + }) + if err != nil { + return nil, nil, err + } + return newActivity, eagerTaskData, nil +} + +func businessIDPolicies(request *workflowservice.StartActivityExecutionRequest) (chasm.BusinessIDReusePolicy, chasm.BusinessIDConflictPolicy, error) { + reusePolicy, ok := businessIDReusePolicyMap[request.GetIdReusePolicy()] + if !ok { + return 0, 0, serviceerror.NewInvalidArgumentf("unsupported ID reuse policy: %v", request.GetIdReusePolicy()) + } + + conflictPolicy, ok := businessIDConflictPolicyMap[request.GetIdConflictPolicy()] + if !ok { + return 0, 0, serviceerror.NewInvalidArgumentf("unsupported ID conflict policy: %v", request.GetIdConflictPolicy()) + } + + return reusePolicy, conflictPolicy, nil +} + +func (h *handler) maybeDisableEagerActivityDispatch( + ctx context.Context, + namespaceID string, + request *workflowservice.StartActivityExecutionRequest, +) { + if request.GetRequestEagerExecution() && !h.eagerActivityDispatchAllowed(ctx, namespaceID, request) { + // Matching grants are best-effort. On a denial or any Matching failure, schedule the + // activity normally instead of failing the start request. + request.RequestEagerExecution = false + } +} + +func startActivityExecutionError(err error) error { + if alreadyStartedErr, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { + return serviceerror.NewActivityExecutionAlreadyStarted("activity execution already started", alreadyStartedErr.CurrentRequestID, alreadyStartedErr.CurrentRunID) + } + return err +} + +// applyOnConflictOptions updates an existing activity after StartExecution returns it unchanged. +func (h *handler) applyOnConflictOptions( + ctx context.Context, + request *workflowservice.StartActivityExecutionRequest, + created bool, + executionKey chasm.ExecutionKey, + maxCallbacks int, +) error { + cbs := request.GetCompletionCallbacks() + links := request.GetLinks() + onConflict := request.GetOnConflictOptions() + attachCallbacks := onConflict.GetAttachCompletionCallbacks() && len(cbs) > 0 + attachLinks := onConflict.GetAttachLinks() && len(links) > 0 + if created || (!attachCallbacks && !attachLinks) { + return nil + } + + // TODO: Use chasm.UpdateWithStartExecution to avoid a second transaction once the engine supports BusinessIDConflictPolicyFail in the updateFn path. + requestID := request.GetRequestId() + _, _, err := chasm.UpdateComponent( + ctx, + chasm.NewComponentRef[*Activity](executionKey), + func(a *Activity, ctx chasm.MutableContext, _ any) (any, error) { + if attachCallbacks { + if err := a.addCompletionCallbacks(ctx, requestID, cbs, maxCallbacks); err != nil { + return nil, err + } + } + if attachLinks { + if err := a.attachLinks(ctx, links, requestID, h.linkValidator, request.GetNamespace()); err != nil { + return nil, err + } + } + return nil, nil + }, + nil, + chasm.WithRequestID(requestID), + ) + if err != nil && !errors.Is(err, chasm.ErrRequestIDAlreadyUsed) { + return err + } + return nil +} + func (h *handler) eagerActivityDispatchAllowed( ctx context.Context, namespaceID string, diff --git a/chasm/lib/activity/validator_test.go b/chasm/lib/activity/validator_test.go index 6e20bfc16a7..0bdd4eaff10 100644 --- a/chasm/lib/activity/validator_test.go +++ b/chasm/lib/activity/validator_test.go @@ -779,16 +779,16 @@ func TestValidateStandaloneUserMetadata(t *testing.T) { }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { h := &frontendHandler{ config: &Config{ BlobSizeLimitError: defaultBlobSizeLimitError, BlobSizeLimitWarn: defaultBlobSizeLimitWarn, DefaultActivityRetryPolicy: getDefaultRetrySettings, MaxIDLengthLimit: func() int { return defaultMaxIDLengthLimit }, - MaxUserMetadataSummarySize: func(string) int { return test.summaryLimit }, - MaxUserMetadataDetailsSize: func(string) int { return test.detailsLimit }, + MaxUserMetadataSummarySize: func(string) int { return testCase.summaryLimit }, + MaxUserMetadataDetailsSize: func(string) int { return testCase.detailsLimit }, }, linkValidator: newLinkValidator( defaultMaxLinksPerRequest, @@ -803,16 +803,16 @@ func TestValidateStandaloneUserMetadata(t *testing.T) { ActivityType: &commonpb.ActivityType{Name: defaultActivityType}, TaskQueue: &taskqueuepb.TaskQueue{Name: defaultTaskQueue}, StartToCloseTimeout: durationpb.New(10 * time.Second), - UserMetadata: test.metadata, + UserMetadata: testCase.metadata, } _, err := h.validateAndPopulateStartRequest(t.Context(), req, namespace.ID(defaultNamespaceID)) - if test.errContains == "" { + if testCase.errContains == "" { require.NoError(t, err) return } var invalidArgument *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgument) - require.ErrorContains(t, err, test.errContains) + require.ErrorContains(t, err, testCase.errContains) }) } } diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 095b4f3dfed..01b2e89eb40 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1275,6 +1275,8 @@ func (s *standaloneActivityTestSuite) TestEagerStartFallsBackWhenMatchingDenies( env := s.newTestEnv( testcore.WithDynamicConfig(dynamicconfig.EnableActivityEagerDispatchCheck, true), testcore.WithDynamicConfig(dynamicconfig.MatchingBacklogNegligibleAge, time.Duration(0)), + testcore.WithDynamicConfig(dynamicconfig.MatchingNumTaskqueueReadPartitions, 1), + testcore.WithDynamicConfig(dynamicconfig.MatchingNumTaskqueueWritePartitions, 1), ) t := s.T() ctx := s.Context() @@ -1314,14 +1316,14 @@ func (s *standaloneActivityTestSuite) TestEagerStartFallsBackWhenMatchingDenies( eager := start(eagerActivityID, true, 5) require.Nil(t, eager.GetEagerActivityTask(), "Matching denial must fall back to normal dispatch") - for _, activity := range []struct { + for _, startedActivity := range []struct { id string runID string }{ {backloggedActivityID, backlogged.GetRunId()}, {eagerActivityID, eager.GetRunId()}, } { - task := env.pollActivityTaskAndValidate(ctx, t, activity.id, taskQueue, activity.runID) + task := env.pollActivityTaskAndValidate(ctx, t, startedActivity.id, taskQueue, startedActivity.runID) _, err := env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ Namespace: env.Namespace().String(), TaskToken: task.GetTaskToken(), @@ -1329,7 +1331,7 @@ func (s *standaloneActivityTestSuite) TestEagerStartFallsBackWhenMatchingDenies( Identity: defaultIdentity, }) require.NoError(t, err) - env.validateCompletion(ctx, t, activity.id, activity.runID, defaultIdentity) + env.validateCompletion(ctx, t, startedActivity.id, startedActivity.runID, defaultIdentity) } } From 1bab344be6dbd42785774edd42f861e6c4b98a61 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Wed, 7 Oct 2026 17:15:14 -0500 Subject: [PATCH 14/22] Remove temporary API dependency scaffolding --- Makefile | 14 ---- cmd/tools/check-dependencies/main.go | 85 ----------------------- cmd/tools/check-dependencies/main_test.go | 55 --------------- go.mod | 2 - go.sum | 4 +- tests/mixedbrain/go.mod | 2 - tests/mixedbrain/go.sum | 4 +- 7 files changed, 4 insertions(+), 162 deletions(-) diff --git a/Makefile b/Makefile index ad25d70ccb7..684900b1a1d 100644 --- a/Makefile +++ b/Makefile @@ -204,17 +204,9 @@ $(BUF): | $(LOCALBIN) GO_API_VER = $(shell go list -m -f '{{.Version}}' go.temporal.io/api \ || (echo "failed to fetch version for go.temporal.io/api" >&2)) -GO_API_REPLACED = $(shell go list -m -f '{{if .Replace}}true{{end}}' go.temporal.io/api \ - || (echo "failed to resolve replacement for go.temporal.io/api" >&2)) PROTOGEN := $(LOCALBIN)/protogen-$(GO_API_VER) -ifeq ($(GO_API_REPLACED),true) -$(PROTOGEN): | $(LOCALBIN) - @printf $(COLOR) "Building protogen through the replaced go.temporal.io/api module..." - @go build -o $@ go.temporal.io/api/cmd/protogen -else $(PROTOGEN): | $(LOCALBIN) $(call go-install-tool,$(PROTOGEN),go.temporal.io/api/cmd/protogen,$(GO_API_VER)) -endif ACTIONLINT_VER := v1.7.7 ACTIONLINT := $(LOCALBIN)/actionlint-$(ACTIONLINT_VER) @@ -293,14 +285,8 @@ $(STAMPDIR)/protoc-gen-go-grpc-$(PROTOC_GEN_GO_GRPC_VER): | $(STAMPDIR) $(LOCALB $(PROTOC_GEN_GO_GRPC): $(STAMPDIR)/protoc-gen-go-grpc-$(PROTOC_GEN_GO_GRPC_VER) PROTOC_GEN_GO_HELPERS := $(LOCALBIN)/protoc-gen-go-helpers-$(GO_API_VER) -ifeq ($(GO_API_REPLACED),true) -$(STAMPDIR)/protoc-gen-go-helpers-$(GO_API_VER): | $(STAMPDIR) $(LOCALBIN) - @printf $(COLOR) "Building protoc-gen-go-helpers through the replaced go.temporal.io/api module..." - @go build -o $(PROTOC_GEN_GO_HELPERS) go.temporal.io/api/cmd/protoc-gen-go-helpers -else $(STAMPDIR)/protoc-gen-go-helpers-$(GO_API_VER): | $(STAMPDIR) $(LOCALBIN) $(call go-install-tool,$(PROTOC_GEN_GO_HELPERS),go.temporal.io/api/cmd/protoc-gen-go-helpers,$(GO_API_VER)) -endif @touch $@ $(PROTOC_GEN_GO_HELPERS): $(STAMPDIR)/protoc-gen-go-helpers-$(GO_API_VER) diff --git a/cmd/tools/check-dependencies/main.go b/cmd/tools/check-dependencies/main.go index e10f4d45c02..2410ba4eb40 100644 --- a/cmd/tools/check-dependencies/main.go +++ b/cmd/tools/check-dependencies/main.go @@ -43,8 +43,6 @@ type moduleSpec struct { // strict rule. const apiModulePath = "go.temporal.io/api" -var replacementContainsCommit = repositoryContainsCommit - var knownModules = []moduleSpec{ { modulePath: apiModulePath, @@ -327,12 +325,6 @@ func validateMainModule( version := modVersion.Version fmt.Printf("Found %s version: %s\n", mod.modulePath, version) - if replacement, ok := findReplacement(modFile, modVersion); ok { - if mod.modulePath != apiModulePath { - return fmt.Errorf("%s@%s: replacements are not allowed", mod.modulePath, version) - } - return validateTemporaryAPIReplacement(ctx, modVersion, replacement) - } if !module.IsPseudoVersion(version) { if !semver.IsValid(version) { @@ -370,83 +362,6 @@ func findRequiredModuleVersion(modFile *modfile.File, modulePath string) (module return module.Version{}, false } -func findReplacement(modFile *modfile.File, required module.Version) (module.Version, bool) { - for _, replacement := range modFile.Replace { - if replacement.Old.Path == required.Path && - (replacement.Old.Version == "" || replacement.Old.Version == required.Version) { - return replacement.New, true - } - } - return module.Version{}, false -} - -func validateTemporaryAPIReplacement( - ctx context.Context, - required module.Version, - replacement module.Version, -) error { - if !module.IsPseudoVersion(required.Version) || !module.IsPseudoVersion(replacement.Version) { - return fmt.Errorf("%s@%s: temporary replacement %s@%s must use matching pseudo-versions", - required.Path, required.Version, replacement.Path, replacement.Version) - } - - requiredRevision, err := module.PseudoVersionRev(required.Version) - if err != nil { - return fmt.Errorf("%s@%s: failed to parse pseudo-version revision: %w", required.Path, required.Version, err) - } - replacementRevision, err := module.PseudoVersionRev(replacement.Version) - if err != nil { - return fmt.Errorf("%s@%s: failed to parse temporary replacement revision: %w", replacement.Path, replacement.Version, err) - } - if requiredRevision != replacementRevision { - return fmt.Errorf("%s@%s: temporary replacement %s@%s names a different commit", - required.Path, required.Version, replacement.Path, replacement.Version) - } - - parts := strings.Split(replacement.Path, "/") - if len(parts) != 3 || parts[0] != "github.com" || parts[2] != "api-go" { - return fmt.Errorf("%s@%s: temporary replacement must point to a GitHub api-go repository, got %s", - required.Path, required.Version, replacement.Path) - } - - repoURL := "https://" + replacement.Path + ".git" - available, err := replacementContainsCommit(ctx, repoURL, replacementRevision) - if err != nil { - return fmt.Errorf("%s@%s: failed to resolve temporary replacement: %w", required.Path, required.Version, err) - } - if !available { - return fmt.Errorf("%s@%s: temporary replacement commit %s is not available from %s", - required.Path, required.Version, replacementRevision, repoURL) - } - - fmt.Printf(" - %s@%s temporarily replaced by %s@%s (ok)\n", - required.Path, required.Version, replacement.Path, replacement.Version) - return nil -} - -func repositoryContainsCommit(ctx context.Context, repoURL string, shortHash string) (bool, error) { - tmpRepo, err := os.MkdirTemp("", "check-dependencies-*") - if err != nil { - return false, fmt.Errorf("failed to create temp repo dir: %w", err) - } - defer func() { _ = os.RemoveAll(tmpRepo) }() - - out, err := exec.CommandContext(ctx, "git", "clone", "--bare", "--filter=blob:none", repoURL, tmpRepo).CombinedOutput() - if err != nil { - return false, fmt.Errorf("git clone failed: %w: %s", err, strings.TrimSpace(string(out))) - } - - out, err = exec.CommandContext(ctx, "git", "-C", tmpRepo, "cat-file", "-e", shortHash+"^{commit}").CombinedOutput() - if err == nil { - return true, nil - } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { - return false, nil - } - return false, fmt.Errorf("git cat-file failed: %w: %s", err, strings.TrimSpace(string(out))) -} - // resolveModuleOriginForSpec reports whether shortHash is reachable from the // default branch of mod's repository. // diff --git a/cmd/tools/check-dependencies/main_test.go b/cmd/tools/check-dependencies/main_test.go index 26189647ccd..2778a2dd2d2 100644 --- a/cmd/tools/check-dependencies/main_test.go +++ b/cmd/tools/check-dependencies/main_test.go @@ -48,61 +48,6 @@ func TestFindRequiredModuleVersion(t *testing.T) { }) } -func TestFindReplacement(t *testing.T) { - f := parseGoMod(t, `module test - -go 1.21 - -require go.temporal.io/api v1.2.4-0.20240101000000-abcdef012345 - -replace go.temporal.io/api => github.com/example/api-go v1.2.4-0.20240101000000-abcdef012345 -`) - - replacement, ok := findReplacement(f, module.Version{ - Path: "go.temporal.io/api", - Version: "v1.2.4-0.20240101000000-abcdef012345", - }) - require.True(t, ok) - require.Equal(t, "github.com/example/api-go", replacement.Path) - require.Equal(t, "v1.2.4-0.20240101000000-abcdef012345", replacement.Version) -} - -func TestValidateTemporaryAPIReplacement(t *testing.T) { - const version = "v1.2.4-0.20240101000000-abcdef012345" - originalResolver := replacementContainsCommit - t.Cleanup(func() { replacementContainsCommit = originalResolver }) - - var gotRepoURL string - var gotRevision string - replacementContainsCommit = func(_ context.Context, repoURL, revision string) (bool, error) { - gotRepoURL = repoURL - gotRevision = revision - return true, nil - } - - required := module.Version{Path: apiModulePath, Version: version} - replacement := module.Version{Path: "github.com/example/api-go", Version: version} - require.NoError(t, validateTemporaryAPIReplacement(context.Background(), required, replacement)) - require.Equal(t, "https://github.com/example/api-go.git", gotRepoURL) - require.Equal(t, "abcdef012345", gotRevision) - - t.Run("rejects a different revision", func(t *testing.T) { - err := validateTemporaryAPIReplacement(context.Background(), required, module.Version{ - Path: replacement.Path, - Version: "v1.2.4-0.20240101000000-fedcba987654", - }) - require.ErrorContains(t, err, "names a different commit") - }) - - t.Run("rejects a non api-go repository", func(t *testing.T) { - err := validateTemporaryAPIReplacement(context.Background(), required, module.Version{ - Path: "github.com/example/not-api-go", - Version: version, - }) - require.ErrorContains(t, err, "must point to a GitHub api-go repository") - }) -} - func TestValidateReleaseBranch(t *testing.T) { tests := []struct { name string diff --git a/go.mod b/go.mod index 299d7067cb6..06285190162 100644 --- a/go.mod +++ b/go.mod @@ -223,6 +223,4 @@ require ( sigs.k8s.io/yaml v1.6.0 // indirect ) -replace go.temporal.io/api => github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed - tool golang.org/x/perf/cmd/benchstat diff --git a/go.sum b/go.sum index dc276bb794a..2c94fc32a0e 100644 --- a/go.sum +++ b/go.sum @@ -425,8 +425,6 @@ github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ= github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= -github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed h1:Jr0wN0pOdf8USIjmOL/v7t8P1dH1oBnnrMsZhaM3klg= -github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= @@ -481,6 +479,8 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= +go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed h1:d09v879yeW0j2iZD0WZnEWqUmLu3RO0ZXtIPxK5nc2g= +go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= go.temporal.io/auto-scaled-workers v0.0.0-20260928183627-719679297cbe h1:BlshPqVI1uh/AsdfGPPjfKc37HEZdGfcmvx+uW+rbe8= go.temporal.io/auto-scaled-workers v0.0.0-20260928183627-719679297cbe/go.mod h1:6Q4guHwlbJldBOZwvc201E0+Jz9Tly1L1Db6C7RJuLw= go.temporal.io/sdk v1.48.0 h1:WDctKDVuh0Z8Nf7euAyqs/EwcPg1JTIIq1Fut8Tq118= diff --git a/tests/mixedbrain/go.mod b/tests/mixedbrain/go.mod index 4e44d0a6674..3ff819e7da9 100644 --- a/tests/mixedbrain/go.mod +++ b/tests/mixedbrain/go.mod @@ -31,8 +31,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace go.temporal.io/api => github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed - replace go.temporal.io/server => ../.. replace go.temporal.io/sdk => go.temporal.io/sdk v1.42.0 diff --git a/tests/mixedbrain/go.sum b/tests/mixedbrain/go.sum index 881af88d9ed..fbfaddc1a2e 100644 --- a/tests/mixedbrain/go.sum +++ b/tests/mixedbrain/go.sum @@ -43,8 +43,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/temporalio/omes v0.0.0-20260915193711-9667582a3051 h1:lJ/bCaq7PVGpSwON+MgqAK8/0oih71Jrq9OP3Gxajns= github.com/temporalio/omes v0.0.0-20260915193711-9667582a3051/go.mod h1:ECa3gdOOjrNXSR1OXjI3s+iO1DhoxDIqcyS4fxK2j0Y= -github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed h1:Jr0wN0pOdf8USIjmOL/v7t8P1dH1oBnnrMsZhaM3klg= -github.com/wenlong-gu/api-go v1.63.7-0.20261002150604-6b55806007ed/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= @@ -57,6 +55,8 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed h1:d09v879yeW0j2iZD0WZnEWqUmLu3RO0ZXtIPxK5nc2g= +go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From 8af210afece25a1c5886b54798b2454a09c5b6b1 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Thu, 8 Oct 2026 16:35:09 -0500 Subject: [PATCH 15/22] Update API dependency after eager activity API merge --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 06285190162..ee0038323e2 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.44.0 go.opentelemetry.io/otel/trace v1.44.0 go.opentelemetry.io/proto/otlp v1.10.0 - go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed + go.temporal.io/api v1.63.7-0.20261008212609-555b97e92c27 go.temporal.io/auto-scaled-workers v0.0.0-20260928183627-719679297cbe go.temporal.io/sdk v1.48.0 go.uber.org/fx v1.24.0 diff --git a/go.sum b/go.sum index 2c94fc32a0e..295241a47c4 100644 --- a/go.sum +++ b/go.sum @@ -479,8 +479,8 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed h1:d09v879yeW0j2iZD0WZnEWqUmLu3RO0ZXtIPxK5nc2g= -go.temporal.io/api v1.63.7-0.20261002150604-6b55806007ed/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= +go.temporal.io/api v1.63.7-0.20261008212609-555b97e92c27 h1:QI4R6Tvtx0s+cCgMnFFsTUuKsw+xDpR/Z9mCx8gs6Os= +go.temporal.io/api v1.63.7-0.20261008212609-555b97e92c27/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= go.temporal.io/auto-scaled-workers v0.0.0-20260928183627-719679297cbe h1:BlshPqVI1uh/AsdfGPPjfKc37HEZdGfcmvx+uW+rbe8= go.temporal.io/auto-scaled-workers v0.0.0-20260928183627-719679297cbe/go.mod h1:6Q4guHwlbJldBOZwvc201E0+Jz9Tly1L1Db6C7RJuLw= go.temporal.io/sdk v1.48.0 h1:WDctKDVuh0Z8Nf7euAyqs/EwcPg1JTIIq1Fut8Tq118= From 8b891d32e9d0988836c2d8adf3c4643bebe44427 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 12 Oct 2026 11:59:13 -0500 Subject: [PATCH 16/22] Persist eager activity acceptance state --- .../gen/activitypb/v1/activity_state.pb.go | 18 ++- chasm/lib/activity/handler.go | 45 +++---- chasm/lib/activity/handler_test.go | 19 --- .../activity/proto/v1/activity_state.proto | 3 + chasm/lib/activity/responses.go | 78 ++++++++++-- chasm/lib/activity/responses_test.go | 117 ++++++++++++++++++ chasm/lib/activity/statemachine.go | 1 + chasm/lib/activity/statemachine_test.go | 1 + tests/activity_standalone_test.go | 17 +++ 9 files changed, 234 insertions(+), 65 deletions(-) diff --git a/chasm/lib/activity/gen/activitypb/v1/activity_state.pb.go b/chasm/lib/activity/gen/activitypb/v1/activity_state.pb.go index 9018035b715..4fe000fabfa 100644 --- a/chasm/lib/activity/gen/activitypb/v1/activity_state.pb.go +++ b/chasm/lib/activity/gen/activitypb/v1/activity_state.pb.go @@ -317,8 +317,10 @@ type ActivityState struct { LastResetRequestId string `protobuf:"bytes,22,opt,name=last_reset_request_id,json=lastResetRequestId,proto3" json:"last_reset_request_id,omitempty"` // Used to de-dupe update requests. LastUpdateOptionsRequestId string `protobuf:"bytes,23,opt,name=last_update_options_request_id,json=lastUpdateOptionsRequestId,proto3" json:"last_update_options_request_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Indicates that eager execution was accepted for this activity's first task. + EagerExecutionAccepted bool `protobuf:"varint,24,opt,name=eager_execution_accepted,json=eagerExecutionAccepted,proto3" json:"eager_execution_accepted,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActivityState) Reset() { @@ -512,6 +514,13 @@ func (x *ActivityState) GetLastUpdateOptionsRequestId() string { return "" } +func (x *ActivityState) GetEagerExecutionAccepted() bool { + if x != nil { + return x.EagerExecutionAccepted + } + return false +} + type ActivityCancelState struct { state protoimpl.MessageState `protogen:"open.v1"` RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` @@ -1254,7 +1263,7 @@ var File_temporal_server_chasm_lib_activity_proto_v1_activity_state_proto protor const file_temporal_server_chasm_lib_activity_proto_v1_activity_state_proto_rawDesc = "" + "\n" + - "@temporal/server/chasm/lib/activity/proto/v1/activity_state.proto\x12+temporal.server.chasm.lib.activity.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&temporal/api/activity/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/deployment/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a'temporal/api/sdk/v1/user_metadata.proto\x1a'temporal/api/taskqueue/v1/message.proto\"\xb9\r\n" + + "@temporal/server/chasm/lib/activity/proto/v1/activity_state.proto\x12+temporal.server.chasm.lib.activity.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&temporal/api/activity/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/deployment/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a'temporal/api/sdk/v1/user_metadata.proto\x1a'temporal/api/taskqueue/v1/message.proto\"\xf3\r\n" + "\rActivityState\x12I\n" + "\ractivity_type\x18\x01 \x01(\v2$.temporal.api.common.v1.ActivityTypeR\factivityType\x12C\n" + "\n" + @@ -1281,7 +1290,8 @@ const file_temporal_server_chasm_lib_activity_proto_v1_activity_state_proto_rawD "\x15reset_restore_options\x18\x14 \x01(\bR\x13resetRestoreOptions\x125\n" + "\x17last_unpause_request_id\x18\x15 \x01(\tR\x14lastUnpauseRequestId\x121\n" + "\x15last_reset_request_id\x18\x16 \x01(\tR\x12lastResetRequestId\x12B\n" + - "\x1elast_update_options_request_id\x18\x17 \x01(\tR\x1alastUpdateOptionsRequestId\"\xa7\x01\n" + + "\x1elast_update_options_request_id\x18\x17 \x01(\tR\x1alastUpdateOptionsRequestId\x128\n" + + "\x18eager_execution_accepted\x18\x18 \x01(\bR\x16eagerExecutionAccepted\"\xa7\x01\n" + "\x13ActivityCancelState\x12\x1d\n" + "\n" + "request_id\x18\x01 \x01(\tR\trequestId\x12=\n" + diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index 1fe8455c3e0..609fff7ca71 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -78,7 +78,13 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St } maxCallbacks := h.config.MaxCallbacksPerExecution(frontendReq.GetNamespace()) - h.maybeDisableEagerActivityDispatch(ctx, req.GetNamespaceId(), frontendReq) + if frontendReq.GetRequestEagerExecution() && + h.config.EnableActivityEagerDispatchCheck(frontendReq.GetNamespace()) && + !h.grantEagerActivityDispatch(ctx, req.GetNamespaceId(), frontendReq) { + // Matching grants are best-effort. On a denial or any Matching failure, schedule the + // activity normally instead of failing the start request. + frontendReq.RequestEagerExecution = false + } result, err := chasm.StartExecution( ctx, @@ -100,7 +106,14 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St ) if err != nil { - return nil, startActivityExecutionError(err) + if alreadyStartedErr, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { + return nil, serviceerror.NewActivityExecutionAlreadyStarted( + "activity execution already started", + alreadyStartedErr.CurrentRequestID, + alreadyStartedErr.CurrentRunID, + ) + } + return nil, err } if result.Created { @@ -246,34 +259,6 @@ func businessIDPolicies(request *workflowservice.StartActivityExecutionRequest) return reusePolicy, conflictPolicy, nil } -func (h *handler) maybeDisableEagerActivityDispatch( - ctx context.Context, - namespaceID string, - request *workflowservice.StartActivityExecutionRequest, -) { - if request.GetRequestEagerExecution() && !h.eagerActivityDispatchAllowed(ctx, namespaceID, request) { - // Matching grants are best-effort. On a denial or any Matching failure, schedule the - // activity normally instead of failing the start request. - request.RequestEagerExecution = false - } -} - -func startActivityExecutionError(err error) error { - if alreadyStartedErr, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { - return serviceerror.NewActivityExecutionAlreadyStarted("activity execution already started", alreadyStartedErr.CurrentRequestID, alreadyStartedErr.CurrentRunID) - } - return err -} - -func (h *handler) eagerActivityDispatchAllowed( - ctx context.Context, - namespaceID string, - request *workflowservice.StartActivityExecutionRequest, -) bool { - return !h.config.EnableActivityEagerDispatchCheck(request.GetNamespace()) || - h.grantEagerActivityDispatch(ctx, namespaceID, request) -} - func (h *handler) grantEagerActivityDispatch( ctx context.Context, namespaceID string, diff --git a/chasm/lib/activity/handler_test.go b/chasm/lib/activity/handler_test.go index 5ab1ab317ef..d4f3a571a23 100644 --- a/chasm/lib/activity/handler_test.go +++ b/chasm/lib/activity/handler_test.go @@ -12,7 +12,6 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/api/matchingservicemock/v1" - "go.temporal.io/server/common/dynamicconfig" "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -76,21 +75,3 @@ func TestGrantEagerActivityDispatch(t *testing.T) { }) } } - -func TestEagerActivityDispatchCheck(t *testing.T) { - request := &workflowservice.StartActivityExecutionRequest{Namespace: "namespace"} - - t.Run("disabled skips Matching", func(t *testing.T) { - handler := &handler{config: &Config{ - EnableActivityEagerDispatchCheck: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(false), - }} - require.True(t, handler.eagerActivityDispatchAllowed(context.Background(), "namespace-id", request)) - }) - - t.Run("enabled falls back when Matching is unavailable", func(t *testing.T) { - handler := &handler{config: &Config{ - EnableActivityEagerDispatchCheck: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(true), - }} - require.False(t, handler.eagerActivityDispatchAllowed(context.Background(), "namespace-id", request)) - }) -} diff --git a/chasm/lib/activity/proto/v1/activity_state.proto b/chasm/lib/activity/proto/v1/activity_state.proto index 700792e6dbd..4b36a1f5e4b 100644 --- a/chasm/lib/activity/proto/v1/activity_state.proto +++ b/chasm/lib/activity/proto/v1/activity_state.proto @@ -162,6 +162,9 @@ message ActivityState { // Used to de-dupe update requests. string last_update_options_request_id = 23; + + // Indicates that eager execution was accepted for this activity's first task. + bool eager_execution_accepted = 24; } message ActivityCancelState { diff --git a/chasm/lib/activity/responses.go b/chasm/lib/activity/responses.go index bafe64144d2..bf023596d5e 100644 --- a/chasm/lib/activity/responses.go +++ b/chasm/lib/activity/responses.go @@ -10,6 +10,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/tasktoken" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" @@ -55,24 +56,76 @@ func (a *Activity) eagerActivityTaskData( key := ctx.ExecutionKey() requestData := a.RequestData.Get(ctx) lastHeartbeat, _ := a.LastHeartbeat.TryGet(ctx) + activityType := a.GetActivityType() + header := requestData.GetHeader() + input := requestData.GetInput() + heartbeatDetails := lastHeartbeat.GetDetails() + scheduledTime := a.GetScheduleTime() + currentAttemptScheduledTime := a.dispatchTimeForAttempt(attempt) + startedTime := attempt.GetStartedTime() + scheduleToCloseTimeout := a.GetScheduleToCloseTimeout() + startToCloseTimeout := a.GetStartToCloseTimeout() + heartbeatTimeout := a.GetHeartbeatTimeout() + retryPolicy := a.GetRetryPolicy() + priority := a.GetPriority() + + // This data is used to construct the eager response after the transaction has completed, so + // it must not retain mutable state owned by the transaction. + if activityType != nil { + activityType = common.CloneProto(activityType) + } + if header != nil { + header = common.CloneProto(header) + } + if input != nil { + input = common.CloneProto(input) + } + if heartbeatDetails != nil { + heartbeatDetails = common.CloneProto(heartbeatDetails) + } + if scheduledTime != nil { + scheduledTime = common.CloneProto(scheduledTime) + } + if currentAttemptScheduledTime != nil { + currentAttemptScheduledTime = common.CloneProto(currentAttemptScheduledTime) + } + if startedTime != nil { + startedTime = common.CloneProto(startedTime) + } + if scheduleToCloseTimeout != nil { + scheduleToCloseTimeout = common.CloneProto(scheduleToCloseTimeout) + } + if startToCloseTimeout != nil { + startToCloseTimeout = common.CloneProto(startToCloseTimeout) + } + if heartbeatTimeout != nil { + heartbeatTimeout = common.CloneProto(heartbeatTimeout) + } + if retryPolicy != nil { + retryPolicy = common.CloneProto(retryPolicy) + } + if priority != nil { + priority = common.CloneProto(priority) + } + return &eagerActivityTaskData{ namespaceID: request.namespaceID, namespace: request.namespace, runID: key.RunID, - activityType: a.GetActivityType(), + activityType: activityType, activityID: key.BusinessID, - header: requestData.GetHeader(), - input: requestData.GetInput(), - heartbeatDetails: lastHeartbeat.GetDetails(), - scheduledTime: a.GetScheduleTime(), - currentAttemptScheduledTime: a.dispatchTimeForAttempt(attempt), - startedTime: attempt.GetStartedTime(), + header: header, + input: input, + heartbeatDetails: heartbeatDetails, + scheduledTime: scheduledTime, + currentAttemptScheduledTime: currentAttemptScheduledTime, + startedTime: startedTime, attempt: attempt.GetCount(), - scheduleToCloseTimeout: a.GetScheduleToCloseTimeout(), - startToCloseTimeout: a.GetStartToCloseTimeout(), - heartbeatTimeout: a.GetHeartbeatTimeout(), - retryPolicy: a.GetRetryPolicy(), - priority: a.GetPriority(), + scheduleToCloseTimeout: scheduleToCloseTimeout, + startToCloseTimeout: startToCloseTimeout, + heartbeatTimeout: heartbeatTimeout, + retryPolicy: retryPolicy, + priority: priority, activityAttemptStartedStamp: attempt.GetStartedStamp(), }, nil } @@ -215,6 +268,7 @@ func (a *Activity) buildActivityExecutionInfo( CanceledReason: a.CancelState.GetReason(), CloseTime: closeTime, CurrentRetryInterval: a.currentRetryInterval(ctx, attempt), + EagerExecutionAccepted: a.GetEagerExecutionAccepted(), ExecutionDuration: executionDuration, ExecutionTime: timestamppb.New(a.firstDispatchTime()), ExpirationTime: expirationTime, diff --git a/chasm/lib/activity/responses_test.go b/chasm/lib/activity/responses_test.go index 106b3e69d27..3373ed75d68 100644 --- a/chasm/lib/activity/responses_test.go +++ b/chasm/lib/activity/responses_test.go @@ -5,11 +5,14 @@ import ( "time" "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" deploymentpb "go.temporal.io/api/deployment/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" "go.temporal.io/server/common/testing/protorequire" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" ) func TestBuildActivityExecutionInfo_IncludeLastDeploymentVersion(t *testing.T) { @@ -43,3 +46,117 @@ func TestBuildActivityExecutionInfo_IncludeLastDeploymentVersion(t *testing.T) { BuildId: "test-build-1", }, resp.FrontendResponse.GetInfo().GetLastDeploymentVersion()) } + +func TestBuildActivityExecutionInfo_EagerExecutionAccepted(t *testing.T) { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, + }, + } + + for _, test := range []struct { + name string + accepted bool + }{ + {name: "accepted", accepted: true}, + {name: "not accepted"}, + } { + t.Run(test.name, func(t *testing.T) { + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + EagerExecutionAccepted: test.accepted, + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{}), + RequestData: chasm.NewDataField(ctx, &activitypb.ActivityRequestData{}), + Visibility: chasm.NewComponentField(ctx, &chasm.Visibility{}), + } + + info := activity.buildActivityExecutionInfo(ctx, &workflowservice.DescribeActivityExecutionRequest{}) + require.Equal(t, test.accepted, info.GetEagerExecutionAccepted()) + }) + } +} + +func TestEagerActivityTaskDataSnapshotsState(t *testing.T) { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleExecutionKey: func() chasm.ExecutionKey { + return chasm.ExecutionKey{BusinessID: "activity-id", RunID: "run-id"} + }, + }, + } + + activityType := &commonpb.ActivityType{Name: "activity-type"} + header := &commonpb.Header{Fields: map[string]*commonpb.Payload{"header": {Data: []byte("header")}}} + input := &commonpb.Payloads{Payloads: []*commonpb.Payload{{Data: []byte("input")}}} + heartbeatDetails := &commonpb.Payloads{Payloads: []*commonpb.Payload{{Data: []byte("heartbeat")}}} + scheduledTime := timestamppb.New(time.Unix(10, 0)) + dispatchTime := timestamppb.New(time.Unix(11, 0)) + startedTime := timestamppb.New(time.Unix(12, 0)) + scheduleToCloseTimeout := durationpb.New(time.Minute) + startToCloseTimeout := durationpb.New(2 * time.Minute) + heartbeatTimeout := durationpb.New(3 * time.Minute) + retryPolicy := &commonpb.RetryPolicy{MaximumAttempts: 2} + priority := &commonpb.Priority{PriorityKey: 1} + + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: activityType, + Status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, + ScheduleTime: scheduledTime, + ScheduleToCloseTimeout: scheduleToCloseTimeout, + StartToCloseTimeout: startToCloseTimeout, + HeartbeatTimeout: heartbeatTimeout, + RetryPolicy: retryPolicy, + Priority: priority, + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ + Count: 1, + StartRequestId: "request-id", + StartedTime: startedTime, + DispatchTime: dispatchTime, + }), + LastHeartbeat: chasm.NewDataField(ctx, &activitypb.ActivityHeartbeatState{Details: heartbeatDetails}), + RequestData: chasm.NewDataField(ctx, &activitypb.ActivityRequestData{ + Header: header, + Input: input, + }), + } + + data, err := activity.eagerActivityTaskData(ctx, eagerActivityTaskRequest{ + namespaceID: "namespace-id", + namespace: "namespace", + requestID: "request-id", + }) + require.NoError(t, err) + require.NotNil(t, data) + + activityType.Name = "changed-activity-type" + header.Fields["header"].Data = []byte("changed-header") + input.Payloads[0].Data = []byte("changed-input") + heartbeatDetails.Payloads[0].Data = []byte("changed-heartbeat") + scheduledTime.Seconds++ + dispatchTime.Seconds++ + startedTime.Seconds++ + scheduleToCloseTimeout.Seconds++ + startToCloseTimeout.Seconds++ + heartbeatTimeout.Seconds++ + retryPolicy.MaximumAttempts++ + priority.PriorityKey++ + + response, err := data.response([]byte("component-ref")) + require.NoError(t, err) + require.Equal(t, "activity-type", response.GetActivityType().GetName()) + require.Equal(t, []byte("header"), response.GetHeader().GetFields()["header"].GetData()) + require.Equal(t, []byte("input"), response.GetInput().GetPayloads()[0].GetData()) + require.Equal(t, []byte("heartbeat"), response.GetHeartbeatDetails().GetPayloads()[0].GetData()) + require.Equal(t, int64(10), response.GetScheduledTime().GetSeconds()) + require.Equal(t, int64(11), response.GetCurrentAttemptScheduledTime().GetSeconds()) + require.Equal(t, int64(12), response.GetStartedTime().GetSeconds()) + require.Equal(t, int64(time.Minute/time.Second), response.GetScheduleToCloseTimeout().GetSeconds()) + require.Equal(t, int64(2*time.Minute/time.Second), response.GetStartToCloseTimeout().GetSeconds()) + require.Equal(t, int64(3*time.Minute/time.Second), response.GetHeartbeatTimeout().GetSeconds()) + require.EqualValues(t, 2, response.GetRetryPolicy().GetMaximumAttempts()) + require.EqualValues(t, 1, response.GetPriority().GetPriorityKey()) +} diff --git a/chasm/lib/activity/statemachine.go b/chasm/lib/activity/statemachine.go index 6aa202a1096..b7f1c9ffaab 100644 --- a/chasm/lib/activity/statemachine.go +++ b/chasm/lib/activity/statemachine.go @@ -414,6 +414,7 @@ func (a *Activity) applyScheduled(ctx chasm.MutableContext, dispatch bool) error func (a *Activity) applyEagerStarted(ctx chasm.MutableContext, event eagerStartEvent) error { attempt := a.LastAttempt.Get(ctx) + a.EagerExecutionAccepted = true attempt.StartedTime = timestamppb.New(ctx.Now(a)) attempt.StartedStamp = attempt.GetStamp() if a.FirstAttemptStartedTime == nil { diff --git a/chasm/lib/activity/statemachine_test.go b/chasm/lib/activity/statemachine_test.go index 68698c9de4f..c4f8a44529f 100644 --- a/chasm/lib/activity/statemachine_test.go +++ b/chasm/lib/activity/statemachine_test.go @@ -225,6 +225,7 @@ func TestTransitionEagerStarted(t *testing.T) { }) require.NoError(t, err) require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, activity.GetStatus()) + require.True(t, activity.GetEagerExecutionAccepted()) require.EqualValues(t, 1, attempt.GetCount()) require.EqualValues(t, 1, attempt.GetStamp()) require.EqualValues(t, 1, attempt.GetStartedStamp()) diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 631a758565f..1416042d68e 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1259,6 +1259,7 @@ func (s *standaloneActivityTestSuite) TestEagerStartResponseAndNoRedelivery() { require.NoError(t, err) require.Equal(t, headers.ClientNameGoSDK, describe.GetInfo().GetSdkName()) require.Equal(t, temporal.SDKVersion, describe.GetInfo().GetSdkVersion()) + require.True(t, describe.GetInfo().GetEagerExecutionAccepted()) retry, err := env.FrontendClient().StartActivityExecution(ctx, request) require.NoError(t, err) @@ -1329,6 +1330,22 @@ func (s *standaloneActivityTestSuite) TestEagerStartFallsBackWhenMatchingDenies( eager := start(eagerActivityID, true, 5) require.Nil(t, eager.GetEagerActivityTask(), "Matching denial must fall back to normal dispatch") + for _, startedActivity := range []struct { + id string + runID string + }{ + {backloggedActivityID, backlogged.GetRunId()}, + {eagerActivityID, eager.GetRunId()}, + } { + describe, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: startedActivity.id, + RunId: startedActivity.runID, + }) + require.NoError(t, err) + require.False(t, describe.GetInfo().GetEagerExecutionAccepted()) + } + for _, startedActivity := range []struct { id string runID string From 0242b8f5b1fa7d776f660daed5ffb68749984d80 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 12 Oct 2026 15:25:24 -0500 Subject: [PATCH 17/22] Add eager dispatch observability --- chasm/lib/activity/handler.go | 7 + chasm/lib/activity/handler_test.go | 21 ++- common/metrics/metric_defs.go | 14 +- service/matching/ratelimit_manager.go | 28 +++- service/matching/ratelimit_manager_test.go | 24 +++- .../matching/task_queue_partition_manager.go | 77 ++++++++++- .../task_queue_partition_manager_test.go | 121 ++++++++++++++++++ 7 files changed, 273 insertions(+), 19 deletions(-) diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index 609fff7ca71..ad83d60fdbf 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -78,6 +78,10 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St } maxCallbacks := h.config.MaxCallbacksPerExecution(frontendReq.GetNamespace()) + // TODO: Reconcile a Matching grant when the subsequent start does not create an eager task. + // This RPC stays outside the CHASM transaction so it does not hold the Activity lock while + // waiting for Matching. A duplicate, conflict, or transaction failure can consume a grant + // without producing an eager task. The eager dispatch metrics record this gap. if frontendReq.GetRequestEagerExecution() && h.config.EnableActivityEagerDispatchCheck(frontendReq.GetNamespace()) && !h.grantEagerActivityDispatch(ctx, req.GetNamespaceId(), frontendReq) { @@ -268,6 +272,9 @@ func (h *handler) grantEagerActivityDispatch( return false } + metrics.EagerDispatchRequestsSent.With( + eagerActivityMetricsHandler(h.metricsHandler, h.config, request), + ).Record(1) response, err := h.matchingClient.GrantEagerDispatch(ctx, &matchingservice.GrantEagerDispatchRequest{ NamespaceId: namespaceID, TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ diff --git a/chasm/lib/activity/handler_test.go b/chasm/lib/activity/handler_test.go index d4f3a571a23..123853cba0c 100644 --- a/chasm/lib/activity/handler_test.go +++ b/chasm/lib/activity/handler_test.go @@ -12,6 +12,9 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/api/matchingservicemock/v1" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -58,6 +61,9 @@ func TestGrantEagerActivityDispatch(t *testing.T) { t.Run(test.name, func(t *testing.T) { controller := gomock.NewController(t) matchingClient := matchingservicemock.NewMockMatchingServiceClient(controller) + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) matchingClient.EXPECT().GrantEagerDispatch(gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, actual *matchingservice.GrantEagerDispatchRequest, _ ...grpc.CallOption) (*matchingservice.GrantEagerDispatchResponse, error) { require.Equal(t, "namespace-id", actual.GetNamespaceId()) @@ -70,8 +76,21 @@ func TestGrantEagerActivityDispatch(t *testing.T) { }, ) - handler := &handler{matchingClient: matchingClient} + handler := &handler{ + config: &Config{ + BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), + }, + matchingClient: matchingClient, + metricsHandler: metricsHandler, + } require.Equal(t, test.granted, handler.grantEagerActivityDispatch(context.Background(), "namespace-id", request)) + + recordings := capture.SnapshotMetric(metrics.EagerDispatchRequestsSent.Name()) + require.Len(t, recordings, 1) + require.EqualValues(t, 1, recordings[0].Value) + require.Equal(t, "namespace", recordings[0].Tags[metrics.NamespaceTag("").Key]) + require.Equal(t, "activity-task-queue", recordings[0].Tags[metrics.UnsafeTaskQueueTag("").Key]) + require.Equal(t, "StartActivityExecution", recordings[0].Tags[metrics.OperationTag("").Key]) }) } } diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 1012a5c916f..2a38fe513e7 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1036,6 +1036,10 @@ var ( SpeculativeWorkflowTaskRollbacks = NewCounterDef("speculative_workflow_task_rollbacks") ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") + EagerDispatchRequestsSent = NewCounterDef( + "eager_dispatch_requests_sent", + WithDescription("Number of task grants sent by History through GrantEagerDispatch, summed from the request item counts."), + ) // WorkflowEagerExecutionCounter is emitted any time eager workflow start is requested. WorkflowEagerExecutionCounter = NewCounterDef("workflow_eager_execution") // WorkflowEagerExecutionDeniedCounter is emitted any time eager workflow start is requested and the serer fell back @@ -1331,7 +1335,15 @@ var ( ConditionFailedErrorPerTaskQueueCounter = NewCounterDef("condition_failed_errors") RespondQueryTaskFailedPerTaskQueueCounter = NewCounterDef("respond_query_failed") RespondNexusTaskFailedPerTaskQueueCounter = NewCounterDef("respond_nexus_failed") - NexusTaskRequests = NewCounterDef( + EagerDispatchRequests = NewCounterDef( + "eager_dispatch_requests", + WithDescription("Number of task grants requested through GrantEagerDispatch, summed from the request item counts."), + ) + EagerDispatchResults = NewCounterDef( + "eager_dispatch_results", + WithDescription("Number of requested task grants by outcome: granted, error, backlog, rate_limit, or per_key_rate_limit."), + ) + NexusTaskRequests = NewCounterDef( "nexus_task_requests", WithDescription("The number of Nexus task poll and respond requests received by the matching service, broken down by namespace, operation, client_name, and is_internal."), ) diff --git a/service/matching/ratelimit_manager.go b/service/matching/ratelimit_manager.go index c925bde2407..68b00ac3bb9 100644 --- a/service/matching/ratelimit_manager.go +++ b/service/matching/ratelimit_manager.go @@ -57,8 +57,14 @@ const ( // 100/second and burst duration of 2 seconds, the capacity of a bucket-type limiting // algorithm would be 200. defaultBurstDuration = time.Second + + eagerDispatchRateLimitNone eagerDispatchRateLimit = iota + eagerDispatchRateLimitGeneral + eagerDispatchRateLimitPerKey ) +type eagerDispatchRateLimit int + // Create a new rate limit manager for the task queue partition. This only allocates; // dynamic config subscriptions are registered in Start, so an unstarted manager holds // no external references and can simply be garbage collected. @@ -340,14 +346,22 @@ func (r *rateLimitManager) consumeTokens(now int64, task *internalTask, tokens i } } -func (r *rateLimitManager) grantTokens(priority *commonpb.Priority, requested int32) int32 { +func (r *rateLimitManager) grantTokens( + priority *commonpb.Priority, + requested int32, +) (int32, eagerDispatchRateLimit) { now := r.timeSource.Now() r.mu.Lock() defer r.mu.Unlock() nowNanos := now.UnixNano() - granted := min(requested, r.wholeQueueReady.Available(r.wholeQueueLimit, nowNanos)) + wholeQueueAvailable := r.wholeQueueReady.Available(r.wholeQueueLimit, nowNanos) + granted := min(requested, wholeQueueAvailable) + limitedBy := eagerDispatchRateLimitNone + if granted < requested { + limitedBy = eagerDispatchRateLimitGeneral + } if r.perKeyLimit.Limited() { key := priority.GetFairnessKey() var ready simplelimiter.Ready @@ -355,9 +369,13 @@ func (r *rateLimitManager) grantTokens(priority *commonpb.Priority, requested in ready = value.(simplelimiter.Ready) // nolint:revive } params := r.perKeyLimit.DivideInterval(getEffectiveWeight(r.perKeyOverrides, priority)) - granted = min(granted, ready.Available(params, nowNanos)) + perKeyAvailable := ready.Available(params, nowNanos) + if perKeyAvailable < granted { + limitedBy = eagerDispatchRateLimitPerKey + } + granted = min(granted, perKeyAvailable) if granted == 0 { - return 0 + return 0, limitedBy } r.perKeyReady.Put(key, ready.Consume(params, nowNanos, int64(granted))) } @@ -365,7 +383,7 @@ func (r *rateLimitManager) grantTokens(priority *commonpb.Priority, requested in if granted > 0 { r.wholeQueueReady = r.wholeQueueReady.Consume(r.wholeQueueLimit, nowNanos, int64(granted)) } - return granted + return granted, limitedBy } // GetFairnessWeightOverrides returns the current fairness weight overrides. diff --git a/service/matching/ratelimit_manager_test.go b/service/matching/ratelimit_manager_test.go index ce30c336b65..5f3b56e70ac 100644 --- a/service/matching/ratelimit_manager_test.go +++ b/service/matching/ratelimit_manager_test.go @@ -72,8 +72,12 @@ func TestRateLimitManagerGrantTokens(t *testing.T) { wholeQueueLimit: simplelimiter.MakeParams(2, 500*time.Millisecond), } - require.Equal(t, int32(2), manager.grantTokens(nil, 5)) - require.Equal(t, int32(0), manager.grantTokens(nil, 1)) + granted, limitedBy := manager.grantTokens(nil, 5) + require.Equal(t, int32(2), granted) + require.Equal(t, eagerDispatchRateLimitGeneral, limitedBy) + granted, limitedBy = manager.grantTokens(nil, 1) + require.Equal(t, int32(0), granted) + require.Equal(t, eagerDispatchRateLimitGeneral, limitedBy) }) t.Run("new matcher isolates fairness keys", func(t *testing.T) { @@ -88,12 +92,20 @@ func TestRateLimitManagerGrantTokens(t *testing.T) { keyOne := &commonpb.Priority{FairnessKey: "one"} keyTwo := &commonpb.Priority{FairnessKey: "two"} - require.Equal(t, int32(1), manager.grantTokens(keyOne, 20)) - require.Equal(t, int32(0), manager.grantTokens(keyOne, 1)) - require.Equal(t, int32(1), manager.grantTokens(keyTwo, 1)) + granted, limitedBy := manager.grantTokens(keyOne, 20) + require.Equal(t, int32(1), granted) + require.Equal(t, eagerDispatchRateLimitPerKey, limitedBy) + granted, limitedBy = manager.grantTokens(keyOne, 1) + require.Equal(t, int32(0), granted) + require.Equal(t, eagerDispatchRateLimitPerKey, limitedBy) + granted, limitedBy = manager.grantTokens(keyTwo, 1) + require.Equal(t, int32(1), granted) + require.Equal(t, eagerDispatchRateLimitNone, limitedBy) timeSource.Advance(time.Second) - require.Equal(t, int32(1), manager.grantTokens(keyOne, 2)) + granted, limitedBy = manager.grantTokens(keyOne, 2) + require.Equal(t, int32(1), granted) + require.Equal(t, eagerDispatchRateLimitPerKey, limitedBy) }) } diff --git a/service/matching/task_queue_partition_manager.go b/service/matching/task_queue_partition_manager.go index b82498ba62c..d8d3345291f 100644 --- a/service/matching/task_queue_partition_manager.go +++ b/service/matching/task_queue_partition_manager.go @@ -660,10 +660,56 @@ type eagerDispatchGrantTarget struct { checkDefaultBacklog bool } +const ( + eagerDispatchOutcomeGranted = "granted" + eagerDispatchOutcomeError = "error" + eagerDispatchOutcomeBacklog = "backlog" + eagerDispatchOutcomeRateLimit = "rate_limit" + eagerDispatchOutcomePerKeyRateLimit = "per_key_rate_limit" +) + +type eagerDispatchOutcomeCounts struct { + granted int64 + backlog int64 + rateLimit int64 + perKeyRateLimit int64 +} + +func (counts eagerDispatchOutcomeCounts) record(metricsHandler metrics.Handler) { + recordEagerDispatchResult(metricsHandler, eagerDispatchOutcomeGranted, counts.granted) + recordEagerDispatchResult(metricsHandler, eagerDispatchOutcomeBacklog, counts.backlog) + recordEagerDispatchResult(metricsHandler, eagerDispatchOutcomeRateLimit, counts.rateLimit) + recordEagerDispatchResult(metricsHandler, eagerDispatchOutcomePerKeyRateLimit, counts.perKeyRateLimit) +} + +func recordEagerDispatchResult(metricsHandler metrics.Handler, outcome string, count int64) { + if count > 0 { + metrics.EagerDispatchResults.With(metricsHandler).Record(count, metrics.OutcomeTag(outcome)) + } +} + +func eagerDispatchRequestCount(items []*matchingservice.GrantEagerDispatchRequest_Item) int64 { + var count int64 + for _, item := range items { + if itemCount := item.GetCount(); itemCount > 0 { + count += int64(itemCount) + } + } + return count +} + func (pm *taskQueuePartitionManagerImpl) GrantEagerDispatch( ctx context.Context, items []*matchingservice.GrantEagerDispatchRequest_Item, -) ([]*matchingservice.GrantEagerDispatchResponse_Item, error) { +) (_ []*matchingservice.GrantEagerDispatchResponse_Item, retErr error) { + requestedCount := eagerDispatchRequestCount(items) + metrics.EagerDispatchRequests.With(pm.metricsHandler).Record(requestedCount) + defer func() { + if retErr != nil { + recordEagerDispatchResult(pm.metricsHandler, eagerDispatchOutcomeError, requestedCount) + } + }() + if _, ok := pm.partition.(*tqid.NormalPartition); !ok { return nil, serviceerror.NewInvalidArgument("eager dispatch grants only support normal task queue partitions") } @@ -678,7 +724,12 @@ func (pm *taskQueuePartitionManagerImpl) GrantEagerDispatch( if err != nil { return nil, err } - return pm.grantEagerDispatch(items, targets) + responseItems, outcomeCounts, err := pm.grantEagerDispatch(items, targets) + if err != nil { + return nil, err + } + outcomeCounts.record(pm.metricsHandler) + return responseItems, nil } func (pm *taskQueuePartitionManagerImpl) resolveEagerDispatchGrantTargets( @@ -738,13 +789,14 @@ func (pm *taskQueuePartitionManagerImpl) getEagerDispatchVersioningInfo() (eager func (pm *taskQueuePartitionManagerImpl) grantEagerDispatch( items []*matchingservice.GrantEagerDispatchRequest_Item, targets []eagerDispatchGrantTarget, -) ([]*matchingservice.GrantEagerDispatchResponse_Item, error) { +) ([]*matchingservice.GrantEagerDispatchResponse_Item, eagerDispatchOutcomeCounts, error) { responseItems := make([]*matchingservice.GrantEagerDispatchResponse_Item, len(items)) + var outcomeCounts eagerDispatchOutcomeCounts backlogPriorities := make(map[physicalTaskQueueManager]priorityKey) for index, item := range items { backlogPriority, err := pm.eagerDispatchBacklogPriority(targets[index], backlogPriorities) if err != nil { - return nil, err + return nil, eagerDispatchOutcomeCounts{}, err } priority := pm.config.clipPriority(priorityKey(item.GetPriority().GetPriorityKey())) @@ -752,13 +804,26 @@ func (pm *taskQueuePartitionManagerImpl) grantEagerDispatch( // backlog priority is higher and prevents eager dispatch when it is <= priority. granted := int32(0) if backlogPriority == 0 || backlogPriority > priority { - granted = pm.rateLimitManager.grantTokens(item.GetPriority(), item.GetCount()) + var limitedBy eagerDispatchRateLimit + granted, limitedBy = pm.rateLimitManager.grantTokens(item.GetPriority(), item.GetCount()) + outcomeCounts.granted += int64(granted) + denied := int64(item.GetCount() - granted) + switch limitedBy { + case eagerDispatchRateLimitGeneral: + outcomeCounts.rateLimit += denied + case eagerDispatchRateLimitPerKey: + outcomeCounts.perKeyRateLimit += denied + default: + // All requested tokens were granted. + } + } else { + outcomeCounts.backlog += int64(item.GetCount()) } responseItems[index] = &matchingservice.GrantEagerDispatchResponse_Item{ GrantedCount: granted, } } - return responseItems, nil + return responseItems, outcomeCounts, nil } func (pm *taskQueuePartitionManagerImpl) eagerDispatchBacklogPriority( diff --git a/service/matching/task_queue_partition_manager_test.go b/service/matching/task_queue_partition_manager_test.go index 14c30a01ca9..f9b81efa9f6 100644 --- a/service/matching/task_queue_partition_manager_test.go +++ b/service/matching/task_queue_partition_manager_test.go @@ -278,6 +278,95 @@ func (s *PartitionManagerTestSuite) TestGrantEagerDispatchReturnsPartialRateLimi }, items) } +func (s *PartitionManagerTestSuite) TestGrantEagerDispatchMetrics() { + tests := []struct { + name string + backlogPriority priorityKey + configure func(*rateLimitManager) + items []*matchingservice.GrantEagerDispatchRequest_Item + expectedError bool + expectedRequest int64 + expectedResults map[string]int64 + }{ + { + name: "granted", + items: []*matchingservice.GrantEagerDispatchRequest_Item{ + {Count: 2}, + }, + expectedRequest: 2, + expectedResults: map[string]int64{eagerDispatchOutcomeGranted: 2}, + }, + { + name: "backlog", + backlogPriority: 2, + items: []*matchingservice.GrantEagerDispatchRequest_Item{ + {Count: 3, Priority: &commonpb.Priority{PriorityKey: 3}}, + }, + expectedRequest: 3, + expectedResults: map[string]int64{eagerDispatchOutcomeBacklog: 3}, + }, + { + name: "general rate limit", + configure: func(rateLimitManager *rateLimitManager) { + rateLimitManager.wholeQueueLimit = simplelimiter.MakeParams(1, 0) + }, + items: []*matchingservice.GrantEagerDispatchRequest_Item{ + {Count: 3}, + }, + expectedRequest: 3, + expectedResults: map[string]int64{ + eagerDispatchOutcomeGranted: 1, + eagerDispatchOutcomeRateLimit: 2, + }, + }, + { + name: "per-key rate limit", + configure: func(rateLimitManager *rateLimitManager) { + rateLimitManager.perKeyLimit = simplelimiter.MakeParams(1, 0) + }, + items: []*matchingservice.GrantEagerDispatchRequest_Item{ + {Count: 3, Priority: &commonpb.Priority{FairnessKey: "key"}}, + }, + expectedRequest: 3, + expectedResults: map[string]int64{ + eagerDispatchOutcomeGranted: 1, + eagerDispatchOutcomePerKeyRateLimit: 2, + }, + }, + { + name: "error", + items: []*matchingservice.GrantEagerDispatchRequest_Item{ + {Count: 3, Version: &deploymentspb.WorkerDeploymentVersion{}}, + }, + expectedError: true, + expectedRequest: 3, + expectedResults: map[string]int64{eagerDispatchOutcomeError: 3}, + }, + } + + for _, test := range tests { + s.Run(test.name, func() { + partitionMgr, capture, cleanup := s.newEagerDispatchPartitionManagerWithMetrics(test.backlogPriority) + defer cleanup() + if test.configure != nil { + test.configure(partitionMgr.rateLimitManager) + } + + _, err := partitionMgr.GrantEagerDispatch(context.Background(), test.items) + if test.expectedError { + s.Require().Error(err) + } else { + s.Require().NoError(err) + } + + requestMetrics := capture.SnapshotMetric(metrics.EagerDispatchRequests.Name()) + s.Require().Len(requestMetrics, 1) + s.Equal(test.expectedRequest, requestMetrics[0].Value) + s.Equal(test.expectedResults, eagerDispatchResultMetrics(capture)) + }) + } +} + func (s *PartitionManagerTestSuite) TestGrantEagerDispatchValidationDoesNotConsumeTokens() { partitionMgr := s.newRateLimitedEagerDispatchPartitionManager() @@ -2812,11 +2901,43 @@ func (s *PartitionManagerTestSuite) newEagerDispatchPartitionManager( userDataManager: s.partitionMgr.userDataManager, logger: s.partitionMgr.logger, throttledLogger: s.partitionMgr.throttledLogger, + metricsHandler: s.partitionMgr.metricsHandler, rateLimitManager: s.partitionMgr.rateLimitManager, defaultQueueFuture: defaultQueueFuture, } } +func (s *PartitionManagerTestSuite) newEagerDispatchPartitionManagerWithMetrics( + defaultBacklogPriority priorityKey, +) (*taskQueuePartitionManagerImpl, *metricstest.Capture, func()) { + partitionMgr := s.newEagerDispatchPartitionManager(defaultBacklogPriority, nil) + captureHandler := metricstest.NewCaptureHandler() + capture := captureHandler.StartCapture() + partitionMgr.metricsHandler = metrics.GetPerTaskQueuePartitionIDScope( + captureHandler, + namespaceName, + partitionMgr.partition, + true, + true, + metrics.OperationTag(metrics.MatchingTaskQueuePartitionManagerScope), + ) + partitionMgr.rateLimitManager = newRateLimitManager( + partitionMgr.userDataManager, + partitionMgr.config, + partitionMgr.partition.TaskQueue().TaskType(), + ) + partitionMgr.rateLimitManager.timeSource = clock.NewEventTimeSource().Update(time.Now()) + return partitionMgr, capture, func() { captureHandler.StopCapture(capture) } +} + +func eagerDispatchResultMetrics(capture *metricstest.Capture) map[string]int64 { + results := make(map[string]int64) + for _, recording := range capture.SnapshotMetric(metrics.EagerDispatchResults.Name()) { + results[recording.Tags[metrics.OutcomeTag("").Key]] += recording.Value.(int64) + } + return results +} + func workerDeploymentVersion(buildID string) *deploymentspb.WorkerDeploymentVersion { return &deploymentspb.WorkerDeploymentVersion{ DeploymentName: "deployment", From 1f944bf68b3c0230eff3ac5073839a68273e0ba3 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 12 Oct 2026 15:29:12 -0500 Subject: [PATCH 18/22] Simplify eager dispatch reconciliation TODO --- chasm/lib/activity/handler.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index ad83d60fdbf..dcfd106f0ee 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -78,10 +78,7 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St } maxCallbacks := h.config.MaxCallbacksPerExecution(frontendReq.GetNamespace()) - // TODO: Reconcile a Matching grant when the subsequent start does not create an eager task. - // This RPC stays outside the CHASM transaction so it does not hold the Activity lock while - // waiting for Matching. A duplicate, conflict, or transaction failure can consume a grant - // without producing an eager task. The eager dispatch metrics record this gap. + // TODO: Reconcile the Matching grant when the subsequent start does not create an eager task. if frontendReq.GetRequestEagerExecution() && h.config.EnableActivityEagerDispatchCheck(frontendReq.GetNamespace()) && !h.grantEagerActivityDispatch(ctx, req.GetNamespaceId(), frontendReq) { From 3ca44940e9e66f2cc22b48f2fe99fce369d6b62c Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Mon, 12 Oct 2026 15:47:48 -0500 Subject: [PATCH 19/22] Align SAA eager dispatch metrics --- chasm/lib/activity/handler_test.go | 10 ++++++++++ common/metrics/metric_defs.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/chasm/lib/activity/handler_test.go b/chasm/lib/activity/handler_test.go index 123853cba0c..1aa5fde40f4 100644 --- a/chasm/lib/activity/handler_test.go +++ b/chasm/lib/activity/handler_test.go @@ -93,4 +93,14 @@ func TestGrantEagerActivityDispatch(t *testing.T) { require.Equal(t, "StartActivityExecution", recordings[0].Tags[metrics.OperationTag("").Key]) }) } + + t.Run("no matching client", func(t *testing.T) { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + handler := &handler{metricsHandler: metricsHandler} + + require.False(t, handler.grantEagerActivityDispatch(context.Background(), "namespace-id", request)) + require.Empty(t, capture.SnapshotMetric(metrics.EagerDispatchRequestsSent.Name())) + }) } diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 2a38fe513e7..19e1c2b91a3 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1038,7 +1038,7 @@ var ( ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") EagerDispatchRequestsSent = NewCounterDef( "eager_dispatch_requests_sent", - WithDescription("Number of task grants sent by History through GrantEagerDispatch, summed from the request item counts."), + WithDescription("Number of task grants sent to Matching through GrantEagerDispatch, summed from the request item counts."), ) // WorkflowEagerExecutionCounter is emitted any time eager workflow start is requested. WorkflowEagerExecutionCounter = NewCounterDef("workflow_eager_execution") From 296e0a0be5e9456c6087912923e1b5393d5162fa Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Tue, 13 Oct 2026 14:56:58 -0500 Subject: [PATCH 20/22] Revert "Add eager dispatch observability" This reverts commit 0242b8f5b1fa7d776f660daed5ffb68749984d80. --- chasm/lib/activity/handler.go | 3 - chasm/lib/activity/handler_test.go | 31 +---- common/metrics/metric_defs.go | 14 +- service/matching/ratelimit_manager.go | 28 +--- service/matching/ratelimit_manager_test.go | 24 +--- .../matching/task_queue_partition_manager.go | 77 +---------- .../task_queue_partition_manager_test.go | 121 ------------------ 7 files changed, 19 insertions(+), 279 deletions(-) diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index dcfd106f0ee..88e61059ff5 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -269,9 +269,6 @@ func (h *handler) grantEagerActivityDispatch( return false } - metrics.EagerDispatchRequestsSent.With( - eagerActivityMetricsHandler(h.metricsHandler, h.config, request), - ).Record(1) response, err := h.matchingClient.GrantEagerDispatch(ctx, &matchingservice.GrantEagerDispatchRequest{ NamespaceId: namespaceID, TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ diff --git a/chasm/lib/activity/handler_test.go b/chasm/lib/activity/handler_test.go index 1aa5fde40f4..d4f3a571a23 100644 --- a/chasm/lib/activity/handler_test.go +++ b/chasm/lib/activity/handler_test.go @@ -12,9 +12,6 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/api/matchingservicemock/v1" - "go.temporal.io/server/common/dynamicconfig" - "go.temporal.io/server/common/metrics" - "go.temporal.io/server/common/metrics/metricstest" "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -61,9 +58,6 @@ func TestGrantEagerActivityDispatch(t *testing.T) { t.Run(test.name, func(t *testing.T) { controller := gomock.NewController(t) matchingClient := matchingservicemock.NewMockMatchingServiceClient(controller) - metricsHandler := metricstest.NewCaptureHandler() - capture := metricsHandler.StartCapture() - defer metricsHandler.StopCapture(capture) matchingClient.EXPECT().GrantEagerDispatch(gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, actual *matchingservice.GrantEagerDispatchRequest, _ ...grpc.CallOption) (*matchingservice.GrantEagerDispatchResponse, error) { require.Equal(t, "namespace-id", actual.GetNamespaceId()) @@ -76,31 +70,8 @@ func TestGrantEagerActivityDispatch(t *testing.T) { }, ) - handler := &handler{ - config: &Config{ - BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), - }, - matchingClient: matchingClient, - metricsHandler: metricsHandler, - } + handler := &handler{matchingClient: matchingClient} require.Equal(t, test.granted, handler.grantEagerActivityDispatch(context.Background(), "namespace-id", request)) - - recordings := capture.SnapshotMetric(metrics.EagerDispatchRequestsSent.Name()) - require.Len(t, recordings, 1) - require.EqualValues(t, 1, recordings[0].Value) - require.Equal(t, "namespace", recordings[0].Tags[metrics.NamespaceTag("").Key]) - require.Equal(t, "activity-task-queue", recordings[0].Tags[metrics.UnsafeTaskQueueTag("").Key]) - require.Equal(t, "StartActivityExecution", recordings[0].Tags[metrics.OperationTag("").Key]) }) } - - t.Run("no matching client", func(t *testing.T) { - metricsHandler := metricstest.NewCaptureHandler() - capture := metricsHandler.StartCapture() - defer metricsHandler.StopCapture(capture) - handler := &handler{metricsHandler: metricsHandler} - - require.False(t, handler.grantEagerActivityDispatch(context.Background(), "namespace-id", request)) - require.Empty(t, capture.SnapshotMetric(metrics.EagerDispatchRequestsSent.Name())) - }) } diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 19e1c2b91a3..1012a5c916f 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1036,10 +1036,6 @@ var ( SpeculativeWorkflowTaskRollbacks = NewCounterDef("speculative_workflow_task_rollbacks") ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") - EagerDispatchRequestsSent = NewCounterDef( - "eager_dispatch_requests_sent", - WithDescription("Number of task grants sent to Matching through GrantEagerDispatch, summed from the request item counts."), - ) // WorkflowEagerExecutionCounter is emitted any time eager workflow start is requested. WorkflowEagerExecutionCounter = NewCounterDef("workflow_eager_execution") // WorkflowEagerExecutionDeniedCounter is emitted any time eager workflow start is requested and the serer fell back @@ -1335,15 +1331,7 @@ var ( ConditionFailedErrorPerTaskQueueCounter = NewCounterDef("condition_failed_errors") RespondQueryTaskFailedPerTaskQueueCounter = NewCounterDef("respond_query_failed") RespondNexusTaskFailedPerTaskQueueCounter = NewCounterDef("respond_nexus_failed") - EagerDispatchRequests = NewCounterDef( - "eager_dispatch_requests", - WithDescription("Number of task grants requested through GrantEagerDispatch, summed from the request item counts."), - ) - EagerDispatchResults = NewCounterDef( - "eager_dispatch_results", - WithDescription("Number of requested task grants by outcome: granted, error, backlog, rate_limit, or per_key_rate_limit."), - ) - NexusTaskRequests = NewCounterDef( + NexusTaskRequests = NewCounterDef( "nexus_task_requests", WithDescription("The number of Nexus task poll and respond requests received by the matching service, broken down by namespace, operation, client_name, and is_internal."), ) diff --git a/service/matching/ratelimit_manager.go b/service/matching/ratelimit_manager.go index 68b00ac3bb9..c925bde2407 100644 --- a/service/matching/ratelimit_manager.go +++ b/service/matching/ratelimit_manager.go @@ -57,14 +57,8 @@ const ( // 100/second and burst duration of 2 seconds, the capacity of a bucket-type limiting // algorithm would be 200. defaultBurstDuration = time.Second - - eagerDispatchRateLimitNone eagerDispatchRateLimit = iota - eagerDispatchRateLimitGeneral - eagerDispatchRateLimitPerKey ) -type eagerDispatchRateLimit int - // Create a new rate limit manager for the task queue partition. This only allocates; // dynamic config subscriptions are registered in Start, so an unstarted manager holds // no external references and can simply be garbage collected. @@ -346,22 +340,14 @@ func (r *rateLimitManager) consumeTokens(now int64, task *internalTask, tokens i } } -func (r *rateLimitManager) grantTokens( - priority *commonpb.Priority, - requested int32, -) (int32, eagerDispatchRateLimit) { +func (r *rateLimitManager) grantTokens(priority *commonpb.Priority, requested int32) int32 { now := r.timeSource.Now() r.mu.Lock() defer r.mu.Unlock() nowNanos := now.UnixNano() - wholeQueueAvailable := r.wholeQueueReady.Available(r.wholeQueueLimit, nowNanos) - granted := min(requested, wholeQueueAvailable) - limitedBy := eagerDispatchRateLimitNone - if granted < requested { - limitedBy = eagerDispatchRateLimitGeneral - } + granted := min(requested, r.wholeQueueReady.Available(r.wholeQueueLimit, nowNanos)) if r.perKeyLimit.Limited() { key := priority.GetFairnessKey() var ready simplelimiter.Ready @@ -369,13 +355,9 @@ func (r *rateLimitManager) grantTokens( ready = value.(simplelimiter.Ready) // nolint:revive } params := r.perKeyLimit.DivideInterval(getEffectiveWeight(r.perKeyOverrides, priority)) - perKeyAvailable := ready.Available(params, nowNanos) - if perKeyAvailable < granted { - limitedBy = eagerDispatchRateLimitPerKey - } - granted = min(granted, perKeyAvailable) + granted = min(granted, ready.Available(params, nowNanos)) if granted == 0 { - return 0, limitedBy + return 0 } r.perKeyReady.Put(key, ready.Consume(params, nowNanos, int64(granted))) } @@ -383,7 +365,7 @@ func (r *rateLimitManager) grantTokens( if granted > 0 { r.wholeQueueReady = r.wholeQueueReady.Consume(r.wholeQueueLimit, nowNanos, int64(granted)) } - return granted, limitedBy + return granted } // GetFairnessWeightOverrides returns the current fairness weight overrides. diff --git a/service/matching/ratelimit_manager_test.go b/service/matching/ratelimit_manager_test.go index 5f3b56e70ac..ce30c336b65 100644 --- a/service/matching/ratelimit_manager_test.go +++ b/service/matching/ratelimit_manager_test.go @@ -72,12 +72,8 @@ func TestRateLimitManagerGrantTokens(t *testing.T) { wholeQueueLimit: simplelimiter.MakeParams(2, 500*time.Millisecond), } - granted, limitedBy := manager.grantTokens(nil, 5) - require.Equal(t, int32(2), granted) - require.Equal(t, eagerDispatchRateLimitGeneral, limitedBy) - granted, limitedBy = manager.grantTokens(nil, 1) - require.Equal(t, int32(0), granted) - require.Equal(t, eagerDispatchRateLimitGeneral, limitedBy) + require.Equal(t, int32(2), manager.grantTokens(nil, 5)) + require.Equal(t, int32(0), manager.grantTokens(nil, 1)) }) t.Run("new matcher isolates fairness keys", func(t *testing.T) { @@ -92,20 +88,12 @@ func TestRateLimitManagerGrantTokens(t *testing.T) { keyOne := &commonpb.Priority{FairnessKey: "one"} keyTwo := &commonpb.Priority{FairnessKey: "two"} - granted, limitedBy := manager.grantTokens(keyOne, 20) - require.Equal(t, int32(1), granted) - require.Equal(t, eagerDispatchRateLimitPerKey, limitedBy) - granted, limitedBy = manager.grantTokens(keyOne, 1) - require.Equal(t, int32(0), granted) - require.Equal(t, eagerDispatchRateLimitPerKey, limitedBy) - granted, limitedBy = manager.grantTokens(keyTwo, 1) - require.Equal(t, int32(1), granted) - require.Equal(t, eagerDispatchRateLimitNone, limitedBy) + require.Equal(t, int32(1), manager.grantTokens(keyOne, 20)) + require.Equal(t, int32(0), manager.grantTokens(keyOne, 1)) + require.Equal(t, int32(1), manager.grantTokens(keyTwo, 1)) timeSource.Advance(time.Second) - granted, limitedBy = manager.grantTokens(keyOne, 2) - require.Equal(t, int32(1), granted) - require.Equal(t, eagerDispatchRateLimitPerKey, limitedBy) + require.Equal(t, int32(1), manager.grantTokens(keyOne, 2)) }) } diff --git a/service/matching/task_queue_partition_manager.go b/service/matching/task_queue_partition_manager.go index d8d3345291f..b82498ba62c 100644 --- a/service/matching/task_queue_partition_manager.go +++ b/service/matching/task_queue_partition_manager.go @@ -660,56 +660,10 @@ type eagerDispatchGrantTarget struct { checkDefaultBacklog bool } -const ( - eagerDispatchOutcomeGranted = "granted" - eagerDispatchOutcomeError = "error" - eagerDispatchOutcomeBacklog = "backlog" - eagerDispatchOutcomeRateLimit = "rate_limit" - eagerDispatchOutcomePerKeyRateLimit = "per_key_rate_limit" -) - -type eagerDispatchOutcomeCounts struct { - granted int64 - backlog int64 - rateLimit int64 - perKeyRateLimit int64 -} - -func (counts eagerDispatchOutcomeCounts) record(metricsHandler metrics.Handler) { - recordEagerDispatchResult(metricsHandler, eagerDispatchOutcomeGranted, counts.granted) - recordEagerDispatchResult(metricsHandler, eagerDispatchOutcomeBacklog, counts.backlog) - recordEagerDispatchResult(metricsHandler, eagerDispatchOutcomeRateLimit, counts.rateLimit) - recordEagerDispatchResult(metricsHandler, eagerDispatchOutcomePerKeyRateLimit, counts.perKeyRateLimit) -} - -func recordEagerDispatchResult(metricsHandler metrics.Handler, outcome string, count int64) { - if count > 0 { - metrics.EagerDispatchResults.With(metricsHandler).Record(count, metrics.OutcomeTag(outcome)) - } -} - -func eagerDispatchRequestCount(items []*matchingservice.GrantEagerDispatchRequest_Item) int64 { - var count int64 - for _, item := range items { - if itemCount := item.GetCount(); itemCount > 0 { - count += int64(itemCount) - } - } - return count -} - func (pm *taskQueuePartitionManagerImpl) GrantEagerDispatch( ctx context.Context, items []*matchingservice.GrantEagerDispatchRequest_Item, -) (_ []*matchingservice.GrantEagerDispatchResponse_Item, retErr error) { - requestedCount := eagerDispatchRequestCount(items) - metrics.EagerDispatchRequests.With(pm.metricsHandler).Record(requestedCount) - defer func() { - if retErr != nil { - recordEagerDispatchResult(pm.metricsHandler, eagerDispatchOutcomeError, requestedCount) - } - }() - +) ([]*matchingservice.GrantEagerDispatchResponse_Item, error) { if _, ok := pm.partition.(*tqid.NormalPartition); !ok { return nil, serviceerror.NewInvalidArgument("eager dispatch grants only support normal task queue partitions") } @@ -724,12 +678,7 @@ func (pm *taskQueuePartitionManagerImpl) GrantEagerDispatch( if err != nil { return nil, err } - responseItems, outcomeCounts, err := pm.grantEagerDispatch(items, targets) - if err != nil { - return nil, err - } - outcomeCounts.record(pm.metricsHandler) - return responseItems, nil + return pm.grantEagerDispatch(items, targets) } func (pm *taskQueuePartitionManagerImpl) resolveEagerDispatchGrantTargets( @@ -789,14 +738,13 @@ func (pm *taskQueuePartitionManagerImpl) getEagerDispatchVersioningInfo() (eager func (pm *taskQueuePartitionManagerImpl) grantEagerDispatch( items []*matchingservice.GrantEagerDispatchRequest_Item, targets []eagerDispatchGrantTarget, -) ([]*matchingservice.GrantEagerDispatchResponse_Item, eagerDispatchOutcomeCounts, error) { +) ([]*matchingservice.GrantEagerDispatchResponse_Item, error) { responseItems := make([]*matchingservice.GrantEagerDispatchResponse_Item, len(items)) - var outcomeCounts eagerDispatchOutcomeCounts backlogPriorities := make(map[physicalTaskQueueManager]priorityKey) for index, item := range items { backlogPriority, err := pm.eagerDispatchBacklogPriority(targets[index], backlogPriorities) if err != nil { - return nil, eagerDispatchOutcomeCounts{}, err + return nil, err } priority := pm.config.clipPriority(priorityKey(item.GetPriority().GetPriorityKey())) @@ -804,26 +752,13 @@ func (pm *taskQueuePartitionManagerImpl) grantEagerDispatch( // backlog priority is higher and prevents eager dispatch when it is <= priority. granted := int32(0) if backlogPriority == 0 || backlogPriority > priority { - var limitedBy eagerDispatchRateLimit - granted, limitedBy = pm.rateLimitManager.grantTokens(item.GetPriority(), item.GetCount()) - outcomeCounts.granted += int64(granted) - denied := int64(item.GetCount() - granted) - switch limitedBy { - case eagerDispatchRateLimitGeneral: - outcomeCounts.rateLimit += denied - case eagerDispatchRateLimitPerKey: - outcomeCounts.perKeyRateLimit += denied - default: - // All requested tokens were granted. - } - } else { - outcomeCounts.backlog += int64(item.GetCount()) + granted = pm.rateLimitManager.grantTokens(item.GetPriority(), item.GetCount()) } responseItems[index] = &matchingservice.GrantEagerDispatchResponse_Item{ GrantedCount: granted, } } - return responseItems, outcomeCounts, nil + return responseItems, nil } func (pm *taskQueuePartitionManagerImpl) eagerDispatchBacklogPriority( diff --git a/service/matching/task_queue_partition_manager_test.go b/service/matching/task_queue_partition_manager_test.go index f9b81efa9f6..14c30a01ca9 100644 --- a/service/matching/task_queue_partition_manager_test.go +++ b/service/matching/task_queue_partition_manager_test.go @@ -278,95 +278,6 @@ func (s *PartitionManagerTestSuite) TestGrantEagerDispatchReturnsPartialRateLimi }, items) } -func (s *PartitionManagerTestSuite) TestGrantEagerDispatchMetrics() { - tests := []struct { - name string - backlogPriority priorityKey - configure func(*rateLimitManager) - items []*matchingservice.GrantEagerDispatchRequest_Item - expectedError bool - expectedRequest int64 - expectedResults map[string]int64 - }{ - { - name: "granted", - items: []*matchingservice.GrantEagerDispatchRequest_Item{ - {Count: 2}, - }, - expectedRequest: 2, - expectedResults: map[string]int64{eagerDispatchOutcomeGranted: 2}, - }, - { - name: "backlog", - backlogPriority: 2, - items: []*matchingservice.GrantEagerDispatchRequest_Item{ - {Count: 3, Priority: &commonpb.Priority{PriorityKey: 3}}, - }, - expectedRequest: 3, - expectedResults: map[string]int64{eagerDispatchOutcomeBacklog: 3}, - }, - { - name: "general rate limit", - configure: func(rateLimitManager *rateLimitManager) { - rateLimitManager.wholeQueueLimit = simplelimiter.MakeParams(1, 0) - }, - items: []*matchingservice.GrantEagerDispatchRequest_Item{ - {Count: 3}, - }, - expectedRequest: 3, - expectedResults: map[string]int64{ - eagerDispatchOutcomeGranted: 1, - eagerDispatchOutcomeRateLimit: 2, - }, - }, - { - name: "per-key rate limit", - configure: func(rateLimitManager *rateLimitManager) { - rateLimitManager.perKeyLimit = simplelimiter.MakeParams(1, 0) - }, - items: []*matchingservice.GrantEagerDispatchRequest_Item{ - {Count: 3, Priority: &commonpb.Priority{FairnessKey: "key"}}, - }, - expectedRequest: 3, - expectedResults: map[string]int64{ - eagerDispatchOutcomeGranted: 1, - eagerDispatchOutcomePerKeyRateLimit: 2, - }, - }, - { - name: "error", - items: []*matchingservice.GrantEagerDispatchRequest_Item{ - {Count: 3, Version: &deploymentspb.WorkerDeploymentVersion{}}, - }, - expectedError: true, - expectedRequest: 3, - expectedResults: map[string]int64{eagerDispatchOutcomeError: 3}, - }, - } - - for _, test := range tests { - s.Run(test.name, func() { - partitionMgr, capture, cleanup := s.newEagerDispatchPartitionManagerWithMetrics(test.backlogPriority) - defer cleanup() - if test.configure != nil { - test.configure(partitionMgr.rateLimitManager) - } - - _, err := partitionMgr.GrantEagerDispatch(context.Background(), test.items) - if test.expectedError { - s.Require().Error(err) - } else { - s.Require().NoError(err) - } - - requestMetrics := capture.SnapshotMetric(metrics.EagerDispatchRequests.Name()) - s.Require().Len(requestMetrics, 1) - s.Equal(test.expectedRequest, requestMetrics[0].Value) - s.Equal(test.expectedResults, eagerDispatchResultMetrics(capture)) - }) - } -} - func (s *PartitionManagerTestSuite) TestGrantEagerDispatchValidationDoesNotConsumeTokens() { partitionMgr := s.newRateLimitedEagerDispatchPartitionManager() @@ -2901,43 +2812,11 @@ func (s *PartitionManagerTestSuite) newEagerDispatchPartitionManager( userDataManager: s.partitionMgr.userDataManager, logger: s.partitionMgr.logger, throttledLogger: s.partitionMgr.throttledLogger, - metricsHandler: s.partitionMgr.metricsHandler, rateLimitManager: s.partitionMgr.rateLimitManager, defaultQueueFuture: defaultQueueFuture, } } -func (s *PartitionManagerTestSuite) newEagerDispatchPartitionManagerWithMetrics( - defaultBacklogPriority priorityKey, -) (*taskQueuePartitionManagerImpl, *metricstest.Capture, func()) { - partitionMgr := s.newEagerDispatchPartitionManager(defaultBacklogPriority, nil) - captureHandler := metricstest.NewCaptureHandler() - capture := captureHandler.StartCapture() - partitionMgr.metricsHandler = metrics.GetPerTaskQueuePartitionIDScope( - captureHandler, - namespaceName, - partitionMgr.partition, - true, - true, - metrics.OperationTag(metrics.MatchingTaskQueuePartitionManagerScope), - ) - partitionMgr.rateLimitManager = newRateLimitManager( - partitionMgr.userDataManager, - partitionMgr.config, - partitionMgr.partition.TaskQueue().TaskType(), - ) - partitionMgr.rateLimitManager.timeSource = clock.NewEventTimeSource().Update(time.Now()) - return partitionMgr, capture, func() { captureHandler.StopCapture(capture) } -} - -func eagerDispatchResultMetrics(capture *metricstest.Capture) map[string]int64 { - results := make(map[string]int64) - for _, recording := range capture.SnapshotMetric(metrics.EagerDispatchResults.Name()) { - results[recording.Tags[metrics.OutcomeTag("").Key]] += recording.Value.(int64) - } - return results -} - func workerDeploymentVersion(buildID string) *deploymentspb.WorkerDeploymentVersion { return &deploymentspb.WorkerDeploymentVersion{ DeploymentName: "deployment", From 7e193e38a3e8638d9f69242fbddb0e64766115d9 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Tue, 13 Oct 2026 15:00:26 -0500 Subject: [PATCH 21/22] Share eager dispatch grant helper --- chasm/lib/activity/handler.go | 17 +- chasm/lib/activity/responses.go | 160 ++++++++---------- chasm/lib/activity/responses_test.go | 9 +- common/eagerdispatch/grant.go | 28 +++ common/eagerdispatch/grant_test.go | 41 +++++ .../workflow_task_completed_handler.go | 17 +- 6 files changed, 163 insertions(+), 109 deletions(-) create mode 100644 common/eagerdispatch/grant.go create mode 100644 common/eagerdispatch/grant_test.go diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index 88e61059ff5..05392b035d9 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -10,10 +10,10 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/api/matchingservice/v1" - taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" "go.temporal.io/server/common/contextutil" + "go.temporal.io/server/common/eagerdispatch" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -269,17 +269,16 @@ func (h *handler) grantEagerActivityDispatch( return false } - response, err := h.matchingClient.GrantEagerDispatch(ctx, &matchingservice.GrantEagerDispatchRequest{ - NamespaceId: namespaceID, - TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ - TaskQueue: request.GetTaskQueue().GetName(), - TaskQueueType: enumspb.TASK_QUEUE_TYPE_ACTIVITY, - }, - Items: []*matchingservice.GrantEagerDispatchRequest_Item{{ + response, err := eagerdispatch.Grant( + ctx, + h.matchingClient, + namespaceID, + request.GetTaskQueue().GetName(), + []*matchingservice.GrantEagerDispatchRequest_Item{{ Count: 1, Priority: request.GetPriority(), }}, - }) + ) return err == nil && len(response.GetItems()) == 1 && response.GetItems()[0].GetGrantedCount() == 1 } diff --git a/chasm/lib/activity/responses.go b/chasm/lib/activity/responses.go index bf023596d5e..8948e7cdda0 100644 --- a/chasm/lib/activity/responses.go +++ b/chasm/lib/activity/responses.go @@ -7,6 +7,7 @@ import ( commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" + nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" @@ -24,24 +25,25 @@ type eagerActivityTaskRequest struct { } type eagerActivityTaskData struct { - namespaceID string - namespace string - runID string - activityType *commonpb.ActivityType - activityID string - header *commonpb.Header - input *commonpb.Payloads - heartbeatDetails *commonpb.Payloads - scheduledTime *timestamppb.Timestamp - currentAttemptScheduledTime *timestamppb.Timestamp - startedTime *timestamppb.Timestamp - attempt int32 - scheduleToCloseTimeout *durationpb.Duration - startToCloseTimeout *durationpb.Duration - heartbeatTimeout *durationpb.Duration - retryPolicy *commonpb.RetryPolicy - priority *commonpb.Priority - activityAttemptStartedStamp int32 + namespaceID string + namespace string + runID string + activityType *commonpb.ActivityType + activityID string + header *commonpb.Header + input *commonpb.Payloads + heartbeatDetails *commonpb.Payloads + scheduledTime *timestamppb.Timestamp + currentAttemptScheduledTime *timestamppb.Timestamp + startedTime *timestamppb.Timestamp + attempt int32 + scheduleToCloseTimeout *durationpb.Duration + startToCloseTimeout *durationpb.Duration + heartbeatTimeout *durationpb.Duration + retryPolicy *commonpb.RetryPolicy + priority *commonpb.Priority + propagatedNexusSerializationContext *nexuspb.PropagatedSerializationContext + activityAttemptStartedStamp int32 } func (a *Activity) eagerActivityTaskData( @@ -68,65 +70,44 @@ func (a *Activity) eagerActivityTaskData( heartbeatTimeout := a.GetHeartbeatTimeout() retryPolicy := a.GetRetryPolicy() priority := a.GetPriority() + propagatedNexusSerializationContext := requestData.GetPropagatedNexusSerializationContext() // This data is used to construct the eager response after the transaction has completed, so // it must not retain mutable state owned by the transaction. - if activityType != nil { - activityType = common.CloneProto(activityType) - } - if header != nil { - header = common.CloneProto(header) - } - if input != nil { - input = common.CloneProto(input) - } - if heartbeatDetails != nil { - heartbeatDetails = common.CloneProto(heartbeatDetails) - } - if scheduledTime != nil { - scheduledTime = common.CloneProto(scheduledTime) - } - if currentAttemptScheduledTime != nil { - currentAttemptScheduledTime = common.CloneProto(currentAttemptScheduledTime) - } - if startedTime != nil { - startedTime = common.CloneProto(startedTime) - } - if scheduleToCloseTimeout != nil { - scheduleToCloseTimeout = common.CloneProto(scheduleToCloseTimeout) - } - if startToCloseTimeout != nil { - startToCloseTimeout = common.CloneProto(startToCloseTimeout) - } - if heartbeatTimeout != nil { - heartbeatTimeout = common.CloneProto(heartbeatTimeout) - } - if retryPolicy != nil { - retryPolicy = common.CloneProto(retryPolicy) - } - if priority != nil { - priority = common.CloneProto(priority) - } + activityType = common.CloneProto(activityType) + header = common.CloneProto(header) + input = common.CloneProto(input) + heartbeatDetails = common.CloneProto(heartbeatDetails) + scheduledTime = common.CloneProto(scheduledTime) + currentAttemptScheduledTime = common.CloneProto(currentAttemptScheduledTime) + startedTime = common.CloneProto(startedTime) + scheduleToCloseTimeout = common.CloneProto(scheduleToCloseTimeout) + startToCloseTimeout = common.CloneProto(startToCloseTimeout) + heartbeatTimeout = common.CloneProto(heartbeatTimeout) + retryPolicy = common.CloneProto(retryPolicy) + priority = common.CloneProto(priority) + propagatedNexusSerializationContext = common.CloneProto(propagatedNexusSerializationContext) return &eagerActivityTaskData{ - namespaceID: request.namespaceID, - namespace: request.namespace, - runID: key.RunID, - activityType: activityType, - activityID: key.BusinessID, - header: header, - input: input, - heartbeatDetails: heartbeatDetails, - scheduledTime: scheduledTime, - currentAttemptScheduledTime: currentAttemptScheduledTime, - startedTime: startedTime, - attempt: attempt.GetCount(), - scheduleToCloseTimeout: scheduleToCloseTimeout, - startToCloseTimeout: startToCloseTimeout, - heartbeatTimeout: heartbeatTimeout, - retryPolicy: retryPolicy, - priority: priority, - activityAttemptStartedStamp: attempt.GetStartedStamp(), + namespaceID: request.namespaceID, + namespace: request.namespace, + runID: key.RunID, + activityType: activityType, + activityID: key.BusinessID, + header: header, + input: input, + heartbeatDetails: heartbeatDetails, + scheduledTime: scheduledTime, + currentAttemptScheduledTime: currentAttemptScheduledTime, + startedTime: startedTime, + attempt: attempt.GetCount(), + scheduleToCloseTimeout: scheduleToCloseTimeout, + startToCloseTimeout: startToCloseTimeout, + heartbeatTimeout: heartbeatTimeout, + retryPolicy: retryPolicy, + priority: priority, + propagatedNexusSerializationContext: propagatedNexusSerializationContext, + activityAttemptStartedStamp: attempt.GetStartedStamp(), }, nil } @@ -150,23 +131,24 @@ func (d *eagerActivityTaskData) response(componentRef []byte) (*workflowservice. } return &workflowservice.PollActivityTaskQueueResponse{ - TaskToken: token, - WorkflowNamespace: d.namespace, - ActivityType: d.activityType, - ActivityId: d.activityID, - Header: d.header, - Input: d.input, - HeartbeatDetails: d.heartbeatDetails, - ScheduledTime: d.scheduledTime, - CurrentAttemptScheduledTime: d.currentAttemptScheduledTime, - StartedTime: d.startedTime, - Attempt: d.attempt, - ScheduleToCloseTimeout: d.scheduleToCloseTimeout, - StartToCloseTimeout: d.startToCloseTimeout, - HeartbeatTimeout: d.heartbeatTimeout, - RetryPolicy: d.retryPolicy, - Priority: d.priority, - ActivityRunId: d.runID, + TaskToken: token, + WorkflowNamespace: d.namespace, + ActivityType: d.activityType, + ActivityId: d.activityID, + Header: d.header, + Input: d.input, + HeartbeatDetails: d.heartbeatDetails, + ScheduledTime: d.scheduledTime, + CurrentAttemptScheduledTime: d.currentAttemptScheduledTime, + StartedTime: d.startedTime, + Attempt: d.attempt, + ScheduleToCloseTimeout: d.scheduleToCloseTimeout, + StartToCloseTimeout: d.startToCloseTimeout, + HeartbeatTimeout: d.heartbeatTimeout, + RetryPolicy: d.retryPolicy, + Priority: d.priority, + PropagatedNexusSerializationContext: d.propagatedNexusSerializationContext, + ActivityRunId: d.runID, }, nil } diff --git a/chasm/lib/activity/responses_test.go b/chasm/lib/activity/responses_test.go index 3373ed75d68..403f6053443 100644 --- a/chasm/lib/activity/responses_test.go +++ b/chasm/lib/activity/responses_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" commonpb "go.temporal.io/api/common/v1" deploymentpb "go.temporal.io/api/deployment/v1" + nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" @@ -99,6 +100,7 @@ func TestEagerActivityTaskDataSnapshotsState(t *testing.T) { heartbeatTimeout := durationpb.New(3 * time.Minute) retryPolicy := &commonpb.RetryPolicy{MaximumAttempts: 2} priority := &commonpb.Priority{PriorityKey: 1} + propagatedNexusSerializationContext := &nexuspb.PropagatedSerializationContext{Endpoint: "endpoint", Service: "service", Operation: "operation"} activity := &Activity{ ActivityState: &activitypb.ActivityState{ @@ -119,8 +121,9 @@ func TestEagerActivityTaskDataSnapshotsState(t *testing.T) { }), LastHeartbeat: chasm.NewDataField(ctx, &activitypb.ActivityHeartbeatState{Details: heartbeatDetails}), RequestData: chasm.NewDataField(ctx, &activitypb.ActivityRequestData{ - Header: header, - Input: input, + Header: header, + Input: input, + PropagatedNexusSerializationContext: propagatedNexusSerializationContext, }), } @@ -144,6 +147,7 @@ func TestEagerActivityTaskDataSnapshotsState(t *testing.T) { heartbeatTimeout.Seconds++ retryPolicy.MaximumAttempts++ priority.PriorityKey++ + propagatedNexusSerializationContext.Endpoint = "changed-endpoint" response, err := data.response([]byte("component-ref")) require.NoError(t, err) @@ -159,4 +163,5 @@ func TestEagerActivityTaskDataSnapshotsState(t *testing.T) { require.Equal(t, int64(3*time.Minute/time.Second), response.GetHeartbeatTimeout().GetSeconds()) require.EqualValues(t, 2, response.GetRetryPolicy().GetMaximumAttempts()) require.EqualValues(t, 1, response.GetPriority().GetPriorityKey()) + require.Equal(t, "endpoint", response.GetPropagatedNexusSerializationContext().GetEndpoint()) } diff --git a/common/eagerdispatch/grant.go b/common/eagerdispatch/grant.go new file mode 100644 index 00000000000..ae3583a2ffc --- /dev/null +++ b/common/eagerdispatch/grant.go @@ -0,0 +1,28 @@ +// Package eagerdispatch contains shared client-side helpers for eager dispatch admission. +package eagerdispatch + +import ( + "context" + + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/api/matchingservice/v1" + taskqueuespb "go.temporal.io/server/api/taskqueue/v1" +) + +// Grant asks Matching whether the activity task queue may eagerly dispatch each item. +func Grant( + ctx context.Context, + client matchingservice.MatchingServiceClient, + namespaceID string, + taskQueue string, + items []*matchingservice.GrantEagerDispatchRequest_Item, +) (*matchingservice.GrantEagerDispatchResponse, error) { + return client.GrantEagerDispatch(ctx, &matchingservice.GrantEagerDispatchRequest{ + NamespaceId: namespaceID, + TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ + TaskQueue: taskQueue, + TaskQueueType: enumspb.TASK_QUEUE_TYPE_ACTIVITY, + }, + Items: items, + }) +} diff --git a/common/eagerdispatch/grant_test.go b/common/eagerdispatch/grant_test.go new file mode 100644 index 00000000000..c2be82f9d4b --- /dev/null +++ b/common/eagerdispatch/grant_test.go @@ -0,0 +1,41 @@ +package eagerdispatch + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/api/matchingservicemock/v1" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" +) + +func TestGrant(t *testing.T) { + ctrl := gomock.NewController(t) + client := matchingservicemock.NewMockMatchingServiceClient(ctrl) + items := []*matchingservice.GrantEagerDispatchRequest_Item{ + {Count: 1, Priority: &commonpb.Priority{PriorityKey: 1}}, + {Count: 1, Priority: &commonpb.Priority{PriorityKey: 2}}, + } + expectedResponse := &matchingservice.GrantEagerDispatchResponse{ + Items: []*matchingservice.GrantEagerDispatchResponse_Item{{GrantedCount: 1}, {GrantedCount: 1}}, + } + + client.EXPECT().GrantEagerDispatch(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, request *matchingservice.GrantEagerDispatchRequest, _ ...grpc.CallOption) (*matchingservice.GrantEagerDispatchResponse, error) { + require.Equal(t, "namespace-id", request.GetNamespaceId()) + require.Equal(t, "task-queue", request.GetTaskQueuePartition().GetTaskQueue()) + require.Equal(t, enumspb.TASK_QUEUE_TYPE_ACTIVITY, request.GetTaskQueuePartition().GetTaskQueueType()) + require.Same(t, items[0], request.GetItems()[0]) + require.Same(t, items[1], request.GetItems()[1]) + return expectedResponse, nil + }, + ) + + response, err := Grant(context.Background(), client, "namespace-id", "task-queue", items) + require.NoError(t, err) + require.Same(t, expectedResponse, response) +} diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index 2d99179e1fa..fb1c55e2ce8 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -22,12 +22,12 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/api/matchingservice/v1" - taskqueuespb "go.temporal.io/server/api/taskqueue/v1" chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/collection" "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/eagerdispatch" "go.temporal.io/server/common/effect" "go.temporal.io/server/common/enums" "go.temporal.io/server/common/log" @@ -600,20 +600,19 @@ func (handler *workflowTaskCompletedHandler) grantEagerActivityDispatch( } executionInfo := handler.mutableState.GetExecutionInfo() - response, err := handler.matchingClient.GrantEagerDispatch(ctx, &matchingservice.GrantEagerDispatchRequest{ - NamespaceId: executionInfo.GetNamespaceId(), - TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ - TaskQueue: attr.GetTaskQueue().GetName(), - TaskQueueType: enumspb.TASK_QUEUE_TYPE_ACTIVITY, - }, - Items: []*matchingservice.GrantEagerDispatchRequest_Item{ + response, err := eagerdispatch.Grant( + ctx, + handler.matchingClient, + executionInfo.GetNamespaceId(), + attr.GetTaskQueue().GetName(), + []*matchingservice.GrantEagerDispatchRequest_Item{ { Count: 1, Priority: attr.GetPriority(), Version: worker_versioning.DeploymentVersionFromDeployment(handler.workflowTaskDeployment), }, }, - }) + ) return err == nil && len(response.GetItems()) == 1 && response.GetItems()[0].GetGrantedCount() == 1 } From e1eddc1ac075eb586bad4a64e82763f4fc996f08 Mon Sep 17 00:00:00 2001 From: Wenlong Gu Date: Tue, 13 Oct 2026 16:45:09 -0500 Subject: [PATCH 22/22] Simplify eager dispatch helpers --- chasm/lib/activity/handler.go | 16 ++---- chasm/lib/activity/responses.go | 53 +++++-------------- common/eagerdispatch/grant.go | 28 ---------- common/eagerdispatch/grant_test.go | 41 -------------- common/util.go | 30 +++++++++++ common/util_test.go | 11 ++++ .../workflow_task_completed_handler.go | 17 ++---- 7 files changed, 61 insertions(+), 135 deletions(-) delete mode 100644 common/eagerdispatch/grant.go delete mode 100644 common/eagerdispatch/grant_test.go diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index 05392b035d9..f45799c4e1f 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -9,11 +9,10 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" - "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/contextutil" - "go.temporal.io/server/common/eagerdispatch" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -265,21 +264,14 @@ func (h *handler) grantEagerActivityDispatch( namespaceID string, request *workflowservice.StartActivityExecutionRequest, ) bool { - if h.matchingClient == nil { - return false - } - - response, err := eagerdispatch.Grant( + return common.GrantEagerDispatch( ctx, h.matchingClient, namespaceID, request.GetTaskQueue().GetName(), - []*matchingservice.GrantEagerDispatchRequest_Item{{ - Count: 1, - Priority: request.GetPriority(), - }}, + request.GetPriority(), + nil, ) - return err == nil && len(response.GetItems()) == 1 && response.GetItems()[0].GetGrantedCount() == 1 } // DescribeActivityExecution queries current activity state, optionally as a long-poll that waits diff --git a/chasm/lib/activity/responses.go b/chasm/lib/activity/responses.go index 8948e7cdda0..eeb2549587e 100644 --- a/chasm/lib/activity/responses.go +++ b/chasm/lib/activity/responses.go @@ -58,55 +58,28 @@ func (a *Activity) eagerActivityTaskData( key := ctx.ExecutionKey() requestData := a.RequestData.Get(ctx) lastHeartbeat, _ := a.LastHeartbeat.TryGet(ctx) - activityType := a.GetActivityType() - header := requestData.GetHeader() - input := requestData.GetInput() - heartbeatDetails := lastHeartbeat.GetDetails() - scheduledTime := a.GetScheduleTime() - currentAttemptScheduledTime := a.dispatchTimeForAttempt(attempt) - startedTime := attempt.GetStartedTime() - scheduleToCloseTimeout := a.GetScheduleToCloseTimeout() - startToCloseTimeout := a.GetStartToCloseTimeout() - heartbeatTimeout := a.GetHeartbeatTimeout() - retryPolicy := a.GetRetryPolicy() - priority := a.GetPriority() - propagatedNexusSerializationContext := requestData.GetPropagatedNexusSerializationContext() // This data is used to construct the eager response after the transaction has completed, so // it must not retain mutable state owned by the transaction. - activityType = common.CloneProto(activityType) - header = common.CloneProto(header) - input = common.CloneProto(input) - heartbeatDetails = common.CloneProto(heartbeatDetails) - scheduledTime = common.CloneProto(scheduledTime) - currentAttemptScheduledTime = common.CloneProto(currentAttemptScheduledTime) - startedTime = common.CloneProto(startedTime) - scheduleToCloseTimeout = common.CloneProto(scheduleToCloseTimeout) - startToCloseTimeout = common.CloneProto(startToCloseTimeout) - heartbeatTimeout = common.CloneProto(heartbeatTimeout) - retryPolicy = common.CloneProto(retryPolicy) - priority = common.CloneProto(priority) - propagatedNexusSerializationContext = common.CloneProto(propagatedNexusSerializationContext) - return &eagerActivityTaskData{ namespaceID: request.namespaceID, namespace: request.namespace, runID: key.RunID, - activityType: activityType, + activityType: common.CloneProto(a.GetActivityType()), activityID: key.BusinessID, - header: header, - input: input, - heartbeatDetails: heartbeatDetails, - scheduledTime: scheduledTime, - currentAttemptScheduledTime: currentAttemptScheduledTime, - startedTime: startedTime, + header: common.CloneProto(requestData.GetHeader()), + input: common.CloneProto(requestData.GetInput()), + heartbeatDetails: common.CloneProto(lastHeartbeat.GetDetails()), + scheduledTime: common.CloneProto(a.GetScheduleTime()), + currentAttemptScheduledTime: common.CloneProto(a.dispatchTimeForAttempt(attempt)), + startedTime: common.CloneProto(attempt.GetStartedTime()), attempt: attempt.GetCount(), - scheduleToCloseTimeout: scheduleToCloseTimeout, - startToCloseTimeout: startToCloseTimeout, - heartbeatTimeout: heartbeatTimeout, - retryPolicy: retryPolicy, - priority: priority, - propagatedNexusSerializationContext: propagatedNexusSerializationContext, + scheduleToCloseTimeout: common.CloneProto(a.GetScheduleToCloseTimeout()), + startToCloseTimeout: common.CloneProto(a.GetStartToCloseTimeout()), + heartbeatTimeout: common.CloneProto(a.GetHeartbeatTimeout()), + retryPolicy: common.CloneProto(a.GetRetryPolicy()), + priority: common.CloneProto(a.GetPriority()), + propagatedNexusSerializationContext: common.CloneProto(requestData.GetPropagatedNexusSerializationContext()), activityAttemptStartedStamp: attempt.GetStartedStamp(), }, nil } diff --git a/common/eagerdispatch/grant.go b/common/eagerdispatch/grant.go deleted file mode 100644 index ae3583a2ffc..00000000000 --- a/common/eagerdispatch/grant.go +++ /dev/null @@ -1,28 +0,0 @@ -// Package eagerdispatch contains shared client-side helpers for eager dispatch admission. -package eagerdispatch - -import ( - "context" - - enumspb "go.temporal.io/api/enums/v1" - "go.temporal.io/server/api/matchingservice/v1" - taskqueuespb "go.temporal.io/server/api/taskqueue/v1" -) - -// Grant asks Matching whether the activity task queue may eagerly dispatch each item. -func Grant( - ctx context.Context, - client matchingservice.MatchingServiceClient, - namespaceID string, - taskQueue string, - items []*matchingservice.GrantEagerDispatchRequest_Item, -) (*matchingservice.GrantEagerDispatchResponse, error) { - return client.GrantEagerDispatch(ctx, &matchingservice.GrantEagerDispatchRequest{ - NamespaceId: namespaceID, - TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ - TaskQueue: taskQueue, - TaskQueueType: enumspb.TASK_QUEUE_TYPE_ACTIVITY, - }, - Items: items, - }) -} diff --git a/common/eagerdispatch/grant_test.go b/common/eagerdispatch/grant_test.go deleted file mode 100644 index c2be82f9d4b..00000000000 --- a/common/eagerdispatch/grant_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package eagerdispatch - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - commonpb "go.temporal.io/api/common/v1" - enumspb "go.temporal.io/api/enums/v1" - "go.temporal.io/server/api/matchingservice/v1" - "go.temporal.io/server/api/matchingservicemock/v1" - "go.uber.org/mock/gomock" - "google.golang.org/grpc" -) - -func TestGrant(t *testing.T) { - ctrl := gomock.NewController(t) - client := matchingservicemock.NewMockMatchingServiceClient(ctrl) - items := []*matchingservice.GrantEagerDispatchRequest_Item{ - {Count: 1, Priority: &commonpb.Priority{PriorityKey: 1}}, - {Count: 1, Priority: &commonpb.Priority{PriorityKey: 2}}, - } - expectedResponse := &matchingservice.GrantEagerDispatchResponse{ - Items: []*matchingservice.GrantEagerDispatchResponse_Item{{GrantedCount: 1}, {GrantedCount: 1}}, - } - - client.EXPECT().GrantEagerDispatch(gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, request *matchingservice.GrantEagerDispatchRequest, _ ...grpc.CallOption) (*matchingservice.GrantEagerDispatchResponse, error) { - require.Equal(t, "namespace-id", request.GetNamespaceId()) - require.Equal(t, "task-queue", request.GetTaskQueuePartition().GetTaskQueue()) - require.Equal(t, enumspb.TASK_QUEUE_TYPE_ACTIVITY, request.GetTaskQueuePartition().GetTaskQueueType()) - require.Same(t, items[0], request.GetItems()[0]) - require.Same(t, items[1], request.GetItems()[1]) - return expectedResponse, nil - }, - ) - - response, err := Grant(context.Background(), client, "namespace-id", "task-queue", items) - require.NoError(t, err) - require.Same(t, expectedResponse, response) -} diff --git a/common/util.go b/common/util.go index 5af646bbf78..b8728ff4cb6 100644 --- a/common/util.go +++ b/common/util.go @@ -15,8 +15,10 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" + deploymentspb "go.temporal.io/server/api/deployment/v1" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/api/matchingservice/v1" + taskqueuespb "go.temporal.io/server/api/taskqueue/v1" workflowspb "go.temporal.io/server/api/workflow/v1" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/log" @@ -691,6 +693,34 @@ func CloneProto[T proto.Message](v T) T { return proto.Clone(v).(T) } +// GrantEagerDispatch asks Matching whether it may eagerly dispatch one activity task. +func GrantEagerDispatch( + ctx context.Context, + client matchingservice.MatchingServiceClient, + namespaceID string, + taskQueue string, + priority *commonpb.Priority, + version *deploymentspb.WorkerDeploymentVersion, +) bool { + if client == nil { + return false + } + + response, err := client.GrantEagerDispatch(ctx, &matchingservice.GrantEagerDispatchRequest{ + NamespaceId: namespaceID, + TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ + TaskQueue: taskQueue, + TaskQueueType: enumspb.TASK_QUEUE_TYPE_ACTIVITY, + }, + Items: []*matchingservice.GrantEagerDispatchRequest_Item{{ + Count: 1, + Priority: priority, + Version: version, + }}, + }) + return err == nil && len(response.GetItems()) == 1 && response.GetItems()[0].GetGrantedCount() == 1 +} + func CloneProtoMap[K comparable, T proto.Message](src map[K]T) map[K]T { if src == nil { return nil diff --git a/common/util_test.go b/common/util_test.go index b67c46cd477..ce9ed0a3844 100644 --- a/common/util_test.go +++ b/common/util_test.go @@ -648,3 +648,14 @@ func TestCreateHistoryStartWorkflowRequestPayloads(t *testing.T) { // ensure the original request object is unmodified require.Equal(t, startRequestClone, startRequest) } + +func TestGrantEagerDispatchWithoutClient(t *testing.T) { + require.False(t, GrantEagerDispatch( + context.Background(), + nil, + "namespace-id", + "task-queue", + nil, + nil, + )) +} diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index fb1c55e2ce8..2549cee7ac7 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -27,7 +27,6 @@ import ( "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/collection" "go.temporal.io/server/common/definition" - "go.temporal.io/server/common/eagerdispatch" "go.temporal.io/server/common/effect" "go.temporal.io/server/common/enums" "go.temporal.io/server/common/log" @@ -595,25 +594,15 @@ func (handler *workflowTaskCompletedHandler) grantEagerActivityDispatch( ctx context.Context, attr *commandpb.ScheduleActivityTaskCommandAttributes, ) bool { - if handler.matchingClient == nil { - return false - } - executionInfo := handler.mutableState.GetExecutionInfo() - response, err := eagerdispatch.Grant( + return common.GrantEagerDispatch( ctx, handler.matchingClient, executionInfo.GetNamespaceId(), attr.GetTaskQueue().GetName(), - []*matchingservice.GrantEagerDispatchRequest_Item{ - { - Count: 1, - Priority: attr.GetPriority(), - Version: worker_versioning.DeploymentVersionFromDeployment(handler.workflowTaskDeployment), - }, - }, + attr.GetPriority(), + worker_versioning.DeploymentVersionFromDeployment(handler.workflowTaskDeployment), ) - return err == nil && len(response.GetItems()) == 1 && response.GetItems()[0].GetGrantedCount() == 1 } func (handler *workflowTaskCompletedHandler) handlePostCommandEagerExecuteActivity(