Files
d5dafc4dfb Store worker attributes needed by server to propagate nexus tasks to worker (#9231)
## What changed?
As part of RecordActivityTaskStarted flow, store
worker_control_task_queue for an activity in the mutable state
(ActivityInfo).

Main changes:
- executions.proto: Added the new worker_control_task_queue field.
- mutable_state_impl.go: Update mutable state.
- matching/forwarder.go: Propagate worker_control_task_queue when polls
get forwarded. Otherwise, RecordActivityTaskStarted request will not
have it set when invoked from a forwarded poll.

## Why?
To support activity cancellation without activity heartbeat.

Overall flow:
- [This PR] Store worker attributes in ActivityInfo as part of
RecordActivityTaskStarted call.
- [#9232] When user cancels a workflow, create 1 or more tasks. Group
all activities belonging to a worker into the task (for efficiency).
- [#9233] Lookup the Nexus task queue for each worker, and send a Nexus
operation for each transfer task.
- [SDK] Worker will receive this cancel task and cancel the running
activities.

## How did you test it?
- [ ] built
- [ ] run locally and tested manually
- [ ] covered by existing tests
- [x] added new unit test(s)
- [ ] added new functional test(s)

---------

Co-authored-by: Hanzo Dev <[email protected]>
2026-04-10 17:51:46 -07:00

150 lines
5.0 KiB
Go

package respondactivitytaskcompleted
import (
"context"
"time"
enumspb "go.temporal.io/api/enums/v1"
"go.temporal.io/server/api/historyservice/v1"
"go.temporal.io/server/common"
"go.temporal.io/server/common/definition"
"go.temporal.io/server/common/metrics"
"go.temporal.io/server/common/namespace"
"go.temporal.io/server/common/primitives/timestamp"
"go.temporal.io/server/common/tasktoken"
"go.temporal.io/server/service/history/api"
"go.temporal.io/server/service/history/consts"
historyi "go.temporal.io/server/service/history/interfaces"
"go.temporal.io/server/service/history/workflow"
)
func Invoke(
ctx context.Context,
req *historyservice.RespondActivityTaskCompletedRequest,
shard historyi.ShardContext,
workflowConsistencyChecker api.WorkflowConsistencyChecker,
) (resp *historyservice.RespondActivityTaskCompletedResponse, retError error) {
tokenSerializer := tasktoken.NewSerializer()
request := req.CompleteRequest
token, err0 := tokenSerializer.Deserialize(request.TaskToken)
if err0 != nil {
return nil, consts.ErrDeserializingToken
}
namespaceEntry, err := api.GetActiveNamespace(shard, namespace.ID(req.GetNamespaceId()), token.WorkflowId)
if err != nil {
return nil, err
}
namespaceName := namespaceEntry.Name()
if err := api.SetActivityTaskRunID(ctx, token, workflowConsistencyChecker); err != nil {
return nil, err
}
var attemptStartedTime time.Time
var firstScheduledTime time.Time
var taskQueue string
var workflowTypeName string
var fabricateStartedEvent bool
var versioningBehavior enumspb.VersioningBehavior
err = api.GetAndUpdateWorkflowWithNew(
ctx,
token.Clock,
definition.NewWorkflowKey(
token.NamespaceId,
token.WorkflowId,
token.RunId,
),
func(workflowLease api.WorkflowLease) (*api.UpdateWorkflowAction, error) {
mutableState := workflowLease.GetMutableState()
workflowTypeName = mutableState.GetWorkflowType().GetName()
if !mutableState.IsWorkflowExecutionRunning() {
return nil, consts.ErrWorkflowCompleted
}
scheduledEventID := token.GetScheduledEventId()
isCompletedByID := false
if scheduledEventID == common.EmptyEventID { // client call CompleteActivityById, so get scheduledEventID by activityID
isCompletedByID = true
scheduledEventID, err0 = api.GetActivityScheduledEventID(token.GetActivityId(), mutableState)
if err0 != nil {
return nil, err0
}
}
ai, isRunning := mutableState.GetActivityInfo(scheduledEventID)
// First check to see if cache needs to be refreshed as we could potentially have stale workflow execution in
// some extreme cassandra failure cases.
if !isRunning && scheduledEventID >= mutableState.GetNextEventID() {
metrics.StaleMutableStateCounter.With(shard.GetMetricsHandler()).Record(
1,
metrics.OperationTag(metrics.HistoryRespondActivityTaskCompletedScope))
return nil, consts.ErrStaleState
}
if !isRunning || api.IsActivityTaskNotFoundForToken(token, ai, &isCompletedByID) {
return nil, consts.ErrActivityTaskNotFound
}
// We fabricate a started event only when the activity is not started yet and
// we need to force complete an activity
fabricateStartedEvent = ai.StartedEventId == common.EmptyEventID
if fabricateStartedEvent {
_, err := mutableState.AddActivityTaskStartedEvent(
ai,
scheduledEventID,
"",
req.GetCompleteRequest().GetIdentity(),
nil,
nil,
// TODO (shahab): do we need to do anything with wf redirect in this case or any
// other case where an activity starts?
nil,
"", // workerControlTaskQueue not available for force complete
)
if err != nil {
return nil, err
}
}
ai, _ = mutableState.GetActivityInfo(scheduledEventID)
if _, err = mutableState.AddActivityTaskCompletedEvent(scheduledEventID, ai.StartedEventId, request); err != nil {
// Unable to add ActivityTaskCompleted event to history
return nil, err
}
if !fabricateStartedEvent {
// leave it zero if the event is fabricated so the latency metrics are not emitted
attemptStartedTime = timestamp.TimeValue(ai.StartedTime)
}
firstScheduledTime = timestamp.TimeValue(ai.FirstScheduledTime)
taskQueue = ai.TaskQueue
versioningBehavior = mutableState.GetEffectiveVersioningBehavior()
return &api.UpdateWorkflowAction{
Noop: false,
CreateWorkflowTask: true,
}, nil
},
nil,
shard,
workflowConsistencyChecker,
)
if err == nil {
workflow.RecordActivityCompletionMetrics(
shard,
namespaceName,
taskQueue,
workflow.ActivityCompletionMetrics{
AttemptStartedTime: attemptStartedTime,
FirstScheduledTime: firstScheduledTime,
Status: workflow.ActivityStatusSucceeded,
Closed: true,
},
metrics.OperationTag(metrics.HistoryRespondActivityTaskCompletedScope),
metrics.WorkflowTypeTag(workflowTypeName),
metrics.ActivityTypeTag(token.ActivityType),
metrics.VersioningBehaviorTag(versioningBehavior),
)
}
return &historyservice.RespondActivityTaskCompletedResponse{}, err
}