Files
f97a858310 Callback for workflow update support (#9614)
## What changed?
Added support for Nexus workflow update completion callbacks via CHASM.
This allows a Nexus caller to be notified when a workflow update
completes by attaching completion callbacks to the update request.

## Why?

Nexus operations that target workflow updates need a way to receive
completion notifications. Without this, a Nexus caller that sends an
update has no async mechanism to learn when the update finishes.
Completion callbacks enable the same async notification pattern that
already exists for workflow-level Nexus operations.

## How did you test it?
- [ ] built
- [x] run locally and tested manually
- [ ] covered by existing tests
- [x] added new unit test(s)
- [x] added new functional test(s)

## Potential risks
Touches speculative workflow updates, they are always hard to reason
about. Tried to compensate with lots of test coverage.

Note: Needs this API PR
https://github.com/temporalio/api/pull/742/changes

---

> [!NOTE]
> **High Risk**
> Touches workflow update state machine and mutable state event handling
to persist/trigger per-update callbacks, including close/retry/reset
paths, which is complex and can affect correctness of update outcomes
and callback delivery.
> 
> **Overview**
> Adds **workflow update completion callbacks** via CHASM so Nexus
callers can register callbacks on `UpdateWorkflowExecution` and have
them fired on update completion or workflow close.
> 
> This introduces a `WorkflowUpdate` CHASM component with new
`UpdateState` protobuf (including persisted `rejection_failure`), stores
update callbacks under `Workflow.Updates`, and extends callback
processing to handle *update-level* callbacks on update completion,
rejection (including reset/reapply), and on run transitions
(retry/timeout/continue-as-new) where update callbacks must fire even if
workflow-level callbacks are inherited.
> 
> It also adds dynamic config gates/limits
(`EnableWorkflowUpdateCallbacks`, `MaxCallbacksPerUpdateID`), updates
`DescribeWorkflow` to surface update callback triggers, extends mutable
state/history builder APIs to carry per-update callback options in
`WorkflowExecutionOptionsUpdated`, and adds `Update.AttachCallbacks`
logic to persist/flush callbacks (including buffering while `stateSent`,
request-id dedup, and stricter validation requiring `request_id` when
callbacks are present).
> 
> <sup>Reviewed by [Cursor Bugbot]() for commit
4484fee104d73344ef5370c468008440c7df1c6a. Bugbot is set up for automated
code reviews on this repo. Configure
[here](https://www.cursor.com/dashboard/bugbot).</sup>

---------

Co-authored-by: long-nt-tran <[email protected]>
2026-05-19 18:31:15 -04:00

300 lines
12 KiB
Go

package updateworkflowoptions
import (
"context"
commonpb "go.temporal.io/api/common/v1"
enumspb "go.temporal.io/api/enums/v1"
"go.temporal.io/api/serviceerror"
workflowpb "go.temporal.io/api/workflow/v1"
"go.temporal.io/server/api/historyservice/v1"
"go.temporal.io/server/api/matchingservice/v1"
"go.temporal.io/server/common/definition"
"go.temporal.io/server/common/namespace"
"go.temporal.io/server/common/util"
"go.temporal.io/server/common/worker_versioning"
"go.temporal.io/server/service/history/api"
"go.temporal.io/server/service/history/consts"
historyi "go.temporal.io/server/service/history/interfaces"
"go.temporal.io/server/service/history/workflow"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/fieldmaskpb"
)
func Invoke(
ctx context.Context,
request *historyservice.UpdateWorkflowExecutionOptionsRequest,
shardCtx historyi.ShardContext,
workflowConsistencyChecker api.WorkflowConsistencyChecker,
matchingClient matchingservice.MatchingServiceClient,
versionCache worker_versioning.VersionMembershipAndReactivationStatusCache,
reactivationSignaler api.VersionReactivationSignalerFn,
) (*historyservice.UpdateWorkflowExecutionOptionsResponse, error) {
req := request.GetUpdateRequest()
ns, err := api.GetActiveNamespace(shardCtx, namespace.ID(request.GetNamespaceId()), req.GetWorkflowExecution().GetWorkflowId())
if err != nil {
return nil, err
}
ret := &historyservice.UpdateWorkflowExecutionOptionsResponse{}
// Store versioning override to send reactivation signal after successful persistence
var versioningOverrideForReactivation *workflowpb.VersioningOverride
var shouldSkipReactivation bool
var revisionNumber int64
err = api.GetAndUpdateWorkflowWithNew(
ctx,
nil,
definition.NewWorkflowKey(
ns.ID().String(),
req.GetWorkflowExecution().GetWorkflowId(),
req.GetWorkflowExecution().GetRunId(),
),
func(workflowLease api.WorkflowLease) (*api.UpdateWorkflowAction, error) {
mutableState := workflowLease.GetMutableState()
if !mutableState.IsWorkflowExecutionRunning() {
// in-memory mutable state is still clean, let updateError=nil in the defer func()
// to prevent clearing and reloading mutable state while releasing the lock
return nil, consts.ErrWorkflowCompleted
}
// If the requested override is pinned and omitted optional pinned version, fill in the current pinned version if it exists,
// or error if no pinned version exists.
// Clone the requested options to avoid mutating the original request but only do the cloning work if needed.
requestedOptions := req.GetWorkflowExecutionOptions()
if requestedOptions.GetVersioningOverride().GetPinned().GetBehavior() != workflowpb.VersioningOverride_PINNED_OVERRIDE_BEHAVIOR_UNSPECIFIED &&
requestedOptions.GetVersioningOverride().GetPinned().GetVersion() == nil {
currentVersion := worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(workflow.GetEffectiveDeployment(mutableState.GetExecutionInfo().GetVersioningInfo()))
if effectiveBevior := workflow.GetEffectiveVersioningBehavior(mutableState.GetExecutionInfo().GetVersioningInfo()); effectiveBevior != enumspb.VERSIONING_BEHAVIOR_PINNED {
return nil, serviceerror.NewFailedPreconditionf("must specify a specific pinned override version because workflow with id %v has behavior %s and is not yet pinned to any version",
mutableState.GetExecutionInfo().GetWorkflowId(),
effectiveBevior.String(),
)
}
var ok bool
requestedOptions, ok = proto.Clone(requestedOptions).(*workflowpb.WorkflowExecutionOptions)
if !ok { // this will never happen, but linter wants me to check the casting, so do it just in case
return nil, serviceerror.NewInternalf("failed to copy workflow options to workflow options: %+v", requestedOptions)
}
requestedOptions.GetVersioningOverride().GetPinned().Version = currentVersion
}
// Validate versioning override, if any.
shouldSkipReactivation, revisionNumber, err = worker_versioning.ValidateVersioningOverrideAndGetReactivationEligibility(ctx, requestedOptions.GetVersioningOverride(), matchingClient, versionCache, mutableState.GetExecutionInfo().GetTaskQueue(), enumspb.TASK_QUEUE_TYPE_WORKFLOW, ns.ID().String())
if err != nil {
return nil, err
}
err = validateTimeSkippingConfig(requestedOptions.GetTimeSkippingConfig(), mutableState)
if err != nil {
return nil, err
}
mergedOpts, hasChanges, err := MergeAndApply(mutableState, requestedOptions, req.GetUpdateMask(), req.GetIdentity())
if err != nil {
return nil, err
}
// Set options for gRPC response
ret.WorkflowExecutionOptions = mergedOpts
// If there is no mutable state change at all, return with no new history event and Noop=true
if !hasChanges {
return &api.UpdateWorkflowAction{
Noop: true,
CreateWorkflowTask: false,
}, nil
}
// Store versioning override to send reactivation signal after successful persistence
versioningOverrideForReactivation = mergedOpts.GetVersioningOverride()
// TODO (carly) part 2: handle safe deployment change --> CreateWorkflowTask=true
return &api.UpdateWorkflowAction{
Noop: false,
CreateWorkflowTask: false,
}, nil
},
nil,
shardCtx,
workflowConsistencyChecker,
)
if err != nil {
return nil, err
}
// Notify version workflow if we're pinning to a potentially drained version.
// This is done after successful persistence to avoid signaling if the update fails.
api.ReactivateVersionWorkflowIfPinned(ctx, ns, versioningOverrideForReactivation, reactivationSignaler, shardCtx.GetConfig().EnableVersionReactivationSignals(), shouldSkipReactivation, revisionNumber)
return ret, nil
}
// validateTimeSkippingConfig rejects an update whose MaxSkippedDuration is
// below what the workflow has already skipped — the new bound would be
// retroactively violated. Validated against current MS state, before merge.
func validateTimeSkippingConfig(cfg *workflowpb.TimeSkippingConfig, ms historyi.MutableState) error {
if !cfg.GetEnabled() {
return nil
}
bound, ok := cfg.GetBound().(*workflowpb.TimeSkippingConfig_MaxSkippedDuration)
if !ok {
return nil
}
maxSkipped := bound.MaxSkippedDuration.AsDuration()
accumulated := ms.GetExecutionInfo().GetTimeSkippingInfo().GetAccumulatedSkippedDuration().AsDuration()
if maxSkipped <= accumulated {
return serviceerror.NewInvalidArgumentf(
"max skipped duration must be greater than skipped duration: %v <= %v",
maxSkipped, accumulated,
)
}
return nil
}
// MergeAndApply merges the requested options mentioned in the field mask with the current options in the mutable state
// and applies the changes to the mutable state. Returns the merged options and a boolean indicating if there were any changes.
func MergeAndApply(
ms historyi.MutableState,
opts *workflowpb.WorkflowExecutionOptions,
updateMask *fieldmaskpb.FieldMask,
identity string,
) (*workflowpb.WorkflowExecutionOptions, bool, error) {
// Merge the requested options mentioned in the field mask with the current options in the mutable state
mergedOpts, err := mergeWorkflowExecutionOptions(
getOptionsFromMutableState(ms),
opts,
updateMask,
)
if err != nil {
return nil, false, serviceerror.NewInvalidArgumentf("error applying update_options: %v", err)
}
// If there is no mutable state change at all, return with no new history event and Noop=true
hasChanges := !proto.Equal(mergedOpts, getOptionsFromMutableState(ms))
if !hasChanges {
return mergedOpts, false, nil
}
unsetOverride := false
if mergedOpts.GetVersioningOverride() == nil {
unsetOverride = true
}
_, err = ms.AddWorkflowExecutionOptionsUpdatedEvent(mergedOpts.GetVersioningOverride(), unsetOverride, "", nil, nil, identity, mergedOpts.GetPriority(), mergedOpts.GetTimeSkippingConfig(), nil)
if err != nil {
return nil, hasChanges, err
}
return mergedOpts, hasChanges, nil
}
func getOptionsFromMutableState(ms historyi.MutableState) *workflowpb.WorkflowExecutionOptions {
opts := &workflowpb.WorkflowExecutionOptions{}
if versioningInfo := ms.GetExecutionInfo().GetVersioningInfo(); versioningInfo != nil {
override, ok := proto.Clone(versioningInfo.GetVersioningOverride()).(*workflowpb.VersioningOverride)
if !ok {
return nil
}
opts.VersioningOverride = override
}
if priority := ms.GetExecutionInfo().GetPriority(); priority != nil {
if cloned, ok := proto.Clone(priority).(*commonpb.Priority); ok {
opts.Priority = cloned
}
}
if tsInfo := ms.GetExecutionInfo().GetTimeSkippingInfo(); tsInfo != nil {
if cloned, ok := proto.Clone(tsInfo.GetConfig()).(*workflowpb.TimeSkippingConfig); ok {
opts.TimeSkippingConfig = cloned
}
}
return opts
}
// mergeWorkflowExecutionOptions copies the given paths in `src` struct to `dst` struct
func mergeWorkflowExecutionOptions(
mergeInto, mergeFrom *workflowpb.WorkflowExecutionOptions,
updateMask *fieldmaskpb.FieldMask,
) (*workflowpb.WorkflowExecutionOptions, error) {
_, err := fieldmaskpb.New(mergeInto, updateMask.GetPaths()...)
if err != nil { // errors if any paths are not valid for the struct we are merging into
return nil, err
}
updateFields := util.ParseFieldMask(updateMask)
if _, ok := updateFields["versioningOverride"]; ok {
mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride()
}
if _, ok := updateFields["versioningOverride.deployment"]; ok {
if _, ok := updateFields["versioningOverride.behavior"]; !ok {
return nil, serviceerror.NewInvalidArgument("versioning_override fields must be updated together")
}
mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride()
}
if _, ok := updateFields["versioningOverride.behavior"]; ok {
if _, ok := updateFields["versioningOverride.deployment"]; !ok {
return nil, serviceerror.NewInvalidArgument("versioning_override fields must be updated together")
}
mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride()
}
// ==== Priority
if _, ok := updateFields["priority"]; ok {
mergeInto.Priority = mergeFrom.GetPriority()
}
if _, ok := updateFields["priority.priorityKey"]; ok {
if mergeInto.Priority == nil {
mergeInto.Priority = &commonpb.Priority{}
}
mergeInto.Priority.PriorityKey = mergeFrom.GetPriority().GetPriorityKey()
}
if _, ok := updateFields["priority.fairnessKey"]; ok {
if mergeInto.Priority == nil {
mergeInto.Priority = &commonpb.Priority{}
}
mergeInto.Priority.FairnessKey = mergeFrom.Priority.GetFairnessKey()
}
if _, ok := updateFields["priority.fairnessWeight"]; ok {
if mergeInto.Priority == nil {
mergeInto.Priority = &commonpb.Priority{}
}
mergeInto.Priority.FairnessWeight = mergeFrom.Priority.GetFairnessWeight()
}
// ==== Time Skipping Config
// nil means "no change" — only update if the caller provided an explicit value.
if _, ok := updateFields["timeSkippingConfig"]; ok {
if mergeFrom.GetTimeSkippingConfig() != nil {
mergeInto.TimeSkippingConfig = mergeFrom.GetTimeSkippingConfig()
}
}
if _, ok := updateFields["timeSkippingConfig.enabled"]; ok {
if mergeInto.TimeSkippingConfig == nil {
mergeInto.TimeSkippingConfig = &workflowpb.TimeSkippingConfig{}
}
mergeInto.TimeSkippingConfig.Enabled = mergeFrom.GetTimeSkippingConfig().GetEnabled()
}
if _, ok := updateFields["timeSkippingConfig.maxSkippedDuration"]; ok {
if mergeInto.TimeSkippingConfig == nil {
mergeInto.TimeSkippingConfig = &workflowpb.TimeSkippingConfig{}
}
mergeInto.TimeSkippingConfig.Bound = &workflowpb.TimeSkippingConfig_MaxSkippedDuration{
MaxSkippedDuration: mergeFrom.GetTimeSkippingConfig().GetMaxSkippedDuration(),
}
}
if _, ok := updateFields["timeSkippingConfig.maxElapsedDuration"]; ok {
if mergeInto.TimeSkippingConfig == nil {
mergeInto.TimeSkippingConfig = &workflowpb.TimeSkippingConfig{}
}
mergeInto.TimeSkippingConfig.Bound = &workflowpb.TimeSkippingConfig_MaxElapsedDuration{
MaxElapsedDuration: mergeFrom.GetTimeSkippingConfig().GetMaxElapsedDuration(),
}
}
return mergeInto, nil
}