Files
f97a858310 Callback for workflow update support (#9614)
## What changed?
Added support for Nexus workflow update completion callbacks via CHASM.
This allows a Nexus caller to be notified when a workflow update
completes by attaching completion callbacks to the update request.

## Why?

Nexus operations that target workflow updates need a way to receive
completion notifications. Without this, a Nexus caller that sends an
update has no async mechanism to learn when the update finishes.
Completion callbacks enable the same async notification pattern that
already exists for workflow-level Nexus operations.

## How did you test it?
- [ ] built
- [x] run locally and tested manually
- [ ] covered by existing tests
- [x] added new unit test(s)
- [x] added new functional test(s)

## Potential risks
Touches speculative workflow updates, they are always hard to reason
about. Tried to compensate with lots of test coverage.

Note: Needs this API PR
https://github.com/temporalio/api/pull/742/changes

---

> [!NOTE]
> **High Risk**
> Touches workflow update state machine and mutable state event handling
to persist/trigger per-update callbacks, including close/retry/reset
paths, which is complex and can affect correctness of update outcomes
and callback delivery.
> 
> **Overview**
> Adds **workflow update completion callbacks** via CHASM so Nexus
callers can register callbacks on `UpdateWorkflowExecution` and have
them fired on update completion or workflow close.
> 
> This introduces a `WorkflowUpdate` CHASM component with new
`UpdateState` protobuf (including persisted `rejection_failure`), stores
update callbacks under `Workflow.Updates`, and extends callback
processing to handle *update-level* callbacks on update completion,
rejection (including reset/reapply), and on run transitions
(retry/timeout/continue-as-new) where update callbacks must fire even if
workflow-level callbacks are inherited.
> 
> It also adds dynamic config gates/limits
(`EnableWorkflowUpdateCallbacks`, `MaxCallbacksPerUpdateID`), updates
`DescribeWorkflow` to surface update callback triggers, extends mutable
state/history builder APIs to carry per-update callback options in
`WorkflowExecutionOptionsUpdated`, and adds `Update.AttachCallbacks`
logic to persist/flush callbacks (including buffering while `stateSent`,
request-id dedup, and stricter validation requiring `request_id` when
callbacks are present).
> 
> <sup>Reviewed by [Cursor Bugbot]() for commit
4484fee104d73344ef5370c468008440c7df1c6a. Bugbot is set up for automated
code reviews on this repo. Configure
[here](https://www.cursor.com/dashboard/bugbot).</sup>

---------

Co-authored-by: long-nt-tran <[email protected]>
2026-05-19 18:31:15 -04:00

238 lines
8.2 KiB
Go

package pollupdate_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
commonpb "go.temporal.io/api/common/v1"
enumspb "go.temporal.io/api/enums/v1"
failurepb "go.temporal.io/api/failure/v1"
protocolpb "go.temporal.io/api/protocol/v1"
"go.temporal.io/api/serviceerror"
updatepb "go.temporal.io/api/update/v1"
"go.temporal.io/api/workflowservice/v1"
clockspb "go.temporal.io/server/api/clock/v1"
"go.temporal.io/server/api/historyservice/v1"
"go.temporal.io/server/common/definition"
"go.temporal.io/server/common/locks"
"go.temporal.io/server/common/namespace"
"go.temporal.io/server/common/testing/protorequire"
"go.temporal.io/server/service/history/api"
"go.temporal.io/server/service/history/api/pollupdate"
historyi "go.temporal.io/server/service/history/interfaces"
"go.temporal.io/server/service/history/tests"
"go.temporal.io/server/service/history/workflow/update"
"go.uber.org/mock/gomock"
"google.golang.org/protobuf/types/known/anypb"
)
type (
mockWFConsistencyChecker struct {
api.WorkflowConsistencyChecker
GetWorkflowContextFunc func(
ctx context.Context,
reqClock *clockspb.VectorClock,
workflowKey definition.WorkflowKey,
lockPriority locks.Priority,
) (api.WorkflowLease, error)
}
mockWorkflowLeaseCtx struct {
api.WorkflowLease
GetContextFn func() historyi.WorkflowContext
GetReleaseFnFn func() historyi.ReleaseWorkflowContextFunc
}
mockReg struct {
update.Registry
FindFunc func(context.Context, string) *update.Update
}
mockUpdateEventStore struct {
update.EventStore
}
)
func (mockUpdateEventStore) OnAfterCommit(f func(context.Context)) { f(context.TODO()) }
func (mockUpdateEventStore) OnAfterRollback(f func(context.Context)) {}
func (mockUpdateEventStore) CanAddEvent() bool { return true }
func (mockUpdateEventStore) RejectWorkflowExecutionUpdate(string, *failurepb.Failure) error {
return nil
}
func (mockUpdateEventStore) HasRequestID(string) bool { return false }
func (m mockWFConsistencyChecker) GetWorkflowLease(
ctx context.Context,
clock *clockspb.VectorClock,
wfKey definition.WorkflowKey,
prio locks.Priority,
) (api.WorkflowLease, error) {
return m.GetWorkflowContextFunc(ctx, clock, wfKey, prio)
}
func (m mockWorkflowLeaseCtx) GetReleaseFn() historyi.ReleaseWorkflowContextFunc {
return m.GetReleaseFnFn()
}
func (m mockWorkflowLeaseCtx) GetContext() historyi.WorkflowContext {
return m.GetContextFn()
}
func (m mockReg) Find(ctx context.Context, updateID string) *update.Update {
return m.FindFunc(ctx, updateID)
}
func TestPollOutcome(t *testing.T) {
namespaceId := t.Name() + "-namespace-id"
workflowId := t.Name() + "-workflow-id"
runId := t.Name() + "-run-id"
updateID := t.Name() + "-update-id"
reg := &mockReg{}
mockController := gomock.NewController(t)
wfCtx := historyi.NewMockWorkflowContext(mockController)
wfCtx.EXPECT().GetWorkflowKey().Return(definition.WorkflowKey{NamespaceID: namespaceId, WorkflowID: workflowId, RunID: runId}).AnyTimes()
wfCtx.EXPECT().UpdateRegistry(gomock.Any()).Return(reg).AnyTimes()
apiCtx := mockWorkflowLeaseCtx{
GetReleaseFnFn: func() historyi.ReleaseWorkflowContextFunc { return func(error) {} },
GetContextFn: func() historyi.WorkflowContext {
return wfCtx
},
}
wfcc := mockWFConsistencyChecker{
GetWorkflowContextFunc: func(
ctx context.Context,
reqClock *clockspb.VectorClock,
workflowKey definition.WorkflowKey,
lockPriority locks.Priority,
) (api.WorkflowLease, error) {
return apiCtx, nil
},
}
serverImposedTimeout := 10 * time.Millisecond
mockNamespaceRegistry := namespace.NewMockRegistry(mockController)
mockNamespaceRegistry.EXPECT().GetNamespaceByID(gomock.Any()).Return(tests.GlobalNamespaceEntry, nil).AnyTimes()
shardContext := historyi.NewMockShardContext(mockController)
mockConfig := tests.NewDynamicConfig()
mockConfig.LongPollExpirationInterval = func(_ string) time.Duration { return serverImposedTimeout }
shardContext.EXPECT().GetConfig().Return(mockConfig).AnyTimes()
shardContext.EXPECT().GetNamespaceRegistry().Return(mockNamespaceRegistry).AnyTimes()
req := historyservice.PollWorkflowExecutionUpdateRequest{
Request: &workflowservice.PollWorkflowExecutionUpdateRequest{
UpdateRef: &updatepb.UpdateRef{
WorkflowExecution: &commonpb.WorkflowExecution{
WorkflowId: workflowId,
RunId: runId,
},
UpdateId: updateID,
},
WaitPolicy: &updatepb.WaitPolicy{
LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED,
},
},
}
t.Run("update not found", func(t *testing.T) {
reg.FindFunc = func(ctx context.Context, updateID string) *update.Update {
return nil
}
_, err := pollupdate.Invoke(context.TODO(), &req, shardContext, wfcc)
var notfound *serviceerror.NotFound
require.ErrorAs(t, err, &notfound)
})
t.Run("context deadline expiry before server-imposed deadline expiry", func(t *testing.T) {
reg.FindFunc = func(ctx context.Context, updateID string) *update.Update {
return update.New(updateID)
}
ctx, cncl := context.WithTimeout(context.Background(), serverImposedTimeout/2)
defer cncl()
_, err := pollupdate.Invoke(ctx, &req, shardContext, wfcc)
require.Error(t, err)
})
t.Run("context deadline expiry after server-imposed deadline expiry", func(t *testing.T) {
reg.FindFunc = func(ctx context.Context, updateID string) *update.Update {
return update.New(updateID)
}
ctx, cncl := context.WithTimeout(context.Background(), serverImposedTimeout*2)
defer cncl()
resp, err := pollupdate.Invoke(ctx, &req, shardContext, wfcc)
require.NoError(t, err)
require.Nil(t, resp.GetResponse().Outcome)
require.Equal(t, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ADMITTED, resp.Response.GetStage())
})
t.Run("non-blocking poll with omitted/unspecified wait policy", func(t *testing.T) {
for _, req := range []*historyservice.PollWorkflowExecutionUpdateRequest{{
Request: &workflowservice.PollWorkflowExecutionUpdateRequest{
UpdateRef: req.Request.UpdateRef,
},
}, {
Request: &workflowservice.PollWorkflowExecutionUpdateRequest{
UpdateRef: &updatepb.UpdateRef{
WorkflowExecution: &commonpb.WorkflowExecution{
WorkflowId: workflowId,
RunId: runId,
},
UpdateId: updateID,
},
WaitPolicy: &updatepb.WaitPolicy{
LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_UNSPECIFIED,
},
},
}} {
reg.FindFunc = func(ctx context.Context, updateID string) *update.Update {
return update.New(updateID)
}
resp, err := pollupdate.Invoke(context.Background(), req, shardContext, wfcc)
require.NoError(t, err)
require.True(t, len(resp.GetResponse().UpdateRef.GetWorkflowExecution().RunId) > 0)
require.Nil(t, resp.GetResponse().Outcome)
require.Equal(t, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ADMITTED, resp.Response.GetStage())
}
})
t.Run("get an outcome", func(t *testing.T) {
upd := update.New(updateID)
reg.FindFunc = func(ctx context.Context, updateID string) *update.Update {
return upd
}
reqMsg := updatepb.Request{
Meta: &updatepb.Meta{UpdateId: updateID},
Input: &updatepb.Input{Name: "not_empty"},
}
fail := failurepb.Failure{Message: "intentional failure in " + t.Name()}
wantOutcome := updatepb.Outcome{Value: &updatepb.Outcome_Failure{Failure: &fail}}
rejBody := &updatepb.Rejection{
RejectedRequestMessageId: updateID + "/request",
RejectedRequest: &reqMsg,
Failure: &fail,
}
var rejBodyAny anypb.Any
require.NoError(t, rejBodyAny.MarshalFrom(rejBody))
rejMsg := protocolpb.Message{Body: &rejBodyAny}
errCh := make(chan error, 1)
respCh := make(chan *historyservice.PollWorkflowExecutionUpdateResponse, 1)
go func() {
resp, err := pollupdate.Invoke(context.TODO(), &req, shardContext, wfcc)
errCh <- err
respCh <- resp
}()
evStore := mockUpdateEventStore{}
require.NoError(t, upd.Admit(&reqMsg, evStore))
upd.Send(false, &protocolpb.Message_EventId{EventId: 2208})
require.NoError(t, upd.OnProtocolMessage(&rejMsg, evStore))
require.NoError(t, <-errCh)
resp := <-respCh
protorequire.ProtoEqual(t, &wantOutcome, resp.GetResponse().Outcome)
require.Equal(t, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, resp.Response.GetStage())
})
}