Files
tasks/tests/nexus_workflow_update_test.go
f97a858310 Callback for workflow update support (#9614)
## What changed?
Added support for Nexus workflow update completion callbacks via CHASM.
This allows a Nexus caller to be notified when a workflow update
completes by attaching completion callbacks to the update request.

## Why?

Nexus operations that target workflow updates need a way to receive
completion notifications. Without this, a Nexus caller that sends an
update has no async mechanism to learn when the update finishes.
Completion callbacks enable the same async notification pattern that
already exists for workflow-level Nexus operations.

## How did you test it?
- [ ] built
- [x] run locally and tested manually
- [ ] covered by existing tests
- [x] added new unit test(s)
- [x] added new functional test(s)

## Potential risks
Touches speculative workflow updates, they are always hard to reason
about. Tried to compensate with lots of test coverage.

Note: Needs this API PR
https://github.com/temporalio/api/pull/742/changes

---

> [!NOTE]
> **High Risk**
> Touches workflow update state machine and mutable state event handling
to persist/trigger per-update callbacks, including close/retry/reset
paths, which is complex and can affect correctness of update outcomes
and callback delivery.
> 
> **Overview**
> Adds **workflow update completion callbacks** via CHASM so Nexus
callers can register callbacks on `UpdateWorkflowExecution` and have
them fired on update completion or workflow close.
> 
> This introduces a `WorkflowUpdate` CHASM component with new
`UpdateState` protobuf (including persisted `rejection_failure`), stores
update callbacks under `Workflow.Updates`, and extends callback
processing to handle *update-level* callbacks on update completion,
rejection (including reset/reapply), and on run transitions
(retry/timeout/continue-as-new) where update callbacks must fire even if
workflow-level callbacks are inherited.
> 
> It also adds dynamic config gates/limits
(`EnableWorkflowUpdateCallbacks`, `MaxCallbacksPerUpdateID`), updates
`DescribeWorkflow` to surface update callback triggers, extends mutable
state/history builder APIs to carry per-update callback options in
`WorkflowExecutionOptionsUpdated`, and adds `Update.AttachCallbacks`
logic to persist/flush callbacks (including buffering while `stateSent`,
request-id dedup, and stricter validation requiring `request_id` when
callbacks are present).
> 
> <sup>Reviewed by [Cursor Bugbot]() for commit
4484fee104d73344ef5370c468008440c7df1c6a. Bugbot is set up for automated
code reviews on this repo. Configure
[here](https://www.cursor.com/dashboard/bugbot).</sup>

---------

Co-authored-by: long-nt-tran <[email protected]>
2026-05-19 18:31:15 -04:00

1360 lines
53 KiB
Go

package tests
import (
"context"
"encoding/json"
"errors"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/nexus-rpc/sdk-go/nexus"
"github.com/stretchr/testify/require"
commonpb "go.temporal.io/api/common/v1"
enumspb "go.temporal.io/api/enums/v1"
updatepb "go.temporal.io/api/update/v1"
"go.temporal.io/api/workflowservice/v1"
"go.temporal.io/sdk/client"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/worker"
"go.temporal.io/sdk/workflow"
"go.temporal.io/server/common"
"go.temporal.io/server/common/dynamicconfig"
"go.temporal.io/server/common/nexus/nexustest"
"go.temporal.io/server/common/testing/await"
"go.temporal.io/server/common/testing/parallelsuite"
"go.temporal.io/server/tests/testcore"
)
type NexusWorkflowUpdateTestSuite struct {
parallelsuite.Suite[*NexusWorkflowUpdateTestSuite]
}
func TestNexusWorkflowUpdateTestSuite(t *testing.T) {
parallelsuite.Run(t, &NexusWorkflowUpdateTestSuite{})
}
// updateNexusTestConfig holds configuration for workflow update + nexus integration tests.
type updateNexusTestConfig struct {
taskQueue string
childWfID string
updateID string
}
// newUpdateNexusTestConfig creates a config with randomized names to avoid collisions.
func newUpdateNexusTestConfig(t *testing.T) updateNexusTestConfig {
return updateNexusTestConfig{
taskQueue: testcore.RandomizeStr(t.Name()),
childWfID: testcore.RandomizeStr("child-workflow-id"),
updateID: "update-id",
}
}
// makeUpdateWithCallbackHandler creates a nexus handler that sends a workflow update with
// completion callbacks to the specified child workflow. onStart is an optional callback
// invoked at the start of each operation (e.g. for counting invocations).
// If the update is already completed (e.g., the workflow has finished), the handler returns
// the result synchronously instead of starting an async operation with callbacks.
func makeUpdateWithCallbackHandler(
env *NexusTestEnv,
t *testing.T,
cfg updateNexusTestConfig,
onStart func(),
) nexustest.Handler {
return nexustest.Handler{
OnStartOperation: func(
ctx context.Context,
service, operation string,
input *nexus.LazyValue,
options nexus.StartOperationOptions,
) (nexus.HandlerStartOperationResult[any], error) {
if onStart != nil {
onStart()
}
resp, err := env.FrontendClient().UpdateWorkflowExecution(
ctx,
&workflowservice.UpdateWorkflowExecutionRequest{
Namespace: env.Namespace().String(),
WorkflowExecution: &commonpb.WorkflowExecution{
WorkflowId: cfg.childWfID,
},
WaitPolicy: &updatepb.WaitPolicy{
LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED,
},
Request: &updatepb.Request{
Meta: &updatepb.Meta{
UpdateId: cfg.updateID,
},
Input: &updatepb.Input{
Name: "update",
Args: &commonpb.Payloads{
Payloads: []*commonpb.Payload{testcore.MustToPayload(t, "test")},
},
},
RequestId: uuid.NewString(),
CompletionCallbacks: []*commonpb.Callback{
{
Variant: &commonpb.Callback_Nexus_{
Nexus: &commonpb.Callback_Nexus{
Url: options.CallbackURL,
Header: options.CallbackHeader,
},
},
},
},
},
},
)
if err != nil {
return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "update call failed: %v", err)
}
// Verify the response contains a link.
link := resp.GetLink()
require.NotNil(t, link, "update response should contain a link")
if workflowEvent := link.GetWorkflowEvent(); workflowEvent != nil {
// Accepted/completed update: link points to the accepted event.
require.Equal(t, cfg.childWfID, workflowEvent.GetWorkflowId())
require.Equal(t, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ACCEPTED, workflowEvent.GetRequestIdRef().GetEventType())
} else if wfLink := link.GetWorkflow(); wfLink != nil {
// Rejected update: link points to the workflow with a reason.
require.Equal(t, cfg.childWfID, wfLink.GetWorkflowId())
require.Equal(t, "Update rejected", wfLink.GetReason())
} else {
require.Fail(t, "link should be a workflow event or workflow link")
}
// If the update is already completed, return the result synchronously.
if outcome := resp.GetOutcome(); outcome != nil {
if failure := outcome.GetFailure(); failure != nil {
return nil, &nexus.OperationError{
State: nexus.OperationStateFailed,
Message: failure.GetMessage(),
}
}
if success := outcome.GetSuccess(); success != nil && len(success.GetPayloads()) > 0 {
var result string
if jsonErr := json.Unmarshal(success.GetPayloads()[0].GetData(), &result); jsonErr == nil {
return &nexus.HandlerStartOperationResultSync[any]{Value: result}, nil
}
}
}
return &nexus.HandlerStartOperationResultAsync{
OperationToken: "test",
}, nil
},
}
}
func enableUpdateCallbacksOpts() []testcore.TestOption {
return []testcore.TestOption{
testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true),
testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true),
testcore.WithDynamicConfig(dynamicconfig.EnableWorkflowUpdateCallbacks, true),
}
}
// newUpdateChildWorkflow returns a child workflow function that registers an "update"
// handler and waits for a "stop" signal. If blockOnSignal is true, the update handler
// blocks on a "complete-update" signal before returning, which is useful for ensuring
// the update goes through the async path.
func newUpdateChildWorkflow(blockOnSignal bool) func(workflow.Context, string) (string, error) {
return func(ctx workflow.Context, input string) (string, error) {
if err := workflow.SetUpdateHandler(ctx, "update", func(ctx workflow.Context, input string) (string, error) {
if blockOnSignal {
signalCh := workflow.GetSignalChannel(ctx, "complete-update")
signalCh.Receive(ctx, nil)
}
return "updated: " + input, nil
}); err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "stop")
signalCh.Receive(ctx, nil)
return "done: " + input, nil
}
}
// getFirstWFTaskCompleteEventID scans the workflow history and returns the event ID
// of the first WorkflowTaskCompleted event.
func (s *NexusWorkflowUpdateTestSuite) getFirstWFTaskCompleteEventID(ctx context.Context, env *NexusTestEnv, workflowID, runID string) int64 {
hist := env.SdkClient().GetWorkflowHistory(ctx, workflowID, runID, false, enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT)
for hist.HasNext() {
event, err := hist.Next()
s.NoError(err)
if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_TASK_COMPLETED {
return event.EventId
}
}
s.FailNow("couldn't find a WorkflowTaskCompleted event", "workflowID=%s runID=%s", workflowID, runID)
return 0
}
// newSimpleCallerWF returns a caller workflow that executes a nexus operation targeting
// childWfID and returns the string result.
func (s *NexusWorkflowUpdateTestSuite) newSimpleCallerWF(endpointName, childWfID string) func(workflow.Context) (string, error) {
return func(ctx workflow.Context) (string, error) {
nexusClient := workflow.NewNexusClient(endpointName, "test")
fut := nexusClient.ExecuteOperation(ctx, "operation", childWfID, workflow.NexusOperationOptions{})
var result string
err := fut.Get(ctx, &result)
return result, err
}
}
// awaitUpdateAccepted polls the workflow history until a WorkflowExecutionUpdateAccepted
// event is found, failing the test if it does not appear within 10 seconds.
func (s *NexusWorkflowUpdateTestSuite) awaitUpdateAccepted(ctx context.Context, env *NexusTestEnv, workflowID, runID string) {
await.Require(env.Context(), s.T(), func(t *await.T) {
hist := env.SdkClient().GetWorkflowHistory(ctx, workflowID, runID, false, enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT)
for hist.HasNext() {
event, err := hist.Next()
require.NoError(t, err)
if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ACCEPTED {
return
}
}
require.Fail(t, "update not yet accepted")
}, 10*time.Second, 500*time.Millisecond)
}
// startWorker creates a worker on the given task queue, registers wfs, starts it,
// and schedules cleanup.
func (s *NexusWorkflowUpdateTestSuite) startWorker(env *NexusTestEnv, taskQueue string, wfs ...any) {
w := worker.New(env.SdkClient(), taskQueue, worker.Options{})
for _, wf := range wfs {
w.RegisterWorkflow(wf)
}
s.NoError(w.Start())
s.T().Cleanup(w.Stop)
}
// requireNexusOperationError asserts that err is a WorkflowExecutionError with an inner NexusOperationError,
// and returns the inner NexusOperationError.
func (s *NexusWorkflowUpdateTestSuite) requireNexusOperationError(err error) *temporal.NexusOperationError {
var wee *temporal.WorkflowExecutionError
s.ErrorAs(err, &wee)
var noe *temporal.NexusOperationError
s.ErrorAs(wee, &noe)
return noe
}
// assertAcceptedUpdateCompletedWorkflowError asserts the full error chain:
// WorkflowExecutionError -> NexusOperationError -> ApplicationError{Type: "AcceptedUpdateCompletedWorkflow"}.
// Used to assert the correct error for completion callbacks that failed because the update didn't complete
// before the workflow finishes.
func (s *NexusWorkflowUpdateTestSuite) assertAcceptedUpdateCompletedWorkflowError(err error) {
noe := s.requireNexusOperationError(err)
var appErr *temporal.ApplicationError
s.ErrorAs(noe, &appErr)
s.Equal("AcceptedUpdateCompletedWorkflow", appErr.Type())
}
// assertReappliedUpdateInNewRun verifies that updateID appears as an UpdateAdmitted event
// in runID's history with completion callbacks preserved.
func (s *NexusWorkflowUpdateTestSuite) assertReappliedUpdateInNewRun(ctx context.Context, env *NexusTestEnv, workflowID, runID, updateID string) {
hist := env.SdkClient().GetWorkflowHistory(ctx, workflowID, runID, false, enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT)
found := false
for hist.HasNext() {
event, err := hist.Next()
s.NoError(err)
if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED {
attrs := event.GetWorkflowExecutionUpdateAdmittedEventAttributes()
if attrs.GetRequest().GetMeta().GetUpdateId() == updateID {
found = true
s.NotEmpty(attrs.GetRequest().GetCompletionCallbacks(), "reapplied update should preserve completion callbacks")
}
}
}
s.True(found, "expected reapplied UpdateAdmitted event in new run")
}
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateAsyncNexusOperation() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
childWF := newUpdateChildWorkflow(false)
callerWF := func(ctx workflow.Context) (string, error) {
cwf := workflow.ExecuteChildWorkflow(
workflow.WithWorkflowID(ctx, cfg.childWfID),
childWF,
"initial input",
)
var childWE workflow.Execution
if err := cwf.GetChildWorkflowExecution().Get(ctx, &childWE); err != nil {
return "", err
}
nexusClient := workflow.NewNexusClient(endpointName, "test")
fut := nexusClient.ExecuteOperation(ctx, "operation", childWE.ID, workflow.NexusOperationOptions{})
var result string
err := fut.Get(ctx, &result)
return result, err
}
s.startWorker(env, cfg.taskQueue, callerWF, childWF)
run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
var result string
s.NoError(run.Get(ctx, &result))
s.Equal("updated: test", result)
// Verify the child workflow's history contains the update accepted event with callbacks.
childHistory := env.SdkClient().GetWorkflowHistory(ctx, cfg.childWfID, "", false, enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT)
foundUpdateAccepted := false
for childHistory.HasNext() {
event, err := childHistory.Next()
s.NoError(err)
if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ACCEPTED {
foundUpdateAccepted = true
attrs := event.GetWorkflowExecutionUpdateAcceptedEventAttributes()
s.NotNil(attrs)
s.Equal(cfg.updateID, attrs.GetAcceptedRequest().GetMeta().GetUpdateId())
s.NotEmpty(attrs.GetAcceptedRequest().GetCompletionCallbacks())
break
}
}
s.True(foundUpdateAccepted, "expected to find WorkflowExecutionUpdateAccepted event in child workflow history")
}
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateAsyncAttachedNexusOperation() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
childWF := newUpdateChildWorkflow(true)
callerWF := func(ctx workflow.Context) (string, error) {
cwf := workflow.ExecuteChildWorkflow(
workflow.WithWorkflowID(ctx, cfg.childWfID),
childWF,
"initial input",
)
var childWE workflow.Execution
if err := cwf.GetChildWorkflowExecution().Get(ctx, &childWE); err != nil {
return "", err
}
nexusClient := workflow.NewNexusClient(endpointName, "test")
fut := nexusClient.ExecuteOperation(ctx, "operation", childWE.ID, workflow.NexusOperationOptions{})
var exec workflow.NexusOperationExecution
if err := fut.GetNexusOperationExecution().Get(ctx, &exec); err != nil {
return "", err
}
// Send a second update to verify attaching after starting works.
afut := nexusClient.ExecuteOperation(ctx, "operation", childWE.ID, workflow.NexusOperationOptions{})
var aexec workflow.NexusOperationExecution
if err := afut.GetNexusOperationExecution().Get(ctx, &aexec); err != nil {
return "", err
}
// Signal the child to complete the update now that both operations are attached.
if err := workflow.SignalExternalWorkflow(ctx, childWE.ID, "", "complete-update", nil).Get(ctx, nil); err != nil {
return "", err
}
var aresult string
if err := afut.Get(ctx, &aresult); err != nil {
return "", err
}
var result string
err := fut.Get(ctx, &result)
return result, err
}
s.startWorker(env, cfg.taskQueue, callerWF, childWF)
run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 10 * time.Second,
}, callerWF)
s.NoError(err)
var result string
s.NoError(run.Get(ctx, &result))
s.Equal("updated: test", result)
}
// TestWorkflowUpdateCallbackOnAlreadyCompletedUpdate verifies that when a second caller
// sends an update request with the same update ID after the update has already completed,
// the second request returns the result synchronously without attaching a new callback.
// The child workflow should only have one update callback (from the first request).
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateNoCallbackAttachedOnAlreadyCompletedUpdate() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "already-completed-update-id"
var operationCount atomic.Int32
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, func() { operationCount.Add(1) })
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
childWF := newUpdateChildWorkflow(false)
// Caller workflow sends two nexus operations targeting the same update.
// The first one triggers the update, the second one arrives after it completes
// and should still get the result via AttachCallbacks.
callerWF := func(ctx workflow.Context) (string, error) {
cwf := workflow.ExecuteChildWorkflow(
workflow.WithWorkflowID(ctx, cfg.childWfID),
childWF,
"initial input",
)
var childWE workflow.Execution
if err := cwf.GetChildWorkflowExecution().Get(ctx, &childWE); err != nil {
return "", err
}
nexusClient := workflow.NewNexusClient(endpointName, "test")
// First nexus operation: triggers the update.
fut1 := nexusClient.ExecuteOperation(ctx, "operation", childWE.ID, workflow.NexusOperationOptions{})
var result1 string
if err := fut1.Get(ctx, &result1); err != nil {
return "", err
}
// Second nexus operation: targets the same already-completed update.
fut2 := nexusClient.ExecuteOperation(ctx, "operation", childWE.ID, workflow.NexusOperationOptions{})
var result2 string
if err := fut2.Get(ctx, &result2); err != nil {
return "", err
}
return result1 + " | " + result2, nil
}
s.startWorker(env, cfg.taskQueue, callerWF, childWF)
run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
var result string
s.NoError(run.Get(ctx, &result))
s.Equal("updated: test | updated: test", result)
s.Equal(int32(2), operationCount.Load(), "expected two nexus operations to be started")
// Verify the child workflow has exactly one update callback (from the first request).
// The second request returns synchronously because the update is already completed,
// so no additional callback is attached.
descResp, err := env.FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{
Namespace: env.Namespace().String(),
Execution: &commonpb.WorkflowExecution{
WorkflowId: cfg.childWfID,
},
})
s.NoError(err)
updateCallbackCount := 0
for _, cb := range descResp.GetCallbacks() {
if cb.GetTrigger().GetUpdateWorkflowExecutionCompleted() != nil {
updateCallbackCount++
}
}
s.Equal(1, updateCallbackCount, "expected exactly one update callback on the child workflow")
// Verify the child workflow has the correct request ID infos.
// Each nexus operation generates a unique request ID. If the second operation
// (targeting the already-completed update) had attached its request ID, we would
// see 3 entries instead of 2, or an OPTIONS_UPDATED entry. The count of 2 with
// only STARTED and UPDATE_ACCEPTED types proves the second request ID was not attached.
sdkDescResp, err := env.SdkClient().DescribeWorkflowExecution(ctx, cfg.childWfID, "")
s.NoError(err)
requestIDInfos := sdkDescResp.GetWorkflowExtendedInfo().GetRequestIdInfos()
s.NotNil(requestIDInfos)
s.Len(requestIDInfos, 2, "expected exactly 2 request ID infos: second operation should not attach")
cntStarted := 0
cntAccepted := 0
for _, info := range requestIDInfos {
s.False(info.Buffered)
s.GreaterOrEqual(info.EventId, common.FirstEventID)
s.NotEqual(
enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED,
info.EventType,
"second operation targeting completed update should not create an OPTIONS_UPDATED request ID",
)
switch info.EventType {
case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED:
cntStarted++
case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ACCEPTED:
cntAccepted++
default:
s.Failf("unexpected event type in request ID info", "got %v", info.EventType)
}
}
s.Equal(1, cntStarted, "expected one STARTED request ID info")
s.Equal(1, cntAccepted, "expected one UPDATE_ACCEPTED request ID info from first update acceptance")
}
// TestDescribeWorkflowShowsUpdateCallbacks verifies that DescribeWorkflowExecution
// returns update-level callbacks after an update with callbacks is sent.
func (s *NexusWorkflowUpdateTestSuite) TestDescribeWorkflowShowsUpdateCallbacks() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
taskQueue := testcore.RandomizeStr(s.T().Name())
updateID := "describe-callback-update-id"
callbackURL := "http://localhost:9999/callback"
wf := func(ctx workflow.Context) (string, error) {
if err := workflow.SetUpdateHandler(ctx, "update", func(ctx workflow.Context, input string) (string, error) {
// Wait for a signal so update stays in-progress while we describe.
signalCh := workflow.GetSignalChannel(ctx, "complete-update")
signalCh.Receive(ctx, nil)
return "updated: " + input, nil
}); err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "stop")
signalCh.Receive(ctx, nil)
return "done", nil
}
s.startWorker(env, taskQueue, wf)
run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: taskQueue,
}, wf)
s.NoError(err)
// Send update with completion callbacks (don't wait for completion).
testPayload := testcore.MustToPayload(s.T(), "test")
updateDone := make(chan struct{})
go func() {
defer close(updateDone)
_, _ = env.FrontendClient().UpdateWorkflowExecution(ctx, &workflowservice.UpdateWorkflowExecutionRequest{
Namespace: env.Namespace().String(),
WorkflowExecution: &commonpb.WorkflowExecution{
WorkflowId: run.GetID(),
RunId: run.GetRunID(),
},
WaitPolicy: &updatepb.WaitPolicy{
LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED,
},
Request: &updatepb.Request{
Meta: &updatepb.Meta{
UpdateId: updateID,
},
Input: &updatepb.Input{
Name: "update",
Args: &commonpb.Payloads{
Payloads: []*commonpb.Payload{testPayload},
},
},
RequestId: uuid.NewString(),
CompletionCallbacks: []*commonpb.Callback{
{
Variant: &commonpb.Callback_Nexus_{
Nexus: &commonpb.Callback_Nexus{
Url: callbackURL,
},
},
},
},
},
})
}()
// Wait until the update is accepted by checking DescribeWorkflowExecution.
await.Require(env.Context(), s.T(), func(t *await.T) {
desc, err := env.SdkClient().DescribeWorkflowExecution(ctx, run.GetID(), run.GetRunID())
require.NoError(t, err)
require.NotNil(t, desc.GetCallbacks(), "callbacks should be present")
found := false
for _, cb := range desc.GetCallbacks() {
if cb.GetCallback().GetNexus().GetUrl() == callbackURL {
found = true
// Verify the trigger references the update.
trigger := cb.GetTrigger()
require.NotNil(t, trigger)
updateTrigger := trigger.GetUpdateWorkflowExecutionCompleted()
if updateTrigger != nil {
require.Equal(t, updateID, updateTrigger.GetUpdateId())
}
}
}
require.True(t, found, "expected to find callback with URL %s", callbackURL)
}, 10*time.Second, 500*time.Millisecond)
// Complete the update and stop the workflow.
s.NoError(env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "complete-update", nil))
<-updateDone
s.NoError(env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "stop", nil))
}
// TestWorkflowUpdateCallbackAfterResetInflightUpdate verifies that when a workflow is
// reset while an update with completion callbacks is in-flight (accepted but not completed),
// the update is reapplied in the new run and the callback fires when the update completes.
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackAfterResetInflightUpdate() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Target workflow: update handler blocks on "complete-update" signal so the update
// stays in-flight while we perform the reset.
targetWF := func(ctx workflow.Context, input string) (string, error) {
if err := workflow.SetUpdateHandler(ctx, "update", func(ctx workflow.Context, input string) (string, error) {
signalCh := workflow.GetSignalChannel(ctx, "complete-update")
signalCh.Receive(ctx, nil)
return "updated: " + input, nil
}); err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "stop")
signalCh.Receive(ctx, nil)
return "done: " + input, nil
}
// Start target workflow independently (not as child) to avoid parent-child
// complications during reset.
s.startWorker(env, targetTaskQueue, targetWF)
targetRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a nexus operation that triggers the update with callbacks.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
// Wait for the update to be accepted on the target workflow.
s.awaitUpdateAccepted(ctx, env, cfg.childWfID, targetRun.GetRunID())
// Reset the target workflow to the first WFT completed event (before the update).
resetResp, err := env.FrontendClient().ResetWorkflowExecution(ctx, &workflowservice.ResetWorkflowExecutionRequest{
Namespace: env.Namespace().String(),
WorkflowExecution: &commonpb.WorkflowExecution{
WorkflowId: cfg.childWfID,
RunId: targetRun.GetRunID(),
},
Reason: "test reset with inflight update",
RequestId: uuid.NewString(),
WorkflowTaskFinishEventId: s.getFirstWFTaskCompleteEventID(ctx, env, cfg.childWfID, targetRun.GetRunID()),
})
s.NoError(err)
// Verify the update was reapplied in the new run's history.
s.assertReappliedUpdateInNewRun(ctx, env, cfg.childWfID, resetResp.RunId, cfg.updateID)
// Signal the new run to complete the update, which should trigger the callback.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, resetResp.RunId, "complete-update", nil))
// The callback fires -> nexus operation completes -> caller gets the result.
var result string
s.NoError(callerRun.Get(ctx, &result))
s.Equal("updated: test", result)
// Clean up: stop the new run of the target workflow.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, resetResp.RunId, "stop", nil))
}
// TestWorkflowUpdateCallbackAfterResetRejectedUpdate verifies that when a workflow is
// reset while an update with completion callbacks is in-flight (accepted but not completed),
// and the new run's workflow code rejects the reapplied update via a validator, the
// completion callback fires with a failure and the caller's nexus operation fails.
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackAfterResetRejectedUpdate() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Use a shared flag to switch behavior between runs. In the first run the
// update is accepted (and blocks); after we flip the flag the validator
// rejects every update.
var shouldReject atomic.Bool
// Single workflow function used for both runs.
targetWF := func(ctx workflow.Context, input string) (string, error) {
err := workflow.SetUpdateHandlerWithOptions(ctx, "update",
func(ctx workflow.Context, input string) (string, error) {
signalCh := workflow.GetSignalChannel(ctx, "complete-update")
signalCh.Receive(ctx, nil)
return "updated: " + input, nil
},
workflow.UpdateHandlerOptions{
Validator: func(ctx workflow.Context, input string) error {
if shouldReject.Load() {
return errors.New("update rejected after reset")
}
return nil
},
},
)
if err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "stop")
signalCh.Receive(ctx, nil)
return "done: " + input, nil
}
s.startWorker(env, targetTaskQueue, targetWF)
targetRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a nexus operation that triggers the update with callbacks.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
// Wait for the update to be accepted on the target workflow.
s.awaitUpdateAccepted(ctx, env, cfg.childWfID, targetRun.GetRunID())
// Flip the flag so the validator rejects updates in the new run.
shouldReject.Store(true)
// Reset the target workflow to the first WFT completed event (before the update).
resetResp, err := env.FrontendClient().ResetWorkflowExecution(ctx, &workflowservice.ResetWorkflowExecutionRequest{
Namespace: env.Namespace().String(),
WorkflowExecution: &commonpb.WorkflowExecution{
WorkflowId: cfg.childWfID,
RunId: targetRun.GetRunID(),
},
Reason: "test reset with inflight update expecting rejection",
RequestId: uuid.NewString(),
WorkflowTaskFinishEventId: s.getFirstWFTaskCompleteEventID(ctx, env, cfg.childWfID, targetRun.GetRunID()),
})
s.NoError(err)
// Verify the update was reapplied in the new run's history.
s.assertReappliedUpdateInNewRun(ctx, env, cfg.childWfID, resetResp.RunId, cfg.updateID)
// The reapplied update is rejected by the validator -> callback fires with failure ->
// nexus operation fails -> caller workflow fails.
var result string
err = callerRun.Get(ctx, &result)
s.Error(err, "expected caller workflow to fail because the reapplied update was rejected")
// Verify it's a NexusOperationError wrapping the rejection failure.
_ = s.requireNexusOperationError(err)
// Clean up: stop the new run of the target workflow.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, resetResp.RunId, "stop", nil))
}
// TestWorkflowUpdateCallbackAfterResetCompletedUpdate verifies that when a workflow is
// reset after an update with callbacks has already completed, the update is reapplied in
// the new run, completes again, and a new nexus operation targeting the same update ID
// receives the result via the AttachCallbacks path.
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackAfterResetCompletedUpdate() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "reset-completed-update-id"
var operationCount atomic.Int32
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, func() { operationCount.Add(1) })
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Target workflow: update handler completes immediately.
targetWF := newUpdateChildWorkflow(false)
s.startWorker(env, targetTaskQueue, targetWF)
targetRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a single nexus operation.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
// First caller: triggers the update, it completes, callback fires.
run1, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
var result1 string
s.NoError(run1.Get(ctx, &result1))
s.Equal("updated: test", result1)
// Reset the target workflow to before the update.
resetResp, err := env.FrontendClient().ResetWorkflowExecution(ctx, &workflowservice.ResetWorkflowExecutionRequest{
Namespace: env.Namespace().String(),
WorkflowExecution: &commonpb.WorkflowExecution{
WorkflowId: cfg.childWfID,
RunId: targetRun.GetRunID(),
},
Reason: "test reset with completed update",
RequestId: uuid.NewString(),
WorkflowTaskFinishEventId: s.getFirstWFTaskCompleteEventID(ctx, env, cfg.childWfID, targetRun.GetRunID()),
})
s.NoError(err)
// The update is reapplied and completes again in the new run.
// Wait for the update to complete in the new run before sending the second operation.
await.Require(env.Context(), s.T(), func(t *await.T) {
hist := env.SdkClient().GetWorkflowHistory(ctx, cfg.childWfID, resetResp.RunId, false, enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT)
for hist.HasNext() {
event, err := hist.Next()
require.NoError(t, err)
if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_COMPLETED {
return
}
}
require.Fail(t, "update not yet completed in new run")
}, 10*time.Second, 500*time.Millisecond)
// Second caller: sends a new nexus operation targeting the same update ID.
// Since the update is already completed in the new run, AttachCallbacks fires the callback.
run2, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
var result2 string
s.NoError(run2.Get(ctx, &result2))
s.Equal("updated: test", result2)
s.Equal(int32(2), operationCount.Load(), "expected two nexus operations to be started")
// Clean up: stop the new run of the target workflow.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, resetResp.RunId, "stop", nil))
}
// TestWorkflowUpdateSyncReturnForCompletedWorkflow verifies that when a second nexus
// operation targets the same update ID on a workflow that has already completed, the
// handler detects the update is already completed and returns the result synchronously
// (instead of starting an async operation with callbacks).
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateSyncReturnForCompletedWorkflow() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "sync-return-completed-wf-update-id"
var operationCount atomic.Int32
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, func() { operationCount.Add(1) })
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Target workflow: update handler completes immediately.
targetWF := newUpdateChildWorkflow(false)
s.startWorker(env, targetTaskQueue, targetWF)
targetRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a single nexus operation.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
// First caller: triggers the update, it completes, callback fires.
run1, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
var result1 string
s.NoError(run1.Get(ctx, &result1))
s.Equal("updated: test", result1)
// Complete the target workflow by sending the "stop" signal.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, targetRun.GetRunID(), "stop", nil))
// Wait for the target workflow to complete.
var targetResult string
s.NoError(targetRun.Get(ctx, &targetResult))
// Second caller: sends a new nexus operation targeting the same update ID.
// Since the workflow is completed and the update was already completed,
// UpdateWorkflowExecution returns the outcome directly -> handler returns sync.
run2, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
var result2 string
s.NoError(run2.Get(ctx, &result2))
s.Equal("updated: test", result2)
s.Equal(int32(2), operationCount.Load(), "expected two nexus operations to be started")
}
// TestWorkflowUpdateCallbackOnFailedUpdate verifies that when an update handler returns
// an error (update completes with a failure outcome), the completion callback fires and
// the caller's nexus operation completes with a failure.
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackOnFailedUpdate() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "failed-update-id"
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Target workflow: update handler returns an error after acceptance.
targetWF := func(ctx workflow.Context, input string) (string, error) {
if err := workflow.SetUpdateHandler(ctx, "update", func(ctx workflow.Context, input string) (string, error) {
return "", temporal.NewApplicationError("update handler failed", "UpdateFailed", nil)
}); err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "stop")
signalCh.Receive(ctx, nil)
return "done: " + input, nil
}
s.startWorker(env, targetTaskQueue, targetWF)
_, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a nexus operation targeting the child.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
// The update is accepted but the handler returns an error -> update completes with
// failure -> callback fires -> nexus operation fails -> caller workflow fails.
var result string
err = callerRun.Get(ctx, &result)
s.Error(err, "expected caller workflow to fail because the update failed")
// Verify it's a NexusOperationError wrapping the update failure.
_ = s.requireNexusOperationError(err)
// Clean up: stop the target workflow.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, "", "stop", nil))
}
// TestWorkflowUpdateCallbackOnWorkflowTerminate verifies that when a workflow is
// terminated while an update with completion callbacks is in-flight (accepted, handler
// blocking), the ProcessCloseCallbacks mechanism fires the callback and the caller's
// nexus operation completes.
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackOnWorkflowTerminate() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "terminate-update-id"
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Target workflow: update handler blocks on a signal so it stays in-flight.
targetWF := func(ctx workflow.Context, input string) (string, error) {
if err := workflow.SetUpdateHandler(ctx, "update", func(ctx workflow.Context, input string) (string, error) {
signalCh := workflow.GetSignalChannel(ctx, "complete-update")
signalCh.Receive(ctx, nil)
return "updated: " + input, nil
}); err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "stop")
signalCh.Receive(ctx, nil)
return "done: " + input, nil
}
s.startWorker(env, targetTaskQueue, targetWF)
_, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a nexus operation targeting the child.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
// Wait for the update to be accepted on the target.
s.awaitUpdateAccepted(ctx, env, cfg.childWfID, "")
// Terminate the target workflow while the update is in-flight.
// ProcessCloseCallbacks should fire the update-level callbacks.
s.NoError(env.SdkClient().TerminateWorkflow(ctx, cfg.childWfID, "", "testing terminate with inflight update callback"))
// The callback fires -> nexus operation completes -> caller workflow finishes.
// The caller should get an error (the nexus operation failed because the
// target was terminated).
var result string
err = callerRun.Get(ctx, &result)
s.Error(err, "expected caller workflow to fail because the target was terminated")
s.assertAcceptedUpdateCompletedWorkflowError(err)
}
// TestWorkflowUpdateCallbackOnWorkflowComplete verifies that when a workflow completes
// normally while an update with completion callbacks is in-flight (accepted, handler
// blocking), the ProcessCloseCallbacks mechanism fires the callback and the caller's
// nexus operation completes with a failure (the run closes without completing the update).
// This exercises mutable_state_impl.go processCloseCallbacksChasm -> wf.ProcessCloseCallbacks.
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackOnWorkflowComplete() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "complete-wf-update-id"
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Update handler blocks on "complete-update" signal so the update stays in-flight
// while the workflow itself completes via the "stop" signal.
targetWF := newUpdateChildWorkflow(true)
s.startWorker(env, targetTaskQueue, targetWF)
_, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a nexus operation targeting the child.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
// Wait for the update to be accepted on the target.
s.awaitUpdateAccepted(ctx, env, cfg.childWfID, "")
// Complete the target workflow normally while the update is still in-flight.
// processCloseCallbacksChasm fires the update-level callbacks on workflow close.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, "", "stop", nil))
// The callback fires -> nexus operation completes with failure -> caller workflow fails.
var result string
err = callerRun.Get(ctx, &result)
s.Error(err, "expected caller workflow to fail because the target completed while update was in-flight")
s.assertAcceptedUpdateCompletedWorkflowError(err)
}
// TestWorkflowUpdateCallbackOnWorkflowContinueAsNew verifies that when a workflow
// continues-as-new while an update with completion callbacks is in-flight (accepted,
// handler blocking), the update callbacks are fired and the caller's nexus operation
// completes with a failure (the old run is closed).
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackOnWorkflowContinueAsNew() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "continue-as-new-update-id"
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Target workflow: update handler blocks on a signal so it stays in-flight.
// When "continue-as-new" signal is received, the workflow continues as new.
var targetWF func(ctx workflow.Context, input string) (string, error)
targetWF = func(ctx workflow.Context, input string) (string, error) {
if err := workflow.SetUpdateHandler(ctx, "update", func(ctx workflow.Context, input string) (string, error) {
signalCh := workflow.GetSignalChannel(ctx, "complete-update")
signalCh.Receive(ctx, nil)
return "updated: " + input, nil
}); err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "continue-as-new")
signalCh.Receive(ctx, nil)
return "", workflow.NewContinueAsNewError(ctx, targetWF, "continued")
}
s.startWorker(env, targetTaskQueue, targetWF)
_, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a nexus operation targeting the child.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
// Wait for the update to be accepted on the target.
s.awaitUpdateAccepted(ctx, env, cfg.childWfID, "")
// Signal the target workflow to continue-as-new while the update is in-flight.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, "", "continue-as-new", nil))
// The callback fires -> nexus operation completes -> caller workflow finishes.
// The caller should get an error (the nexus operation failed because the
// target continued as new and the update was aborted).
var result string
err = callerRun.Get(ctx, &result)
s.Error(err, "expected caller workflow to fail because the target continued as new")
s.assertAcceptedUpdateCompletedWorkflowError(err)
}
// TestWorkflowUpdateCallbackOnWorkflowFailedWithRetry verifies that when a workflow
// fails with a retry policy (RetryState=IN_PROGRESS) while an update with completion
// callbacks is in-flight (accepted, handler blocking), the update callbacks are fired
// and the caller's nexus operation completes with a failure (the old run is closed).
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackOnWorkflowFailedWithRetry() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "failed-retry-update-id"
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Target workflow: update handler blocks on a signal so it stays in-flight.
// When "fail" signal is received, the workflow returns an error (which will
// be retried due to the retry policy).
targetWF := func(ctx workflow.Context, input string) (string, error) {
if err := workflow.SetUpdateHandler(ctx, "update", func(ctx workflow.Context, input string) (string, error) {
signalCh := workflow.GetSignalChannel(ctx, "complete-update")
signalCh.Receive(ctx, nil)
return "updated: " + input, nil
}); err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "fail")
signalCh.Receive(ctx, nil)
return "", errors.New("intentional failure for retry test")
}
s.startWorker(env, targetTaskQueue, targetWF)
_, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
RetryPolicy: &temporal.RetryPolicy{
InitialInterval: 1 * time.Second,
MaximumAttempts: 3,
BackoffCoefficient: 1,
},
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a nexus operation targeting the child.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
// Wait for the update to be accepted on the target.
s.awaitUpdateAccepted(ctx, env, cfg.childWfID, "")
// Signal the target workflow to fail while the update is in-flight.
// The retry policy will cause a new run to be created.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, "", "fail", nil))
// The callback fires -> nexus operation completes -> caller workflow finishes.
// The caller should get an error (the nexus operation failed because the
// target failed and the update was aborted).
var result string
err = callerRun.Get(ctx, &result)
s.Error(err, "expected caller workflow to fail because the target workflow failed with retry")
s.assertAcceptedUpdateCompletedWorkflowError(err)
}
// TestWorkflowUpdateCallbackOnRejectedUpdate verifies that when an update is rejected
// by the workflow's validator, the nexus handler detects the rejection (which is returned
// as a completed update with a failure outcome) and returns a synchronous failure to the
// caller. This tests the proper handling of rejection in the callback flow.
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateCallbackOnRejectedUpdate() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
cfg := newUpdateNexusTestConfig(s.T())
cfg.updateID = "rejected-update-id"
h := makeUpdateWithCallbackHandler(env, s.T(), cfg, nil)
endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h)
targetTaskQueue := testcore.RandomizeStr("target-" + s.T().Name())
// Target workflow: validator rejects all updates.
targetWF := func(ctx workflow.Context, input string) (string, error) {
err := workflow.SetUpdateHandlerWithOptions(ctx, "update",
func(ctx workflow.Context, input string) (string, error) {
return "updated: " + input, nil
},
workflow.UpdateHandlerOptions{
Validator: func(ctx workflow.Context, input string) error {
return errors.New("update rejected by validator")
},
},
)
if err != nil {
return "", err
}
signalCh := workflow.GetSignalChannel(ctx, "stop")
signalCh.Receive(ctx, nil)
return "done: " + input, nil
}
s.startWorker(env, targetTaskQueue, targetWF)
_, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
ID: cfg.childWfID,
TaskQueue: targetTaskQueue,
}, targetWF, "initial input")
s.NoError(err)
// Caller workflow sends a nexus operation targeting the child.
callerWF := s.newSimpleCallerWF(endpointName, cfg.childWfID)
s.startWorker(env, cfg.taskQueue, callerWF)
callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: cfg.taskQueue,
WorkflowExecutionTimeout: 30 * time.Second,
}, callerWF)
s.NoError(err)
// The update is rejected by the validator -> nexus handler detects rejection and
// returns sync failure -> nexus operation fails -> caller workflow fails.
var result string
err = callerRun.Get(ctx, &result)
s.Error(err, "expected caller workflow to fail because the update was rejected")
// Verify it's a NexusOperationError containing the rejection message.
noe := s.requireNexusOperationError(err)
s.Contains(noe.Error(), "update rejected by validator")
// Clean up: stop the target workflow.
s.NoError(env.SdkClient().SignalWorkflow(ctx, cfg.childWfID, "", "stop", nil))
}
// TestWorkflowUpdateRequestIDInAcceptedEvent verifies that when an update request includes
// a RequestId, it is preserved in the WorkflowExecutionUpdateAccepted event's AcceptedRequest.
func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateRequestIDInAcceptedEvent() {
env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...)
ctx := testcore.NewContext()
taskQueue := testcore.RandomizeStr(s.T().Name())
updateID := "request-id-accepted-test"
requestID := uuid.NewString()
wf := newUpdateChildWorkflow(false)
s.startWorker(env, taskQueue, wf)
run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{
TaskQueue: taskQueue,
}, wf, "initial input")
s.NoError(err)
// Send an update with a specific RequestId and wait for completion.
_, err = env.FrontendClient().UpdateWorkflowExecution(ctx, &workflowservice.UpdateWorkflowExecutionRequest{
Namespace: env.Namespace().String(),
WorkflowExecution: &commonpb.WorkflowExecution{
WorkflowId: run.GetID(),
RunId: run.GetRunID(),
},
WaitPolicy: &updatepb.WaitPolicy{
LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED,
},
Request: &updatepb.Request{
Meta: &updatepb.Meta{
UpdateId: updateID,
},
Input: &updatepb.Input{
Name: "update",
Args: &commonpb.Payloads{
Payloads: []*commonpb.Payload{testcore.MustToPayload(s.T(), "test")},
},
},
RequestId: requestID,
},
})
s.NoError(err)
// Verify the accepted event contains the request ID in the AcceptedRequest.
hist := env.SdkClient().GetWorkflowHistory(ctx, run.GetID(), run.GetRunID(), false, enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT)
foundAccepted := false
for hist.HasNext() {
event, err := hist.Next()
s.NoError(err)
if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ACCEPTED {
foundAccepted = true
attrs := event.GetWorkflowExecutionUpdateAcceptedEventAttributes()
s.NotNil(attrs)
s.Equal(updateID, attrs.GetAcceptedRequest().GetMeta().GetUpdateId())
s.Equal(requestID, attrs.GetAcceptedRequest().GetRequestId())
break
}
}
s.True(foundAccepted, "expected to find WorkflowExecutionUpdateAccepted event")
// Clean up.
s.NoError(env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "stop", nil))
}