mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-04 02:04:03 +00:00
## What changed? Added support for Nexus workflow update completion callbacks via CHASM. This allows a Nexus caller to be notified when a workflow update completes by attaching completion callbacks to the update request. ## Why? Nexus operations that target workflow updates need a way to receive completion notifications. Without this, a Nexus caller that sends an update has no async mechanism to learn when the update finishes. Completion callbacks enable the same async notification pattern that already exists for workflow-level Nexus operations. ## How did you test it? - [ ] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) ## Potential risks Touches speculative workflow updates, they are always hard to reason about. Tried to compensate with lots of test coverage. Note: Needs this API PR https://github.com/temporalio/api/pull/742/changes --- > [!NOTE] > **High Risk** > Touches workflow update state machine and mutable state event handling to persist/trigger per-update callbacks, including close/retry/reset paths, which is complex and can affect correctness of update outcomes and callback delivery. > > **Overview** > Adds **workflow update completion callbacks** via CHASM so Nexus callers can register callbacks on `UpdateWorkflowExecution` and have them fired on update completion or workflow close. > > This introduces a `WorkflowUpdate` CHASM component with new `UpdateState` protobuf (including persisted `rejection_failure`), stores update callbacks under `Workflow.Updates`, and extends callback processing to handle *update-level* callbacks on update completion, rejection (including reset/reapply), and on run transitions (retry/timeout/continue-as-new) where update callbacks must fire even if workflow-level callbacks are inherited. > > It also adds dynamic config gates/limits (`EnableWorkflowUpdateCallbacks`, `MaxCallbacksPerUpdateID`), updates `DescribeWorkflow` to surface update callback triggers, extends mutable state/history builder APIs to carry per-update callback options in `WorkflowExecutionOptionsUpdated`, and adds `Update.AttachCallbacks` logic to persist/flush callbacks (including buffering while `stateSent`, request-id dedup, and stricter validation requiring `request_id` when callbacks are present). > > <sup>Reviewed by [Cursor Bugbot]() for commit 4484fee104d73344ef5370c468008440c7df1c6a. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup> --------- Co-authored-by: long-nt-tran <[email protected]>
320 lines
12 KiB
Go
320 lines
12 KiB
Go
package workflow
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
commonpb "go.temporal.io/api/common/v1"
|
|
failurepb "go.temporal.io/api/failure/v1"
|
|
historypb "go.temporal.io/api/history/v1"
|
|
"go.temporal.io/api/serviceerror"
|
|
"go.temporal.io/server/chasm"
|
|
"go.temporal.io/server/chasm/lib/callback"
|
|
callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1"
|
|
"go.temporal.io/server/chasm/lib/nexusoperation"
|
|
chasmworkflowpb "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb/v1"
|
|
"go.temporal.io/server/service/history/historybuilder"
|
|
"google.golang.org/protobuf/types/known/emptypb"
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
)
|
|
|
|
type Workflow struct {
|
|
chasm.UnimplementedComponent
|
|
|
|
// For now, workflow state is managed by mutable_state_impl, not CHASM engine, leaving it empty as CHASM expects a
|
|
// state object.
|
|
*emptypb.Empty
|
|
|
|
// MSPointer is a special in-memory field for accessing the underlying mutable state.
|
|
chasm.MSPointer
|
|
|
|
// Callbacks map is used to store the callbacks for the workflow.
|
|
Callbacks chasm.Map[string, *callback.Callback]
|
|
|
|
// Operations map is used to store the Nexus operations for the workflow, keyed by scheduled event ID.
|
|
Operations chasm.Map[int64, *nexusoperation.Operation]
|
|
|
|
// IncomingSignals map is used to track incoming signals, keyed by request ID,
|
|
// to allow DescribeWorkflow to resolve RequestIDRef signal backlinks.
|
|
IncomingSignals chasm.Map[string, *chasmworkflowpb.IncomingSignalData]
|
|
|
|
// Updates indexed by update ID, used to store the update components.
|
|
Updates chasm.Map[string, *WorkflowUpdate]
|
|
}
|
|
|
|
func NewWorkflow(
|
|
_ chasm.MutableContext,
|
|
msPointer chasm.MSPointer,
|
|
) *Workflow {
|
|
return &Workflow{
|
|
MSPointer: msPointer,
|
|
}
|
|
}
|
|
|
|
func (w *Workflow) LifecycleState(
|
|
_ chasm.Context,
|
|
) chasm.LifecycleState {
|
|
// NOTE: closeTransactionHandleRootLifecycleChange() is bypassed in tree.go
|
|
//
|
|
// NOTE: detached mode is not implemented yet, so always return Running here.
|
|
// Otherwise, tasks for callback component can't be executed after workflow is closed.
|
|
return chasm.LifecycleStateRunning
|
|
}
|
|
|
|
func (w *Workflow) ContextMetadata(_ chasm.Context) map[string]string {
|
|
// TODO: Export workflow metadata from the CHASM workflow root instead of CloseTransaction().
|
|
return nil
|
|
}
|
|
|
|
func (w *Workflow) Terminate(
|
|
_ chasm.MutableContext,
|
|
_ chasm.TerminateComponentRequest,
|
|
) (chasm.TerminateComponentResponse, error) {
|
|
return chasm.TerminateComponentResponse{}, serviceerror.NewInternal("workflow root Terminate should not be called")
|
|
}
|
|
|
|
// ProcessCloseCallbacks triggers "WorkflowClosed" callbacks using the CHASM implementation.
|
|
// It schedules all workflow-level and update-level callbacks that are in STANDBY state.
|
|
func (w *Workflow) ProcessCloseCallbacks(ctx chasm.MutableContext) error {
|
|
if err := callback.ScheduleStandbyCallbacks(ctx, w.Callbacks); err != nil {
|
|
return err
|
|
}
|
|
return w.ProcessAllUpdateCloseCallbacks(ctx)
|
|
}
|
|
|
|
// ProcessAllUpdateCloseCallbacks triggers callbacks for all updates without touching
|
|
// workflow-level callbacks. This is used when the workflow is continuing to a new run
|
|
// (ContinueAsNew, retry, cron): workflow-level callbacks are inherited by the new run,
|
|
// but update callbacks must fire now because the update was aborted on the old run.
|
|
func (w *Workflow) ProcessAllUpdateCloseCallbacks(ctx chasm.MutableContext) error {
|
|
for _, updateField := range w.Updates {
|
|
if err := callback.ScheduleStandbyCallbacks(ctx, updateField.Get(ctx).Callbacks); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ProcessUpdateCallbacks triggers callbacks for a single updateID if exists.
|
|
func (w *Workflow) ProcessUpdateCallbacks(ctx chasm.MutableContext, updateID string) error {
|
|
update, exists := w.Updates[updateID]
|
|
if !exists {
|
|
return serviceerror.NewNotFoundf("update with ID %s not found", updateID)
|
|
}
|
|
return callback.ScheduleStandbyCallbacks(ctx, update.Get(ctx).Callbacks)
|
|
}
|
|
|
|
// RejectUpdate stores the rejection failure on the WorkflowUpdate component and
|
|
// fires any pending callbacks. This is used when a reapplied update (after reset)
|
|
// is rejected by the worker's validator - the callbacks need to deliver the
|
|
// rejection failure to the caller.
|
|
func (w *Workflow) RejectUpdate(ctx chasm.MutableContext, updateID string, rejectionFailure *failurepb.Failure) error {
|
|
updateField, exists := w.Updates[updateID]
|
|
if !exists {
|
|
return nil // no callbacks registered for this update
|
|
}
|
|
|
|
upd := updateField.Get(ctx)
|
|
upd.RejectionFailure = rejectionFailure
|
|
|
|
return callback.ScheduleStandbyCallbacks(ctx, upd.Callbacks)
|
|
}
|
|
|
|
// totalCallbackCount returns the total number of callbacks across workflow-level
|
|
// and all update-level callback maps.
|
|
func (w *Workflow) totalCallbackCount(ctx chasm.Context) int {
|
|
count := len(w.Callbacks)
|
|
for _, updateField := range w.Updates {
|
|
count += len(updateField.Get(ctx).Callbacks)
|
|
}
|
|
return count
|
|
}
|
|
|
|
// checkWorkflowCallbackLimit returns an error if adding newCount callbacks would
|
|
// exceed the per-workflow maximum.
|
|
func (w *Workflow) checkWorkflowCallbackLimit(ctx chasm.Context, newCount, maxCallbacksPerWorkflow int) error {
|
|
current := w.totalCallbackCount(ctx)
|
|
if newCount+current > maxCallbacksPerWorkflow {
|
|
return serviceerror.NewFailedPreconditionf(
|
|
"cannot attach more than %d callbacks to a workflow (%d callbacks already attached)",
|
|
maxCallbacksPerWorkflow,
|
|
current,
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addCallbacksToMap converts common callbacks to CHASM callback components and
|
|
// inserts them into the target map, keyed by "<requestID>-<index>".
|
|
//
|
|
// All callbacks are validated up front, so target is not mutated unless every
|
|
// callback can be converted successfully (atomic from the caller's POV).
|
|
func addCallbacksToMap(
|
|
ctx chasm.MutableContext,
|
|
target chasm.Map[string, *callback.Callback],
|
|
requestID string,
|
|
eventTime *timestamppb.Timestamp,
|
|
completionCallbacks []*commonpb.Callback,
|
|
) error {
|
|
chasmCBs := make([]*callbackspb.Callback, len(completionCallbacks))
|
|
for i, cb := range completionCallbacks {
|
|
chasmCB := &callbackspb.Callback{Links: cb.GetLinks()}
|
|
switch variant := cb.Variant.(type) {
|
|
case *commonpb.Callback_Nexus_:
|
|
chasmCB.Variant = &callbackspb.Callback_Nexus_{
|
|
Nexus: &callbackspb.Callback_Nexus{
|
|
Url: variant.Nexus.GetUrl(),
|
|
Header: variant.Nexus.GetHeader(),
|
|
},
|
|
}
|
|
default:
|
|
return serviceerror.NewInvalidArgumentf("unsupported callback variant: %T", variant)
|
|
}
|
|
chasmCBs[i] = chasmCB
|
|
}
|
|
|
|
for idx, chasmCB := range chasmCBs {
|
|
// requestID (unique per API call) + idx (position within the request) ensures unique, idempotent callback IDs.
|
|
// Unlike HSM callbacks, CHASM replicates entire trees rather than replaying events, so deterministic
|
|
// cross-cluster IDs based on event version are not needed.
|
|
id := fmt.Sprintf("%s-%d", requestID, idx)
|
|
if _, exists := target[id]; exists {
|
|
// Already registered, skip to avoid overwriting.
|
|
continue
|
|
}
|
|
callbackObj := callback.NewCallback(requestID, eventTime, &callbackspb.CallbackState{}, chasmCB)
|
|
target[id] = chasm.NewComponentField(ctx, callbackObj)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AddCompletionCallbacks creates completion callbacks using the CHASM implementation.
|
|
// maxCallbacksPerWorkflow is the configured maximum number of callbacks allowed per workflow.
|
|
func (w *Workflow) AddCompletionCallbacks(
|
|
ctx chasm.MutableContext,
|
|
eventTime *timestamppb.Timestamp,
|
|
requestID string,
|
|
completionCallbacks []*commonpb.Callback,
|
|
maxCallbacksPerWorkflow int,
|
|
) error {
|
|
if err := w.checkWorkflowCallbackLimit(ctx, len(completionCallbacks), maxCallbacksPerWorkflow); err != nil {
|
|
return err
|
|
}
|
|
|
|
if w.Callbacks == nil {
|
|
w.Callbacks = make(chasm.Map[string, *callback.Callback], len(completionCallbacks))
|
|
}
|
|
|
|
return addCallbacksToMap(ctx, w.Callbacks, requestID, eventTime, completionCallbacks)
|
|
}
|
|
|
|
// AddUpdateCompletionCallbacks creates completion callbacks using the CHASM implementation.
|
|
// maxCallbacksPerWorkflow is the configured maximum number of callbacks allowed per workflow.
|
|
// maxCallbacksPerUpdateID is the configured maximum number of callbacks allowed per update ID.
|
|
func (w *Workflow) AddUpdateCompletionCallbacks(
|
|
ctx chasm.MutableContext,
|
|
eventTime *timestamppb.Timestamp,
|
|
updateID string,
|
|
requestID string,
|
|
completionCallbacks []*commonpb.Callback,
|
|
maxCallbacksPerWorkflow int,
|
|
maxCallbacksPerUpdateID int,
|
|
) error {
|
|
if err := w.checkWorkflowCallbackLimit(ctx, len(completionCallbacks), maxCallbacksPerWorkflow); err != nil {
|
|
return err
|
|
}
|
|
|
|
if w.Updates == nil {
|
|
w.Updates = make(chasm.Map[string, *WorkflowUpdate], 1)
|
|
}
|
|
if _, ok := w.Updates[updateID]; !ok {
|
|
workflowUpdateObj := NewWorkflowUpdate(ctx, updateID, w.MSPointer)
|
|
workflowUpdateObj.Callbacks = make(chasm.Map[string, *callback.Callback], len(completionCallbacks))
|
|
w.Updates[updateID] = chasm.NewComponentField(ctx, workflowUpdateObj)
|
|
}
|
|
|
|
update := w.Updates[updateID].Get(ctx)
|
|
|
|
currentCallbackCount := len(update.Callbacks)
|
|
if len(completionCallbacks)+currentCallbackCount > maxCallbacksPerUpdateID {
|
|
return serviceerror.NewFailedPreconditionf(
|
|
"cannot attach more than %d callbacks to update %q (%d callbacks already attached)",
|
|
maxCallbacksPerUpdateID,
|
|
updateID,
|
|
currentCallbackCount,
|
|
)
|
|
}
|
|
|
|
return addCallbacksToMap(ctx, update.Callbacks, requestID, eventTime, completionCallbacks)
|
|
}
|
|
|
|
// addAndApplyHistoryEvent adds a history event to the workflow and applies the corresponding event definition,
|
|
// looked up by Go type. This is the preferred way to add and apply events as it provides go-to-definition navigation.
|
|
func addAndApplyHistoryEvent[D EventDefinition](
|
|
w *Workflow,
|
|
ctx chasm.MutableContext,
|
|
setAttributes func(*historypb.HistoryEvent),
|
|
) (*historypb.HistoryEvent, error) {
|
|
def, ok := eventDefinitionByGoType[D](workflowContextFromChasm(ctx).registry)
|
|
if !ok {
|
|
return nil, serviceerror.NewInternalf("no event definition registered for Go type %T", (*D)(nil))
|
|
}
|
|
event := w.AddHistoryEvent(def.Type(), setAttributes)
|
|
return event, def.Apply(ctx, w, event)
|
|
}
|
|
|
|
// AddIncomingSignalEvent adds an entry for the signal requestID -> eventID mapping to
|
|
// track all signals that have been received by the workflow.
|
|
// Note that since signals are buffered, the eventID may the common.BufferedEventID, which
|
|
// will be updated to a concrete eventID once this signal is flushed to the DB.
|
|
// If caller tries to add an already-existing eventID, this function will ignore and silently return
|
|
// instead of overwriting -- use UpdateIncomingSignalEvent to update existing entries.
|
|
func (w *Workflow) AddIncomingSignalEvent(
|
|
ctx chasm.MutableContext,
|
|
requestID string,
|
|
eventID int64,
|
|
) error {
|
|
if w.IncomingSignals == nil {
|
|
w.IncomingSignals = make(chasm.Map[string, *chasmworkflowpb.IncomingSignalData])
|
|
}
|
|
if w.HasIncomingSignalEvent(ctx, requestID) {
|
|
return nil
|
|
}
|
|
w.IncomingSignals[requestID] = chasm.NewDataField(ctx, &chasmworkflowpb.IncomingSignalData{
|
|
// This might be common.BufferedEventID, which will be updated via UpdateIncomingSignalEvent
|
|
// once this signal is flushed to DB.
|
|
EventId: eventID,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// UpdateIncomingSignalEvent updates the eventID for an existing signal requestID in the map.
|
|
// If the requestID is not in the map, this is a no-op (e.g. when called for non-signal request IDs
|
|
// during buffer flush).
|
|
func (w *Workflow) UpdateIncomingSignalEvent(
|
|
ctx chasm.MutableContext,
|
|
requestID string,
|
|
eventID int64,
|
|
) error {
|
|
if w.HasIncomingSignalEvent(ctx, requestID) {
|
|
w.IncomingSignals[requestID].Get(ctx).EventId = eventID
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// HasIncomingSignalEvent returns true if a signal with this requestID is already persisted
|
|
// in this CHASM tree.
|
|
func (w *Workflow) HasIncomingSignalEvent(_ chasm.Context, requestID string) bool {
|
|
_, exists := w.IncomingSignals[requestID]
|
|
return exists
|
|
}
|
|
|
|
// HasAnyBufferedEvent returns true if the workflow has any buffered event matching the given filter.
|
|
func (w *Workflow) HasAnyBufferedEvent(filter historybuilder.BufferedEventFilter) bool {
|
|
return w.MSPointer.HasAnyBufferedEvent(filter)
|
|
}
|
|
|
|
func (w *Workflow) WorkflowTypeName() string {
|
|
return w.GetWorkflowTypeName()
|
|
}
|