mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-30 14:56:19 +00:00
## What changed? Added support for Nexus workflow update completion callbacks via CHASM. This allows a Nexus caller to be notified when a workflow update completes by attaching completion callbacks to the update request. ## Why? Nexus operations that target workflow updates need a way to receive completion notifications. Without this, a Nexus caller that sends an update has no async mechanism to learn when the update finishes. Completion callbacks enable the same async notification pattern that already exists for workflow-level Nexus operations. ## How did you test it? - [ ] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) ## Potential risks Touches speculative workflow updates, they are always hard to reason about. Tried to compensate with lots of test coverage. Note: Needs this API PR https://github.com/temporalio/api/pull/742/changes --- > [!NOTE] > **High Risk** > Touches workflow update state machine and mutable state event handling to persist/trigger per-update callbacks, including close/retry/reset paths, which is complex and can affect correctness of update outcomes and callback delivery. > > **Overview** > Adds **workflow update completion callbacks** via CHASM so Nexus callers can register callbacks on `UpdateWorkflowExecution` and have them fired on update completion or workflow close. > > This introduces a `WorkflowUpdate` CHASM component with new `UpdateState` protobuf (including persisted `rejection_failure`), stores update callbacks under `Workflow.Updates`, and extends callback processing to handle *update-level* callbacks on update completion, rejection (including reset/reapply), and on run transitions (retry/timeout/continue-as-new) where update callbacks must fire even if workflow-level callbacks are inherited. > > It also adds dynamic config gates/limits (`EnableWorkflowUpdateCallbacks`, `MaxCallbacksPerUpdateID`), updates `DescribeWorkflow` to surface update callback triggers, extends mutable state/history builder APIs to carry per-update callback options in `WorkflowExecutionOptionsUpdated`, and adds `Update.AttachCallbacks` logic to persist/flush callbacks (including buffering while `stateSent`, request-id dedup, and stricter validation requiring `request_id` when callbacks are present). > > <sup>Reviewed by [Cursor Bugbot]() for commit 4484fee104d73344ef5370c468008440c7df1c6a. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup> --------- Co-authored-by: long-nt-tran <[email protected]>
852 lines
32 KiB
Go
852 lines
32 KiB
Go
package update
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"time"
|
|
|
|
commonpb "go.temporal.io/api/common/v1"
|
|
enumspb "go.temporal.io/api/enums/v1"
|
|
failurepb "go.temporal.io/api/failure/v1"
|
|
historypb "go.temporal.io/api/history/v1"
|
|
protocolpb "go.temporal.io/api/protocol/v1"
|
|
"go.temporal.io/api/serviceerror"
|
|
updatepb "go.temporal.io/api/update/v1"
|
|
"go.temporal.io/server/common"
|
|
"go.temporal.io/server/common/effect"
|
|
"go.temporal.io/server/common/future"
|
|
"go.temporal.io/server/common/metrics"
|
|
"google.golang.org/protobuf/proto"
|
|
"google.golang.org/protobuf/types/known/anypb"
|
|
)
|
|
|
|
// pendingCallback holds a AttachCallbacks request that arrived while the Update
|
|
// was in stateSent. These are flushed to the event store on acceptance
|
|
// in onAcceptanceMsg. In-memory only; lost on registry clear/lock release.
|
|
type pendingCallback struct {
|
|
requestID string
|
|
completionCallbacks []*commonpb.Callback
|
|
}
|
|
|
|
type (
|
|
// Update docs are at /docs/architecture/workflow-update.md.
|
|
Update struct {
|
|
// These fields must be accessed only while holding workflow lock.
|
|
id string
|
|
state state
|
|
// The `request` field holds the Update payload submitted with the original request. It is stored in the
|
|
// Registry to be sent to the worker and then written to history in an UpdateAccepted event.
|
|
// Therefore, it is nil when the Update in the Registry is in stateAccepted, stateCompleted etc., since then the
|
|
// request has already been written to an UpdateAccepted event.
|
|
// In addition, it is a nil when the Update in the Registry is in stateAdmitted AND it was created from an
|
|
// UpdateInfo.AdmissionInfo entry in MutableState. In this case, the reason it is a nil is the following:
|
|
// 1. The presence of the AdmissionInfo entry in MutableState implies that there is an UpdateAdmitted event in
|
|
// history. This event always contains the original request payload.
|
|
// 2. Therefore, it is not necessary to write a second copy of the request payload to an UpdateAccepted event,
|
|
// so we don't *need* to load the request into the Registry.
|
|
// 3. Furthermore, it is possible that many UpdateAdmitted events were created after a Reset or during conflict
|
|
// resolution. In that situation, we *must not* attempt to load all the payloads into the Registry.
|
|
request *anypb.Any // of type *updatepb.Request
|
|
acceptedEventID int64
|
|
onComplete func()
|
|
checkLimits func(*updatepb.Request) error
|
|
instrumentation *instrumentation
|
|
admittedTime time.Time
|
|
// pendingCallbacks buffers AttachCallbacks requests that arrive while
|
|
// the Update is in stateSent. Flushed to the event store in onAcceptanceMsg.
|
|
// Cleared on rejection, abort, or rollback. In-memory only; lost on lock release.
|
|
pendingCallbacks []pendingCallback
|
|
|
|
// These fields might be accessed while not holding the workflow lock.
|
|
accepted future.Future[*failurepb.Failure]
|
|
outcome future.Future[*updatepb.Outcome]
|
|
}
|
|
|
|
updateOpt func(*Update)
|
|
)
|
|
|
|
// New creates a new Update instance with the provided ID and set of options.
|
|
func New(id string, opts ...updateOpt) *Update {
|
|
upd := &Update{
|
|
id: id,
|
|
state: stateCreated,
|
|
onComplete: func() {},
|
|
checkLimits: func(request *updatepb.Request) error { return nil },
|
|
instrumentation: &noopInstrumentation,
|
|
accepted: future.NewFuture[*failurepb.Failure](),
|
|
outcome: future.NewFuture[*updatepb.Outcome](),
|
|
}
|
|
for _, opt := range opts {
|
|
opt(upd)
|
|
}
|
|
return upd
|
|
}
|
|
|
|
func newAdmitted(id string, request *anypb.Any, opts ...updateOpt) *Update {
|
|
upd := &Update{
|
|
id: id,
|
|
state: stateAdmitted,
|
|
request: request,
|
|
onComplete: func() {},
|
|
instrumentation: &noopInstrumentation,
|
|
accepted: future.NewFuture[*failurepb.Failure](),
|
|
outcome: future.NewFuture[*updatepb.Outcome](),
|
|
admittedTime: time.Now().UTC(),
|
|
}
|
|
for _, opt := range opts {
|
|
opt(upd)
|
|
}
|
|
return upd
|
|
}
|
|
|
|
func newAccepted(id string, acceptedEventID int64, opts ...updateOpt) *Update {
|
|
upd := &Update{
|
|
id: id,
|
|
state: stateAccepted,
|
|
request: nil,
|
|
acceptedEventID: acceptedEventID,
|
|
onComplete: func() {},
|
|
instrumentation: &noopInstrumentation,
|
|
accepted: future.NewReadyFuture[*failurepb.Failure](nil, nil),
|
|
outcome: future.NewFuture[*updatepb.Outcome](),
|
|
}
|
|
for _, opt := range opts {
|
|
opt(upd)
|
|
}
|
|
return upd
|
|
}
|
|
|
|
func newCompleted(
|
|
id string,
|
|
outcomeFuture *future.ReadyFutureImpl[*updatepb.Outcome],
|
|
opts ...updateOpt,
|
|
) *Update {
|
|
upd := &Update{
|
|
id: id,
|
|
state: stateCompleted,
|
|
onComplete: func() {},
|
|
instrumentation: &noopInstrumentation,
|
|
accepted: future.NewReadyFuture[*failurepb.Failure](nil, nil),
|
|
outcome: outcomeFuture,
|
|
}
|
|
for _, opt := range opts {
|
|
opt(upd)
|
|
}
|
|
return upd
|
|
}
|
|
|
|
func withCompletionCallback(cb func()) updateOpt {
|
|
return func(u *Update) {
|
|
u.onComplete = cb
|
|
}
|
|
}
|
|
|
|
func withLimitChecker(cb func(req *updatepb.Request) error) updateOpt {
|
|
return func(u *Update) {
|
|
u.checkLimits = cb
|
|
}
|
|
}
|
|
|
|
func withInstrumentation(i *instrumentation) updateOpt {
|
|
return func(u *Update) {
|
|
u.instrumentation = i
|
|
}
|
|
}
|
|
|
|
// WaitLifecycleStage waits until the Update has reached waitStage or a timeout.
|
|
// If the Update reaches waitStage with no timeout, the outcome (if any) is returned.
|
|
// If there is a timeout due to the supplied soft timeout,
|
|
// then the most advanced stage known to have been reached is returned together with an empty outcome.
|
|
// If there is a timeout due to supplied context deadline expiry, then the error is returned.
|
|
// If waitStage is UNSPECIFIED, current, reached Status is returned immediately (even if ctx is expired).
|
|
func (u *Update) WaitLifecycleStage(
|
|
ctx context.Context,
|
|
waitStage enumspb.UpdateWorkflowExecutionLifecycleStage,
|
|
softTimeout time.Duration,
|
|
) (*Status, error) {
|
|
|
|
stCtx, stCancel := context.WithTimeout(ctx, softTimeout)
|
|
defer stCancel()
|
|
|
|
var err error
|
|
|
|
if u.outcome.Ready() || waitStage == enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED {
|
|
var outcome *updatepb.Outcome
|
|
outcome, err = u.outcome.Get(stCtx)
|
|
if err == nil {
|
|
return statusCompleted(outcome), nil
|
|
}
|
|
|
|
// If err is coming from the user provided context (context.DeadlineExceeded or context.Canceled), then return it to the caller.
|
|
if errors.Is(err, ctx.Err()) {
|
|
metrics.WorkflowExecutionUpdateClientTimeout.With(u.instrumentation.metrics).Record(1)
|
|
return nil, err
|
|
}
|
|
|
|
// If err is not coming from stCtx, then it means that the error is coming from the future itself.
|
|
// If this err is not registryClearedErr, then it needs to be returned to the caller.
|
|
if !errors.Is(err, stCtx.Err()) && !errors.Is(err, registryClearedErr) {
|
|
return nil, err
|
|
}
|
|
|
|
// Only get here if there is an error, and this error is coming from stCtx or is registryClearedErr.
|
|
// In both cases, check if the Update has reached ACCEPTED stage.
|
|
}
|
|
|
|
// Update is not completed but maybe it is accepted.
|
|
if u.accepted.Ready() || waitStage == enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED {
|
|
// Using the same context which might be already expired, but if accepted future is ready,
|
|
// then it will return immediately without checking context deadline.
|
|
var rejection *failurepb.Failure
|
|
rejection, err = u.accepted.Get(stCtx)
|
|
if err == nil {
|
|
if rejection != nil {
|
|
return statusRejected(rejection), nil
|
|
}
|
|
// Even if only ACCEPTED stage was requested, check if Update was completed on the same WFT,
|
|
// and return Update result if it was.
|
|
if u.outcome.Ready() {
|
|
var outcome *updatepb.Outcome
|
|
outcome, err = u.outcome.Get(stCtx)
|
|
if err == nil {
|
|
return statusCompleted(outcome), nil
|
|
}
|
|
// If outcome future returned an error, then ACCEPTED is the most advanced stage reached,
|
|
// and it should be returned to the caller (because it was requested).
|
|
// This can happen when Workflow completes after accepting but not completing Update.
|
|
}
|
|
return statusAccepted(), nil
|
|
}
|
|
|
|
// If err is coming from the user provided context (context.DeadlineExceeded or context.Canceled), then return it to the caller.
|
|
if errors.Is(err, ctx.Err()) {
|
|
metrics.WorkflowExecutionUpdateClientTimeout.With(u.instrumentation.metrics).Record(1)
|
|
return nil, ctx.Err()
|
|
}
|
|
|
|
// If err is not coming from stCtx, then it means that the error is coming from the future itself.
|
|
// If this err is not registryClearedErr, then it needs to be returned to the caller.
|
|
if !errors.Is(err, stCtx.Err()) && !errors.Is(err, registryClearedErr) {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// Because of the checks above err (if is not nil) can be only registryClearedErr here.
|
|
// It is converted to Unavailable (retryable) error and returned to the caller.
|
|
// This error will be retried (by history service handler, or history service client in frontend,
|
|
// or SDK, or user client). This will recreate Update in the Registry.
|
|
if errors.Is(err, registryClearedErr) {
|
|
return nil, AbortedByServerErr
|
|
}
|
|
|
|
// TODO: assert(err == nil)
|
|
|
|
// If waitStage=COMPLETED or ACCEPTED and neither has been reached before the softTimeout has expired.
|
|
if stCtx.Err() != nil {
|
|
metrics.WorkflowExecutionUpdateServerTimeout.With(u.instrumentation.metrics).Record(1)
|
|
return statusAdmitted(), nil
|
|
}
|
|
|
|
// Only get here if waitStage=ADMITTED or UNSPECIFIED and neither ACCEPTED nor COMPLETED are reached.
|
|
// Return ADMITTED (as the most advanced stage reached) and empty outcome.
|
|
return statusAdmitted(), nil
|
|
}
|
|
|
|
// abort set Update futures with error or failure (which is passed to all waiters)
|
|
// and set state to stateAborted. It is a terminal state. Update can't be changed after it is aborted.
|
|
// abort uses effects and intermediate stateProvisionallyAborted to delay actual aborting until effects are applied.
|
|
func (u *Update) abort(
|
|
reason AbortReason,
|
|
effects effect.Controller,
|
|
) {
|
|
abortFailure, abortErr := reason.FailureError(u.state)
|
|
if abortFailure == nil && abortErr == nil {
|
|
// If both failure and err are nil, then it means that Update in this state can't be aborted.
|
|
return
|
|
}
|
|
|
|
// Clear any buffered AttachCallbacks callbacks defensively. Abort is called during
|
|
// cleanup (e.g., registry clear, workflow close) where a hard error would be
|
|
// worse than silently clearing.
|
|
u.pendingCallbacks = nil
|
|
|
|
u.instrumentation.countAborted(u.id, reason)
|
|
prevState := u.setState(stateProvisionallyAborted)
|
|
|
|
effects.OnAfterCommit(func(context.Context) {
|
|
if !u.state.Matches(stateSet(stateProvisionallyAborted | stateProvisionallyCompletedAfterAccepted)) {
|
|
return
|
|
}
|
|
var abortOutcome *updatepb.Outcome
|
|
if abortFailure != nil {
|
|
abortOutcome = &updatepb.Outcome{Value: &updatepb.Outcome_Failure{Failure: abortFailure}}
|
|
}
|
|
|
|
beforeCommitState := u.setState(stateAborted)
|
|
u.outcome.(*future.FutureImpl[*updatepb.Outcome]).Set(abortOutcome, abortErr)
|
|
if beforeCommitState == stateProvisionallyCompletedAfterAccepted {
|
|
// If the Update is accepted *and* aborted in the same WFT (because WF was completed in the same WFT),
|
|
// then its state is ProvisionallyCompletedAfterAccepted here, set by onAcceptance.OnAfterCommit.
|
|
//
|
|
// To prevent a race condition in WaitLifecycleStage, the accepted future
|
|
// has not been set by OnAcceptance earlier, as it must be set *after* the outcome future.
|
|
// Now is the time to set it.
|
|
//
|
|
// Note that the Accepted state is skipped, and it transitions straight to Aborted.
|
|
u.accepted.(*future.FutureImpl[*failurepb.Failure]).Set(nil, nil)
|
|
return
|
|
}
|
|
|
|
// If Update was aborted without being accepted,
|
|
// then accepted future must be also set with failure/error.
|
|
const preAcceptedStates = stateSet(stateCreated | stateProvisionallyAdmitted | stateAdmitted | stateSent | stateProvisionallyAccepted)
|
|
if prevState.Matches(preAcceptedStates) {
|
|
u.accepted.(*future.FutureImpl[*failurepb.Failure]).Set(abortFailure, abortErr)
|
|
}
|
|
})
|
|
effects.OnAfterRollback(func(context.Context) {
|
|
if u.state != stateProvisionallyAborted {
|
|
return
|
|
}
|
|
u.setState(prevState)
|
|
})
|
|
}
|
|
|
|
// Admit works if the Update is in any state, but if the state is anything
|
|
// other than stateCreated then it just early returns a nil error. This
|
|
// effectively gives us Update request deduplication by updateID. If the Update
|
|
// is in stateCreated then it builds a protocolpb.Message that will be sent
|
|
// when Send is called.
|
|
func (u *Update) Admit(
|
|
req *updatepb.Request,
|
|
eventStore EventStore, // Will be useful for durable admitted.
|
|
) error {
|
|
if u.state != stateCreated {
|
|
return nil
|
|
}
|
|
if err := u.checkLimits(req); err != nil {
|
|
// Remove the update from the registry immediately.
|
|
u.onComplete()
|
|
return err
|
|
}
|
|
if err := validateRequestMsg(u.id, req); err != nil {
|
|
return err
|
|
}
|
|
if !eventStore.CanAddEvent() {
|
|
// There shouldn't be any waiters before Update is admitted (this func returns).
|
|
// Call abort to seal the Update.
|
|
u.abort(AbortReasonWorkflowCompleted, eventStore)
|
|
// This error must be not nil.
|
|
_, abortErr := AbortReasonWorkflowCompleted.FailureError(stateCreated)
|
|
return abortErr
|
|
}
|
|
|
|
u.instrumentation.countRequestMsg()
|
|
|
|
// Marshal Update request here to return InvalidArgument to the API caller if it can't be marshaled.
|
|
reqAny, err := anypb.New(req)
|
|
if err != nil {
|
|
return serviceerror.NewInvalidArgumentf("unable to unmarshal request: %v", err)
|
|
}
|
|
u.request = reqAny
|
|
|
|
prevState := u.setState(stateProvisionallyAdmitted)
|
|
eventStore.OnAfterCommit(func(context.Context) {
|
|
if u.state != stateProvisionallyAdmitted {
|
|
return
|
|
}
|
|
u.setState(stateAdmitted)
|
|
u.admittedTime = time.Now().UTC()
|
|
})
|
|
eventStore.OnAfterRollback(func(context.Context) {
|
|
if u.state != stateProvisionallyAdmitted {
|
|
return
|
|
}
|
|
u.setState(prevState)
|
|
var timeZero time.Time
|
|
u.admittedTime = timeZero
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// AttachCallbacks attaches completion callbacks from a second caller to an update
|
|
// that has already progressed past admission. If the update is accepted, it writes
|
|
// a WorkflowExecutionOptionsUpdatedEvent with the caller's callbacks and request ID.
|
|
// If the update is in stateSent (sent to worker, not yet accepted), callbacks are
|
|
// buffered in memory and flushed when the update is accepted. If the update is
|
|
// already completed, returns true without attaching callbacks since the caller
|
|
// receives the result synchronously.
|
|
//
|
|
// Returns (true, nil) if the caller should proceed (callbacks attached or update already completed),
|
|
// (false, nil) if the update is in an early state where attachment does not apply,
|
|
// or (false, error) if the update is in a transient state where the caller should retry.
|
|
func (u *Update) AttachCallbacks(
|
|
req *updatepb.Request,
|
|
eventStore EventStore,
|
|
) (isCallbackAttached bool, err error) {
|
|
// Only attach callbacks if the request actually has something to attach.
|
|
// This preserves existing behavior for callers that don't set callbacks.
|
|
if len(req.GetCompletionCallbacks()) == 0 {
|
|
return false, nil
|
|
}
|
|
if req.GetRequestId() == "" {
|
|
return false, serviceerror.NewInvalidArgumentf("invalid %T: request_id is required when completion_callbacks are set", req)
|
|
}
|
|
|
|
switch u.state {
|
|
case stateProvisionallyAccepted,
|
|
stateProvisionallyCompleted,
|
|
stateProvisionallyCompletedAfterAccepted,
|
|
stateProvisionallyAborted:
|
|
// Provisional states are transient — they exist only between an event write
|
|
// and its OnAfterCommit callback within a single workflow task completion
|
|
// transaction. In practice, AttachCallbacks should never see these states because
|
|
// a new UpdateWorkflowExecution API call must acquire the workflow lock,
|
|
// which means the previous transaction has already committed and provisional
|
|
// states have resolved. This guard is kept defensively in case future code
|
|
// paths call AttachCallbacks within the same transaction.
|
|
return false, serviceerror.NewResourceExhausted(enumspb.RESOURCE_EXHAUSTED_CAUSE_BUSY_WORKFLOW, "workflow update is not yet accepted, please retry")
|
|
|
|
case stateSent:
|
|
// stateSent: the update has been sent to the worker but not yet accepted.
|
|
// Buffer the callbacks in memory; they will be flushed to the event store
|
|
// when the update is accepted in onAcceptanceMsg.
|
|
// Returning (true, nil) is safe because:
|
|
// - The caller already holds the workflow lock
|
|
// - A workflow task already exists (the update was sent via one)
|
|
// - No new workflow task is needed — just buffer until acceptance
|
|
// - The event will be written atomically with acceptance
|
|
// If the Update struct is lost (registry cleared), the abort mechanism fires
|
|
// registryClearedErr on the caller's future, prompting an immediate retry.
|
|
if req.GetRequestId() != "" {
|
|
for _, pc := range u.pendingCallbacks {
|
|
if pc.requestID == req.GetRequestId() {
|
|
return true, nil
|
|
}
|
|
}
|
|
}
|
|
u.pendingCallbacks = append(u.pendingCallbacks, pendingCallback{
|
|
requestID: req.GetRequestId(),
|
|
completionCallbacks: req.GetCompletionCallbacks(),
|
|
})
|
|
return true, nil
|
|
|
|
case stateAccepted:
|
|
// Persist immediately as its own event, u.persistCallback(...) will dedup against requestIDs
|
|
// that are already recorded on the workflow.
|
|
return u.persistCallback(eventStore, req.GetRequestId(), req.GetCompletionCallbacks())
|
|
|
|
case stateCompleted:
|
|
// If the update is already completed, the result is returned synchronously
|
|
// in the UpdateWorkflowExecution response — no callback needed.
|
|
return true, nil
|
|
|
|
default:
|
|
// All other states are too early or not applicable for callback attachment.
|
|
return false, nil
|
|
}
|
|
}
|
|
|
|
// persistPendingCallbacks writes one WorkflowExecutionOptionsUpdatedEvent per
|
|
// buffered AttachCallbacks callback, skipping any whose requestID is already persisted.
|
|
// Called from onAcceptanceMsg after the acceptance event has been written.
|
|
//
|
|
// NOTE: Each pending callback requires its own event because the API proto's
|
|
// WorkflowUpdateOptionsUpdate carries a singular AttachedRequestId, and the
|
|
// WorkflowUpdateOptions map is keyed by update ID (all entries here share u.id,
|
|
// so only one map entry is possible per event). Each requestID must be durably
|
|
// recorded in the event so that ApplyWorkflowExecutionOptionsUpdatedEvent can
|
|
// call AttachRequestID during replay for correct deduplication.
|
|
//
|
|
// In practice, the number of buffered callbacks is very small (1-2): it requires
|
|
// multiple concurrent callers to call AttachCallbacks while the update is in
|
|
// stateSent. The per-update callback limit (MaxCallbacksPerUpdateID) bounds the
|
|
// worst case.
|
|
func (u *Update) persistPendingCallbacks(eventStore EventStore) error {
|
|
for _, pc := range u.pendingCallbacks {
|
|
if _, err := u.persistCallback(eventStore, pc.requestID, pc.completionCallbacks); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
u.pendingCallbacks = nil
|
|
return nil
|
|
}
|
|
|
|
// persistCallback writes a single WorkflowExecutionOptionsUpdatedEvent to attach the
|
|
// given requestID and completion callbacks, and deduplicate on requestID if already attached.
|
|
func (u *Update) persistCallback(
|
|
eventStore EventStore,
|
|
requestID string,
|
|
completionCallbacks []*commonpb.Callback,
|
|
) (isCallbackAttached bool, err error) {
|
|
// Callback is already attached, no need to update.
|
|
if requestID != "" && eventStore.HasRequestID(requestID) {
|
|
return true, nil
|
|
}
|
|
_, err = eventStore.AddWorkflowExecutionOptionsUpdatedEvent(
|
|
nil, false, "", nil, nil, "", nil, nil,
|
|
[]*historypb.WorkflowExecutionOptionsUpdatedEventAttributes_WorkflowUpdateOptionsUpdate{{
|
|
UpdateId: u.id,
|
|
AttachedRequestId: requestID,
|
|
AttachedCompletionCallbacks: completionCallbacks,
|
|
}},
|
|
)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// OnProtocolMessage delivers a message to the Update state machine. The Body field of
|
|
// *protocolpb.Message parameter is expected to be one of *updatepb.Response,
|
|
// *updatepb.Rejection, *updatepb.Acceptance. Writes to the EventStore
|
|
// occur synchronously but externally observable effects on this Update (e.g.,
|
|
// emitting an Outcome or an Accepted) are registered with the EventStore to be
|
|
// applied after the durable Updates are committed. If the EventStore rolls back
|
|
// its effects, this state machine does the same.
|
|
//
|
|
// If you modify the state machine, please update the diagram in /docs/architecture/workflow-update.md.
|
|
func (u *Update) OnProtocolMessage(
|
|
protocolMsg *protocolpb.Message,
|
|
eventStore EventStore,
|
|
) error {
|
|
if protocolMsg == nil {
|
|
return serviceerror.NewInvalidArgumentf("Update %s received nil message", u.id)
|
|
}
|
|
|
|
if protocolMsg.Body == nil {
|
|
return serviceerror.NewInvalidArgumentf("Update %s received message with nil body", u.id)
|
|
}
|
|
|
|
body, err := protocolMsg.Body.UnmarshalNew()
|
|
if err != nil {
|
|
return serviceerror.NewInvalidArgumentf("unable to unmarshal request: %v", err)
|
|
}
|
|
|
|
// If no new events can be added to the event store (e.g., workflow is completed),
|
|
// then only Rejection messages can be processed, because they don't create new events in the history.
|
|
// All other message types abort Update.
|
|
_, isRejection := body.(*updatepb.Rejection)
|
|
shouldAbort := !(eventStore.CanAddEvent() || isRejection)
|
|
if shouldAbort {
|
|
u.abort(AbortReasonWorkflowCompleted, eventStore)
|
|
return nil
|
|
}
|
|
|
|
switch updMsg := body.(type) {
|
|
case *updatepb.Acceptance:
|
|
return u.onAcceptanceMsg(updMsg, eventStore)
|
|
case *updatepb.Rejection:
|
|
return u.onRejectionMsg(updMsg, eventStore)
|
|
case *updatepb.Response:
|
|
return u.onResponseMsg(updMsg, eventStore)
|
|
default:
|
|
return serviceerror.NewInvalidArgumentf("Message type %T not supported", body)
|
|
}
|
|
}
|
|
|
|
// needToSend returns true if outgoing message can be generated for the current Update state.
|
|
// If includeAlreadySent is set to true, then it will return true even if Update was already sent but not processed by worker.
|
|
func (u *Update) needToSend(includeAlreadySent bool) bool {
|
|
if includeAlreadySent {
|
|
return u.state.Matches(stateSet(stateAdmitted | stateSent))
|
|
}
|
|
return u.state.Matches(stateSet(stateAdmitted))
|
|
}
|
|
|
|
// Send move Update from stateAdmitted to stateSent and returns the message to be sent to worker.
|
|
// If Update is not in expected stateAdmitted, Send does nothing and returns nil.
|
|
// If includeAlreadySent is set to true, then Send will return a message even if Update was already sent but not processed by worker.
|
|
// If Update lacks a request, then return nil; the request will be communicated to the worker via an UpdateAdmitted event.
|
|
// Note: once Update moved to stateSent it never moves back to stateAdmitted.
|
|
func (u *Update) Send(
|
|
includeAlreadySent bool,
|
|
sequencingID *protocolpb.Message_EventId,
|
|
) *protocolpb.Message {
|
|
if !u.needToSend(includeAlreadySent) {
|
|
return nil
|
|
}
|
|
|
|
u.instrumentation.countSent()
|
|
if u.state == stateSent {
|
|
u.instrumentation.countSentAgain()
|
|
}
|
|
|
|
if u.state == stateAdmitted {
|
|
u.setState(stateSent)
|
|
}
|
|
|
|
if u.request == nil {
|
|
// This implies that the Update in the Registry derives from an UpdateAdmitted event exists;
|
|
// this event (which contains the request payload) is how the Update request will be communicated to the worker.
|
|
return nil
|
|
}
|
|
return &protocolpb.Message{
|
|
ProtocolInstanceId: u.id,
|
|
Id: u.outgoingMessageID(),
|
|
SequencingId: sequencingID,
|
|
Body: u.request,
|
|
}
|
|
}
|
|
|
|
// isSent checks if Update was sent to worker.
|
|
func (u *Update) isSent() bool {
|
|
return u.state.Matches(stateSet(stateSent))
|
|
}
|
|
|
|
// outgoingMessageID returns the ID of the message that is used to Send the Update to the worker.
|
|
func (u *Update) outgoingMessageID() string {
|
|
return u.id + "/request"
|
|
}
|
|
|
|
// onAcceptanceMsg expects the Update to be in stateSent (or stateAdmitted) and returns an
|
|
// error if it finds otherwise. An event is written to the provided EventStore
|
|
// and on commit the accepted future is completed and the Update transitions to
|
|
// stateAccepted.
|
|
func (u *Update) onAcceptanceMsg(
|
|
acpt *updatepb.Acceptance,
|
|
eventStore EventStore,
|
|
) error {
|
|
// Normally Update goes from stateAdmitted to stateSent and then to stateAccepted,
|
|
// therefore, the only valid state here is stateSent.
|
|
// But if Update Registry is cleared after Update was sent to the worker,
|
|
// it will be recreated by retries in stateAdmitted, and then the worker can accept the previous (cleared) Update
|
|
// with the same updateID. Because it is, in fact, the same Update, server should process this accepts message w/o error.
|
|
// Therefore, stateAdmitted is also a valid state.
|
|
if err := u.checkStateSet(acpt, stateSet(stateSent|stateAdmitted)); err != nil {
|
|
return err
|
|
}
|
|
if err := validateAcceptanceMsg(acpt); err != nil {
|
|
return err
|
|
}
|
|
u.instrumentation.countAcceptanceMsg()
|
|
|
|
// If the in-registry Update lacks a request payload, this implies that there is an UpdateAdmitted event in
|
|
// history. In this case, we write the UpdateAccepted event without a request payload, since the UpdateAdmitted
|
|
// event has it.
|
|
//
|
|
// Thus, the following sequences of events are all possible, and SDK workers must handle them correctly:
|
|
// UpdateAdmitted(requestPayload)
|
|
// UpdateAdmitted(requestPayload)
|
|
// ...
|
|
// UpdateAccepted(nil)
|
|
// UpdateAccepted(requestPayload)
|
|
var acceptedRequest *updatepb.Request
|
|
if u.request != nil {
|
|
acceptedRequest = &updatepb.Request{}
|
|
if err := u.request.UnmarshalTo(acceptedRequest); err != nil {
|
|
return serviceerror.NewInternalf("unable to unmarshal original request: %v", err)
|
|
}
|
|
}
|
|
|
|
event, err := eventStore.AddWorkflowExecutionUpdateAcceptedEvent(
|
|
u.id,
|
|
u.outgoingMessageID(),
|
|
acpt.AcceptedRequestSequencingEventId,
|
|
acceptedRequest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
u.acceptedEventID = event.EventId
|
|
|
|
// Persist any callbacks that were buffered by AttachCallbacks while in stateSent or stateAdmitted.
|
|
// See persistPendingCallbacks for why this writes one event per pending entry.
|
|
if err := u.persistPendingCallbacks(eventStore); err != nil {
|
|
return err
|
|
}
|
|
|
|
prevState := u.setState(stateProvisionallyAccepted)
|
|
eventStore.OnAfterCommit(func(context.Context) {
|
|
if !u.state.Matches(stateSet(stateProvisionallyAccepted | stateProvisionallyCompleted | stateProvisionallyAborted)) {
|
|
return
|
|
}
|
|
u.request = nil
|
|
|
|
// If the Update is accepted *and* completed in the same WFT, then its state has transitioned
|
|
// from ProvisionallyAccepted to ProvisionallyCompleted in onResponseMsg by the
|
|
// time we get here.
|
|
//
|
|
// Now, to prevent a race condition in WaitLifecycleStage, the accepted future
|
|
// cannot be set here right now, as it must be set *after* the outcome future.
|
|
//
|
|
// So instead, the state is set to ProvisionallyCompletedAfterAccepted here,
|
|
// and onResponseMsg.OnAfterCommit callback will set the futures in the correct order.
|
|
if u.state == stateProvisionallyCompleted {
|
|
u.state = stateProvisionallyCompletedAfterAccepted
|
|
return
|
|
}
|
|
// If the Update is accepted *and* WF is completed in the same WFT, then its state has transitioned
|
|
// from ProvisionallyAccepted to ProvisionallyAborted in abort function by the
|
|
// time we get here.
|
|
//
|
|
// Now, to prevent a race condition in WaitLifecycleStage, the accepted future
|
|
// cannot be set here right now, as it must be set *after* the outcome future.
|
|
//
|
|
// Same ProvisionallyCompletedAfterAccepted is reused here (although it is
|
|
// technically ProvisionallyAbortedAfterAccepted), and abort.OnAfterCommit callback
|
|
// will set the futures in the correct order.
|
|
if u.state == stateProvisionallyAborted {
|
|
u.state = stateProvisionallyCompletedAfterAccepted
|
|
return
|
|
}
|
|
u.setState(stateAccepted)
|
|
u.accepted.(*future.FutureImpl[*failurepb.Failure]).Set(nil, nil)
|
|
})
|
|
eventStore.OnAfterRollback(func(context.Context) {
|
|
if u.state != stateProvisionallyAccepted {
|
|
return
|
|
}
|
|
u.acceptedEventID = common.EmptyEventID
|
|
u.pendingCallbacks = nil
|
|
u.setState(prevState)
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// onRejectionMsg expects the Update state to be stateSent (or stateAdmitted) and returns
|
|
// an error otherwise. On commit of buffered effects, the state
|
|
// machine transitions to stateCompleted and the accepted and outcome futures
|
|
// are both completed with the failurepb.Failure value from the updatepb.Rejection input message.
|
|
func (u *Update) onRejectionMsg(
|
|
rej *updatepb.Rejection,
|
|
eventStore EventStore,
|
|
) error {
|
|
// See comment in onAcceptanceMsg about stateAdmitted.
|
|
if err := u.checkStateSet(rej, stateSet(stateSent|stateAdmitted)); err != nil {
|
|
return err
|
|
}
|
|
if err := validateRejectionMsg(rej); err != nil {
|
|
return err
|
|
}
|
|
u.instrumentation.countRejectionMsg()
|
|
// Notify the event store so it can fire any completion callbacks that were
|
|
// registered at admission time (e.g., after reset/reapply) and clean up
|
|
// the update's mutable-state entry.
|
|
if err := eventStore.RejectWorkflowExecutionUpdate(u.id, rej.Failure); err != nil {
|
|
return err
|
|
}
|
|
// Clear any buffered AttachCallbacks callbacks — they cannot be delivered for a rejected update.
|
|
u.pendingCallbacks = nil
|
|
return u.reject(rej.Failure, eventStore)
|
|
}
|
|
|
|
// rejects an Update with provided failure.
|
|
func (u *Update) reject(
|
|
rejectionFailure *failurepb.Failure,
|
|
effects effect.Controller,
|
|
) error {
|
|
if len(u.pendingCallbacks) > 0 {
|
|
// Invariant: buffer must be cleared before reject. If we reach here,
|
|
// there is a bug in the caller (onRejectionMsg should clear the buffer).
|
|
return serviceerror.NewInternalf(
|
|
"update %s: reject called with %d pending AttachCallbacks callbacks",
|
|
u.id, len(u.pendingCallbacks),
|
|
)
|
|
}
|
|
prevState := u.setState(stateProvisionallyCompleted)
|
|
effects.OnAfterCommit(func(context.Context) {
|
|
if u.state != stateProvisionallyCompleted {
|
|
return
|
|
}
|
|
|
|
u.request = nil
|
|
u.setState(stateCompleted)
|
|
outcome := updatepb.Outcome{
|
|
Value: &updatepb.Outcome_Failure{Failure: rejectionFailure},
|
|
}
|
|
u.accepted.(*future.FutureImpl[*failurepb.Failure]).Set(rejectionFailure, nil)
|
|
u.outcome.(*future.FutureImpl[*updatepb.Outcome]).Set(&outcome, nil)
|
|
u.onComplete()
|
|
})
|
|
effects.OnAfterRollback(func(context.Context) {
|
|
if u.state != stateProvisionallyCompleted {
|
|
return
|
|
}
|
|
u.setState(prevState)
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// onResponseMsg expects the Update to be in either stateProvisionallyAccepted
|
|
// or stateAccepted and returns an error if it finds otherwise. On commit of
|
|
// buffered effects, the state machine will transition to stateCompleted, and the
|
|
// outcome future is completed with the updatepb.Outcome from the updatepb.Response input message.
|
|
func (u *Update) onResponseMsg(
|
|
res *updatepb.Response,
|
|
eventStore EventStore,
|
|
) error {
|
|
if err := u.checkStateSet(res, stateSet(stateProvisionallyAccepted|stateAccepted)); err != nil {
|
|
return err
|
|
}
|
|
if err := validateResponseMsg(u.id, res); err != nil {
|
|
return err
|
|
}
|
|
if _, err := eventStore.AddWorkflowExecutionUpdateCompletedEvent(u.acceptedEventID, res); err != nil {
|
|
return err
|
|
}
|
|
u.instrumentation.countResponseMsg()
|
|
prevState := u.setState(stateProvisionallyCompleted)
|
|
eventStore.OnAfterCommit(func(context.Context) {
|
|
if !u.state.Matches(stateSet(stateProvisionallyCompleted | stateProvisionallyCompletedAfterAccepted)) {
|
|
return
|
|
}
|
|
beforeCommitState := u.setState(stateCompleted)
|
|
u.outcome.(*future.FutureImpl[*updatepb.Outcome]).Set(res.GetOutcome(), nil)
|
|
if beforeCommitState == stateProvisionallyCompletedAfterAccepted {
|
|
// If the Update is accepted *and* completed in the same WFT,
|
|
// then its state is ProvisionallyCompletedAfterAccepted here, set by onAcceptance.OnAfterCommit.
|
|
//
|
|
// To prevent a race condition in WaitLifecycleStage, the accepted future
|
|
// has not been set by OnAcceptance earlier, as it must be set *after* the outcome future.
|
|
// Now is the time to set it.
|
|
//
|
|
// Note that the Accepted state is skipped, and it transitions straight to Completed.
|
|
u.accepted.(*future.FutureImpl[*failurepb.Failure]).Set(nil, nil)
|
|
}
|
|
u.onComplete()
|
|
})
|
|
eventStore.OnAfterRollback(func(context.Context) {
|
|
if u.state != stateProvisionallyCompleted {
|
|
return
|
|
}
|
|
u.setState(prevState)
|
|
})
|
|
return nil
|
|
}
|
|
|
|
func (u *Update) checkStateSet(msg proto.Message, allowed stateSet) error {
|
|
if u.state.Matches(allowed) {
|
|
return nil
|
|
}
|
|
u.instrumentation.invalidStateTransition(u.id, msg, u.state)
|
|
return serviceerror.NewInvalidArgumentf("invalid state transition attempted for Update %s: "+
|
|
"received %T message while in state %s", u.id, msg, u.state)
|
|
}
|
|
|
|
// setState assigns the current state to a new value returning the original value.
|
|
func (u *Update) setState(newState state) state {
|
|
prevState := u.state
|
|
u.state = newState
|
|
u.instrumentation.stateChange(u.id, prevState, newState)
|
|
return prevState
|
|
}
|
|
|
|
func (u *Update) GetSize() int {
|
|
size := len(u.id)
|
|
size += proto.Size(u.request)
|
|
if u.accepted.Ready() {
|
|
res, _ := u.accepted.Get(context.Background())
|
|
size += res.Size()
|
|
}
|
|
if u.outcome.Ready() {
|
|
res, _ := u.outcome.Get(context.Background())
|
|
size += res.Size()
|
|
}
|
|
return size
|
|
}
|
|
|
|
func (u *Update) AcceptedEventID() int64 {
|
|
return u.acceptedEventID
|
|
}
|