mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-30 14:56:19 +00:00
## What changed? Cherry-pick versioning PRs - #9168 - Cache for system protection - #9262 - Cache for system protection - #9239 - Critical PR to enable sending `TargetVersionChanged` flag for Upgrade-on-CaN feature - #9147 - Tracks version drainage properly when version receives workflows via `VersioningOverride`. Needed for automated worker controllers to correctly scale versioned workers that received workflows via `VersioningOverride`. - #9300 - Needed for `approximate_backlog_count` metric to track Current and Ramping version tasks correctly - #9316 - Needed for `approximate_backlog_count` metric to track Current and Ramping version tasks correctly - #8957 - Contains minor metric improvement. Included because it adds a test harness that is used in the two metrics PRs above - #9250 - Bug fix of task rescheduling edge case during AutoUpgrade Transition ## Why? For OSS v1.30.2 ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) --- > [!NOTE] > **Medium Risk** > Touches history/matching worker-versioning paths, adding new caches and changing workflow task/start handling and backlog metric emission; incorrect caching or signaling could affect dispatch/upgrade behavior and observability. > > **Overview** > Adds new worker-versioning protections and upgrade signaling: workflow task started events now persist a `workflow_task_target_worker_deployment_version_changed` flag (and emit a new `workflow_target_version_changed_count` metric) under a new `EnableSendTargetVersionChanged` dynamic config. > > Introduces two new caches with metrics and dynamic config knobs: a `RoutingInfoCache` to avoid repeated `GetTaskQueueUserData` lookups during activity start/transition logic, and a `ReactivationSignalCache` plus `EnableVersionReactivationSignals` to dedupe and asynchronously send “reactivation” signals when workflows are pinned (via start/signal-with-start/reset/update-options) to potentially drained/inactive worker versions. > > Extends matching backlog metrics to support version-attributed reporting by adding `BacklogMetricsEmitInterval` and switching queue DB emission to *physical* backlog gauges (`physical_approximate_backlog_*`) when attribution is enabled, while keeping legacy gauges when disabled. > > Adds frontend scaffolding for a new visibility RPC `CountSchedules` (client plumbing, interception/metadata/quota wiring) but leaves the frontend handler unimplemented, and bumps `go.temporal.io/api` to `v1.62.2`. > > <sup>Written by [Cursor Bugbot]() for commit cb8ae14c86666654c51dd66ae91cb225722e1ba7. This will update automatically on new commits. Configure [here]().</sup> --------- Co-authored-by: Shahab Tajik <[email protected]> Co-authored-by: Shivam <[email protected]>
244 lines
8.5 KiB
Go
244 lines
8.5 KiB
Go
package resetworkflow
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/google/uuid"
|
|
enumspb "go.temporal.io/api/enums/v1"
|
|
"go.temporal.io/api/serviceerror"
|
|
workflowpb "go.temporal.io/api/workflow/v1"
|
|
"go.temporal.io/server/api/historyservice/v1"
|
|
"go.temporal.io/server/api/matchingservice/v1"
|
|
"go.temporal.io/server/common"
|
|
"go.temporal.io/server/common/definition"
|
|
"go.temporal.io/server/common/locks"
|
|
"go.temporal.io/server/common/log/tag"
|
|
"go.temporal.io/server/common/metrics"
|
|
"go.temporal.io/server/common/namespace"
|
|
"go.temporal.io/server/common/persistence/versionhistory"
|
|
"go.temporal.io/server/common/worker_versioning"
|
|
"go.temporal.io/server/service/history/api"
|
|
historyi "go.temporal.io/server/service/history/interfaces"
|
|
"go.temporal.io/server/service/history/ndc"
|
|
)
|
|
|
|
func Invoke(
|
|
ctx context.Context,
|
|
resetRequest *historyservice.ResetWorkflowExecutionRequest,
|
|
shardContext historyi.ShardContext,
|
|
workflowConsistencyChecker api.WorkflowConsistencyChecker,
|
|
matchingClient matchingservice.MatchingServiceClient,
|
|
versionMembershipCache worker_versioning.VersionMembershipCache,
|
|
reactivationSignalCache worker_versioning.ReactivationSignalCache,
|
|
reactivationSignaler api.VersionReactivationSignalerFn,
|
|
) (_ *historyservice.ResetWorkflowExecutionResponse, retError error) {
|
|
namespaceID := namespace.ID(resetRequest.GetNamespaceId())
|
|
err := api.ValidateNamespaceUUID(namespaceID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
request := resetRequest.ResetRequest
|
|
workflowID := request.WorkflowExecution.GetWorkflowId()
|
|
baseRunID := request.WorkflowExecution.GetRunId()
|
|
|
|
baseWorkflowLease, err := workflowConsistencyChecker.GetWorkflowLease(
|
|
ctx,
|
|
nil,
|
|
definition.NewWorkflowKey(
|
|
namespaceID.String(),
|
|
workflowID,
|
|
baseRunID,
|
|
),
|
|
locks.PriorityHigh,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { baseWorkflowLease.GetReleaseFn()(retError) }()
|
|
|
|
baseMutableState := baseWorkflowLease.GetMutableState()
|
|
if request.GetWorkflowTaskFinishEventId() <= common.FirstEventID ||
|
|
request.GetWorkflowTaskFinishEventId() >= baseMutableState.GetNextEventID() {
|
|
return nil, serviceerror.NewInvalidArgument("Workflow task finish ID must be > 1 && <= workflow last event ID.")
|
|
}
|
|
|
|
// Validate versioning override, if any.
|
|
err = validatePostResetOperationInputs(ctx, request.GetPostResetOperations(), matchingClient, versionMembershipCache,
|
|
baseMutableState.GetExecutionInfo().GetTaskQueue(), namespaceID.String())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// also load the current run of the workflow, it can be different from the base runID
|
|
currentRunID, err := workflowConsistencyChecker.GetCurrentWorkflowRunID(
|
|
ctx,
|
|
namespaceID.String(),
|
|
request.WorkflowExecution.GetWorkflowId(),
|
|
locks.PriorityHigh,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if baseRunID == "" {
|
|
baseRunID = currentRunID
|
|
}
|
|
|
|
var currentWorkflowLease api.WorkflowLease
|
|
if currentRunID == baseRunID {
|
|
currentWorkflowLease = baseWorkflowLease
|
|
} else {
|
|
currentWorkflowLease, err = workflowConsistencyChecker.GetWorkflowLease(
|
|
ctx,
|
|
nil,
|
|
definition.NewWorkflowKey(
|
|
namespaceID.String(),
|
|
workflowID,
|
|
currentRunID,
|
|
),
|
|
locks.PriorityHigh,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { currentWorkflowLease.GetReleaseFn()(retError) }()
|
|
}
|
|
|
|
// dedup by requestID
|
|
if currentWorkflowLease.GetMutableState().GetExecutionState().CreateRequestId == request.GetRequestId() {
|
|
shardContext.GetLogger().Info("Duplicated reset request",
|
|
tag.WorkflowID(workflowID),
|
|
tag.WorkflowRunID(currentRunID),
|
|
tag.WorkflowNamespaceID(namespaceID.String()))
|
|
return &historyservice.ResetWorkflowExecutionResponse{
|
|
RunId: currentRunID,
|
|
}, nil
|
|
}
|
|
|
|
resetRunID := uuid.New().String()
|
|
baseRebuildLastEventID := request.GetWorkflowTaskFinishEventId() - 1
|
|
baseVersionHistories := baseMutableState.GetExecutionInfo().GetVersionHistories()
|
|
baseCurrentVersionHistory, err := versionhistory.GetCurrentVersionHistory(baseVersionHistories)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
baseRebuildLastEventVersion, err := versionhistory.GetVersionHistoryEventVersion(baseCurrentVersionHistory, baseRebuildLastEventID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
baseCurrentBranchToken := baseCurrentVersionHistory.GetBranchToken()
|
|
baseNextEventID := baseMutableState.GetNextEventID()
|
|
baseWorkflow := ndc.NewWorkflow(
|
|
shardContext.GetClusterMetadata(),
|
|
baseWorkflowLease.GetContext(),
|
|
baseWorkflowLease.GetMutableState(),
|
|
baseWorkflowLease.GetReleaseFn(),
|
|
)
|
|
|
|
namespaceEntry, err := api.GetActiveNamespace(shardContext, namespaceID, workflowID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
metrics.WorkflowResetCount.With(
|
|
shardContext.GetMetricsHandler().WithTags(
|
|
metrics.NamespaceTag(namespaceEntry.Name().String()),
|
|
metrics.OperationTag(metrics.HistoryResetWorkflowScope),
|
|
metrics.VersioningBehaviorTag(baseMutableState.GetEffectiveVersioningBehavior()),
|
|
),
|
|
).Record(1)
|
|
|
|
allowResetWithPendingChildren := shardContext.GetConfig().AllowResetWithPendingChildren(namespaceEntry.Name().String())
|
|
if err := ndc.NewWorkflowResetter(
|
|
shardContext,
|
|
workflowConsistencyChecker.GetWorkflowCache(),
|
|
shardContext.GetLogger(),
|
|
).ResetWorkflow(
|
|
ctx,
|
|
namespaceID,
|
|
workflowID,
|
|
baseRunID,
|
|
baseCurrentBranchToken,
|
|
baseRebuildLastEventID,
|
|
baseRebuildLastEventVersion,
|
|
baseNextEventID,
|
|
resetRunID,
|
|
request.GetRequestId(),
|
|
baseWorkflow,
|
|
ndc.NewWorkflow(
|
|
shardContext.GetClusterMetadata(),
|
|
currentWorkflowLease.GetContext(),
|
|
currentWorkflowLease.GetMutableState(),
|
|
currentWorkflowLease.GetReleaseFn(),
|
|
),
|
|
request.GetReason(),
|
|
nil,
|
|
GetResetReapplyExcludeTypes(request.GetResetReapplyExcludeTypes(), request.GetResetReapplyType()),
|
|
allowResetWithPendingChildren,
|
|
resetRequest.ResetRequest.PostResetOperations,
|
|
); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Notify version workflow if we're pinning to a potentially drained version via post-reset operations
|
|
for _, operation := range request.GetPostResetOperations() {
|
|
if updateOpts, ok := operation.GetVariant().(*workflowpb.PostResetOperation_UpdateWorkflowOptions_); ok {
|
|
api.ReactivateVersionWorkflowIfPinned(ctx, namespaceEntry,
|
|
updateOpts.UpdateWorkflowOptions.GetWorkflowExecutionOptions().GetVersioningOverride(), reactivationSignalCache, reactivationSignaler, shardContext.GetConfig().EnableVersionReactivationSignals())
|
|
}
|
|
}
|
|
|
|
return &historyservice.ResetWorkflowExecutionResponse{
|
|
RunId: resetRunID,
|
|
}, nil
|
|
}
|
|
|
|
// GetResetReapplyExcludeTypes computes the set of requested exclude types. It
|
|
// uses the reset_reapply_exclude_types request field (a set of event types to
|
|
// exclude from reapply), as well as the deprecated reset_reapply_type request
|
|
// field (a specification of what to include).
|
|
func GetResetReapplyExcludeTypes(
|
|
excludeTypes []enumspb.ResetReapplyExcludeType,
|
|
includeType enumspb.ResetReapplyType,
|
|
) map[enumspb.ResetReapplyExcludeType]struct{} {
|
|
// A client who wishes to have reapplication of all supported event types should omit the deprecated
|
|
// reset_reapply_type field (since its default value is RESET_REAPPLY_TYPE_ALL_ELIGIBLE).
|
|
exclude := map[enumspb.ResetReapplyExcludeType]struct{}{}
|
|
switch includeType {
|
|
case enumspb.RESET_REAPPLY_TYPE_SIGNAL:
|
|
// A client sending this value of the deprecated reset_reapply_type field will not have any events other than
|
|
// signal reapplied.
|
|
exclude[enumspb.RESET_REAPPLY_EXCLUDE_TYPE_UPDATE] = struct{}{}
|
|
case enumspb.RESET_REAPPLY_TYPE_NONE:
|
|
exclude[enumspb.RESET_REAPPLY_EXCLUDE_TYPE_SIGNAL] = struct{}{}
|
|
exclude[enumspb.RESET_REAPPLY_EXCLUDE_TYPE_UPDATE] = struct{}{}
|
|
case enumspb.RESET_REAPPLY_TYPE_UNSPECIFIED, enumspb.RESET_REAPPLY_TYPE_ALL_ELIGIBLE:
|
|
// Do nothing.
|
|
}
|
|
for _, e := range excludeTypes {
|
|
exclude[e] = struct{}{}
|
|
}
|
|
return exclude
|
|
}
|
|
|
|
// validatePostResetOperationInputs validates the optional post reset operation inputs.
|
|
func validatePostResetOperationInputs(ctx context.Context,
|
|
postResetOperations []*workflowpb.PostResetOperation,
|
|
matchingClient matchingservice.MatchingServiceClient,
|
|
versionMembershipCache worker_versioning.VersionMembershipCache,
|
|
taskQueue string,
|
|
namespaceID string) error {
|
|
for _, operation := range postResetOperations {
|
|
switch op := operation.GetVariant().(type) {
|
|
case *workflowpb.PostResetOperation_UpdateWorkflowOptions_:
|
|
opts := op.UpdateWorkflowOptions.GetWorkflowExecutionOptions()
|
|
if err := worker_versioning.ValidateVersioningOverride(ctx, opts.GetVersioningOverride(), matchingClient, versionMembershipCache, taskQueue, enumspb.TASK_QUEUE_TYPE_WORKFLOW, namespaceID); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
return serviceerror.NewInvalidArgumentf("unsupported post reset operation: %T", op)
|
|
}
|
|
}
|
|
return nil
|
|
}
|