Files
b5351e8607 fix: make errInvalidComputeConfig non-retryable in deployment activities (#10127)
## Summary

- Changed `UpdateWorkerControllerInstanceFromDeployment` (activities.go)
and `UpdateWorkerControllerInstance` (version_activities.go) to return
`NewNonRetryableApplicationError` instead of `NewApplicationError` when
wrapping `InvalidArgument` errors from the WCI client
- Added functional test
`TestCreateWorkerDeploymentVersion_InvalidComputeConfig_ReturnsPromptly`
that verifies the error returns promptly when creating a version with
invalid compute config on a busy deployment

## Context

The `TestCreateWorkerDeploymentVersion_Errors` test in the CLI repo
(temporalio/cli) is flaky. When `create-version` is called with invalid
AWS Lambda credentials, the expected `"failed to assume role"` error is
sometimes replaced by `"too many requests"`.

Root cause: both activities wrap `InvalidArgument` errors as
**retryable** `ApplicationError`. The activity retries up to 5 times
(each attempt timing out trying to reach AWS IMDS), keeping the
deployment workflow busy. The deployment client's retry loop then times
out after 1 minute and `convertAndRecordError` masks the actual error as
`"too many requests"`.

The existing unit tests (`workflow_test.go:1543`,
`version_workflow_test.go:2383`) already mock the activity returning
`NewNonRetryableApplicationError` -- the activities just weren't
matching that expectation.

## Test plan

- [x] Existing unit tests pass (`go test
./service/worker/workerdeployment/...`)
- [x] Existing functional tests pass
(`TestCreateWorkerDeploymentVersion_InvalidScalingGroups`,
`TestCreateWorkerDeploymentVersion_InvalidArgs`)
- [x] New functional test passes 5/5 runs, consistently under 3 seconds
- [ ] Verify CLI `TestCreateWorkerDeploymentVersion_Errors` passes after
bumping server dependency

Co-authored-by: Jay Pipes <[email protected]>
2026-04-30 16:26:18 -07:00

301 lines
10 KiB
Go

package workerdeployment
import (
"cmp"
"context"
"errors"
"sync"
computepb "go.temporal.io/api/compute/v1"
enumspb "go.temporal.io/api/enums/v1"
"go.temporal.io/api/serviceerror"
updatepb "go.temporal.io/api/update/v1"
"go.temporal.io/sdk/activity"
"go.temporal.io/sdk/temporal"
deploymentspb "go.temporal.io/server/api/deployment/v1"
"go.temporal.io/server/api/matchingservice/v1"
"go.temporal.io/server/common/namespace"
"go.temporal.io/server/common/sdk"
"go.temporal.io/server/common/worker_versioning"
)
type (
Activities struct {
activityDeps
namespace *namespace.Namespace
}
)
func (a *Activities) SyncWorkerDeploymentVersion(ctx context.Context, args *deploymentspb.SyncVersionStateActivityArgs) (*deploymentspb.SyncVersionStateActivityResult, error) {
identity := "worker-deployment workflow " + activity.GetInfo(ctx).WorkflowExecution.ID
res, err := a.WorkerDeploymentClient.SyncVersionWorkflowFromWorkerDeployment(
ctx,
a.namespace,
args.DeploymentName,
args.Version,
args.UpdateArgs,
identity,
args.RequestId,
)
if err != nil {
return nil, err
}
return &deploymentspb.SyncVersionStateActivityResult{
//nolint:staticcheck // SA1019
VersionState: res.GetVersionState(),
Summary: res.GetSummary(),
}, nil
}
func (a *Activities) SyncUnversionedRamp(
ctx context.Context,
input *deploymentspb.SyncUnversionedRampActivityArgs,
) (*deploymentspb.SyncDeploymentVersionUserDataResponse, error) {
logger := activity.GetLogger(ctx)
// Get all the task queues in the current version and put them into SyncUserData format
currVersionInfo, _, err := a.WorkerDeploymentClient.DescribeVersion(ctx, a.namespace, input.CurrentVersion, false)
if err != nil {
return nil, err
}
data := &deploymentspb.DeploymentVersionData{
Version: nil,
RoutingUpdateTime: input.UpdateArgs.RoutingUpdateTime,
RampingSinceTime: input.UpdateArgs.RampingSinceTime,
RampPercentage: input.UpdateArgs.RampPercentage,
}
var taskQueueSyncs []*deploymentspb.SyncDeploymentVersionUserDataRequest_SyncUserData
for _, tqInfo := range currVersionInfo.GetTaskQueueInfos() {
taskQueueSyncs = append(taskQueueSyncs, &deploymentspb.SyncDeploymentVersionUserDataRequest_SyncUserData{
Name: tqInfo.GetName(),
Types: []enumspb.TaskQueueType{tqInfo.GetType()},
Data: data,
})
}
// For each task queue, sync the unversioned ramp data
errs := make(chan error)
var lock sync.Mutex
maxVersionByTQName := make(map[string]int64)
for _, e := range taskQueueSyncs {
go func(syncData *deploymentspb.SyncDeploymentVersionUserDataRequest_SyncUserData) {
logger.Info("syncing unversioned ramp to task queue userdata", "taskQueue", syncData.Name, "types", syncData.Types)
var res *matchingservice.SyncDeploymentUserDataResponse
var err error
res, err = a.MatchingClient.SyncDeploymentUserData(ctx, &matchingservice.SyncDeploymentUserDataRequest{
NamespaceId: a.namespace.ID().String(),
TaskQueue: syncData.Name,
TaskQueueTypes: syncData.Types,
Operation: &matchingservice.SyncDeploymentUserDataRequest_UpdateVersionData{UpdateVersionData: syncData.Data},
})
if err != nil {
logger.Error("syncing task queue userdata", "taskQueue", syncData.Name, "types", syncData.Types, "error", err)
} else {
lock.Lock()
maxVersionByTQName[syncData.Name] = max(maxVersionByTQName[syncData.Name], res.Version)
lock.Unlock()
}
errs <- err
}(e)
}
for range taskQueueSyncs {
err = cmp.Or(err, <-errs)
}
if err != nil {
return nil, err
}
return &deploymentspb.SyncDeploymentVersionUserDataResponse{TaskQueueMaxVersions: maxVersionByTQName}, nil
}
func (a *Activities) CheckUnversionedRampUserDataPropagation(ctx context.Context, input *deploymentspb.CheckWorkerDeploymentUserDataPropagationRequest) error {
logger := activity.GetLogger(ctx)
errs := make(chan error)
for n, v := range input.TaskQueueMaxVersions {
go func(name string, version int64) {
logger.Info("waiting for unversioned ramp userdata propagation", "taskQueue", name, "version", version)
_, err := a.MatchingClient.CheckTaskQueueUserDataPropagation(ctx, &matchingservice.CheckTaskQueueUserDataPropagationRequest{
NamespaceId: a.namespace.ID().String(),
TaskQueue: name,
Version: version,
})
if err != nil {
logger.Error("waiting for unversioned ramp userdata propagation", "taskQueue", name, "type", version, "error", err)
}
errs <- err
}(n, v)
}
var err error
for range input.TaskQueueMaxVersions {
err = cmp.Or(err, <-errs)
}
return err
}
func (a *Activities) IsVersionMissingTaskQueues(ctx context.Context, args *deploymentspb.IsVersionMissingTaskQueuesArgs) (*deploymentspb.IsVersionMissingTaskQueuesResult, error) {
res, err := a.WorkerDeploymentClient.IsVersionMissingTaskQueues(
ctx,
a.namespace,
args.PrevCurrentVersion,
args.NewCurrentVersion,
)
if err != nil {
return nil, err
}
return &deploymentspb.IsVersionMissingTaskQueuesResult{
IsMissingTaskQueues: res,
}, nil
}
func (a *Activities) DeleteWorkerDeploymentVersion(ctx context.Context, args *deploymentspb.DeleteVersionActivityArgs) error {
identity := "worker-deployment workflow " + activity.GetInfo(ctx).WorkflowExecution.ID
versionObj, err := worker_versioning.WorkerDeploymentVersionFromStringV31(args.Version)
if err != nil {
return err
}
workflowID := GenerateVersionWorkflowID(args.DeploymentName, versionObj.GetBuildId())
updatePayload, err := sdk.PreferProtoDataConverter.ToPayloads(&deploymentspb.DeleteVersionArgs{
Identity: identity,
Version: args.Version,
SkipDrainage: args.SkipDrainage,
AsyncPropagation: args.AsyncPropagation,
})
if err != nil {
return err
}
outcome, err := updateWorkflow(
ctx,
a.HistoryClient,
a.namespace,
workflowID,
&updatepb.Request{
Input: &updatepb.Input{Name: DeleteVersion, Args: updatePayload},
Meta: &updatepb.Meta{UpdateId: args.RequestId, Identity: identity},
},
)
if err != nil {
return err
}
err = extractApplicationErrorOrInternal(outcome.GetFailure())
if err != nil {
return err
}
return nil
}
func (a *Activities) RegisterWorkerInVersion(ctx context.Context, args *deploymentspb.RegisterWorkerInVersionArgs) error {
identity := "worker-deployment workflow " + activity.GetInfo(ctx).WorkflowExecution.ID
err := a.WorkerDeploymentClient.RegisterWorkerInVersion(
ctx,
a.namespace,
args,
identity,
)
if err != nil {
return err
}
return nil
}
func (a *Activities) DescribeVersionFromWorkerDeployment(ctx context.Context, args *deploymentspb.DescribeVersionFromWorkerDeploymentActivityArgs) (*deploymentspb.DescribeVersionFromWorkerDeploymentActivityResult, error) {
res, _, err := a.WorkerDeploymentClient.DescribeVersion(ctx, a.namespace, args.Version, false)
if err != nil {
return nil, err
}
return &deploymentspb.DescribeVersionFromWorkerDeploymentActivityResult{
TaskQueueInfos: res.TaskQueueInfos,
}, nil
}
func (a *Activities) SyncDeploymentVersionUserDataFromWorkerDeployment(
ctx context.Context,
input *deploymentspb.SyncDeploymentVersionUserDataRequest,
) (*deploymentspb.SyncDeploymentVersionUserDataResponse, error) {
logger := activity.GetLogger(ctx)
errs := make(chan error)
var lock sync.Mutex
maxVersionByName := make(map[string]int64)
for _, e := range input.Sync {
go func(syncData *deploymentspb.SyncDeploymentVersionUserDataRequest_SyncUserData) {
logger.Info("syncing task queue userdata for deployment version", "taskQueue", syncData.Name, "types", syncData.Types)
var res *matchingservice.SyncDeploymentUserDataResponse
var err error
if input.ForgetVersion {
res, err = a.MatchingClient.SyncDeploymentUserData(ctx, &matchingservice.SyncDeploymentUserDataRequest{
NamespaceId: a.namespace.ID().String(),
DeploymentName: input.GetDeploymentName(),
TaskQueue: syncData.Name,
TaskQueueTypes: syncData.Types,
Operation: &matchingservice.SyncDeploymentUserDataRequest_ForgetVersion{
ForgetVersion: input.Version,
},
})
} else {
res, err = a.MatchingClient.SyncDeploymentUserData(ctx, &matchingservice.SyncDeploymentUserDataRequest{
NamespaceId: a.namespace.ID().String(),
DeploymentName: input.GetDeploymentName(),
TaskQueue: syncData.Name,
TaskQueueTypes: syncData.Types,
Operation: &matchingservice.SyncDeploymentUserDataRequest_UpdateVersionData{
UpdateVersionData: syncData.Data,
},
})
}
if err != nil {
logger.Error("syncing task queue userdata", "taskQueue", syncData.Name, "types", syncData.Types, "error", err)
} else {
lock.Lock()
maxVersionByName[syncData.Name] = max(maxVersionByName[syncData.Name], res.Version)
lock.Unlock()
}
errs <- err
}(e)
}
var err error
for range input.Sync {
err = cmp.Or(err, <-errs)
}
if err != nil {
return nil, err
}
return &deploymentspb.SyncDeploymentVersionUserDataResponse{TaskQueueMaxVersions: maxVersionByName}, nil
}
func (a *Activities) StartWorkerDeploymentVersionWorkflow(
ctx context.Context,
input *deploymentspb.StartWorkerDeploymentVersionRequest,
) error {
logger := activity.GetLogger(ctx)
logger.Info("starting worker deployment version workflow", "deploymentName", input.DeploymentName, "buildID", input.BuildId)
startIdentity := "deployment workflow " + activity.GetInfo(ctx).WorkflowExecution.ID
return a.WorkerDeploymentClient.StartWorkerDeploymentVersion(ctx, a.namespace, input.DeploymentName, input.BuildId, startIdentity, input.RequestId, input.GetIdentity(), input.GetComputeConfig())
}
func (a *Activities) UpdateWorkerControllerInstanceFromDeployment(ctx context.Context, input *deploymentspb.UpdateWorkerControllerInstanceInput) (*computepb.ComputeConfigSummary, error) {
upserts := scalingGroupUpdatesToWCI(input.GetUpsertScalingGroups())
resp, err := a.WorkerControllerInstanceClient.UpdateWorkerControllerInstance(ctx, a.namespace, input.GetVersion(), nil, input.GetIdentity(), upserts, input.GetRemoveScalingGroups())
if err != nil {
var invalidArgs *serviceerror.InvalidArgument
if errors.As(err, &invalidArgs) {
return nil, temporal.NewNonRetryableApplicationError(err.Error(), errInvalidComputeConfig, nil)
}
return nil, err
}
if resp == nil {
return nil, nil
}
return wciSpecToComputeConfigSummary(resp.Spec), nil
}
func (a *Activities) DeleteWorkerControllerInstanceFromDeployment(ctx context.Context, input *deploymentspb.DeleteWorkerControllerInstanceInput) error {
return a.WorkerControllerInstanceClient.DeleteWorkerControllerInstance(ctx, a.namespace, input.GetVersion(), input.GetIdentity())
}