mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-30 14:56:19 +00:00
Lina JodoinandGitHub
a646b2d5ef
I'm sending the review for the mechanism (and an example callsite), and I'll send out a review that makes use of it throughout the scheduler code afterwards. ## What changed? - CHASM scheduler components each parent a new EventLog component, which we'll use to record decision logic details that are normally too spammy for logs. - No plan to expose this through any APIs just yet; it isn't meant to be customer-facing (though, I am thinking about how we can make it easy for dev success to get at) ## Why? - V1 debugging was often done through use of the workflow's event history; without an analogue for V2, it's difficult to get a clear picture of how the logic arrived at a certain output state. - Info logging in the same capacity would likely be too spammy for our observability system. - It's difficult to trace a complete picture for an individual schedule *over time* using just Loki. Schedules are often long-lived, with state transitions taking place in short blips over periods of days or weeks. Being able to get a complete list on demand for a relevant scheduler will greatly improve debugging time. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks - This could increase scheduler state size, however: - V2 already measures notably lighter on storage than V1 (not to mention, no event history data whatsoever) - Log is bounded to max 30 messages per component (usually scheduler, generator, invoker) of max 1000 characters each (e.g., ~100KB), as a safety mechanism - Most messages logged will be small and informational, not large state dumps (e.g., "scheduler unpaused", "generator scheduled for ..."). - Backfillers each get their own log, which shares a Backfiller's lifecycle.
88 lines
3.3 KiB
Go
88 lines
3.3 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"time"
|
|
|
|
"go.temporal.io/server/common/backoff"
|
|
"go.temporal.io/server/common/dynamicconfig"
|
|
)
|
|
|
|
type (
|
|
Tweakables struct {
|
|
DefaultCatchupWindow time.Duration // Default for catchup window
|
|
MinCatchupWindow time.Duration // Minimum for catchup window
|
|
MaxBufferSize int // MaxBufferSize limits the number of buffered actions pending execution in total
|
|
GeneratorBufferReserveSize int // Minimum number of spaces in `BufferedStarts` reserved for automated actions.
|
|
CanceledTerminatedCountAsFailures bool // Whether cancelled+terminated count for pause-on-failure
|
|
MaxActionsPerExecution int // Limits the number of actions (startWorkflow, terminate/cancel) taken by ExecuteTask in a single iteration
|
|
IdleTime time.Duration // How long to keep schedules after they're done
|
|
EventLogMaxEntries int // Maximum EventLog entries retained per component; the earliest entries are dropped beyond this.
|
|
EventLogMaxMessageLen int // Maximum byte length of an EventLog message; longer messages are truncated at a UTF-8 boundary.
|
|
}
|
|
|
|
// Config is the CHASM Scheduler dynamic config, shared among all sub-components.
|
|
Config struct {
|
|
Tweakables dynamicconfig.TypedPropertyFnWithNamespaceFilter[Tweakables]
|
|
ServiceCallTimeout dynamicconfig.DurationPropertyFn
|
|
RetryPolicy func() backoff.RetryPolicy
|
|
}
|
|
)
|
|
|
|
var (
|
|
CurrentTweakables = dynamicconfig.NewNamespaceTypedSetting(
|
|
"scheduler.tweakables",
|
|
DefaultTweakables,
|
|
"A set of tweakable parameters for the CHASM scheduler.")
|
|
|
|
RetryPolicyInitialInterval = dynamicconfig.NewGlobalDurationSetting(
|
|
"scheduler.retryPolicy.initialInterval",
|
|
time.Second,
|
|
`The initial backoff interval when retrying a failed task.`,
|
|
)
|
|
|
|
RetryPolicyMaximumInterval = dynamicconfig.NewGlobalDurationSetting(
|
|
"scheduler.retryPolicy.maxInterval",
|
|
time.Minute,
|
|
`The maximum backoff interval when retrying a failed task.`,
|
|
)
|
|
|
|
ServiceCallTimeout = dynamicconfig.NewGlobalDurationSetting(
|
|
"scheduler.serviceCallTimeout",
|
|
2*time.Second,
|
|
`The upper bound on how long a service call can take before being timed out.`,
|
|
)
|
|
|
|
// SentinelIdleTime is how long a CHASM sentinel reserves the schedule ID
|
|
// before auto-closing via the idle task mechanism. Matches the dummy
|
|
// workflow's duration.
|
|
SentinelIdleTime = 15 * time.Minute
|
|
|
|
DefaultTweakables = Tweakables{
|
|
DefaultCatchupWindow: 365 * 24 * time.Hour,
|
|
MinCatchupWindow: 10 * time.Second,
|
|
MaxBufferSize: 1000,
|
|
GeneratorBufferReserveSize: 50,
|
|
CanceledTerminatedCountAsFailures: false,
|
|
MaxActionsPerExecution: 5,
|
|
IdleTime: 7 * 24 * time.Hour,
|
|
EventLogMaxEntries: 30,
|
|
EventLogMaxMessageLen: 1000,
|
|
}
|
|
)
|
|
|
|
func ConfigProvider(dc *dynamicconfig.Collection) *Config {
|
|
return &Config{
|
|
Tweakables: CurrentTweakables.Get(dc),
|
|
ServiceCallTimeout: ServiceCallTimeout.Get(dc),
|
|
RetryPolicy: func() backoff.RetryPolicy {
|
|
return backoff.NewExponentialRetryPolicy(
|
|
RetryPolicyInitialInterval.Get(dc)(),
|
|
).WithMaximumInterval(
|
|
RetryPolicyMaximumInterval.Get(dc)(),
|
|
).WithExpirationInterval(
|
|
backoff.NoInterval,
|
|
)
|
|
},
|
|
}
|
|
}
|