mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-30 14:56:19 +00:00
I'm sending the review for the mechanism (and an example callsite), and I'll send out a review that makes use of it throughout the scheduler code afterwards. ## What changed? - CHASM scheduler components each parent a new EventLog component, which we'll use to record decision logic details that are normally too spammy for logs. - No plan to expose this through any APIs just yet; it isn't meant to be customer-facing (though, I am thinking about how we can make it easy for dev success to get at) ## Why? - V1 debugging was often done through use of the workflow's event history; without an analogue for V2, it's difficult to get a clear picture of how the logic arrived at a certain output state. - Info logging in the same capacity would likely be too spammy for our observability system. - It's difficult to trace a complete picture for an individual schedule *over time* using just Loki. Schedules are often long-lived, with state transitions taking place in short blips over periods of days or weeks. Being able to get a complete list on demand for a relevant scheduler will greatly improve debugging time. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks - This could increase scheduler state size, however: - V2 already measures notably lighter on storage than V1 (not to mention, no event history data whatsoever) - Log is bounded to max 30 messages per component (usually scheduler, generator, invoker) of max 1000 characters each (e.g., ~100KB), as a safety mechanism - Most messages logged will be small and informational, not large state dumps (e.g., "scheduler unpaused", "generator scheduled for ..."). - Backfillers each get their own log, which shares a Backfiller's lifecycle.
135 lines
4.7 KiB
Protocol Buffer
135 lines
4.7 KiB
Protocol Buffer
syntax = "proto3";
|
|
|
|
package temporal.server.chasm.lib.scheduler.proto.v1;
|
|
|
|
import "google/protobuf/timestamp.proto";
|
|
import "temporal/api/common/v1/message.proto";
|
|
import "temporal/api/failure/v1/message.proto";
|
|
import "temporal/api/schedule/v1/message.proto";
|
|
import "temporal/server/api/schedule/v1/message.proto";
|
|
|
|
option go_package = "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb;schedulerpb";
|
|
|
|
// CHASM scheduler top-level state.
|
|
message SchedulerState {
|
|
// Scheduler request parameters and metadata.
|
|
temporal.api.schedule.v1.Schedule schedule = 2;
|
|
temporal.api.schedule.v1.ScheduleInfo info = 3;
|
|
|
|
// State common to all generators is stored in the top-level machine.
|
|
string namespace = 5;
|
|
string namespace_id = 6;
|
|
string schedule_id = 7;
|
|
|
|
// Implemented as a sequence number. Used for optimistic locking against
|
|
// update requests.
|
|
int64 conflict_token = 8;
|
|
|
|
// The closed flag is set true after a schedule completes, and the idle timer
|
|
// expires.
|
|
bool closed = 9;
|
|
|
|
// When true, this scheduler is a sentinel that exists only to reserve the
|
|
// schedule ID. All API operations return NotFound.
|
|
bool sentinel = 10;
|
|
|
|
// Set when a migration to workflow-backed scheduler (V1) is pending.
|
|
// Unpause operations are blocked while this is set.
|
|
WorkflowMigrationState workflow_migration = 11;
|
|
}
|
|
|
|
// WorkflowMigrationState tracks the state of an in-progress V2-to-V1 migration.
|
|
message WorkflowMigrationState {
|
|
// The schedule's paused state before migration was initiated. Used to
|
|
// restore the correct paused state when passing state to the V1 workflow.
|
|
bool pre_migration_paused = 1;
|
|
|
|
// The schedule's notes before migration was initiated.
|
|
string pre_migration_notes = 2;
|
|
}
|
|
|
|
// CHASM scheduler's Generator internal state.
|
|
message GeneratorState {
|
|
// High water mark.
|
|
google.protobuf.Timestamp last_processed_time = 3;
|
|
|
|
// A list of upcoming times an action will be triggered.
|
|
repeated google.protobuf.Timestamp future_action_times = 4;
|
|
}
|
|
|
|
// CHASM scheduler's Invoker internal state.
|
|
message InvokerState {
|
|
// Buffered starts that will be started by the Invoker.
|
|
repeated temporal.server.api.schedule.v1.BufferedStart buffered_starts = 2;
|
|
|
|
// Workflow executions that will be cancelled due to overlap policy.
|
|
repeated temporal.api.common.v1.WorkflowExecution cancel_workflows = 3;
|
|
|
|
// Workflow executions that will be terminated due to overlap policy.
|
|
repeated temporal.api.common.v1.WorkflowExecution terminate_workflows = 4;
|
|
|
|
// High water mark, used for evaluating when to fire tasks that are backing
|
|
// off from a retry. LastProcessedTime is stored as state so that task
|
|
// generation will be consistent, regardless of when generation occurs, such
|
|
// as after applying a replicated state (as opposed to evaluating based on
|
|
// present time).
|
|
google.protobuf.Timestamp last_processed_time = 5;
|
|
|
|
reserved 6;
|
|
}
|
|
|
|
// CHASM scheduler's Backfiller internal state. Backfill requests are 1:1
|
|
// with Backfiller nodes. Backfiller nodes also handle immediate trigger requests.
|
|
message BackfillerState {
|
|
oneof request {
|
|
temporal.api.schedule.v1.BackfillRequest backfill_request = 1;
|
|
|
|
// When set, immediately buffer a single manual action.
|
|
temporal.api.schedule.v1.TriggerImmediatelyRequest trigger_request = 2;
|
|
}
|
|
|
|
// Every Backfiller should be assigned a unique ID upon creation, used
|
|
// for deduplication.
|
|
string backfill_id = 6;
|
|
|
|
// High water mark.
|
|
google.protobuf.Timestamp last_processed_time = 7;
|
|
|
|
// Attempt count, incremented when the buffer is full and the Backfiller
|
|
// needs to back off before retrying to fill.
|
|
int64 attempt = 8;
|
|
}
|
|
|
|
// CHASM scheduler retains the payload data for the last completed workflow. Both
|
|
// last success and failure are stored simultaneously.
|
|
message LastCompletionResult {
|
|
temporal.api.common.v1.Payload success = 1;
|
|
temporal.api.failure.v1.Failure failure = 2;
|
|
}
|
|
|
|
// SchedulerMigrationState is a stack-agnostic interchange format for migrating
|
|
// scheduler state between V1 (workflow-backed) and V2 (CHASM) implementations.
|
|
message SchedulerMigrationState {
|
|
SchedulerState scheduler_state = 1;
|
|
GeneratorState generator_state = 2;
|
|
InvokerState invoker_state = 3;
|
|
map<string, BackfillerState> backfillers = 4;
|
|
LastCompletionResult last_completion_result = 5;
|
|
|
|
// Visibility data.
|
|
map<string, temporal.api.common.v1.Payload> search_attributes = 6;
|
|
map<string, temporal.api.common.v1.Payload> memo = 7;
|
|
}
|
|
|
|
// EventLog keeps a bounded record of state changes, including automation
|
|
// decisions, in lieu of a workflow event history. Entries are meant for
|
|
// human consumption, and aren't used for any scheduler computation.
|
|
message EventLog {
|
|
repeated Event events = 1;
|
|
}
|
|
|
|
message Event {
|
|
google.protobuf.Timestamp time = 1;
|
|
string message = 2;
|
|
}
|