mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-04 02:04:03 +00:00
Replace the coroutine-push activity path with deterministic, history-backed
replay. One way: the old push-result mechanism is removed.
Server (pkg/tasks):
- durable.go: seq-keyed activity records (wfact/ns/wf/run/seq) — the
exactly-once anchor. applyScheduleActivity is idempotent per (run,seq);
completeWorkflowActivity records the terminal outcome and schedules a new
workflow task so the worker replays and advances. Per-run keyedMutex
serializes advancing a run. Server-side retry+backoff per RetryPolicy
(temporal.RetryPolicy, finite default bound). Recover() rebuilds in-flight
dispatch from the store at boot (called from Embed).
- workflow tasks now carry the run's full event-sourced history.
- dispatch.go: ScheduleActivity/CompleteActivityTask(push) -> DispatchActivity
(seq-bound token) + ResolveActivityToken. workflowPeer/pendingActivity and
OpcodeDeliverActivityResult retired.
- respondWorkflowHandler applies ScheduleActivity{seq} commands under the run
lock; respondActivity* resolve the token and advance the run.
Worker (pkg/sdk/worker):
- workerEnv is now a replay decider: ExecuteActivity resolves from history by
seq (recorded result -> no re-dispatch; else emit ScheduleActivity{seq} and
end the episode via a blockedFuture). Stateless between episodes.
- removed OnActivityResult/pendingActivities push machinery.
Client (pkg/sdk/client): drop OnActivityResult + ScheduleActivity from
WorkerTransport (dead in the replay model).
Tests: e2e (exactly-once, real workers) green; new worker replay determinism
tests (unresolved->schedule+block, resolved->recorded result, 2-activity
determinism). Full 'go test -race ./pkg/tasks/... ./pkg/sdk/...' green.
253 lines
8.7 KiB
Go
253 lines
8.7 KiB
Go
// Copyright © 2026 Hanzo AI. MIT License.
|
|
|
|
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/tasks/pkg/sdk/client"
|
|
"github.com/hanzoai/tasks/pkg/sdk/workflow"
|
|
)
|
|
|
|
// Phase-2a replay decider. A workflow task carries the run's event-sourced
|
|
// history; ExecuteActivity resolves from history by seq. These tests assert
|
|
// the decider (a) NEVER runs the activity function in-process (the server
|
|
// dispatches it to an activity worker), (b) emits a ScheduleActivity{seq}
|
|
// command when the activity's result is not yet in history, (c) returns the
|
|
// recorded result without re-dispatch when it is, and (d) is deterministic:
|
|
// same history ⇒ same command sequence.
|
|
|
|
// activityLocalCallCount tracks whether an activity fn was invoked
|
|
// in-process. For the replay decider this must stay at zero.
|
|
var activityLocalCallCount atomic.Int32
|
|
|
|
func wireTargetActivity(ctx context.Context, n int) (int, error) {
|
|
activityLocalCallCount.Add(1)
|
|
return n * 2, nil
|
|
}
|
|
|
|
// wireCallingWorkflow calls workflow.ExecuteActivity exactly once and
|
|
// returns whatever the Future settles with.
|
|
func wireCallingWorkflow(ctx workflow.Context, n int) (int, error) {
|
|
ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
|
|
StartToCloseTimeout: 30 * time.Second,
|
|
TaskQueue: "test-queue",
|
|
})
|
|
fut := workflow.ExecuteActivity(ctx, wireTargetActivity, n)
|
|
var out int
|
|
if err := fut.Get(ctx, &out); err != nil {
|
|
return 0, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// evStarted builds a WORKFLOW_EXECUTION_STARTED event carrying the workflow
|
|
// input arguments.
|
|
func evStarted(args ...any) map[string]any {
|
|
return map[string]any{
|
|
"eventId": 1,
|
|
"eventType": evtWorkflowStarted,
|
|
"attributes": map[string]any{"input": args},
|
|
}
|
|
}
|
|
|
|
// evScheduled builds an ACTIVITY_TASK_SCHEDULED{seq} event.
|
|
func evScheduled(seq int) map[string]any {
|
|
return map[string]any{
|
|
"eventType": evtActivityScheduled,
|
|
"attributes": map[string]any{"seq": seq},
|
|
}
|
|
}
|
|
|
|
// evCompleted builds an ACTIVITY_TASK_COMPLETED{seq,result} event. result is
|
|
// stored as the raw JSON of the activity's return value, matching the engine.
|
|
func evCompleted(t *testing.T, seq int, result any) map[string]any {
|
|
return map[string]any{
|
|
"eventType": evtActivityCompleted,
|
|
"attributes": map[string]any{"seq": seq, "result": json.RawMessage(mustJSON(t, result))},
|
|
}
|
|
}
|
|
|
|
func historyJSON(t *testing.T, events ...map[string]any) []byte {
|
|
t.Helper()
|
|
return mustJSON(t, events)
|
|
}
|
|
|
|
// replayEpisode runs one decision episode for wfName over history against a
|
|
// fresh worker with fns registered by register, and returns the decoded
|
|
// command envelope the worker responded with.
|
|
func replayEpisode(t *testing.T, register func(w *workerImpl), wfName string, history []byte) commandsEnvelope {
|
|
t.Helper()
|
|
ft := &fakeTransport{}
|
|
w := newTestWorker(t, ft)
|
|
register(w)
|
|
w.dispatchWorkflowTask(context.Background(), &client.WorkflowTask{
|
|
TaskToken: []byte{0x01},
|
|
WorkflowID: "wf-replay",
|
|
RunID: "run-replay",
|
|
WorkflowTypeName: wfName,
|
|
History: history,
|
|
})
|
|
if ft.workflowCompleted.Load() != 1 || ft.lastWorkflowResp == nil {
|
|
t.Fatalf("expected exactly one RespondWorkflowTaskCompleted; got %d", ft.workflowCompleted.Load())
|
|
}
|
|
return decodeCommandsEnvelope(t, ft.lastWorkflowResp.Commands)
|
|
}
|
|
|
|
// TestReplay_UnresolvedActivity_SchedulesAndBlocks: with no activity result
|
|
// in history the decider emits ScheduleActivity{seq=0} and does NOT complete
|
|
// the workflow or run the activity in-process.
|
|
func TestReplay_UnresolvedActivity_SchedulesAndBlocks(t *testing.T) {
|
|
activityLocalCallCount.Store(0)
|
|
|
|
env := replayEpisode(t, func(w *workerImpl) {
|
|
w.RegisterWorkflow(wireCallingWorkflow)
|
|
w.RegisterActivity(wireTargetActivity)
|
|
}, "wireCallingWorkflow", historyJSON(t, evStarted(42)))
|
|
|
|
if len(env.Commands) != 1 {
|
|
t.Fatalf("commands = %d, want 1 (scheduleActivity)", len(env.Commands))
|
|
}
|
|
c := env.Commands[0]
|
|
if c.Kind != commandKindScheduleActivity {
|
|
t.Fatalf("command kind = %d, want %d (scheduleActivity)", c.Kind, commandKindScheduleActivity)
|
|
}
|
|
if c.Seq != 0 {
|
|
t.Errorf("seq = %d, want 0", c.Seq)
|
|
}
|
|
if c.ActivityType != "wireTargetActivity" {
|
|
t.Errorf("activityType = %q, want wireTargetActivity", c.ActivityType)
|
|
}
|
|
// input must be the JSON array of args, [42].
|
|
var args []int
|
|
if err := json.Unmarshal(c.Input, &args); err != nil || len(args) != 1 || args[0] != 42 {
|
|
t.Errorf("scheduled input = %q, want [42]", c.Input)
|
|
}
|
|
if got := activityLocalCallCount.Load(); got != 0 {
|
|
t.Fatalf("activity fn invoked in-process %d times; the decider must not run it", got)
|
|
}
|
|
}
|
|
|
|
// TestReplay_ResolvedActivity_ReturnsRecordedResult: with the activity's
|
|
// result in history the decider returns it WITHOUT re-dispatch and the
|
|
// workflow completes carrying the wire-recorded value.
|
|
func TestReplay_ResolvedActivity_ReturnsRecordedResult(t *testing.T) {
|
|
activityLocalCallCount.Store(0)
|
|
|
|
env := replayEpisode(t, func(w *workerImpl) {
|
|
w.RegisterWorkflow(wireCallingWorkflow)
|
|
w.RegisterActivity(wireTargetActivity)
|
|
}, "wireCallingWorkflow", historyJSON(t,
|
|
evStarted(42),
|
|
evScheduled(0),
|
|
evCompleted(t, 0, 84), // the activity worker (elsewhere) doubled n=42
|
|
))
|
|
|
|
if len(env.Commands) != 1 {
|
|
t.Fatalf("commands = %d, want 1 (completeWorkflow)", len(env.Commands))
|
|
}
|
|
c := env.Commands[0]
|
|
if c.Kind != commandKindCompleteWorkflow {
|
|
t.Fatalf("command kind = %d, want %d (completeWorkflow); failure=%q", c.Kind, commandKindCompleteWorkflow, c.Failure)
|
|
}
|
|
var got int
|
|
if err := json.Unmarshal(c.Result, &got); err != nil {
|
|
t.Fatalf("decode result: %v", err)
|
|
}
|
|
if got != 84 {
|
|
t.Errorf("workflow result = %d, want 84 (recorded, not recomputed)", got)
|
|
}
|
|
if n := activityLocalCallCount.Load(); n != 0 {
|
|
t.Fatalf("activity fn invoked in-process %d times; recorded result must be reused", n)
|
|
}
|
|
}
|
|
|
|
// -------- 2-activity determinism ------------------------------------------
|
|
|
|
var stepCalls atomic.Int32
|
|
|
|
func stepAActivity(ctx context.Context) (string, error) { stepCalls.Add(1); return "A", nil }
|
|
func stepBActivity(ctx context.Context) (string, error) { stepCalls.Add(1); return "B", nil }
|
|
|
|
func twoStepWorkflow(ctx workflow.Context) (string, error) {
|
|
ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{StartToCloseTimeout: time.Second})
|
|
var a string
|
|
if err := workflow.ExecuteActivity(ctx, stepAActivity).Get(ctx, &a); err != nil {
|
|
return "", err
|
|
}
|
|
var b string
|
|
if err := workflow.ExecuteActivity(ctx, stepBActivity).Get(ctx, &b); err != nil {
|
|
return "", err
|
|
}
|
|
return a + b, nil
|
|
}
|
|
|
|
// TestReplay_Determinism_TwoActivities: with only activity 0 resolved in
|
|
// history, replay schedules ONLY activity 1 (activity 0 returns its recorded
|
|
// result, no re-dispatch); and the same history yields an identical command
|
|
// sequence across repeated replays.
|
|
func TestReplay_Determinism_TwoActivities(t *testing.T) {
|
|
stepCalls.Store(0)
|
|
|
|
history := historyJSON(t,
|
|
evStarted(),
|
|
evScheduled(0),
|
|
evCompleted(t, 0, "A"),
|
|
)
|
|
reg := func(w *workerImpl) {
|
|
w.RegisterWorkflow(twoStepWorkflow)
|
|
w.RegisterActivity(stepAActivity)
|
|
w.RegisterActivity(stepBActivity)
|
|
}
|
|
|
|
first := replayEpisode(t, reg, "twoStepWorkflow", history)
|
|
if len(first.Commands) != 1 {
|
|
t.Fatalf("commands = %d, want 1 (only activity 1 scheduled)", len(first.Commands))
|
|
}
|
|
c := first.Commands[0]
|
|
if c.Kind != commandKindScheduleActivity || c.Seq != 1 || c.ActivityType != "stepBActivity" {
|
|
t.Fatalf("first replay command = %+v; want scheduleActivity seq=1 stepBActivity", c)
|
|
}
|
|
|
|
// Determinism: replay the identical history again → identical command seq.
|
|
second := replayEpisode(t, reg, "twoStepWorkflow", history)
|
|
if len(second.Commands) != len(first.Commands) {
|
|
t.Fatalf("non-deterministic: first %d commands, second %d", len(first.Commands), len(second.Commands))
|
|
}
|
|
for i := range first.Commands {
|
|
if first.Commands[i].Kind != second.Commands[i].Kind || first.Commands[i].Seq != second.Commands[i].Seq ||
|
|
first.Commands[i].ActivityType != second.Commands[i].ActivityType {
|
|
t.Fatalf("non-deterministic command %d: %+v vs %+v", i, first.Commands[i], second.Commands[i])
|
|
}
|
|
}
|
|
|
|
if stepCalls.Load() != 0 {
|
|
t.Fatalf("activity fns invoked in-process %d times; the decider must not run them", stepCalls.Load())
|
|
}
|
|
}
|
|
|
|
// mustJSON is a test helper that json.Marshals v or fails the test.
|
|
func mustJSON(t *testing.T, v any) []byte {
|
|
t.Helper()
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
t.Fatalf("marshal: %v", err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
// decodeCommandsEnvelope parses the producer-side wire shape for
|
|
// assertion. Shared across the worker dispatch tests.
|
|
func decodeCommandsEnvelope(t *testing.T, b []byte) commandsEnvelope {
|
|
t.Helper()
|
|
var e commandsEnvelope
|
|
if err := json.Unmarshal(b, &e); err != nil {
|
|
t.Fatalf("decode envelope: %v (raw=%q)", err, b)
|
|
}
|
|
return e
|
|
}
|