base/plugins/tasks/durable.go

package tasks

import (
	"context"
	"fmt"
	"log/slog"
	"os"
	"regexp"
	"sync"
	"time"

	"go.temporal.io/api/workflowservice/v1"
	"go.temporal.io/sdk/client"
	"go.temporal.io/sdk/temporal"
)

// validIDPattern restricts space/org IDs to safe characters, preventing
// injection into Temporal visibility queries.
var validIDPattern = regexp.MustCompile(`^[a-zA-Z0-9_.-]{1,128}$`)

// DurableConfig holds connection settings for durable task execution.
// When enabled, tasks are submitted as Temporal workflows for crash-safe execution.
// Supports both local Temporal (localhost:7233) and cloud (tasks.hanzo.ai).
type DurableConfig struct {
	// Enabled activates durable execution. Default false (SQLite-only mode).
	Enabled bool `json:"enabled" yaml:"enabled"`

	// Address is the Temporal frontend address. Default "tasks.hanzo.ai:7233".
	Address string `json:"address" yaml:"address"`

	// Namespace is the Temporal namespace. For multi-tenant, use org ID.
	// Default "default".
	Namespace string `json:"namespace" yaml:"namespace"`

	// DefaultQueue is the task queue name. Default "default".
	DefaultQueue string `json:"default_queue" yaml:"default_queue"`

	// RunWorker starts an embedded Temporal worker in this process.
	// If false, tasks are submitted but executed by external workers.
	// Default true when enabled.
	RunWorker bool `json:"run_worker" yaml:"run_worker"`
}

// DefaultDurableConfig returns production defaults with env var overrides.
//
// Env vars:
//   - TASKS_ENABLED (or HANZO_TASKS_ENABLED): "true" to enable
//   - TASKS_ADDRESS (or HANZO_TASKS_ADDRESS): Temporal frontend address
//   - TASKS_NAMESPACE (or HANZO_TASKS_NAMESPACE): Temporal namespace (org ID for multi-tenant)
//   - TASKS_QUEUE: default task queue name
//   - TASKS_WORKER: "false" to disable embedded worker
func DefaultDurableConfig() DurableConfig {
	cfg := DurableConfig{
		Enabled:      false,
		Address:      "tasks.hanzo.ai:7233",
		Namespace:    "default",
		DefaultQueue: "default",
		RunWorker:    true,
	}

	for _, key := range []string{"TASKS_ENABLED", "HANZO_TASKS_ENABLED"} {
		if v := os.Getenv(key); v == "true" || v == "1" {
			cfg.Enabled = true
			break
		}
	}
	for _, key := range []string{"TASKS_ADDRESS", "HANZO_TASKS_ADDRESS"} {
		if v := os.Getenv(key); v != "" {
			cfg.Address = v
			break
		}
	}
	for _, key := range []string{"TASKS_NAMESPACE", "HANZO_TASKS_NAMESPACE"} {
		if v := os.Getenv(key); v != "" {
			cfg.Namespace = v
			break
		}
	}
	if v := os.Getenv("TASKS_QUEUE"); v != "" {
		cfg.DefaultQueue = v
	}
	if v := os.Getenv("TASKS_WORKER"); v == "false" || v == "0" {
		cfg.RunWorker = false
	}

	return cfg
}

// DurableStore implements durable task execution via Temporal.
// Supports multi-tenant org isolation via per-namespace client connections.
type DurableStore struct {
	Client    client.Client // default namespace client
	addr      string
	namespace string
	connected bool

	// orgClients caches per-org Temporal client connections.
	// Key is org ID (= Temporal namespace).
	mu         sync.RWMutex
	orgClients map[string]client.Client
}

// NewDurableStore connects to the Temporal service.
func NewDurableStore(addr, namespace string) (*DurableStore, error) {
	c, err := client.Dial(client.Options{
		HostPort:  addr,
		Namespace: namespace,
	})
	if err != nil {
		return nil, fmt.Errorf("tasks: failed to connect to %s: %w", addr, err)
	}
	return &DurableStore{
		Client:     c,
		addr:       addr,
		namespace:  namespace,
		connected:  true,
		orgClients: make(map[string]client.Client),
	}, nil
}

// ClientForOrg returns a Temporal client scoped to the given org namespace.
// Connections are cached and protected by a mutex. Falls back to the default
// client if org is empty or matches the default namespace.
func (ds *DurableStore) ClientForOrg(orgID string) (client.Client, error) {
	if orgID == "" || orgID == ds.namespace {
		return ds.Client, nil
	}
	if !validIDPattern.MatchString(orgID) {
		return nil, fmt.Errorf("tasks: invalid org ID: %q", orgID)
	}

	// Fast path: read lock.
	ds.mu.RLock()
	if c, ok := ds.orgClients[orgID]; ok {
		ds.mu.RUnlock()
		return c, nil
	}
	ds.mu.RUnlock()

	// Slow path: write lock, double-check, dial.
	ds.mu.Lock()
	defer ds.mu.Unlock()

	if c, ok := ds.orgClients[orgID]; ok {
		return c, nil
	}

	c, err := client.Dial(client.Options{
		HostPort:  ds.addr,
		Namespace: orgID,
	})
	if err != nil {
		return nil, fmt.Errorf("tasks: failed to connect to namespace %s: %w", orgID, err)
	}
	ds.orgClients[orgID] = c
	return c, nil
}

// Close shuts down all client connections.
func (ds *DurableStore) Close() {
	ds.mu.Lock()
	for _, c := range ds.orgClients {
		c.Close()
	}
	ds.orgClients = nil
	ds.mu.Unlock()

	if ds.Client != nil {
		ds.Client.Close()
		ds.connected = false
	}
}

// IsConnected reports whether the durable store has an active connection.
func (ds *DurableStore) IsConnected() bool {
	return ds != nil && ds.connected
}

// SubmitTask starts a durable workflow execution for a task.
// Uses task.OrgID as the Temporal namespace for multi-tenant isolation.
// Falls back to default namespace if OrgID is empty.
func (ds *DurableStore) SubmitTask(ctx context.Context, task *Task) error {
	c, err := ds.ClientForOrg(task.OrgID)
	if err != nil {
		return err
	}

	queue := task.SpaceID
	if queue == "" {
		queue = "default"
	}

	opts := client.StartWorkflowOptions{
		ID:        task.ID,
		TaskQueue: queue,
	}
	if task.Timeout > 0 {
		opts.WorkflowExecutionTimeout = task.Timeout
	}
	if task.MaxRetries > 0 {
		opts.RetryPolicy = &temporal.RetryPolicy{
			MaximumAttempts:    int32(task.MaxRetries),
			InitialInterval:    2 * time.Second,
			MaximumInterval:    time.Minute,
			BackoffCoefficient: 2.0,
		}
	}

	_, err = c.ExecuteWorkflow(ctx, opts, AgentTaskWorkflow, task)
	if err != nil {
		return fmt.Errorf("tasks: failed to submit workflow: %w", err)
	}
	task.State = TaskRunning
	return nil
}

// SubmitWorkflow starts a pipeline or fan-out workflow.
func (ds *DurableStore) SubmitWorkflow(ctx context.Context, wf *Workflow, tasks []*Task, parallel bool) error {
	c, err := ds.ClientForOrg(wf.OrgID)
	if err != nil {
		return err
	}

	queue := wf.SpaceID
	if queue == "" {
		queue = "default"
	}

	opts := client.StartWorkflowOptions{
		ID:        wf.ID,
		TaskQueue: queue,
	}

	var wfFunc interface{}
	if parallel {
		wfFunc = FanOutWorkflow
	} else {
		wfFunc = PipelineWorkflow
	}

	_, err = c.ExecuteWorkflow(ctx, opts, wfFunc, wf, tasks)
	if err != nil {
		return fmt.Errorf("tasks: failed to submit workflow: %w", err)
	}
	wf.State = TaskRunning
	return nil
}

// GetTaskStatus queries a running workflow for its current state.
// orgID selects the Temporal namespace. Empty string uses the default.
func (ds *DurableStore) GetTaskStatus(ctx context.Context, taskID string, orgID ...string) (TaskState, string, error) {
	c, err := ds.clientForOrgVar(orgID)
	if err != nil {
		return TaskPending, "", err
	}
	desc, err := c.DescribeWorkflowExecution(ctx, taskID, "")
	if err != nil {
		return TaskPending, "", err
	}

	info := desc.WorkflowExecutionInfo
	switch info.GetStatus().String() {
	case "Running":
		return TaskRunning, "", nil
	case "Completed":
		return TaskCompleted, "", nil
	case "Failed":
		return TaskFailed, "workflow failed", nil
	case "Canceled", "Cancelled":
		return TaskCancelled, "", nil
	case "TimedOut":
		return TaskFailed, "timed out", nil
	default:
		return TaskPending, "", nil
	}
}

// CancelTask cancels a running workflow.
func (ds *DurableStore) CancelTask(ctx context.Context, taskID string, orgID ...string) error {
	c, err := ds.clientForOrgVar(orgID)
	if err != nil {
		return err
	}
	return c.CancelWorkflow(ctx, taskID, "")
}

// SignalTask sends a signal to a running workflow.
func (ds *DurableStore) SignalTask(ctx context.Context, taskID, signalName string, data interface{}, orgID ...string) error {
	c, err := ds.clientForOrgVar(orgID)
	if err != nil {
		return err
	}
	return c.SignalWorkflow(ctx, taskID, "", signalName, data)
}

// clientForOrgVar extracts the first orgID from a variadic param.
func (ds *DurableStore) clientForOrgVar(orgID []string) (client.Client, error) {
	if len(orgID) > 0 && orgID[0] != "" {
		return ds.ClientForOrg(orgID[0])
	}
	return ds.Client, nil
}

// ListTasks returns tasks in a space by querying workflow visibility.
func (ds *DurableStore) ListTasks(ctx context.Context, spaceID string, orgID ...string) ([]*Task, error) {
	if spaceID != "" && !validIDPattern.MatchString(spaceID) {
		return nil, fmt.Errorf("tasks: invalid space ID: %q", spaceID)
	}
	c, err := ds.clientForOrgVar(orgID)
	if err != nil {
		return nil, err
	}
	query := fmt.Sprintf(`TaskQueue = "%s"`, spaceID)
	resp, err := c.ListWorkflow(ctx, &workflowservice.ListWorkflowExecutionsRequest{
		Query: query,
	})
	if err != nil {
		return nil, fmt.Errorf("tasks: list failed: %w", err)
	}

	var result []*Task
	for _, exec := range resp.GetExecutions() {
		task := &Task{
			ID:      exec.Execution.GetWorkflowId(),
			SpaceID: spaceID,
		}
		switch exec.GetStatus().String() {
		case "Running":
			task.State = TaskRunning
		case "Completed":
			task.State = TaskCompleted
		case "Failed":
			task.State = TaskFailed
		case "Canceled", "Cancelled":
			task.State = TaskCancelled
		default:
			task.State = TaskPending
		}
		if exec.GetStartTime() != nil {
			t := exec.GetStartTime().AsTime()
			task.StartedAt = &t
		}
		result = append(result, task)
	}
	return result, nil
}

// GetNextTask finds the next pending task in a space and claims it for the agent.
func (ds *DurableStore) GetNextTask(ctx context.Context, spaceID, agentID string, orgID ...string) (*Task, error) {
	tasks, err := ds.ListTasks(ctx, spaceID, orgID...)
	if err != nil {
		return nil, err
	}
	for _, t := range tasks {
		if t.State == TaskRunning || t.State == TaskPending {
			_ = ds.SignalTask(ctx, t.ID, "claim", map[string]string{"agent_id": agentID})
			return t, nil
		}
	}
	return nil, nil
}

// ListWorkflows returns workflows in a space by querying visibility.
func (ds *DurableStore) ListWorkflows(ctx context.Context, spaceID string, orgID ...string) ([]*Workflow, error) {
	if spaceID != "" && !validIDPattern.MatchString(spaceID) {
		return nil, fmt.Errorf("tasks: invalid space ID: %q", spaceID)
	}
	c, err := ds.clientForOrgVar(orgID)
	if err != nil {
		return nil, err
	}
	query := fmt.Sprintf(`TaskQueue = "%s" AND WorkflowType IN ("PipelineWorkflow", "FanOutWorkflow")`, spaceID)
	resp, err := c.ListWorkflow(ctx, &workflowservice.ListWorkflowExecutionsRequest{
		Query: query,
	})
	if err != nil {
		return nil, fmt.Errorf("tasks: list workflows failed: %w", err)
	}

	var result []*Workflow
	for _, exec := range resp.GetExecutions() {
		wf := &Workflow{
			ID:      exec.Execution.GetWorkflowId(),
			SpaceID: spaceID,
			Name:    exec.GetType().GetName(),
		}
		switch exec.GetStatus().String() {
		case "Running":
			wf.State = TaskRunning
		case "Completed":
			wf.State = TaskCompleted
		case "Failed":
			wf.State = TaskFailed
		default:
			wf.State = TaskPending
		}
		result = append(result, wf)
	}
	return result, nil
}

// logDurableError logs a durable store error without failing the operation.
func logDurableError(logger *slog.Logger, op string, taskID string, err error) {
	if logger != nil {
		logger.Warn("tasks: durable "+op+" failed, SQLite state is authoritative",
			slog.String("task_id", taskID),
			slog.String("error", err.Error()),
		)
	}
}