tasks/service/history/replication/fx.go

package replication

import (
	"context"
	"math/rand"
	"strconv"

	"github.com/dgryski/go-farm"
	persistencespb "github.com/hanzoai/tasks/api/persistence/v1"
	"github.com/hanzoai/tasks/client"
	"github.com/hanzoai/tasks/common/clock"
	"github.com/hanzoai/tasks/common/cluster"
	"github.com/hanzoai/tasks/common/definition"
	"github.com/hanzoai/tasks/common/headers"
	"github.com/hanzoai/tasks/common/log"
	"github.com/hanzoai/tasks/common/metrics"
	"github.com/hanzoai/tasks/common/namespace"
	"github.com/hanzoai/tasks/common/namespace/nsreplication"
	"github.com/hanzoai/tasks/common/persistence"
	"github.com/hanzoai/tasks/common/persistence/serialization"
	"github.com/hanzoai/tasks/common/quotas"
	ctasks "github.com/hanzoai/tasks/common/tasks"
	"github.com/hanzoai/tasks/service/history/configs"
	historyi "github.com/hanzoai/tasks/service/history/interfaces"
	"github.com/hanzoai/tasks/service/history/queues"
	"github.com/hanzoai/tasks/service/history/replication/eventhandler"
	"github.com/hanzoai/tasks/service/history/shard"
	"github.com/hanzoai/tasks/service/history/tasks"
	"go.uber.org/fx"
)

type (
	ClusterChannelKey struct {
		ClusterName string
	}
	TaskSerializer interface {
		SerializeReplicationTask(task tasks.Task) (*persistencespb.ReplicationTaskInfo, error)
		DeserializeReplicationTask(replicationTask *persistencespb.ReplicationTaskInfo) (tasks.Task, error)
	}
)

var Module = fx.Provide(
	NewTaskFetcherFactory,
	func(m persistence.ExecutionManager) ExecutionManager {
		return m
	},
	nsreplication.NewNoopDataMerger,
	NewExecutionManagerDLQWriter,
	ClientSchedulerRateLimiterProvider,
	ServerSchedulerRateLimiterProvider,
	PersistenceRateLimiterProvider,
	func(serializer serialization.Serializer) TaskSerializer {
		return serializer
	},
	replicationTaskConverterFactoryProvider,
	replicationTaskExecutorProvider,
	fx.Annotated{
		Name:   "HighPriorityTaskScheduler",
		Target: replicationStreamHighPrioritySchedulerProvider,
	},
	fx.Annotated{
		Name:   "LowPriorityTaskScheduler",
		Target: replicationStreamLowPrioritySchedulerProvider,
	},
	executableTaskConverterProvider,
	streamReceiverMonitorProvider,
	eagerNamespaceRefresherProvider,
	sequentialTaskQueueFactoryProvider,
	dlqWriterAdapterProvider,
	newDLQWriterToggle,
	historyPaginatedFetcherProvider,
	resendHandlerProvider,
	eventImporterProvider,
	historyEventsHandlerProvider,
)

func eagerNamespaceRefresherProvider(
	metadataManager persistence.MetadataManager,
	namespaceRegistry namespace.Registry,
	logger log.Logger,
	clientBean client.Bean,
	clusterMetadata cluster.Metadata,
	dataMerger nsreplication.NamespaceDataMerger,
	metricsHandler metrics.Handler,
) EagerNamespaceRefresher {
	return NewEagerNamespaceRefresher(
		metadataManager,
		namespaceRegistry,
		logger,
		clientBean,
		nsreplication.NewTaskExecutor(
			clusterMetadata.GetCurrentClusterName(),
			metadataManager,
			dataMerger,
			logger,
		),
		clusterMetadata.GetCurrentClusterName(),
		metricsHandler,
	)
}

func replicationTaskConverterFactoryProvider(
	config *configs.Config,
	replicationTaskSerializer TaskSerializer,
) SourceTaskConverterProvider {
	return func(
		historyEngine historyi.Engine,
		shardContext historyi.ShardContext,
		clientClusterName string,
		serializer serialization.Serializer,
	) SourceTaskConverter {
		return NewSourceTaskConverter(
			historyEngine,
			shardContext.GetNamespaceRegistry(),
			serializer,
			replicationTaskSerializer,
			config)
	}
}

func replicationTaskExecutorProvider() TaskExecutorProvider {
	return func(params TaskExecutorParams) TaskExecutor {
		return NewTaskExecutor(
			params.RemoteCluster,
			params.Shard,
			params.RemoteHistoryFetcher,
			params.DeleteManager,
			params.WorkflowCache,
		)
	}
}

func replicationStreamHighPrioritySchedulerProvider(
	config *configs.Config,
	logger log.Logger,
	queueFactory ctasks.SequentialTaskQueueFactory[TrackableExecutableTask],
	lc fx.Lifecycle,
) ctasks.Scheduler[TrackableExecutableTask] {
	// SequentialScheduler has panic wrapper when executing task,
	// if changing the executor, please make sure other executor has panic wrapper
	scheduler := ctasks.NewSequentialScheduler[TrackableExecutableTask](
		&ctasks.SequentialSchedulerOptions{
			QueueSize:   config.ReplicationProcessorSchedulerQueueSize(),
			WorkerCount: config.ReplicationProcessorSchedulerWorkerCount,
		},
		WorkflowKeyHashFn,
		queueFactory,
		logger,
	)
	taskChannelKeyFn := func(e TrackableExecutableTask) ClusterChannelKey {
		return ClusterChannelKey{
			ClusterName: e.SourceClusterName(),
		}
	}
	channelWeightFn := func(key ClusterChannelKey) int {
		return 1
	}
	// This creates a per cluster channel.
	// They share the same weight so it just does a round-robin on all clusters' tasks.
	rrScheduler := ctasks.NewInterleavedWeightedRoundRobinScheduler(
		ctasks.InterleavedWeightedRoundRobinSchedulerOptions[TrackableExecutableTask, ClusterChannelKey]{
			TaskChannelKeyFn: taskChannelKeyFn,
			ChannelWeightFn:  channelWeightFn,
		},
		scheduler,
		logger,
	)
	lc.Append(fx.StartStopHook(rrScheduler.Start, rrScheduler.Stop))
	return rrScheduler
}

func replicationStreamLowPrioritySchedulerProvider(
	rateLimiter ClientSchedulerRateLimiter,
	timeSource clock.TimeSource,
	config *configs.Config,
	nsRegistry namespace.Registry,
	logger log.Logger,
	metricsHandler metrics.Handler,
	lc fx.Lifecycle,
) ctasks.Scheduler[TrackableExecutableTask] {
	queueFactory := func(task TrackableExecutableTask) ctasks.SequentialTaskQueue[TrackableExecutableTask] {
		item := task.QueueID()
		workflowKey, ok := item.(definition.WorkflowKey)
		if !ok {
			return NewSequentialTaskQueueWithID(item)
		}
		return NewSequentialTaskQueueWithID(workflowKey.NamespaceID + "_" + workflowKey.WorkflowID)
	}
	taskQueueHashFunc := func(item any) uint32 {
		workflowKey, ok := item.(definition.WorkflowKey)
		if !ok {
			return 0
		}

		idBytes := []byte(workflowKey.NamespaceID + "_" + workflowKey.WorkflowID + "_" + strconv.Itoa(rand.Intn(config.ReplicationLowPriorityTaskParallelism())))
		return farm.Fingerprint32(idBytes)
	}
	// SequentialScheduler has panic wrapper when executing task,
	// if changing the executor, please make sure other executor has panic wrapper
	scheduler := ctasks.NewSequentialScheduler[TrackableExecutableTask](
		&ctasks.SequentialSchedulerOptions{
			QueueSize:   config.ReplicationProcessorSchedulerQueueSize(),
			WorkerCount: config.ReplicationLowPriorityProcessorSchedulerWorkerCount,
		},
		taskQueueHashFunc,
		queueFactory,
		logger,
	)
	taskChannelKeyFn := func(e TrackableExecutableTask) ClusterChannelKey {
		return ClusterChannelKey{
			ClusterName: e.SourceClusterName(),
		}
	}
	channelWeightFn := func(key ClusterChannelKey) int {
		return 1
	}
	taskQuotaRequestFn := func(t TrackableExecutableTask) quotas.Request {
		var taskType string
		var nsName namespace.Name
		replicationTask := t.ReplicationTask()
		if replicationTask != nil {
			taskType = replicationTask.TaskType.String()

			rawTaskInfo := replicationTask.GetRawTaskInfo()
			if rawTaskInfo != nil {
				var err error
				nsName, err = nsRegistry.GetNamespaceName(namespace.ID(replicationTask.GetRawTaskInfo().NamespaceId))
				if err != nil {
					nsName = namespace.EmptyName
				}
			}
		}
		return quotas.NewRequest(
			taskType,
			taskSchedulerToken,
			nsName.String(),
			headers.CallerTypePreemptable,
			0,
			"")
	}
	taskMetricsTagsFn := func(t TrackableExecutableTask) []metrics.Tag {
		replicationTask := t.ReplicationTask()
		var taskType string
		namespaceTag := metrics.NamespaceUnknownTag()
		if replicationTask != nil {
			taskType = replicationTask.TaskType.String()
			rawTaskInfo := replicationTask.GetRawTaskInfo()
			if rawTaskInfo != nil {
				nsName, err := nsRegistry.GetNamespaceName(namespace.ID(replicationTask.GetRawTaskInfo().NamespaceId))
				if err != nil {
					namespaceTag = metrics.NamespaceTag(nsName.String())
				}
			}
		}
		return []metrics.Tag{
			namespaceTag,
			metrics.TaskTypeTag(taskType),
			metrics.OperationTag(taskType), // for backward compatibility
			metrics.TaskPriorityTag(ctasks.PriorityPreemptable.String()),
		}
	}
	// This creates a per cluster channel.
	// They share the same weight so it just does a round-robin on all clusters' tasks.
	rrScheduler := ctasks.NewInterleavedWeightedRoundRobinScheduler(
		ctasks.InterleavedWeightedRoundRobinSchedulerOptions[TrackableExecutableTask, ClusterChannelKey]{
			TaskChannelKeyFn: taskChannelKeyFn,
			ChannelWeightFn:  channelWeightFn,
		},
		scheduler,
		logger,
	)
	ts := ctasks.NewRateLimitedScheduler[TrackableExecutableTask](
		rrScheduler,
		rateLimiter,
		timeSource,
		taskQuotaRequestFn,
		taskMetricsTagsFn,
		ctasks.RateLimitedSchedulerOptions{
			Enabled:          config.ReplicationEnableRateLimit,
			EnableShadowMode: config.ReplicationEnableRateLimitShadowMode,
		},
		logger,
		metricsHandler,
	)
	lc.Append(fx.StartStopHook(ts.Start, ts.Stop))
	return ts
}

func sequentialTaskQueueFactoryProvider(
	logger log.Logger,
	metricsHandler metrics.Handler,
	config *configs.Config,
) ctasks.SequentialTaskQueueFactory[TrackableExecutableTask] {
	return func(task TrackableExecutableTask) ctasks.SequentialTaskQueue[TrackableExecutableTask] {
		if config.EnableReplicationTaskBatching() {
			return NewSequentialBatchableTaskQueue(task, nil, logger, metricsHandler)
		}
		item := task.QueueID()
		workflowKey, ok := item.(definition.WorkflowKey)
		if !ok {
			return NewSequentialTaskQueueWithID(item)
		}
		return NewSequentialTaskQueueWithID(workflowKey.NamespaceID + "_" + workflowKey.WorkflowID)
	}
}

func executableTaskConverterProvider(
	processToolBox ProcessToolBox,
) ExecutableTaskConverter {
	return NewExecutableTaskConverter(processToolBox)
}

func streamReceiverMonitorProvider(
	processToolBox ProcessToolBox,
	taskConverter ExecutableTaskConverter,
) StreamReceiverMonitor {
	return NewStreamReceiverMonitor(
		processToolBox,
		taskConverter,
		processToolBox.Config.EnableReplicationStream(),
	)
}

func resendHandlerProvider(
	namespaceRegistry namespace.Registry,
	clientBean client.Bean,
	serializer serialization.Serializer,
	clusterMetadata cluster.Metadata,
	shardController shard.Controller,
	config *configs.Config,
	remoteHistoryFetcher eventhandler.HistoryPaginatedFetcher,
	logger log.Logger,
	importer eventhandler.EventImporter,
) eventhandler.ResendHandler {
	return eventhandler.NewResendHandler(
		namespaceRegistry,
		clientBean,
		serializer,
		clusterMetadata,
		func(ctx context.Context, namespaceId namespace.ID, workflowId string) (historyi.Engine, error) {
			shardContext, err := shardController.GetShardByNamespaceWorkflow(
				namespaceId,
				workflowId,
			)
			if err != nil {
				return nil, err
			}
			return shardContext.GetEngine(ctx)
		},
		remoteHistoryFetcher,
		importer,
		logger,
		config,
	)
}

func eventImporterProvider(
	historyFetcher eventhandler.HistoryPaginatedFetcher,
	shardController shard.Controller,
	serializer serialization.Serializer,
	logger log.Logger,
) eventhandler.EventImporter {
	return eventhandler.NewEventImporter(
		historyFetcher,
		func(ctx context.Context, namespaceId namespace.ID, workflowId string) (historyi.Engine, error) {
			shardContext, err := shardController.GetShardByNamespaceWorkflow(
				namespaceId,
				workflowId,
			)
			if err != nil {
				return nil, err
			}
			return shardContext.GetEngine(ctx)
		},
		serializer,
		logger,
	)
}

func dlqWriterAdapterProvider(
	dlqWriter *queues.DLQWriter,
	replicationTaskSerializer TaskSerializer,
	clusterMetadata cluster.Metadata,
) *DLQWriterAdapter {
	return NewDLQWriterAdapter(dlqWriter, replicationTaskSerializer, clusterMetadata.GetCurrentClusterName())
}

func historyEventsHandlerProvider(
	clusterMetadata cluster.Metadata,
	importer eventhandler.EventImporter,
	shardController shard.Controller,
	logger log.Logger,
) eventhandler.HistoryEventsHandler {
	return eventhandler.NewHistoryEventsHandler(
		clusterMetadata,
		importer,
		shardController,
		logger,
	)
}

func historyPaginatedFetcherProvider(
	namespaceRegistry namespace.Registry,
	clientBean client.Bean,
	serializer serialization.Serializer,
	logger log.Logger,
) eventhandler.HistoryPaginatedFetcher {
	return eventhandler.NewHistoryPaginatedFetcher(
		namespaceRegistry,
		clientBean,
		serializer,
		logger,
	)
}