mirror of
https://github.com/hanzo-fi/webhooks.git
synced 2026-10-30 20:32:55 +00:00
Maxence MaireauxandGitHub
bc0e5976c2
Redesign the retry worker to support multi-worker scaling with continuous batch processing instead of all-at-once polling. Key changes: - Add "retrying" intermediate status for atomic claim pattern via CTE - Batch processing with configurable LIMIT (--retry-batch-size, default 50) - Reduce tick interval from 1min to 3s (--retry-period) for continuous flow - Parallel batch processing via pond worker pool (bounded concurrency) - Crash recovery for stale "retrying" attempts (runs once per minute) - Partial indexes for retry polling, claimed attempts, and recovery - UpdateAttemptsStatus scoped to "retrying" rows only (preserves history) - Worker resilient to per-webhook errors (log + continue, no crash) - HTTP client timeout (30s) prevents blocking on slow endpoints - Oldest retries prioritized via ORDER BY next_retry_after ASC
88 lines
2.4 KiB
Go
88 lines
2.4 KiB
Go
package cmd
|
|
|
|
import (
|
|
"net/http"
|
|
|
|
"github.com/formancehq/go-libs/v2/otlp"
|
|
|
|
"github.com/formancehq/go-libs/v2/auth"
|
|
"github.com/formancehq/go-libs/v2/aws/iam"
|
|
"github.com/formancehq/go-libs/v2/publish"
|
|
|
|
"github.com/formancehq/webhooks/pkg/storage/postgres"
|
|
|
|
"github.com/formancehq/go-libs/v2/bun/bunconnect"
|
|
"github.com/formancehq/go-libs/v2/licence"
|
|
|
|
"github.com/formancehq/go-libs/v2/otlp/otlptraces"
|
|
|
|
"github.com/formancehq/go-libs/v2/httpserver"
|
|
"github.com/formancehq/go-libs/v2/service"
|
|
"github.com/formancehq/webhooks/cmd/flag"
|
|
"github.com/formancehq/webhooks/pkg/backoff"
|
|
innerotlp "github.com/formancehq/webhooks/pkg/otlp"
|
|
"github.com/formancehq/webhooks/pkg/worker"
|
|
"github.com/spf13/cobra"
|
|
"go.uber.org/fx"
|
|
)
|
|
|
|
func newWorkerCommand() *cobra.Command {
|
|
ret := &cobra.Command{
|
|
Use: "worker",
|
|
Short: "Run webhooks worker",
|
|
RunE: runWorker,
|
|
PreRunE: handleAutoMigrate,
|
|
}
|
|
otlp.AddFlags(ret.Flags())
|
|
otlptraces.AddFlags(ret.Flags())
|
|
publish.AddFlags(ServiceName, ret.Flags())
|
|
auth.AddFlags(ret.Flags())
|
|
flag.Init(ret.Flags())
|
|
bunconnect.AddFlags(ret.Flags())
|
|
iam.AddFlags(ret.Flags())
|
|
service.AddFlags(ret.Flags())
|
|
licence.AddFlags(ret.Flags())
|
|
|
|
return ret
|
|
}
|
|
|
|
func runWorker(cmd *cobra.Command, _ []string) error {
|
|
connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(cmd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
retryPeriod, _ := cmd.Flags().GetDuration(flag.RetryPeriod)
|
|
retryBatchSize, _ := cmd.Flags().GetInt(flag.RetryBatchSize)
|
|
minBackOffDelay, _ := cmd.Flags().GetDuration(flag.MinBackoffDelay)
|
|
maxBackOffDelay, _ := cmd.Flags().GetDuration(flag.MaxBackoffDelay)
|
|
abortAfter, _ := cmd.Flags().GetDuration(flag.AbortAfter)
|
|
topics, _ := cmd.Flags().GetStringSlice(flag.KafkaTopics)
|
|
listen, _ := cmd.Flags().GetString(flag.Listen)
|
|
|
|
return service.New(
|
|
cmd.OutOrStdout(),
|
|
innerotlp.HttpClientModule(),
|
|
licence.FXModuleFromFlags(cmd, ServiceName),
|
|
postgres.NewModule(*connectionOptions, service.IsDebug(cmd)),
|
|
fx.Provide(worker.NewWorkerHandler),
|
|
fx.Invoke(func(lc fx.Lifecycle, h http.Handler) {
|
|
lc.Append(httpserver.NewHook(h, httpserver.WithAddress(listen)))
|
|
}),
|
|
otlp.FXModuleFromFlags(cmd),
|
|
otlptraces.FXModuleFromFlags(cmd),
|
|
worker.StartModule(
|
|
cmd,
|
|
retryPeriod,
|
|
backoff.NewExponential(
|
|
minBackOffDelay,
|
|
maxBackOffDelay,
|
|
abortAfter,
|
|
),
|
|
retryBatchSize,
|
|
service.IsDebug(cmd),
|
|
topics,
|
|
),
|
|
).Run(cmd)
|
|
}
|