Files
fgueryandGitHub a63e3a20a8 feat(EN-214): create events via storage instead of manually calling event activity (#594)
* feat(EN-214): add outbox table migration

* feat(EN-214): add outbox models and storage layer

* feat(EN-214): implement outbox publisher activity

* feat(EN-214): implement outbox schedule activity

* feat(EN-214): add outbox publisher workflow

* feat(EN-214): migrate accounts to outbox pattern

* feat(EN-214): migrate balances to outbox pattern

* feat(EN-214): migrate payments to outbox pattern

* feat(EN-214): migrate bank accounts to outbox pattern

* feat(EN-214): migrate pools to outbox pattern

* feat(EN-214): migrate payment initiations to outbox pattern

* feat(EN-214): migrate PSU and connections to outbox pattern

* feat(EN-214): migrate connectors to outbox pattern

* feat(EN-214): remove legacy send_events workflow

* feat(EN-214): update workflows to use outbox pattern

* feat(EN-214): improve outbox idempotency and error handling

* feat(EN-214): update storage tasks and tests

* test(EN-214): update e2e tests for outbox pattern

* test(EN-214): clean up workflow tests

* chore(EN-214): update docker compose configuration

* feat(EN-214): add e2e run config in ide

* fix: add back docker-compose-dev run config

* chore: improve connector uninstall comments & remove connector id being passed

* fix: ensure events are not pushed if balances are not created

* fix: ensure err is not overwritten when using rollback

* fix: ensure we use v3 provider format

* chore: apply the nitpick comments from coderabbit

* chore: consistentely use e() error handling

* feat: optionally disable publisher schedule to make e2e test more consistent

* fix: improve error handling on uninstall connector

* avoid using custom payload and rely on existing interfaces; also avoid using different types for events

* fix: remove unecessary unmarshalling

* fix: ensure that rollbackOnTxError picks up errors on tx.Commit()

* fix: add extra validation on iban & account number in the test

* test: cover marshall/unmarshalling of event payload

* fix: increase test timeout

* try to make test more resiliant

* fix: wrong pointer use in loop

* fix: error formatting issue

* fix: move oubtox schedule creation out of an activity and do it in the worker itself

* feat: OutboxEventDeleteAndSend as batch

* change outbox_event to have id only instead of idempotency key

* make e2e test less brittle, save eventID as idempotency key in publication

* feat: make outbox_event keep an history

* feat(EN-214): added a schedule to cleanup outbox events

* test: make cleanup actually clean

* ignore linting rules for now

* feat: do not add psu metadata to event anymore as it's irrelevant

* fix: create local variable instead of relying on pointers, even if not 100% necessary

* fix: remove temporal schedule describe calls
2025-11-24 16:05:42 +00:00

164 lines
4.2 KiB
Go

package storage
import (
"context"
"encoding/json"
"github.com/formancehq/go-libs/v3/pointer"
"github.com/formancehq/go-libs/v3/time"
internalEvents "github.com/formancehq/payments/internal/events"
"github.com/formancehq/payments/internal/models"
"github.com/formancehq/payments/pkg/events"
"github.com/pkg/errors"
"github.com/uptrace/bun"
)
type task struct {
bun.BaseModel `bun:"table:tasks"`
// Mandatory fields
ID models.TaskID `bun:"id,pk,type:character varying,notnull"`
Status models.TaskStatus `bun:"status,type:text,notnull"`
CreatedAt time.Time `bun:"created_at,type:timestamp without time zone,notnull"`
UpdatedAt time.Time `bun:"updated_at,type:timestamp without time zone,notnull"`
// Optional fields
ConnectorID *models.ConnectorID `bun:"connector_id,type:character varying"`
CreatedObjectID *string `bun:"created_object_id,type:character varying"`
Error *string `bun:"error,type:text"`
}
func (s *store) TasksUpsert(ctx context.Context, task models.Task) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return e("failed to begin transaction", err)
}
defer func() {
rollbackOnTxError(ctx, &tx, err)
}()
t := fromTaskModel(task)
query := tx.NewInsert().
Model(&t).
On("CONFLICT (id) DO UPDATE").
Set("status = EXCLUDED.status").
Set("updated_at = EXCLUDED.updated_at")
if task.CreatedObjectID != nil {
query.Set("created_object_id = EXCLUDED.created_object_id")
}
if task.Error != nil {
query.Set("error = EXCLUDED.error")
} else {
query.Set("error = NULL")
}
_, err = query.Exec(ctx)
if err != nil {
return e("failed to insert task", err)
}
// Create outbox event for task update
payload := internalEvents.TaskMessagePayload{
ID: task.ID.String(),
Status: string(task.Status),
CreatedAt: task.CreatedAt,
UpdatedAt: task.UpdatedAt,
CreatedObjectID: task.CreatedObjectID,
}
if task.ConnectorID != nil {
connectorIDStr := task.ConnectorID.String()
payload.ConnectorID = &connectorIDStr
}
if task.Error != nil {
errorStr := task.Error.Error()
payload.Error = &errorStr
}
var payloadBytes []byte
payloadBytes, err = json.Marshal(payload)
if err != nil {
return e("failed to marshal task event payload", err)
}
outboxEvent := models.OutboxEvent{
ID: models.EventID{
EventIdempotencyKey: task.IdempotencyKey(),
ConnectorID: task.ConnectorID,
},
EventType: events.EventTypeUpdatedTask,
EntityID: task.ID.String(),
Payload: payloadBytes,
CreatedAt: time.Now().UTC().Time,
Status: models.OUTBOX_STATUS_PENDING,
ConnectorID: task.ConnectorID,
}
if err = s.OutboxEventsInsert(ctx, tx, []models.OutboxEvent{outboxEvent}); err != nil {
return err
}
if err = tx.Commit(); err != nil {
return e("failed to commit transaction", err)
}
return nil
}
func (s *store) TasksGet(ctx context.Context, id models.TaskID) (*models.Task, error) {
var t task
err := s.db.NewSelect().
Model(&t).
Where("id = ?", id).
Scan(ctx)
if err != nil {
return nil, e("failed to fetch task", err)
}
return pointer.For(toTaskModel(t)), nil
}
func (s *store) TasksDeleteFromConnectorID(ctx context.Context, connectorID models.ConnectorID) error {
_, err := s.db.NewDelete().
Model((*task)(nil)).
Where("connector_id = ?", connectorID).
Exec(ctx)
return e("failed to delete tasks", err)
}
func fromTaskModel(from models.Task) task {
return task{
ID: from.ID,
ConnectorID: from.ConnectorID,
Status: from.Status,
CreatedAt: time.New(from.CreatedAt),
UpdatedAt: time.New(from.UpdatedAt),
CreatedObjectID: from.CreatedObjectID,
Error: func() *string {
if from.Error == nil {
return nil
}
return pointer.For(from.Error.Error())
}(),
}
}
func toTaskModel(from task) models.Task {
return models.Task{
ID: from.ID,
ConnectorID: from.ConnectorID,
Status: from.Status,
CreatedAt: from.CreatedAt.Time,
UpdatedAt: from.UpdatedAt.Time,
CreatedObjectID: from.CreatedObjectID,
Error: func() error {
if from.Error == nil {
return nil
}
return errors.New(*from.Error)
}(),
}
}