mirror of
https://github.com/hanzo-fi/payments.git
synced 2026-10-30 20:31:20 +00:00
* feat(EN-214): add outbox table migration * feat(EN-214): add outbox models and storage layer * feat(EN-214): implement outbox publisher activity * feat(EN-214): implement outbox schedule activity * feat(EN-214): add outbox publisher workflow * feat(EN-214): migrate accounts to outbox pattern * feat(EN-214): migrate balances to outbox pattern * feat(EN-214): migrate payments to outbox pattern * feat(EN-214): migrate bank accounts to outbox pattern * feat(EN-214): migrate pools to outbox pattern * feat(EN-214): migrate payment initiations to outbox pattern * feat(EN-214): migrate PSU and connections to outbox pattern * feat(EN-214): migrate connectors to outbox pattern * feat(EN-214): remove legacy send_events workflow * feat(EN-214): update workflows to use outbox pattern * feat(EN-214): improve outbox idempotency and error handling * feat(EN-214): update storage tasks and tests * test(EN-214): update e2e tests for outbox pattern * test(EN-214): clean up workflow tests * chore(EN-214): update docker compose configuration * feat(EN-214): add e2e run config in ide * fix: add back docker-compose-dev run config * chore: improve connector uninstall comments & remove connector id being passed * fix: ensure events are not pushed if balances are not created * fix: ensure err is not overwritten when using rollback * fix: ensure we use v3 provider format * chore: apply the nitpick comments from coderabbit * chore: consistentely use e() error handling * feat: optionally disable publisher schedule to make e2e test more consistent * fix: improve error handling on uninstall connector * avoid using custom payload and rely on existing interfaces; also avoid using different types for events * fix: remove unecessary unmarshalling * fix: ensure that rollbackOnTxError picks up errors on tx.Commit() * fix: add extra validation on iban & account number in the test * test: cover marshall/unmarshalling of event payload * fix: increase test timeout * try to make test more resiliant * fix: wrong pointer use in loop * fix: error formatting issue * fix: move oubtox schedule creation out of an activity and do it in the worker itself * feat: OutboxEventDeleteAndSend as batch * change outbox_event to have id only instead of idempotency key * make e2e test less brittle, save eventID as idempotency key in publication * feat: make outbox_event keep an history * feat(EN-214): added a schedule to cleanup outbox events * test: make cleanup actually clean * ignore linting rules for now * feat: do not add psu metadata to event anymore as it's irrelevant * fix: create local variable instead of relying on pointers, even if not 100% necessary * fix: remove temporal schedule describe calls
164 lines
4.2 KiB
Go
164 lines
4.2 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
|
|
"github.com/formancehq/go-libs/v3/pointer"
|
|
"github.com/formancehq/go-libs/v3/time"
|
|
internalEvents "github.com/formancehq/payments/internal/events"
|
|
"github.com/formancehq/payments/internal/models"
|
|
"github.com/formancehq/payments/pkg/events"
|
|
"github.com/pkg/errors"
|
|
"github.com/uptrace/bun"
|
|
)
|
|
|
|
type task struct {
|
|
bun.BaseModel `bun:"table:tasks"`
|
|
|
|
// Mandatory fields
|
|
ID models.TaskID `bun:"id,pk,type:character varying,notnull"`
|
|
Status models.TaskStatus `bun:"status,type:text,notnull"`
|
|
CreatedAt time.Time `bun:"created_at,type:timestamp without time zone,notnull"`
|
|
UpdatedAt time.Time `bun:"updated_at,type:timestamp without time zone,notnull"`
|
|
|
|
// Optional fields
|
|
ConnectorID *models.ConnectorID `bun:"connector_id,type:character varying"`
|
|
CreatedObjectID *string `bun:"created_object_id,type:character varying"`
|
|
Error *string `bun:"error,type:text"`
|
|
}
|
|
|
|
func (s *store) TasksUpsert(ctx context.Context, task models.Task) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return e("failed to begin transaction", err)
|
|
}
|
|
defer func() {
|
|
rollbackOnTxError(ctx, &tx, err)
|
|
}()
|
|
|
|
t := fromTaskModel(task)
|
|
|
|
query := tx.NewInsert().
|
|
Model(&t).
|
|
On("CONFLICT (id) DO UPDATE").
|
|
Set("status = EXCLUDED.status").
|
|
Set("updated_at = EXCLUDED.updated_at")
|
|
|
|
if task.CreatedObjectID != nil {
|
|
query.Set("created_object_id = EXCLUDED.created_object_id")
|
|
}
|
|
|
|
if task.Error != nil {
|
|
query.Set("error = EXCLUDED.error")
|
|
} else {
|
|
query.Set("error = NULL")
|
|
}
|
|
|
|
_, err = query.Exec(ctx)
|
|
if err != nil {
|
|
return e("failed to insert task", err)
|
|
}
|
|
|
|
// Create outbox event for task update
|
|
payload := internalEvents.TaskMessagePayload{
|
|
ID: task.ID.String(),
|
|
Status: string(task.Status),
|
|
CreatedAt: task.CreatedAt,
|
|
UpdatedAt: task.UpdatedAt,
|
|
CreatedObjectID: task.CreatedObjectID,
|
|
}
|
|
if task.ConnectorID != nil {
|
|
connectorIDStr := task.ConnectorID.String()
|
|
payload.ConnectorID = &connectorIDStr
|
|
}
|
|
if task.Error != nil {
|
|
errorStr := task.Error.Error()
|
|
payload.Error = &errorStr
|
|
}
|
|
|
|
var payloadBytes []byte
|
|
payloadBytes, err = json.Marshal(payload)
|
|
if err != nil {
|
|
return e("failed to marshal task event payload", err)
|
|
}
|
|
|
|
outboxEvent := models.OutboxEvent{
|
|
ID: models.EventID{
|
|
EventIdempotencyKey: task.IdempotencyKey(),
|
|
ConnectorID: task.ConnectorID,
|
|
},
|
|
EventType: events.EventTypeUpdatedTask,
|
|
EntityID: task.ID.String(),
|
|
Payload: payloadBytes,
|
|
CreatedAt: time.Now().UTC().Time,
|
|
Status: models.OUTBOX_STATUS_PENDING,
|
|
ConnectorID: task.ConnectorID,
|
|
}
|
|
|
|
if err = s.OutboxEventsInsert(ctx, tx, []models.OutboxEvent{outboxEvent}); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = tx.Commit(); err != nil {
|
|
return e("failed to commit transaction", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *store) TasksGet(ctx context.Context, id models.TaskID) (*models.Task, error) {
|
|
var t task
|
|
|
|
err := s.db.NewSelect().
|
|
Model(&t).
|
|
Where("id = ?", id).
|
|
Scan(ctx)
|
|
if err != nil {
|
|
return nil, e("failed to fetch task", err)
|
|
}
|
|
|
|
return pointer.For(toTaskModel(t)), nil
|
|
}
|
|
|
|
func (s *store) TasksDeleteFromConnectorID(ctx context.Context, connectorID models.ConnectorID) error {
|
|
_, err := s.db.NewDelete().
|
|
Model((*task)(nil)).
|
|
Where("connector_id = ?", connectorID).
|
|
Exec(ctx)
|
|
return e("failed to delete tasks", err)
|
|
}
|
|
|
|
func fromTaskModel(from models.Task) task {
|
|
return task{
|
|
ID: from.ID,
|
|
ConnectorID: from.ConnectorID,
|
|
Status: from.Status,
|
|
CreatedAt: time.New(from.CreatedAt),
|
|
UpdatedAt: time.New(from.UpdatedAt),
|
|
CreatedObjectID: from.CreatedObjectID,
|
|
Error: func() *string {
|
|
if from.Error == nil {
|
|
return nil
|
|
}
|
|
return pointer.For(from.Error.Error())
|
|
}(),
|
|
}
|
|
}
|
|
|
|
func toTaskModel(from task) models.Task {
|
|
return models.Task{
|
|
ID: from.ID,
|
|
ConnectorID: from.ConnectorID,
|
|
Status: from.Status,
|
|
CreatedAt: from.CreatedAt.Time,
|
|
UpdatedAt: from.UpdatedAt.Time,
|
|
CreatedObjectID: from.CreatedObjectID,
|
|
Error: func() error {
|
|
if from.Error == nil {
|
|
return nil
|
|
}
|
|
return errors.New(*from.Error)
|
|
}(),
|
|
}
|
|
}
|