Files
zeekayandHanzo Dev 43e315f18c a backup that cannot hold the database refuses, and a deadlock is retried
Three things, all the same shape: SQLite's guarantees were being assumed on
an engine that does not make them.

A backup archives the data directory, which is where the database lives on
SQLite and not on Postgres. There it returned 204 and produced an archive
with no database in it, and restore then failed inside a goroutine after the
204 had gone out — so an operator saw successful backups, a populated list,
plausible sizes, and had nothing recoverable. It refuses on an engine whose
database is not in that directory, before extracting anything, and says where
to take the backup instead. Refusing beats shelling out to pg_dump: that adds
a binary that has to out-version the server, and its output cannot be restored
by the path we have, so it would be a second restore mechanism.

What stopped a restore is now retrievable. It is recorded in core, so a CLI
or a hook caller sees it too, and GET on the restore address renders it: 202
while running, 200 and the reason when stopped, 404 when this process has not
tried. POST still answers 204, because the admin client reads a body on any
other 2xx.

A serialization failure or a deadlock is a transient and is retried at the
transaction boundary, which is the only place a retry means anything — the
second statement of a deadlocked transaction reports 25P02, so retrying a
statement spends the attempt and loses the evidence. The transaction watches
its own statements for the two SQLSTATEs rather than reading the returned
error, which by then has been wrapped down to a message string. Batch had to
become re-runnable for this: it accumulated results outside the transaction.

Retrying turns a transient into a success and does not pretend to fix a
convoy: with a few concurrent opposing batches the loss goes to zero, and
with thirty pairs over six rows every pair collides and no budget clears it.

SQLite keeps the same executions at the same intervals.

Co-authored-by: Hanzo Dev <[email protected]>
2026-08-13 13:09:18 -07:00

179 lines
5.2 KiB
Go

package apis
import (
"context"
"net/http"
"path/filepath"
"time"
"github.com/hanzoai/base/core"
"github.com/hanzoai/base/tools/router"
"github.com/hanzoai/base/tools/routine"
"github.com/hanzoai/base/tools/types"
"github.com/spf13/cast"
)
// bindBackupApi registers the file api endpoints and the corresponding handlers.
func bindBackupApi(app core.App, rg *router.RouterGroup[*core.RequestEvent]) {
sub := rg.Group("/backups")
sub.GET("", backupsList).Bind(RequireSuperuserAuth())
sub.POST("", backupCreate).Bind(RequireSuperuserAuth())
sub.POST("/upload", backupUpload).Bind(BodyLimit(0), RequireSuperuserAuth())
sub.GET("/{key}", backupDownload) // relies on superuser file token
sub.DELETE("/{key}", backupDelete).Bind(RequireSuperuserAuth())
sub.POST("/{key}/restore", backupRestore).Bind(RequireSuperuserAuth())
sub.GET("/{key}/restore", backupRestoreOutcome).Bind(RequireSuperuserAuth())
}
type backupFileInfo struct {
Modified types.DateTime `json:"modified"`
Key string `json:"key"`
Size int64 `json:"size"`
}
func backupsList(e *core.RequestEvent) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
fsys, err := e.App.NewBackupsFilesystem()
if err != nil {
return e.BadRequestError("Failed to load backups filesystem.", err)
}
defer fsys.Close()
fsys.SetContext(ctx)
backups, err := fsys.List("")
if err != nil {
return e.BadRequestError("Failed to retrieve backup items. Raw error: \n"+err.Error(), nil)
}
result := make([]backupFileInfo, len(backups))
for i, obj := range backups {
modified, _ := types.ParseDateTime(obj.ModTime)
result[i] = backupFileInfo{
Key: obj.Key,
Size: obj.Size,
Modified: modified,
}
}
return e.JSON(http.StatusOK, result)
}
func backupDownload(e *core.RequestEvent) error {
fileToken := e.Request.URL.Query().Get("token")
authRecord, err := e.App.FindAuthRecordByToken(fileToken, core.TokenTypeFile)
if err != nil || !authRecord.IsSuperuser() {
return e.ForbiddenError("Insufficient permissions to access the resource.", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
fsys, err := e.App.NewBackupsFilesystem()
if err != nil {
return e.InternalServerError("Failed to load backups filesystem.", err)
}
defer fsys.Close()
fsys.SetContext(ctx)
key := e.Request.PathValue("key")
return fsys.Serve(
e.Response,
e.Request,
key,
filepath.Base(key), // without the path prefix (if any)
)
}
func backupDelete(e *core.RequestEvent) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
fsys, err := e.App.NewBackupsFilesystem()
if err != nil {
return e.InternalServerError("Failed to load backups filesystem.", err)
}
defer fsys.Close()
fsys.SetContext(ctx)
key := e.Request.PathValue("key")
if key != "" && cast.ToString(e.App.Store().Get(core.StoreKeyActiveBackup)) == key {
return e.BadRequestError("The backup is currently being used and cannot be deleted.", nil)
}
if err := fsys.Delete(key); err != nil {
return e.BadRequestError("Invalid or already deleted backup file. Raw error: \n"+err.Error(), nil)
}
return e.NoContent(http.StatusNoContent)
}
func backupRestore(e *core.RequestEvent) error {
if e.App.Store().Has(core.StoreKeyActiveBackup) {
return e.BadRequestError("Try again later - another backup/restore process has already been started.", nil)
}
key := e.Request.PathValue("key")
existsCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
fsys, err := e.App.NewBackupsFilesystem()
if err != nil {
return e.InternalServerError("Failed to load backups filesystem.", err)
}
defer fsys.Close()
fsys.SetContext(existsCtx)
if exists, err := fsys.Exists(key); !exists {
return e.BadRequestError("Missing or invalid backup file.", err)
}
routine.FireAndForget(func() {
// give some optimistic time to write the response before restarting the app
time.Sleep(1 * time.Second)
// wait max 10 minutes to fetch the backup
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
if err := e.App.RestoreBackup(ctx, key); err != nil {
e.App.Logger().Error("Failed to restore backup", "key", key, "error", err.Error())
}
})
return e.NoContent(http.StatusNoContent)
}
// backupRestoreOutcome answers how the restore of this backup ended.
//
// A restore runs past the response that starts it, because a restore that
// completes replaces the data directory and restarts the process. So the reply
// that starts one says only that it started, and this is where the answer is:
// 202 while it runs, 200 with the reason when it stopped, 404 when this process
// has not restored that backup.
func backupRestoreOutcome(e *core.RequestEvent) error {
key := e.Request.PathValue("key")
if key != "" && cast.ToString(e.App.Store().Get(core.StoreKeyActiveBackup)) == key {
return e.NoContent(http.StatusAccepted)
}
failure, ok := e.App.Store().Get(core.StoreKeyRestoreFailure).(core.RestoreFailure)
if !ok || failure.Name != key {
return e.NotFoundError("No restore of this backup has been attempted.", nil)
}
return e.JSON(http.StatusOK, failure)
}