base/core/base_backup.go

package core

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"time"

	"github.com/hanzoai/base/tools/archive"
	"github.com/hanzoai/base/tools/filesystem"
	"github.com/hanzoai/base/tools/inflector"
	"github.com/hanzoai/base/tools/osutils"
	"github.com/hanzoai/base/tools/security"
	"github.com/hanzoai/base/tools/types"
	"github.com/hanzoai/orm/dialect"
)

const (
	StoreKeyActiveBackup = "@activeBackup"

	// StoreKeyRestoreFailure holds the [RestoreFailure] of the most recent
	// restore this process attempted, if it stopped.
	StoreKeyRestoreFailure = "@restoreFailure"
)

// RestoreFailure is a restore that stopped, and why.
//
// A restore that completes replaces the data directory and restarts the process,
// so the attempt a running process can still be asked about is the one that did
// not complete.
type RestoreFailure struct {
	Time  types.DateTime `json:"time"`
	Name  string         `json:"name"`
	Error string         `json:"error"`
}

// checkLocalDatabase reports whether d keeps the database in the data directory.
//
// A backup is an archive of that directory and a restore replaces it, so both
// carry the whole database exactly when the engine keeps it in a file there. A
// server engine keeps it in the server, which is also where it is backed up and
// restored.
func checkLocalDatabase(d dialect.Dialect) error {
	if LocalDatabase(d) {
		return nil
	}

	return fmt.Errorf(
		"a backup archives the data directory, which holds the database on sqlite; on %s back up and restore the database at the server",
		d.Name(),
	)
}

// CreateBackup creates a new backup of the current app data directory.
//
// If name is empty, it will be autogenerated.
// If backup with the same name exists, the new backup file will replace it.
//
// The backup is executed within a transaction, meaning that new writes
// will be temporary "blocked" until the backup file is generated.
//
// To safely perform the backup, it is recommended to have free disk space
// for at least 2x the size of the data directory.
//
// By default backups are stored in data/backups
// (the backups directory itself is excluded from the generated backup).
//
// When using S3 storage for the uploaded collection files, you have to
// take care manually to backup those since they are not part of the data.
//
// Backups can be stored on S3 if it is configured in app.Settings().Backups.
//
// The archive is the data directory, so a backup is a backup of the database on
// an engine that keeps the database there. A server engine keeps it in the
// server, which is where it is backed up.
func (app *BaseApp) CreateBackup(ctx context.Context, name string) error {
	if app.Store().Has(StoreKeyActiveBackup) {
		return errors.New("try again later - another backup/restore operation has already been started")
	}

	app.Store().Set(StoreKeyActiveBackup, name)
	defer app.Store().Remove(StoreKeyActiveBackup)

	event := new(BackupEvent)
	event.App = app
	event.Context = ctx
	event.Name = name
	// default root dir entries to exclude from the backup generation
	event.Exclude = []string{LocalBackupsDirName, LocalTempDirName, LocalAutocertCacheDirName, LocalNotifyDirName, lostFoundDirName}

	return app.OnBackupCreate().Trigger(event, func(e *BackupEvent) error {
		if err := checkLocalDatabase(e.App.Dialect()); err != nil {
			return err
		}

		// generate a default name if missing
		if e.Name == "" {
			e.Name = generateBackupName(e.App, "hz_backup_")
		}

		// make sure that the special temp directory exists
		// note: it needs to be inside the current data to avoid "cross-device link" errors
		localTempDir := filepath.Join(e.App.DataDir(), LocalTempDirName)
		if err := os.MkdirAll(localTempDir, os.ModePerm); err != nil {
			return fmt.Errorf("failed to create a temp dir: %w", err)
		}

		// archive data in a temp directory, exluding the "backups" and the temp dirs
		//
		// run in transaction to temporary block other writes (transactions uses the NonconcurrentDB connection)
		// ---
		tempPath := filepath.Join(localTempDir, "hz_backup_"+security.PseudorandomString(6))
		createErr := e.App.RunInTransaction(func(txApp App) error {
			return txApp.AuxRunInTransaction(func(txApp App) error {
				// fold the write-ahead log back in so the archived files are
				// whole (errors are ignored — the archive is still usable and
				// an engine without a log has nothing to fold)
				if checkpoint := txApp.Dialect().Checkpoint(); checkpoint != "" {
					txApp.DB().NewQuery(checkpoint).Execute()
					txApp.AuxDB().NewQuery(checkpoint).Execute()
				}

				return archive.Create(txApp.DataDir(), tempPath, e.Exclude...)
			})
		})
		if createErr != nil {
			return createErr
		}
		defer os.Remove(tempPath)

		// persist the backup in the backups filesystem
		// ---
		fsys, err := e.App.NewBackupsFilesystem()
		if err != nil {
			return err
		}
		defer fsys.Close()

		fsys.SetContext(e.Context)

		file, err := filesystem.NewFileFromPath(tempPath)
		if err != nil {
			return err
		}
		file.OriginalName = e.Name
		file.Name = file.OriginalName

		if err := fsys.UploadFile(file, file.Name); err != nil {
			return err
		}

		return nil
	})
}

// RestoreBackup restores the backup with the specified name and restarts
// the current running application process.
//
// NB! This feature is experimental and currently is expected to work only on UNIX based systems.
//
// To safely perform the restore it is recommended to have free disk space
// for at least 2x the size of the restored data backup.
//
// The performed steps are:
//
//  1. Download the backup with the specified name in a temp location
//     (this is in case of S3; otherwise it creates a temp copy of the zip)
//
//  2. Extract the backup in a temp directory inside the app "data"
//     (eg. "data/.temp_to_delete/hz_restore").
//
//  3. Move the current app "data" content (excluding the local backups and the special temp dir)
//     under another temp sub dir that will be deleted on the next app start up
//     (eg. "data/.temp_to_delete/old_data").
//     This is because on some environments it may not be allowed
//     to delete the currently open "data" files.
//
//  4. Move the extracted dir content to the app "data".
//
//  5. Restart the app (on successful app bootstap it will also remove the old data).
//
// If a failure occur during the restore process the dir changes are reverted.
// If for whatever reason the revert is not possible, it panics.
//
// Note that if your data has custom network mounts as subdirectories, then
// it is possible the restore to fail during the `os.Rename` operations
// (see https://github.com/hanzoai/base/issues/4647).
//
// A restore replaces the data directory, so it restores the database on an
// engine that keeps the database there. A server engine keeps it in the server,
// which is where it is restored. What stopped a restore is recorded under
// [StoreKeyRestoreFailure], because a restore that completes restarts the
// process and so cannot report back.
func (app *BaseApp) RestoreBackup(ctx context.Context, name string) error {
	if app.Store().Has(StoreKeyActiveBackup) {
		return errors.New("try again later - another backup/restore operation has already been started")
	}

	app.Store().Set(StoreKeyActiveBackup, name)
	defer app.Store().Remove(StoreKeyActiveBackup)

	event := new(BackupEvent)
	event.App = app
	event.Context = ctx
	event.Name = name
	// default root dir entries to exclude from the backup restore
	event.Exclude = []string{LocalBackupsDirName, LocalTempDirName, LocalAutocertCacheDirName, LocalNotifyDirName, lostFoundDirName}

	// a restore ends either by replacing the data directory and restarting the
	// process or by stopping, so the outcome a still-running process can be asked
	// about is recorded rather than only said once
	restoreErr := app.OnBackupRestore().Trigger(event, func(e *BackupEvent) error {
		if runtime.GOOS == "windows" {
			return errors.New("restore is not supported on Windows")
		}

		if err := checkLocalDatabase(e.App.Dialect()); err != nil {
			return err
		}

		// make sure that the special temp directory exists
		// note: it needs to be inside the current data to avoid "cross-device link" errors
		localTempDir := filepath.Join(e.App.DataDir(), LocalTempDirName)
		if err := os.MkdirAll(localTempDir, os.ModePerm); err != nil {
			return fmt.Errorf("failed to create a temp dir: %w", err)
		}

		fsys, err := e.App.NewBackupsFilesystem()
		if err != nil {
			return err
		}
		defer fsys.Close()

		fsys.SetContext(e.Context)

		if ok, _ := fsys.Exists(name); !ok {
			return fmt.Errorf("missing or invalid backup file %q to restore", name)
		}

		extractedDataDir := filepath.Join(localTempDir, "hz_restore_"+security.PseudorandomString(8))
		defer os.RemoveAll(extractedDataDir)

		// extract the zip
		if e.App.Settings().Backups.S3.Enabled {
			br, err := fsys.GetReader(name)
			if err != nil {
				return err
			}
			defer br.Close()

			// create a temp zip file from the blob.Reader and try to extract it
			tempZip, err := os.CreateTemp(localTempDir, "hz_restore_zip")
			if err != nil {
				return err
			}
			defer os.Remove(tempZip.Name())
			defer tempZip.Close() // note: this technically shouldn't be necessary but it is here to workaround platforms discrepancies

			_, err = io.Copy(tempZip, br)
			if err != nil {
				return err
			}

			err = archive.Extract(tempZip.Name(), extractedDataDir)
			if err != nil {
				return err
			}

			// remove the temp zip file since we no longer need it
			// (this is in case the app restarts and the defer calls are not called)
			_ = tempZip.Close()
			err = os.Remove(tempZip.Name())
			if err != nil {
				e.App.Logger().Warn(
					"[RestoreBackup] Failed to remove the temp zip backup file",
					"file", tempZip.Name(),
					"error", err.Error(),
				)
			}
		} else {
			// manually construct the local path to avoid creating a copy of the zip file
			// since the blob reader currently doesn't implement ReaderAt
			zipPath := filepath.Join(e.App.DataDir(), LocalBackupsDirName, filepath.Base(name))

			err = archive.Extract(zipPath, extractedDataDir)
			if err != nil {
				return err
			}
		}

		// ensure that at least a database file exists
		extractedDB := filepath.Join(extractedDataDir, dataFile)
		if _, err := os.Stat(extractedDB); err != nil {
			return fmt.Errorf("data.db file is missing or invalid: %w", err)
		}

		oldTempDataDir := filepath.Join(localTempDir, "old_data_"+security.PseudorandomString(8))

		replaceErr := e.App.RunInTransaction(func(txApp App) error {
			return txApp.AuxRunInTransaction(func(txApp App) error {
				// move the current data dir content to a special temp location
				// that will hold the old data between dirs replace
				// (the temp dir will be automatically removed on the next app start)
				if err := osutils.MoveDirContent(txApp.DataDir(), oldTempDataDir, e.Exclude...); err != nil {
					return fmt.Errorf("failed to move the current data dir content to a temp location: %w", err)
				}

				// move the extracted archive content to the app's data dir
				if err := osutils.MoveDirContent(extractedDataDir, txApp.DataDir(), e.Exclude...); err != nil {
					return fmt.Errorf("failed to move the extracted archive content to data dir: %w", err)
				}

				return nil
			})
		})
		if replaceErr != nil {
			return replaceErr
		}

		revertDataDirChanges := func() error {
			return e.App.RunInTransaction(func(txApp App) error {
				return txApp.AuxRunInTransaction(func(txApp App) error {
					if err := osutils.MoveDirContent(txApp.DataDir(), extractedDataDir, e.Exclude...); err != nil {
						return fmt.Errorf("failed to revert the extracted dir change: %w", err)
					}

					if err := osutils.MoveDirContent(oldTempDataDir, txApp.DataDir(), e.Exclude...); err != nil {
						return fmt.Errorf("failed to revert old data dir dir change: %w", err)
					}

					return nil
				})
			})
		}

		// restart the app
		if err := e.App.Restart(); err != nil {
			if revertErr := revertDataDirChanges(); revertErr != nil {
				panic(revertErr)
			}

			return fmt.Errorf("failed to restart the app process: %w", err)
		}

		return nil
	})
	if restoreErr != nil {
		app.Store().Set(StoreKeyRestoreFailure, RestoreFailure{
			Time:  types.NowDateTime(),
			Name:  name,
			Error: restoreErr.Error(),
		})
	} else {
		app.Store().Remove(StoreKeyRestoreFailure)
	}

	return restoreErr
}

// registerAutobackupHooks registers the autobackup app serve hooks.
func (app *BaseApp) registerAutobackupHooks() {
	const jobId = "__hzAutoBackup__"

	loadJob := func() {
		rawSchedule := app.Settings().Backups.Cron
		if rawSchedule == "" {
			app.Cron().Remove(jobId)
			return
		}

		app.Cron().Add(jobId, rawSchedule, func() {
			const autoPrefix = "@auto_backup_"

			name := generateBackupName(app, autoPrefix)

			if err := app.CreateBackup(context.Background(), name); err != nil {
				app.Logger().Error(
					"[Backup cron] Failed to create backup",
					"name", name,
					"error", err.Error(),
				)
			}

			maxKeep := app.Settings().Backups.CronMaxKeep

			if maxKeep == 0 {
				return // no explicit limit
			}

			fsys, err := app.NewBackupsFilesystem()
			if err != nil {
				app.Logger().Error(
					"[Backup cron] Failed to initialize the backup filesystem",
					"error", err.Error(),
				)
				return
			}
			defer fsys.Close()

			files, err := fsys.List(autoPrefix)
			if err != nil {
				app.Logger().Error(
					"[Backup cron] Failed to list autogenerated backups",
					"error", err.Error(),
				)
				return
			}

			if maxKeep >= len(files) {
				return // nothing to remove
			}

			// sort desc
			sort.Slice(files, func(i, j int) bool {
				return files[i].ModTime.After(files[j].ModTime)
			})

			// keep only the most recent n auto backup files
			toRemove := files[maxKeep:]

			for _, f := range toRemove {
				if err := fsys.Delete(f.Key); err != nil {
					app.Logger().Error(
						"[Backup cron] Failed to remove old autogenerated backup",
						"key", f.Key,
						"error", err.Error(),
					)
				}
			}
		})
	}

	app.OnBootstrap().BindFunc(func(e *BootstrapEvent) error {
		if err := e.Next(); err != nil {
			return err
		}

		loadJob()

		return nil
	})

	app.OnSettingsReload().BindFunc(func(e *SettingsReloadEvent) error {
		if err := e.Next(); err != nil {
			return err
		}

		loadJob()

		return nil
	})
}

func generateBackupName(app App, prefix string) string {
	appName := inflector.Snakecase(app.Settings().Meta.AppName)
	if len(appName) > 50 {
		appName = appName[:50]
	}

	return fmt.Sprintf(
		"%s%s_%s.zip",
		prefix,
		appName,
		time.Now().UTC().Format("20060102150405"),
	)
}