Add database driver for sqlcipher

This is a copy of the existing sqlite3 driver, except that it imports `_
"github.com/mutecomm/go-sqlcipher/v4"` as database driver.  This allows
to use migrate with encrypted sqlite databases.

database/sqlcipher/README.md

@@ -0,0 +1,3 @@
+# sqlcipher
+
+This is just a copy of the sqlite3 driver except that it imports `github.com/mutecomm/go-sqlcipher`.

database/sqlcipher/examples/migrations/33_create_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS pets;

database/sqlcipher/examples/migrations/33_create_table.up.sql

@@ -0,0 +1,3 @@
+CREATE TABLE pets (
+  name string
+);

database/sqlcipher/examples/migrations/44_alter_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS pets;

database/sqlcipher/examples/migrations/44_alter_table.up.sql

@@ -0,0 +1 @@
+ALTER TABLE pets ADD predator bool;

database/sqlcipher/sqlcipher.go

@@ -0,0 +1,265 @@
+package sqlcipher
+
+import (
+	"database/sql"
+	"fmt"
+	"io"
+	"io/ioutil"
+	nurl "net/url"
+	"strconv"
+	"strings"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/hashicorp/go-multierror"
+	_ "github.com/mutecomm/go-sqlcipher/v4"
+)
+
+func init() {
+	database.Register("sqlcipher", &Sqlite{})
+}
+
+var DefaultMigrationsTable = "schema_migrations"
+var (
+	ErrDatabaseDirty  = fmt.Errorf("database is dirty")
+	ErrNilConfig      = fmt.Errorf("no config")
+	ErrNoDatabaseName = fmt.Errorf("no database name")
+)
+
+type Config struct {
+	MigrationsTable string
+	DatabaseName    string
+	NoTxWrap        bool
+}
+
+type Sqlite struct {
+	db       *sql.DB
+	isLocked bool
+
+	config *Config
+}
+
+func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	}
+
+	if err := instance.Ping(); err != nil {
+		return nil, err
+	}
+
+	if len(config.MigrationsTable) == 0 {
+		config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	mx := &Sqlite{
+		db:     instance,
+		config: config,
+	}
+	if err := mx.ensureVersionTable(); err != nil {
+		return nil, err
+	}
+	return mx, nil
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the Sqlite type.
+func (m *Sqlite) ensureVersionTable() (err error) {
+	if err = m.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := m.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	query := fmt.Sprintf(`
+	CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool);
+  CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version);
+  `, m.config.MigrationsTable, m.config.MigrationsTable)
+
+	if _, err := m.db.Exec(query); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *Sqlite) Open(url string) (database.Driver, error) {
+	purl, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+	dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite3://", "", 1)
+	db, err := sql.Open("sqlite3", dbfile)
+	if err != nil {
+		return nil, err
+	}
+
+	qv := purl.Query()
+
+	migrationsTable := qv.Get("x-migrations-table")
+	if len(migrationsTable) == 0 {
+		migrationsTable = DefaultMigrationsTable
+	}
+
+	noTxWrap := false
+	if v := qv.Get("x-no-tx-wrap"); v != "" {
+		noTxWrap, err = strconv.ParseBool(v)
+		if err != nil {
+			return nil, fmt.Errorf("x-no-tx-wrap: %s", err)
+		}
+	}
+
+	mx, err := WithInstance(db, &Config{
+		DatabaseName:    purl.Path,
+		MigrationsTable: migrationsTable,
+		NoTxWrap:        noTxWrap,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return mx, nil
+}
+
+func (m *Sqlite) Close() error {
+	return m.db.Close()
+}
+
+func (m *Sqlite) Drop() (err error) {
+	query := `SELECT name FROM sqlite_master WHERE type = 'table';`
+	tables, err := m.db.Query(query)
+	if err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
+	tableNames := make([]string, 0)
+	for tables.Next() {
+		var tableName string
+		if err := tables.Scan(&tableName); err != nil {
+			return err
+		}
+		if len(tableName) > 0 {
+			tableNames = append(tableNames, tableName)
+		}
+	}
+	if len(tableNames) > 0 {
+		for _, t := range tableNames {
+			query := "DROP TABLE " + t
+			err = m.executeQuery(query)
+			if err != nil {
+				return &database.Error{OrigErr: err, Query: []byte(query)}
+			}
+		}
+		query := "VACUUM"
+		_, err = m.db.Query(query)
+		if err != nil {
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+
+	return nil
+}
+
+func (m *Sqlite) Lock() error {
+	if m.isLocked {
+		return database.ErrLocked
+	}
+	m.isLocked = true
+	return nil
+}
+
+func (m *Sqlite) Unlock() error {
+	if !m.isLocked {
+		return nil
+	}
+	m.isLocked = false
+	return nil
+}
+
+func (m *Sqlite) Run(migration io.Reader) error {
+	migr, err := ioutil.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+	query := string(migr[:])
+
+	if m.config.NoTxWrap {
+		return m.executeQueryNoTx(query)
+	}
+	return m.executeQuery(query)
+}
+
+func (m *Sqlite) executeQuery(query string) error {
+	tx, err := m.db.Begin()
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction start failed"}
+	}
+	if _, err := tx.Exec(query); err != nil {
+		if errRollback := tx.Rollback(); errRollback != nil {
+			err = multierror.Append(err, errRollback)
+		}
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	if err := tx.Commit(); err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction commit failed"}
+	}
+	return nil
+}
+
+func (m *Sqlite) executeQueryNoTx(query string) error {
+	if _, err := m.db.Exec(query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	return nil
+}
+
+func (m *Sqlite) SetVersion(version int, dirty bool) error {
+	tx, err := m.db.Begin()
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction start failed"}
+	}
+
+	query := "DELETE FROM " + m.config.MigrationsTable
+	if _, err := tx.Exec(query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	// Also re-write the schema version for nil dirty versions to prevent
+	// empty schema version for failed down migration on the first migration
+	// See: https://github.com/golang-migrate/migrate/issues/330
+	if version >= 0 || (version == database.NilVersion && dirty) {
+		query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable)
+		if _, err := tx.Exec(query, version, dirty); err != nil {
+			if errRollback := tx.Rollback(); errRollback != nil {
+				err = multierror.Append(err, errRollback)
+			}
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction commit failed"}
+	}
+
+	return nil
+}
+
+func (m *Sqlite) Version() (version int, dirty bool, err error) {
+	query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1"
+	err = m.db.QueryRow(query).Scan(&version, &dirty)
+	if err != nil {
+		return database.NilVersion, false, nil
+	}
+	return version, dirty, nil
+}

database/sqlcipher/sqlcipher_test.go

@@ -0,0 +1,162 @@
+package sqlcipher
+
+import (
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/golang-migrate/migrate/v4"
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+	_ "github.com/mutecomm/go-sqlcipher/v4"
+)
+
+func Test(t *testing.T) {
+	dir, err := ioutil.TempDir("", "sqlite3-driver-test")
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := os.RemoveAll(dir); err != nil {
+			t.Error(err)
+		}
+	}()
+	t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
+	p := &Sqlite{}
+	addr := fmt.Sprintf("sqlite3://%s", filepath.Join(dir, "sqlite3.db"))
+	d, err := p.Open(addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);"))
+}
+
+func TestMigrate(t *testing.T) {
+	dir, err := ioutil.TempDir("", "sqlite3-driver-test")
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := os.RemoveAll(dir); err != nil {
+			t.Error(err)
+		}
+	}()
+	t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
+
+	db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db"))
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			return
+		}
+	}()
+	driver, err := WithInstance(db, &Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m, err := migrate.NewWithDatabaseInstance(
+		"file://./examples/migrations",
+		"ql", driver)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dt.TestMigrate(t, m)
+}
+
+func TestMigrationTable(t *testing.T) {
+	dir, err := ioutil.TempDir("", "sqlite3-driver-test-migration-table")
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := os.RemoveAll(dir); err != nil {
+			t.Error(err)
+		}
+	}()
+
+	t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
+
+	db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db"))
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			return
+		}
+	}()
+
+	config := &Config{
+		MigrationsTable: "my_migration_table",
+	}
+	driver, err := WithInstance(db, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m, err := migrate.NewWithDatabaseInstance(
+		"file://./examples/migrations",
+		"ql", driver)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("UP")
+	err = m.Up()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.Query(fmt.Sprintf("SELECT * FROM %s", config.MigrationsTable))
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestNoTxWrap(t *testing.T) {
+	dir, err := ioutil.TempDir("", "sqlite3-driver-test")
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := os.RemoveAll(dir); err != nil {
+			t.Error(err)
+		}
+	}()
+	t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
+	p := &Sqlite{}
+	addr := fmt.Sprintf("sqlite3://%s?x-no-tx-wrap=true", filepath.Join(dir, "sqlite3.db"))
+	d, err := p.Open(addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// An explicit BEGIN statement would ordinarily fail without x-no-tx-wrap.
+	// (Transactions in sqlite may not be nested.)
+	dt.Test(t, d, []byte("BEGIN; CREATE TABLE t (Qty int, Name string); COMMIT;"))
+}
+
+func TestNoTxWrapInvalidValue(t *testing.T) {
+	dir, err := ioutil.TempDir("", "sqlite3-driver-test")
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := os.RemoveAll(dir); err != nil {
+			t.Error(err)
+		}
+	}()
+	t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
+	p := &Sqlite{}
+	addr := fmt.Sprintf("sqlite3://%s?x-no-tx-wrap=yeppers", filepath.Join(dir, "sqlite3.db"))
+	_, err = p.Open(addr)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "x-no-tx-wrap")
+		assert.Contains(t, err.Error(), "invalid syntax")
+	}
+}

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/lib/pq v1.3.0
 	github.com/markbates/pkger v0.15.1
 	github.com/mattn/go-sqlite3 v1.10.0
+	github.com/mutecomm/go-sqlcipher/v4 v4.4.0
 	github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8
 	github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba
 	github.com/snowflakedb/gosnowflake v1.3.5

go.sum

@@ -288,6 +288,8 @@ github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK86
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/mutecomm/go-sqlcipher/v4 v4.4.0 h1:sV1tWCWGAVlPhNGT95Q+z/txFxuhAYWwHD1afF5bMZg=
+github.com/mutecomm/go-sqlcipher/v4 v4.4.0/go.mod h1:PyN04SaWalavxRGH9E8ZftG6Ju7rsPrGmQRjrEaVpiY=
 github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 h1:P48LjvUQpTReR3TQRbxSeSBsMXzfK0uol7eRcr7VBYQ=
 github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA=
 github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba h1:fhFP5RliM2HW/8XdcO5QngSfFli9GcRIpMXvypTQt6E=
@@ -423,6 +425,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190225153610-fe579d43d832/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=