Checksum VFS. (#176)

conn.go

@@ -521,10 +521,3 @@ func (c *Conn) stmtsIter(yield func(*Stmt) bool) {
 		}
 	}
 }
-
-// DriverConn is implemented by the SQLite [database/sql] driver connection.
-//
-// Deprecated: use [github.com/ncruces/go-sqlite3/driver.Conn] instead.
-type DriverConn interface {
-	Raw() *Conn
-}

tests/wal_test.go

@@ -77,7 +77,7 @@ func TestWAL_readonly(t *testing.T) {
 
 	// Select the data using the second (readonly) connection.
 	var name string
-	err = db2.QueryRow("SELECT name FROM t").Scan(&name)
+	err = db2.QueryRow(`SELECT name FROM t`).Scan(&name)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -95,7 +95,7 @@ func TestWAL_readonly(t *testing.T) {
 	}
 
 	// Select the data using the second (readonly) connection.
-	err = db2.QueryRow("SELECT name FROM t").Scan(&name)
+	err = db2.QueryRow(`SELECT name FROM t`).Scan(&name)
 	if err != nil {
 		t.Fatal(err)
 	}

txn.go

@@ -143,7 +143,7 @@ func (c *Conn) Savepoint() Savepoint {
 	// Names can be reused, but this makes catching bugs more likely.
 	name = QuoteIdentifier(name + "_" + strconv.Itoa(int(rand.Int31())))
 
-	err := c.txnExecInterrupted("SAVEPOINT " + name)
+	err := c.txnExecInterrupted(`SAVEPOINT ` + name)
 	if err != nil {
 		panic(err)
 	}
@@ -187,7 +187,7 @@ func (s Savepoint) Release(errp *error) {
 		if s.c.GetAutocommit() { // There is nothing to commit.
 			return
 		}
-		*errp = s.c.Exec("RELEASE " + s.name)
+		*errp = s.c.Exec(`RELEASE ` + s.name)
 		if *errp == nil {
 			return
 		}
@@ -199,8 +199,7 @@ func (s Savepoint) Release(errp *error) {
 		return
 	}
 	// ROLLBACK and RELEASE even if interrupted.
-	err := s.c.txnExecInterrupted("ROLLBACK TO " +
-		s.name + "; RELEASE " + s.name)
+	err := s.c.txnExecInterrupted(`ROLLBACK TO ` + s.name + `; RELEASE ` + s.name)
 	if err != nil {
 		panic(err)
 	}
@@ -213,7 +212,7 @@ func (s Savepoint) Release(errp *error) {
 // https://sqlite.org/lang_transaction.html
 func (s Savepoint) Rollback() error {
 	// ROLLBACK even if interrupted.
-	return s.c.txnExecInterrupted("ROLLBACK TO " + s.name)
+	return s.c.txnExecInterrupted(`ROLLBACK TO ` + s.name)
 }
 
 func (c *Conn) txnExecInterrupted(sql string) error {

util/vfsutil/wrap.go

@@ -22,6 +22,14 @@ func UnwrapFile[T vfs.File](f vfs.File) (_ T, _ bool) {
 	}
 }
 
+// WrapOpenFilename helps wrap [vfs.VFSFilename].
+func WrapOpenFilename(f vfs.VFS, name *vfs.Filename, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
+	if f, ok := f.(vfs.VFSFilename); ok {
+		return f.OpenFilename(name, flags)
+	}
+	return f.Open(name.String(), flags)
+}
+
 // WrapLockState helps wrap [vfs.FileLockState].
 func WrapLockState(f vfs.File) vfs.LockLevel {
 	if f, ok := f.(vfs.FileLockState); ok {

vfs/adiantum/adiantum_test.go

@@ -21,7 +21,7 @@ var testDB string
 
 func Test_fileformat(t *testing.T) {
 	readervfs.Create("test.db", ioutil.NewSizeReaderAt(strings.NewReader(testDB)))
-	adiantum.Register("radiantum", vfs.Find("reader"), nil)
+	vfs.Register("radiantum", adiantum.Wrap(vfs.Find("reader"), nil))
 
 	db, err := driver.Open("file:test.db?vfs=radiantum")
 	if err != nil {

vfs/adiantum/api.go

@@ -40,24 +40,25 @@ import (
 )
 
 func init() {
-	Register("adiantum", vfs.Find(""), nil)
+	vfs.Register("adiantum", Wrap(vfs.Find(""), nil))
 }
 
-// Register registers an encrypting VFS, wrapping a base VFS,
-// and possibly using a custom HBSH cipher construction.
+// Wrap wraps a base VFS to create an encrypting VFS,
+// possibly using a custom HBSH cipher construction.
+//
 // To use the default Adiantum construction, set cipher to nil.
 //
 // The default construction uses a 32 byte key/hexkey.
 // If a textkey is provided, the default KDF is Argon2id
 // with 64 MiB of memory, 3 iterations, and 4 threads.
-func Register(name string, base vfs.VFS, cipher HBSHCreator) {
+func Wrap(base vfs.VFS, cipher HBSHCreator) vfs.VFS {
 	if cipher == nil {
 		cipher = adiantumCreator{}
 	}
-	vfs.Register(name, &hbshVFS{
+	return &hbshVFS{
 		VFS:  base,
 		init: cipher,
-	})
+	}
 }
 
 // HBSHCreator creates an [hbsh.HBSH] cipher

vfs/adiantum/example_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 func ExampleRegister_hpolyc() {
-	adiantum.Register("hpolyc", vfs.Find(""), hpolycCreator{})
+	vfs.Register("hpolyc", adiantum.Wrap(vfs.Find(""), hpolycCreator{}))
 
 	db, err := sqlite3.Open("file:demo.db?vfs=hpolyc" +
 		"&textkey=correct+horse+battery+staple")

vfs/adiantum/hbsh.go

@@ -24,11 +24,7 @@ func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag,
 }
 
 func (h *hbshVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
-	if hf, ok := h.VFS.(vfs.VFSFilename); ok {
-		file, flags, err = hf.OpenFilename(name, flags)
-	} else {
-		file, flags, err = h.VFS.Open(name.String(), flags)
-	}
+	file, flags, err = vfsutil.WrapOpenFilename(h.VFS, name, flags)
 
 	// Encrypt everything except super journals and memory files.
 	if err != nil || flags&(vfs.OPEN_SUPER_JOURNAL|vfs.OPEN_MEMORY) != 0 {
@@ -49,13 +45,14 @@ func (h *hbshVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs
 		} else if t, ok := params["textkey"]; ok && len(t[0]) > 0 {
 			key = h.init.KDF(t[0])
 		} else if flags&vfs.OPEN_MAIN_DB != 0 {
-			// Main datatabases may have their key specified as a PRAGMA.
+			// Main databases may have their key specified as a PRAGMA.
 			return &hbshFile{File: file, init: h.init}, flags, nil
 		}
 		hbsh = h.init.HBSH(key)
 	}
 
 	if hbsh == nil {
+		file.Close()
 		return nil, flags, sqlite3.CANTOPEN
 	}
 	return &hbshFile{File: file, hbsh: hbsh, init: h.init}, flags, nil

vfs/cksmvfs/README.md

@@ -0,0 +1,20 @@
+# Go `cksmvfs` SQLite VFS
+
+This package wraps an SQLite VFS to help detect database corruption.
+
+The `"cksmvfs"` VFS wraps the default SQLite VFS adding an 8-byte checksum
+to the end of every page in an SQLite database.\
+The checksum is added as each page is written
+and verified as each page is read.\
+The checksum is intended to help detect database corruption
+caused by random bit-flips in the mass storage device.
+
+This implementation is compatible with SQLite's
+[Checksum VFS Shim](https://sqlite.org/cksumvfs.html).
+
+> [!IMPORTANT]
+> [Checksums](https://en.wikipedia.org/wiki/Checksum)
+> are meant to protect against _silent data corruption_ (bit rot).
+> They do not offer _authenticity_ (i.e. protect against _forgery_),
+> nor prevent _silent loss of durability_.
+> Checkpoint WAL mode databases to improve durabiliy.

vfs/cksmvfs/api.go

@@ -0,0 +1,75 @@
+// Package cksmvfs wraps an SQLite VFS to help detect database corruption.
+//
+// The "cksmvfs" [vfs.VFS] wraps the default VFS adding an 8-byte checksum
+// to the end of every page in an SQLite database.
+// The checksum is added as each page is written
+// and verified as each page is read.
+// The checksum is intended to help detect database corruption
+// caused by random bit-flips in the mass storage device.
+//
+// This implementation is compatible with SQLite's
+// [Checksum VFS Shim].
+//
+// [Checksum VFS Shim]: https://sqlite.org/cksumvfs.html
+package cksmvfs
+
+import (
+	"fmt"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/vfs"
+)
+
+func init() {
+	vfs.Register("cksmvfs", Wrap(vfs.Find("")))
+}
+
+// Wrap wraps a base VFS to create a checksumming VFS.
+func Wrap(base vfs.VFS) vfs.VFS {
+	return &cksmVFS{VFS: base}
+}
+
+// EnableChecksums enables checksums on a database.
+func EnableChecksums(db *sqlite3.Conn, schema string) error {
+	if f, ok := db.Filename("").DatabaseFile().(*cksmFile); !ok {
+		return fmt.Errorf("cksmvfs: incorrect type: %T", f)
+	}
+
+	r, err := db.FileControl(schema, sqlite3.FCNTL_RESERVE_BYTES)
+	if err != nil {
+		return err
+	}
+	if r == 8 {
+		// Correct value, enabled.
+		return nil
+	}
+	if r == 0 {
+		// Default value, enable.
+		_, err = db.FileControl(schema, sqlite3.FCNTL_RESERVE_BYTES, 8)
+		if err != nil {
+			return err
+		}
+		r, err = db.FileControl(schema, sqlite3.FCNTL_RESERVE_BYTES)
+		if err != nil {
+			return err
+		}
+	}
+	if r != 8 {
+		// Invalid value.
+		return fmt.Errorf("cksmvfs: reserve bytes must be 8, is: %d", r)
+	}
+
+	// VACUUM the database.
+	if schema != "" {
+		err = db.Exec(`VACUUM ` + sqlite3.QuoteIdentifier(schema))
+	} else {
+		err = db.Exec(`VACUUM`)
+	}
+	if err != nil {
+		return err
+	}
+
+	// Checkpoint the WAL.
+	_, _, err = db.WALCheckpoint(schema, sqlite3.CHECKPOINT_RESTART)
+	return err
+}

vfs/cksmvfs/api_test.go

@@ -0,0 +1,133 @@
+package cksmvfs_test
+
+import (
+	_ "embed"
+	"log"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/driver"
+	_ "github.com/ncruces/go-sqlite3/embed"
+	_ "github.com/ncruces/go-sqlite3/internal/testcfg"
+	"github.com/ncruces/go-sqlite3/util/ioutil"
+	"github.com/ncruces/go-sqlite3/vfs"
+	"github.com/ncruces/go-sqlite3/vfs/cksmvfs"
+	"github.com/ncruces/go-sqlite3/vfs/memdb"
+	"github.com/ncruces/go-sqlite3/vfs/readervfs"
+)
+
+//go:embed testdata/cksm.db
+var cksmDB string
+
+func Test_fileformat(t *testing.T) {
+	readervfs.Create("test.db", ioutil.NewSizeReaderAt(strings.NewReader(cksmDB)))
+	vfs.Register("rcksm", cksmvfs.Wrap(vfs.Find("reader")))
+
+	db, err := driver.Open("file:test.db?vfs=rcksm")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	var enabled bool
+	err = db.QueryRow(`PRAGMA checksum_verification`).Scan(&enabled)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !enabled {
+		t.Error("want true")
+	}
+
+	db.SetMaxIdleConns(0) // Clears the page cache.
+
+	_, err = db.Exec(`PRAGMA integrity_check`)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+//go:embed testdata/test.db
+var testDB []byte
+
+func Test_enable(t *testing.T) {
+	memdb.Create("nockpt.db", testDB)
+	vfs.Register("mcksm", cksmvfs.Wrap(vfs.Find("memdb")))
+
+	db, err := driver.Open("file:/nockpt.db?vfs=mcksm",
+		func(db *sqlite3.Conn) error {
+			return cksmvfs.EnableChecksums(db, "")
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	var enabled bool
+	err = db.QueryRow(`PRAGMA checksum_verification`).Scan(&enabled)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !enabled {
+		t.Error("want true")
+	}
+
+	db.SetMaxIdleConns(0) // Clears the page cache.
+
+	_, err = db.Exec(`PRAGMA integrity_check`)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func Test_new(t *testing.T) {
+	if !vfs.SupportsFileLocking {
+		t.Skip("skipping without locks")
+	}
+
+	name := "file:" +
+		filepath.ToSlash(filepath.Join(t.TempDir(), "test.db")) +
+		"?vfs=cksmvfs&_pragma=journal_mode(wal)"
+
+	db, err := driver.Open(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	var enabled bool
+	err = db.QueryRow(`PRAGMA checksum_verification`).Scan(&enabled)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !enabled {
+		t.Error("want true")
+	}
+
+	var size int
+	err = db.QueryRow(`PRAGMA page_size=1024`).Scan(&size)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != 4096 {
+		t.Errorf("got %d, want 4096", size)
+	}
+
+	_, err = db.Exec(`CREATE TABLE users (id INT, name VARCHAR(10))`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_, err = db.Exec(`INSERT INTO users (id, name) VALUES (0, 'go'), (1, 'zig'), (2, 'whatever')`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	db.SetMaxIdleConns(0) // Clears the page cache.
+
+	_, err = db.Exec(`PRAGMA integrity_check`)
+	if err != nil {
+		t.Fatal(err)
+	}
+}