Windows shared memory. (#181)

.github/workflows/build-test.sh

@@ -6,6 +6,6 @@ echo 'set -eu' > test.sh
 for p in $(go list ./...); do
   dir=".${p#github.com/ncruces/go-sqlite3}"
   name="$(basename "$p").test"
-  (cd ${dir}; go test -c)
-  [ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} ${TESTFLAGS})" >> test.sh
+  (cd ${dir}; go test -c ${BUILDFLAGS:-})
+  [ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} ${TESTFLAGS:-})" >> test.sh
 done

.github/workflows/test.yml

@@ -52,17 +52,17 @@ jobs:
     - name: Test
       run: go test -v ./... -bench . -benchtime=1x
 
+    - name: Test no locks
+      run: go test -v -tags sqlite3_nosys ./...
+      if: matrix.os == 'ubuntu-latest'
+
     - name: Test BSD locks
       run: go test -v -tags sqlite3_flock ./...
       if: matrix.os == 'macos-latest'
 
     - name: Test dot locks
       run: go test -v -tags sqlite3_dotlk ./...
-      if: matrix.os == 'macos-latest'
-
-    - name: Test no locks
-      run: go test -v -tags sqlite3_nosys ./...
-      if: matrix.os == 'ubuntu-latest'
+      if: matrix.os != 'windows-latest'
 
     - name: Test GORM
       shell: bash
@@ -194,6 +194,7 @@ jobs:
       env:
         GOOS: solaris
         TESTFLAGS: '-test.v -test.short'
+        BUILDFLAGS: '-tags sqlite3_dotlk'
       run: .github/workflows/build-test.sh
 
     - name: Test Solaris

internal/testcfg/testcfg.go

@@ -19,7 +19,8 @@ func init() {
 		path := filepath.Join(os.TempDir(), "wazero")
 		if err := os.MkdirAll(path, 0777); err == nil {
 			if cache, err := wazero.NewCompilationCacheWithDir(path); err == nil {
-				sqlite3.RuntimeConfig.WithCompilationCache(cache)
+				sqlite3.RuntimeConfig = sqlite3.RuntimeConfig.
+					WithCompilationCache(cache)
 			}
 		}
 	}

internal/util/mmap.go

@@ -55,10 +55,10 @@ type MappedRegion struct {
 	used bool
 }
 
-func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, prot int) (*MappedRegion, error) {
+func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, readOnly bool) (*MappedRegion, error) {
 	s := ctx.Value(moduleKey{}).(*moduleState)
 	r := s.new(ctx, mod, size)
-	err := r.mmap(f, offset, prot)
+	err := r.mmap(f, offset, readOnly)
 	if err != nil {
 		return nil, err
 	}
@@ -75,7 +75,11 @@ func (r *MappedRegion) Unmap() error {
 	return err
 }
 
-func (r *MappedRegion) mmap(f *os.File, offset int64, prot int) error {
+func (r *MappedRegion) mmap(f *os.File, offset int64, readOnly bool) error {
+	prot := unix.PROT_READ
+	if !readOnly {
+		prot |= unix.PROT_WRITE
+	}
 	_, err := unix.MmapPtr(int(f.Fd()), offset, r.addr, uintptr(r.size),
 		prot, unix.MAP_SHARED|unix.MAP_FIXED)
 	r.used = err == nil

internal/util/mmap_windows.go

@@ -0,0 +1,53 @@
+//go:build !sqlite3_nosys
+
+package util
+
+import (
+	"context"
+	"os"
+	"reflect"
+	"unsafe"
+
+	"github.com/tetratelabs/wazero/api"
+	"golang.org/x/sys/windows"
+)
+
+type MappedRegion struct {
+	windows.Handle
+	Data []byte
+	addr uintptr
+}
+
+func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32) (*MappedRegion, error) {
+	h, err := windows.CreateFileMapping(windows.Handle(f.Fd()), nil, windows.PAGE_READWRITE, 0, 0, nil)
+	if h == 0 {
+		return nil, err
+	}
+
+	a, err := windows.MapViewOfFile(h, windows.FILE_MAP_WRITE,
+		uint32(offset>>32), uint32(offset), uintptr(size))
+	if a == 0 {
+		windows.CloseHandle(h)
+		return nil, err
+	}
+
+	res := &MappedRegion{Handle: h, addr: a}
+	// SliceHeader, although deprecated, avoids a go vet warning.
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&res.Data))
+	sh.Len = int(size)
+	sh.Cap = int(size)
+	sh.Data = a
+	return res, nil
+}
+
+func (r *MappedRegion) Unmap() error {
+	if r.Data == nil {
+		return nil
+	}
+	err := windows.UnmapViewOfFile(r.addr)
+	if err != nil {
+		return err
+	}
+	r.Data = nil
+	return windows.CloseHandle(r.Handle)
+}

vfs/shm.go

@@ -1,4 +1,4 @@
-//go:build ((darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
+//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
 
 package vfs
 
@@ -22,8 +22,5 @@ func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
 	if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 {
 		return nil
 	}
-	return &vfsShm{
-		path:     path,
-		readOnly: flags&OPEN_READONLY != 0,
-	}
+	return &vfsShm{path: path}
 }

vfs/shm_bsd.go

@@ -18,11 +18,9 @@ type vfsShmFile struct {
 	*os.File
 	info os.FileInfo
 
-	// +checklocks:vfsShmFilesMtx
-	refs int
+	refs int // +checklocks:vfsShmFilesMtx
 
-	// +checklocks:Mutex
-	lock [_SHM_NLOCK]int16
+	lock [_SHM_NLOCK]int16 // +checklocks:Mutex
 	sync.Mutex
 }
 
@@ -34,10 +32,9 @@ var (
 
 type vfsShm struct {
 	*vfsShmFile
-	path     string
-	lock     [_SHM_NLOCK]bool
-	regions  []*util.MappedRegion
-	readOnly bool
+	path    string
+	lock    [_SHM_NLOCK]bool
+	regions []*util.MappedRegion
 }
 
 func (s *vfsShm) Close() error {
@@ -69,7 +66,7 @@ func (s *vfsShm) Close() error {
 	panic(util.AssertErr())
 }
 
-func (s *vfsShm) shmOpen() (rc _ErrorCode) {
+func (s *vfsShm) shmOpen() _ErrorCode {
 	if s.vfsShmFile != nil {
 		return _OK
 	}
@@ -100,17 +97,13 @@ func (s *vfsShm) shmOpen() (rc _ErrorCode) {
 		}
 	}
 
-	// Lock and truncate the file, if not readonly.
+	// Lock and truncate the file.
 	// The lock is only released by closing the file.
-	if s.readOnly {
-		rc = _READONLY_CANTINIT
-	} else {
-		if rc := osLock(f, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK); rc != _OK {
-			return rc
-		}
-		if err := f.Truncate(0); err != nil {
-			return _IOERR_SHMOPEN
-		}
+	if rc := osLock(f, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK); rc != _OK {
+		return rc
+	}
+	if err := f.Truncate(0); err != nil {
+		return _IOERR_SHMOPEN
 	}
 
 	// Add the new shared file.
@@ -122,11 +115,11 @@ func (s *vfsShm) shmOpen() (rc _ErrorCode) {
 	for i, g := range vfsShmFiles {
 		if g == nil {
 			vfsShmFiles[i] = s.vfsShmFile
-			return rc
+			return _OK
 		}
 	}
 	vfsShmFiles = append(vfsShmFiles, s.vfsShmFile)
-	return rc
+	return _OK
 }
 
 func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
@@ -148,25 +141,16 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
 		if !extend {
 			return 0, _OK
 		}
-		if s.readOnly || osAllocate(s.File, n) != nil {
+		if osAllocate(s.File, n) != nil {
 			return 0, _IOERR_SHMSIZE
 		}
 	}
 
-	var prot int
-	if s.readOnly {
-		prot = unix.PROT_READ
-	} else {
-		prot = unix.PROT_READ | unix.PROT_WRITE
-	}
-	r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot)
+	r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, false)
 	if err != nil {
 		return 0, _IOERR_SHMMAP
 	}
 	s.regions = append(s.regions, r)
-	if s.readOnly {
-		return r.Ptr, _READONLY
-	}
 	return r.Ptr, _OK
 }
 

vfs/shm_copy.go → vfs/shm_dotlk.go

@@ -4,15 +4,16 @@ package vfs
 
 import (
 	"context"
+	"errors"
+	"io/fs"
+	"os"
 	"sync"
 	"unsafe"
 
 	"github.com/ncruces/go-sqlite3/internal/util"
 	"github.com/tetratelabs/wazero/api"
 )
 
-const _WALINDEX_PGSZ = 32768
-
 type vfsShmBuffer struct {
 	shared []byte // +checklocks:Mutex
 	refs   int    // +checklocks:vfsShmBuffersMtx
@@ -29,15 +30,14 @@ var (
 
 type vfsShm struct {
 	*vfsShmBuffer
-	mod      api.Module
-	alloc    api.Function
-	free     api.Function
-	path     string
-	shadow   []byte
-	ptrs     []uint32
-	stack    [1]uint64
-	lock     [_SHM_NLOCK]bool
-	readOnly bool
+	mod    api.Module
+	alloc  api.Function
+	free   api.Function
+	path   string
+	shadow []byte
+	ptrs   []uint32
+	stack  [1]uint64
+	lock   [_SHM_NLOCK]bool
 }
 
 func (s *vfsShm) Close() error {
@@ -58,13 +58,18 @@ func (s *vfsShm) Close() error {
 		return nil
 	}
 
+	err := os.Remove(s.path)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return _IOERR_UNLOCK
+	}
 	delete(vfsShmBuffers, s.path)
+	s.vfsShmBuffer = nil
 	return nil
 }
 
-func (s *vfsShm) shmOpen() {
+func (s *vfsShm) shmOpen() _ErrorCode {
 	if s.vfsShmBuffer != nil {
-		return
+		return _OK
 	}
 
 	vfsShmBuffersMtx.Lock()
@@ -74,12 +79,23 @@ func (s *vfsShm) shmOpen() {
 	if g, ok := vfsShmBuffers[s.path]; ok {
 		s.vfsShmBuffer = g
 		g.refs++
-		return
+		return _OK
+	}
+
+	// Create a directory on disk to ensure only this process
+	// uses this path to register a shared memory.
+	err := os.Mkdir(s.path, 0777)
+	if errors.Is(err, fs.ErrExist) {
+		return _BUSY
+	}
+	if err != nil {
+		return _IOERR_LOCK
 	}
 
 	// Add the new shared buffer.
 	s.vfsShmBuffer = &vfsShmBuffer{}
 	vfsShmBuffers[s.path] = s.vfsShmBuffer
+	return _OK
 }
 
 func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
@@ -91,8 +107,10 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
 		s.free = mod.ExportedFunction("sqlite3_free")
 		s.alloc = mod.ExportedFunction("sqlite3_malloc64")
 	}
+	if rc := s.shmOpen(); rc != _OK {
+		return 0, rc
+	}
 
-	s.shmOpen()
 	s.Lock()
 	defer s.Unlock()
 	defer s.shmAcquire()
@@ -131,7 +149,7 @@ func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
 
 	switch {
 	case flags&_SHM_LOCK != 0:
-		s.shmAcquire()
+		defer s.shmAcquire()
 	case flags&_SHM_EXCLUSIVE != 0:
 		s.shmRelease()
 	}
@@ -228,6 +246,8 @@ func (s *vfsShm) shmBarrier() {
 //
 // https://sqlite.org/walformat.html#the_wal_index_file_format
 
+const _WALINDEX_PGSZ = 32768
+
 // +checklocks:s.Mutex
 func (s *vfsShm) shmAcquire() {
 	// Copies modified words from shared to private memory.