Skip to content

Commit

Permalink
Refactor, speed.
Browse files Browse the repository at this point in the history
  • Loading branch information
ncruces committed Nov 6, 2024
1 parent 3215376 commit a946c00
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 149 deletions.
84 changes: 84 additions & 0 deletions vfs/shm_copy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
//go:build (windows && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys) || sqlite3_dotlk

package vfs

import (
"unsafe"

"github.com/ncruces/go-sqlite3/internal/util"
)

const (
_WALINDEX_HDR_SIZE = 136
_WALINDEX_PGSZ = 32768
)

// This looks like a safe way of keeping the WAL-index in sync.
//
// The WAL-index file starts with a header,
// and the index doesn't meaningfully change if the header doesn't change.
//
// The header starts with two 48 byte, checksummed, copies of the same information,
// which are accessed independently between memory barriers.
// The checkpoint information that follows uses 4 byte aligned words.
//
// Finally, we have the WAL-index hash tables,
// which are only modified holding the exclusive WAL_WRITE_LOCK.
//
// Since all the data is either redundant+checksummed,
// 4 byte aligned, or modified under an exclusive lock,
// the copies below should correctly keep copies in sync.
//
// https://sqlite.org/walformat.html#the_wal_index_file_format

func (s *vfsShm) shmAcquire() {
if len(s.ptrs) == 0 || shmUnmodified(s.shadow[0][:], s.shared[0][:]) {
return
}
// Copies modified words from shared to private memory.
for id, p := range s.ptrs {
shared := shmPage(s.shared[id][:])
shadow := shmPage(s.shadow[id][:])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
for i, shared := range shared {
if shadow[i] != shared {
shadow[i] = shared
privat[i] = shared
}
}
}
}

func (s *vfsShm) shmRelease() {
if len(s.ptrs) == 0 || shmUnmodified(s.shadow[0][:], util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE)) {
return
}
// Copies modified words from private to shared memory.
for id, p := range s.ptrs {
shared := shmPage(s.shared[id][:])
shadow := shmPage(s.shadow[id][:])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
for i, privat := range privat {
if shadow[i] != privat {
shadow[i] = privat
shared[i] = privat
}
}
}
}

func (s *vfsShm) shmBarrier() {
s.Lock()
s.shmAcquire()
s.shmRelease()
s.Unlock()
}

func shmPage(s []byte) *[_WALINDEX_PGSZ / 4]uint32 {
p := (*uint32)(unsafe.Pointer(unsafe.SliceData(s)))
return (*[_WALINDEX_PGSZ / 4]uint32)(unsafe.Slice(p, _WALINDEX_PGSZ/4))
}

func shmUnmodified(v1, v2 []byte) bool {
return *(*[_WALINDEX_HDR_SIZE]byte)(v1[:]) == *(*[_WALINDEX_HDR_SIZE]byte)(v2[:])
}
94 changes: 7 additions & 87 deletions vfs/shm_dotlk.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,14 @@ import (
"io/fs"
"os"
"sync"
"unsafe"

"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
)

type vfsShmBuffer struct {
shared []byte // +checklocks:Mutex
refs int // +checklocks:vfsShmBuffersMtx
shared [][_WALINDEX_PGSZ]byte
refs int // +checklocks:vfsShmBuffersMtx

lock [_SHM_NLOCK]int16 // +checklocks:Mutex
sync.Mutex
Expand All @@ -34,7 +33,7 @@ type vfsShm struct {
alloc api.Function
free api.Function
path string
shadow []byte
shadow [][_WALINDEX_PGSZ]byte
ptrs []uint32
stack [1]uint64
lock [_SHM_NLOCK]bool
Expand Down Expand Up @@ -115,17 +114,15 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
defer s.Unlock()
defer s.shmAcquire()

n := (int(id) + 1) * int(size)

if n > len(s.shared) {
if int(id) >= len(s.shared) {
if !extend {
return 0, _OK
}
s.shared = append(s.shared, make([]byte, n-len(s.shared))...)
s.shared = append(s.shared, make([][_WALINDEX_PGSZ]byte, int(id)-len(s.shared)+1)...)
}

if n > len(s.shadow) {
s.shadow = append(s.shadow, make([]byte, n-len(s.shadow))...)
if int(id) >= len(s.shadow) {
s.shadow = append(s.shadow, make([][_WALINDEX_PGSZ]byte, int(id)-len(s.shadow)+1)...)
}

for int(id) >= len(s.ptrs) {
Expand Down Expand Up @@ -221,80 +218,3 @@ func (s *vfsShm) shmUnmap(delete bool) {
s.ptrs = nil
s.shadow = nil
}

func (s *vfsShm) shmBarrier() {
s.Lock()
s.shmAcquire()
s.shmRelease()
s.Unlock()
}

// This looks like a safe, if inefficient, way of keeping memory in sync.
//
// The WAL-index file starts with a header.
// This header starts with two 48 byte, checksummed, copies of the same information,
// which are accessed independently between memory barriers.
// The checkpoint information that follows uses 4 byte aligned words.
//
// Finally, we have the WAL-index hash tables,
// which are only modified holding the exclusive WAL_WRITE_LOCK.
// Also, aHash isn't modified unless aPgno changes.
//
// Since all the data is either redundant+checksummed,
// 4 byte aligned, or modified under an exclusive lock,
// the copies below should correctly keep memory in sync.
//
// https://sqlite.org/walformat.html#the_wal_index_file_format

const _WALINDEX_PGSZ = 32768

// +checklocks:s.Mutex
func (s *vfsShm) shmAcquire() {
// Copies modified words from shared to private memory.
for id, p := range s.ptrs {
i0 := id * _WALINDEX_PGSZ
i1 := i0 + _WALINDEX_PGSZ
shared := shmPage(s.shared[i0:i1])
shadow := shmPage(s.shadow[i0:i1])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
if shmPageEq(shadow, shared) {
continue
}
for i, shared := range shared {
if shadow[i] != shared {
shadow[i] = shared
privat[i] = shared
}
}
}
}

// +checklocks:s.Mutex
func (s *vfsShm) shmRelease() {
// Copies modified words from private to shared memory.
for id, p := range s.ptrs {
i0 := id * _WALINDEX_PGSZ
i1 := i0 + _WALINDEX_PGSZ
shared := shmPage(s.shared[i0:i1])
shadow := shmPage(s.shadow[i0:i1])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
if shmPageEq(shadow, privat) {
continue
}
for i, privat := range privat {
if shadow[i] != privat {
shadow[i] = privat
shared[i] = privat
}
}
}
}

func shmPage(s []byte) *[_WALINDEX_PGSZ / 4]uint32 {
p := (*uint32)(unsafe.Pointer(unsafe.SliceData(s)))
return (*[_WALINDEX_PGSZ / 4]uint32)(unsafe.Slice(p, _WALINDEX_PGSZ/4))
}

func shmPageEq(p1, p2 *[_WALINDEX_PGSZ / 4]uint32) bool {
return *(*[_WALINDEX_PGSZ / 8]uint32)(p1[:]) == *(*[_WALINDEX_PGSZ / 8]uint32)(p2[:])
}
65 changes: 3 additions & 62 deletions vfs/shm_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"os"
"sync"
"time"
"unsafe"

"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/windows"
Expand All @@ -25,7 +24,7 @@ type vfsShm struct {
path string
regions []*util.MappedRegion
shared [][]byte
shadow []byte
shadow [][_WALINDEX_PGSZ]byte
ptrs []uint32
stack [1]uint64
blocking bool
Expand Down Expand Up @@ -108,8 +107,8 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
s.shared[id] = r.Data

// Allocate shadow memory.
if n := (int(id) + 1) * int(size); n > len(s.shadow) {
s.shadow = append(s.shadow, make([]byte, n-len(s.shadow))...)
if int(id) >= len(s.shadow) {
s.shadow = append(s.shadow, make([][_WALINDEX_PGSZ]byte, int(id)-len(s.shadow)+1)...)
}

// Allocate local memory.
Expand Down Expand Up @@ -179,64 +178,6 @@ func (s *vfsShm) shmUnmap(delete bool) {
}
}

func (s *vfsShm) shmBarrier() {
s.Lock()
s.shmAcquire()
s.shmRelease()
s.Unlock()
}

const _WALINDEX_PGSZ = 32768

func (s *vfsShm) shmAcquire() {
// Copies modified words from shared to private memory.
for id, p := range s.ptrs {
i0 := id * _WALINDEX_PGSZ
i1 := i0 + _WALINDEX_PGSZ
shared := shmPage(s.shared[id])
shadow := shmPage(s.shadow[i0:i1])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
if shmPageEq(shadow, shared) {
continue
}
for i, shared := range shared {
if shadow[i] != shared {
shadow[i] = shared
privat[i] = shared
}
}
}
}

func (s *vfsShm) shmRelease() {
// Copies modified words from private to shared memory.
for id, p := range s.ptrs {
i0 := id * _WALINDEX_PGSZ
i1 := i0 + _WALINDEX_PGSZ
shared := shmPage(s.shared[id])
shadow := shmPage(s.shadow[i0:i1])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
if shmPageEq(shadow, privat) {
continue
}
for i, privat := range privat {
if shadow[i] != privat {
shadow[i] = privat
shared[i] = privat
}
}
}
}

func shmPage(s []byte) *[_WALINDEX_PGSZ / 4]uint32 {
p := (*uint32)(unsafe.Pointer(unsafe.SliceData(s)))
return (*[_WALINDEX_PGSZ / 4]uint32)(unsafe.Slice(p, _WALINDEX_PGSZ/4))
}

func shmPageEq(p1, p2 *[_WALINDEX_PGSZ / 4]uint32) bool {
return *(*[_WALINDEX_PGSZ / 8]uint32)(p1[:]) == *(*[_WALINDEX_PGSZ / 8]uint32)(p2[:])
}

func (s *vfsShm) shmEnableBlocking(block bool) {
s.blocking = block
}

0 comments on commit a946c00

Please sign in to comment.