diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5cc4c2e..9b71d2a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -160,7 +160,7 @@ jobs: - name: Test ppc64le (interpreter) run: GOARCH=ppc64le go test -v -short ./... - - name: Test s390x (big-endian, z/OS demo) + - name: Test s390x (big-endian, z/OS like) run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./... test-vm: diff --git a/internal/alloc/alloc_other.go b/internal/alloc/alloc_other.go index ded8da10..d9a3de22 100644 --- a/internal/alloc/alloc_other.go +++ b/internal/alloc/alloc_other.go @@ -4,6 +4,21 @@ package alloc import "github.com/tetratelabs/wazero/experimental" -func Virtual(cap, max uint64) experimental.LinearMemory { - return Slice(cap, max) +func NewMemory(cap, max uint64) experimental.LinearMemory { + return &sliceMemory{make([]byte, 0, cap)} +} + +type sliceMemory struct { + buf []byte +} + +func (b *sliceMemory) Free() {} + +func (b *sliceMemory) Reallocate(size uint64) []byte { + if cap := uint64(cap(b.buf)); size > cap { + b.buf = append(b.buf[:cap], make([]byte, size-cap)...) + } else { + b.buf = b.buf[:size] + } + return b.buf } diff --git a/internal/alloc/alloc_slice.go b/internal/alloc/alloc_slice.go deleted file mode 100644 index 5fc725c6..00000000 --- a/internal/alloc/alloc_slice.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys - -package alloc - -import "github.com/tetratelabs/wazero/experimental" - -func Slice(cap, _ uint64) experimental.LinearMemory { - return &sliceMemory{make([]byte, 0, cap)} -} - -type sliceMemory struct { - buf []byte -} - -func (b *sliceMemory) Free() {} - -func (b *sliceMemory) Reallocate(size uint64) []byte { - if cap := uint64(cap(b.buf)); size > cap { - b.buf = append(b.buf[:cap], make([]byte, size-cap)...) - } else { - b.buf = b.buf[:size] - } - return b.buf -} diff --git a/internal/alloc/alloc_test.go b/internal/alloc/alloc_test.go index e41b8fb9..b3a19aaa 100644 --- a/internal/alloc/alloc_test.go +++ b/internal/alloc/alloc_test.go @@ -9,6 +9,6 @@ import ( func TestVirtual(t *testing.T) { defer func() { _ = recover() }() - alloc.Virtual(math.MaxInt+2, math.MaxInt+2) + alloc.NewMemory(math.MaxInt+2, math.MaxInt+2) t.Error("want panic") } diff --git a/internal/alloc/alloc_unix.go b/internal/alloc/alloc_unix.go index c05cfa73..2948487f 100644 --- a/internal/alloc/alloc_unix.go +++ b/internal/alloc/alloc_unix.go @@ -9,7 +9,7 @@ import ( "golang.org/x/sys/unix" ) -func Virtual(_, max uint64) experimental.LinearMemory { +func NewMemory(_, max uint64) experimental.LinearMemory { // Round up to the page size. rnd := uint64(unix.Getpagesize() - 1) max = (max + rnd) &^ rnd diff --git a/internal/alloc/alloc_windows.go b/internal/alloc/alloc_windows.go index 46181b11..8e67e031 100644 --- a/internal/alloc/alloc_windows.go +++ b/internal/alloc/alloc_windows.go @@ -11,7 +11,7 @@ import ( "golang.org/x/sys/windows" ) -func Virtual(_, max uint64) experimental.LinearMemory { +func NewMemory(_, max uint64) experimental.LinearMemory { // Round up to the page size. rnd := uint64(windows.Getpagesize() - 1) max = (max + rnd) &^ rnd diff --git a/internal/testcfg/testcfg.go b/internal/testcfg/testcfg.go index f1b07f86..0e86a2bb 100644 --- a/internal/testcfg/testcfg.go +++ b/internal/testcfg/testcfg.go @@ -1,7 +1,6 @@ package testcfg import ( - "math/bits" "os" "path/filepath" @@ -12,13 +11,8 @@ import ( // notest func init() { - if bits.UintSize < 64 { - return - } - sqlite3.RuntimeConfig = wazero.NewRuntimeConfig(). - WithMemoryCapacityFromMax(true). - WithMemoryLimitPages(1024) + WithMemoryLimitPages(512) if os.Getenv("CI") != "" { path := filepath.Join(os.TempDir(), "wazero") diff --git a/internal/util/mmap.go b/internal/util/mmap.go index 81377267..5788eeb2 100644 --- a/internal/util/mmap.go +++ b/internal/util/mmap.go @@ -1,4 +1,4 @@ -//go:build unix && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys) +//go:build unix && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys) package util @@ -7,17 +7,10 @@ import ( "os" "unsafe" - "github.com/ncruces/go-sqlite3/internal/alloc" "github.com/tetratelabs/wazero/api" - "github.com/tetratelabs/wazero/experimental" "golang.org/x/sys/unix" ) -func withAllocator(ctx context.Context) context.Context { - return experimental.WithMemoryAllocator(ctx, - experimental.MemoryAllocatorFunc(alloc.Virtual)) -} - type mmapState struct { regions []*MappedRegion } diff --git a/internal/util/mmap_other.go b/internal/util/mmap_other.go index b6cd4c55..a2fbf24d 100644 --- a/internal/util/mmap_other.go +++ b/internal/util/mmap_other.go @@ -1,22 +1,5 @@ -//go:build !unix || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys +//go:build !unix || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys package util -import ( - "context" - - "github.com/ncruces/go-sqlite3/internal/alloc" - "github.com/tetratelabs/wazero/experimental" -) - type mmapState struct{} - -func withAllocator(ctx context.Context) context.Context { - return experimental.WithMemoryAllocator(ctx, - experimental.MemoryAllocatorFunc(func(cap, max uint64) experimental.LinearMemory { - if cap == max { - return alloc.Virtual(cap, max) - } - return alloc.Slice(cap, max) - })) -} diff --git a/internal/util/module.go b/internal/util/module.go index 22793e97..4aab0f94 100644 --- a/internal/util/module.go +++ b/internal/util/module.go @@ -3,6 +3,7 @@ package util import ( "context" + "github.com/ncruces/go-sqlite3/internal/alloc" "github.com/tetratelabs/wazero/experimental" ) @@ -14,7 +15,7 @@ type moduleState struct { func NewContext(ctx context.Context) context.Context { state := new(moduleState) - ctx = withAllocator(ctx) + ctx = experimental.WithMemoryAllocator(ctx, experimental.MemoryAllocatorFunc(alloc.NewMemory)) ctx = experimental.WithCloseNotifier(ctx, state) ctx = context.WithValue(ctx, moduleKey{}, state) return ctx diff --git a/sqlite.go b/sqlite.go index a5ff1363..7d7b66ce 100644 --- a/sqlite.go +++ b/sqlite.go @@ -49,10 +49,15 @@ func compileSQLite() { cfg := RuntimeConfig if cfg == nil { cfg = wazero.NewRuntimeConfig() + if bits.UintSize >= 64 { + cfg = cfg.WithMemoryLimitPages(4096) // 256MB + } else { + cfg = cfg.WithMemoryLimitPages(512) // 32MB + } } + cfg = cfg.WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads) - instance.runtime = wazero.NewRuntimeWithConfig(ctx, - cfg.WithCoreFeatures(api.CoreFeaturesV2|experimental.CoreFeaturesThreads)) + instance.runtime = wazero.NewRuntimeWithConfig(ctx, cfg) env := instance.runtime.NewHostModuleBuilder("env") env = vfs.ExportHostFunctions(env) diff --git a/sqlite_test.go b/sqlite_test.go index b5ce4647..fbcd069b 100644 --- a/sqlite_test.go +++ b/sqlite_test.go @@ -6,12 +6,10 @@ import ( "testing" "github.com/ncruces/go-sqlite3/internal/util" - "github.com/tetratelabs/wazero" ) func init() { Path = "./embed/sqlite3.wasm" - RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024) } func Test_sqlite_error_OOM(t *testing.T) { diff --git a/tests/time_test.go b/tests/time_test.go index 611a6f5c..d2683158 100644 --- a/tests/time_test.go +++ b/tests/time_test.go @@ -223,6 +223,9 @@ func TestDB_timeCollation(t *testing.T) { } func TestDB_isoWeek(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } t.Parallel() db, err := sqlite3.Open(":memory:") diff --git a/vfs/README.md b/vfs/README.md index 21e90222..1cbf9409 100644 --- a/vfs/README.md +++ b/vfs/README.md @@ -46,18 +46,14 @@ to check if your build supports file locking. ### Write-Ahead Logging -On 64-bit little-endian Unix, this module uses `mmap` to implement +On little-endian Unix, this module uses `mmap` to implement [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index), like SQLite. -To allow `mmap` to work, each connection needs to reserve up to 4GB of address space. -To limit the address space each connection reserves, -use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go). - With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2) a WAL database can only be accessed by a single proccess. Other processes that attempt to access a database locked with BSD locks, -will fail with the `SQLITE_PROTOCOL` error code. +will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code. Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm), and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases. diff --git a/vfs/os_f2fs_linux.go b/vfs/os_f2fs_linux.go index 6ecb60ed..07bf0a04 100644 --- a/vfs/os_f2fs_linux.go +++ b/vfs/os_f2fs_linux.go @@ -1,4 +1,4 @@ -//go:build (amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys +//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys package vfs @@ -13,7 +13,7 @@ const ( _F2FS_IOC_START_ATOMIC_WRITE = 62721 _F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722 _F2FS_IOC_ABORT_ATOMIC_WRITE = 62725 - _F2FS_IOC_GET_FEATURES = 2147808524 + _F2FS_IOC_GET_FEATURES = 2147808524 // -2147158772 _F2FS_FEATURE_ATOMIC_WRITE = 4 ) diff --git a/vfs/os_std_atomic.go b/vfs/os_std_atomic.go index c3590a7d..ecaff024 100644 --- a/vfs/os_std_atomic.go +++ b/vfs/os_std_atomic.go @@ -1,4 +1,4 @@ -//go:build !linux || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_nosys +//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys package vfs diff --git a/vfs/shm.go b/vfs/shm.go index 0fbd09d0..88306ef8 100644 --- a/vfs/shm.go +++ b/vfs/shm.go @@ -1,4 +1,4 @@ -//go:build (darwin || linux) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys) +//go:build (darwin || linux) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys) package vfs diff --git a/vfs/shm_bsd.go b/vfs/shm_bsd.go index 52ffeacb..8849cc74 100644 --- a/vfs/shm_bsd.go +++ b/vfs/shm_bsd.go @@ -1,4 +1,4 @@ -//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys) +//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys) package vfs diff --git a/vfs/shm_other.go b/vfs/shm_other.go index 4d0f6a2c..12012033 100644 --- a/vfs/shm_other.go +++ b/vfs/shm_other.go @@ -1,4 +1,4 @@ -//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys +//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys package vfs diff --git a/vfs/tests/mptest/mptest_test.go b/vfs/tests/mptest/mptest_test.go index a47133e5..9e16509f 100644 --- a/vfs/tests/mptest/mptest_test.go +++ b/vfs/tests/mptest/mptest_test.go @@ -43,10 +43,9 @@ func TestMain(m *testing.M) { ctx := context.Background() cfg := wazero.NewRuntimeConfig(). WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads). - WithMemoryLimitPages(1024) + WithMemoryLimitPages(512) rt = wazero.NewRuntimeWithConfig(ctx, cfg) wasi_snapshot_preview1.MustInstantiate(ctx, rt) - env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env")) env.NewFunctionBuilder().WithFunc(system).Export("system") _, err := env.Instantiate(ctx) diff --git a/vfs/tests/speedtest1/speedtest1_test.go b/vfs/tests/speedtest1/speedtest1_test.go index 4aa0fd31..de953d1c 100644 --- a/vfs/tests/speedtest1/speedtest1_test.go +++ b/vfs/tests/speedtest1/speedtest1_test.go @@ -42,7 +42,8 @@ func TestMain(m *testing.M) { ctx := context.Background() cfg := wazero.NewRuntimeConfig(). - WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads) + WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads). + WithMemoryLimitPages(512) rt = wazero.NewRuntimeWithConfig(ctx, cfg) wasi_snapshot_preview1.MustInstantiate(ctx, rt) env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))