Configure memory, 32-bit WAL. (#170)

.github/workflows/test.yml

@@ -160,7 +160,7 @@ jobs:
     - name: Test ppc64le (interpreter)
       run: GOARCH=ppc64le go test -v -short ./...
 
-    - name: Test s390x (big-endian, z/OS demo)
+    - name: Test s390x (big-endian, z/OS like)
       run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./...
 
   test-vm:

internal/alloc/alloc_other.go

@@ -4,6 +4,21 @@ package alloc
 
 import "github.com/tetratelabs/wazero/experimental"
 
-func Virtual(cap, max uint64) experimental.LinearMemory {
-	return Slice(cap, max)
+func NewMemory(cap, max uint64) experimental.LinearMemory {
+	return &sliceMemory{make([]byte, 0, cap)}
+}
+
+type sliceMemory struct {
+	buf []byte
+}
+
+func (b *sliceMemory) Free() {}
+
+func (b *sliceMemory) Reallocate(size uint64) []byte {
+	if cap := uint64(cap(b.buf)); size > cap {
+		b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
+	} else {
+		b.buf = b.buf[:size]
+	}
+	return b.buf
 }

internal/alloc/alloc_test.go

@@ -9,6 +9,6 @@ import (
 
 func TestVirtual(t *testing.T) {
 	defer func() { _ = recover() }()
-	alloc.Virtual(math.MaxInt+2, math.MaxInt+2)
+	alloc.NewMemory(math.MaxInt+2, math.MaxInt+2)
 	t.Error("want panic")
 }

internal/alloc/alloc_unix.go

@@ -9,7 +9,7 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func Virtual(_, max uint64) experimental.LinearMemory {
+func NewMemory(_, max uint64) experimental.LinearMemory {
 	// Round up to the page size.
 	rnd := uint64(unix.Getpagesize() - 1)
 	max = (max + rnd) &^ rnd

internal/alloc/alloc_windows.go

@@ -11,7 +11,7 @@ import (
 	"golang.org/x/sys/windows"
 )
 
-func Virtual(_, max uint64) experimental.LinearMemory {
+func NewMemory(_, max uint64) experimental.LinearMemory {
 	// Round up to the page size.
 	rnd := uint64(windows.Getpagesize() - 1)
 	max = (max + rnd) &^ rnd

internal/testcfg/testcfg.go

@@ -1,7 +1,6 @@
 package testcfg
 
 import (
-	"math/bits"
 	"os"
 	"path/filepath"
 
@@ -12,13 +11,8 @@ import (
 // notest
 
 func init() {
-	if bits.UintSize < 64 {
-		return
-	}
-
 	sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().
-		WithMemoryCapacityFromMax(true).
-		WithMemoryLimitPages(1024)
+		WithMemoryLimitPages(512)
 
 	if os.Getenv("CI") != "" {
 		path := filepath.Join(os.TempDir(), "wazero")

internal/util/mmap.go

@@ -1,4 +1,4 @@
-//go:build unix && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
+//go:build unix && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
 
 package util
 
@@ -7,17 +7,10 @@ import (
 	"os"
 	"unsafe"
 
-	"github.com/ncruces/go-sqlite3/internal/alloc"
 	"github.com/tetratelabs/wazero/api"
-	"github.com/tetratelabs/wazero/experimental"
 	"golang.org/x/sys/unix"
 )
 
-func withAllocator(ctx context.Context) context.Context {
-	return experimental.WithMemoryAllocator(ctx,
-		experimental.MemoryAllocatorFunc(alloc.Virtual))
-}
-
 type mmapState struct {
 	regions []*MappedRegion
 }

internal/util/mmap_other.go

@@ -1,22 +1,5 @@
-//go:build !unix || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
+//go:build !unix || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
 
 package util
 
-import (
-	"context"
-
-	"github.com/ncruces/go-sqlite3/internal/alloc"
-	"github.com/tetratelabs/wazero/experimental"
-)
-
 type mmapState struct{}
-
-func withAllocator(ctx context.Context) context.Context {
-	return experimental.WithMemoryAllocator(ctx,
-		experimental.MemoryAllocatorFunc(func(cap, max uint64) experimental.LinearMemory {
-			if cap == max {
-				return alloc.Virtual(cap, max)
-			}
-			return alloc.Slice(cap, max)
-		}))
-}

internal/util/module.go

@@ -3,6 +3,7 @@ package util
 import (
 	"context"
 
+	"github.com/ncruces/go-sqlite3/internal/alloc"
 	"github.com/tetratelabs/wazero/experimental"
 )
 
@@ -14,7 +15,7 @@ type moduleState struct {
 
 func NewContext(ctx context.Context) context.Context {
 	state := new(moduleState)
-	ctx = withAllocator(ctx)
+	ctx = experimental.WithMemoryAllocator(ctx, experimental.MemoryAllocatorFunc(alloc.NewMemory))
 	ctx = experimental.WithCloseNotifier(ctx, state)
 	ctx = context.WithValue(ctx, moduleKey{}, state)
 	return ctx

sqlite.go

@@ -49,10 +49,15 @@ func compileSQLite() {
 	cfg := RuntimeConfig
 	if cfg == nil {
 		cfg = wazero.NewRuntimeConfig()
+		if bits.UintSize >= 64 {
+			cfg = cfg.WithMemoryLimitPages(4096) // 256MB
+		} else {
+			cfg = cfg.WithMemoryLimitPages(512) // 32MB
+		}
 	}
+	cfg = cfg.WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads)
 
-	instance.runtime = wazero.NewRuntimeWithConfig(ctx,
-		cfg.WithCoreFeatures(api.CoreFeaturesV2|experimental.CoreFeaturesThreads))
+	instance.runtime = wazero.NewRuntimeWithConfig(ctx, cfg)
 
 	env := instance.runtime.NewHostModuleBuilder("env")
 	env = vfs.ExportHostFunctions(env)

sqlite_test.go

@@ -6,12 +6,10 @@ import (
 	"testing"
 
 	"github.com/ncruces/go-sqlite3/internal/util"
-	"github.com/tetratelabs/wazero"
 )
 
 func init() {
 	Path = "./embed/sqlite3.wasm"
-	RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
 }
 
 func Test_sqlite_error_OOM(t *testing.T) {

tests/time_test.go

@@ -223,6 +223,9 @@ func TestDB_timeCollation(t *testing.T) {
 }
 
 func TestDB_isoWeek(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
 	t.Parallel()
 
 	db, err := sqlite3.Open(":memory:")

vfs/README.md

@@ -46,18 +46,14 @@ to check if your build supports file locking.
 
 ### Write-Ahead Logging
 
-On 64-bit little-endian Unix, this module uses `mmap` to implement
+On little-endian Unix, this module uses `mmap` to implement
 [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
 like SQLite.
 
-To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.
-To limit the address space each connection reserves,
-use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go).
-
 With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2)
 a WAL database can only be accessed by a single proccess.
 Other processes that attempt to access a database locked with BSD locks,
-will fail with the `SQLITE_PROTOCOL` error code.
+will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code.
 
 Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm),
 and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases.

vfs/os_f2fs_linux.go

@@ -1,4 +1,4 @@
-//go:build (amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys
+//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys
 
 package vfs
 
@@ -13,7 +13,7 @@ const (
 	_F2FS_IOC_START_ATOMIC_WRITE  = 62721
 	_F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722
 	_F2FS_IOC_ABORT_ATOMIC_WRITE  = 62725
-	_F2FS_IOC_GET_FEATURES        = 2147808524
+	_F2FS_IOC_GET_FEATURES        = 2147808524 // -2147158772
 	_F2FS_FEATURE_ATOMIC_WRITE    = 4
 )
 

vfs/os_std_atomic.go

@@ -1,4 +1,4 @@
-//go:build !linux || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_nosys
+//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys
 
 package vfs
 

vfs/shm.go

@@ -1,4 +1,4 @@
-//go:build (darwin || linux) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
+//go:build (darwin || linux) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
 
 package vfs
 

vfs/shm_bsd.go

@@ -1,4 +1,4 @@
-//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
+//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
 
 package vfs
 

vfs/shm_other.go

@@ -1,4 +1,4 @@
-//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
+//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
 
 package vfs
 

vfs/tests/mptest/mptest_test.go

@@ -43,10 +43,9 @@ func TestMain(m *testing.M) {
 	ctx := context.Background()
 	cfg := wazero.NewRuntimeConfig().
 		WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads).
-		WithMemoryLimitPages(1024)
+		WithMemoryLimitPages(512)
 	rt = wazero.NewRuntimeWithConfig(ctx, cfg)
 	wasi_snapshot_preview1.MustInstantiate(ctx, rt)
-
 	env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))
 	env.NewFunctionBuilder().WithFunc(system).Export("system")
 	_, err := env.Instantiate(ctx)

vfs/tests/speedtest1/speedtest1_test.go

@@ -42,7 +42,8 @@ func TestMain(m *testing.M) {
 
 	ctx := context.Background()
 	cfg := wazero.NewRuntimeConfig().
-		WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads)
+		WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads).
+		WithMemoryLimitPages(512)
 	rt = wazero.NewRuntimeWithConfig(ctx, cfg)
 	wasi_snapshot_preview1.MustInstantiate(ctx, rt)
 	env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))