From 9f8c19210b73704f67b64439c76837ce698cd1af Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Nov 2022 15:31:41 -0700 Subject: [PATCH 1/8] std.heap: extract PageAllocator, WasmPageAllocator --- lib/std/heap.zig | 315 ++--------------------------- lib/std/heap/PageAllocator.zig | 110 ++++++++++ lib/std/heap/WasmPageAllocator.zig | 193 ++++++++++++++++++ 3 files changed, 315 insertions(+), 303 deletions(-) create mode 100644 lib/std/heap/PageAllocator.zig create mode 100644 lib/std/heap/WasmPageAllocator.zig diff --git a/lib/std/heap.zig b/lib/std/heap.zig index d46a9ff104df..71933bc18657 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -1,13 +1,13 @@ const std = @import("std.zig"); const builtin = @import("builtin"); const root = @import("root"); -const debug = std.debug; -const assert = debug.assert; +const assert = std.debug.assert; const testing = std.testing; const mem = std.mem; const os = std.os; const c = std.c; const maxInt = std.math.maxInt; +const Allocator = std.mem.Allocator; pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator; pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator; @@ -16,8 +16,11 @@ pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").Log pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator; pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator; +pub const WasmPageAllocator = @import("heap/WasmPageAllocator.zig"); +pub const PageAllocator = @import("heap/PageAllocator.zig"); -const Allocator = mem.Allocator; +/// TODO Utilize this on Windows. +pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null; const CAllocator = struct { comptime { @@ -227,303 +230,6 @@ pub fn alignPageAllocLen(full_len: usize, len: usize) usize { return aligned_len; } -/// TODO Utilize this on Windows. -pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null; - -const PageAllocator = struct { - const vtable = Allocator.VTable{ - .alloc = alloc, - .resize = resize, - .free = free, - }; - - fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { - _ = ra; - _ = log2_align; - assert(n > 0); - if (n > maxInt(usize) - (mem.page_size - 1)) return null; - const aligned_len = mem.alignForward(n, mem.page_size); - - if (builtin.os.tag == .windows) { - const w = os.windows; - const addr = w.VirtualAlloc( - null, - aligned_len, - w.MEM_COMMIT | w.MEM_RESERVE, - w.PAGE_READWRITE, - ) catch return null; - return @ptrCast([*]align(mem.page_size) u8, @alignCast(mem.page_size, addr)); - } - - const hint = @atomicLoad(@TypeOf(next_mmap_addr_hint), &next_mmap_addr_hint, .Unordered); - const slice = os.mmap( - hint, - aligned_len, - os.PROT.READ | os.PROT.WRITE, - os.MAP.PRIVATE | os.MAP.ANONYMOUS, - -1, - 0, - ) catch return null; - assert(mem.isAligned(@ptrToInt(slice.ptr), mem.page_size)); - const new_hint = @alignCast(mem.page_size, slice.ptr + aligned_len); - _ = @cmpxchgStrong(@TypeOf(next_mmap_addr_hint), &next_mmap_addr_hint, hint, new_hint, .Monotonic, .Monotonic); - return slice.ptr; - } - - fn resize( - _: *anyopaque, - buf_unaligned: []u8, - log2_buf_align: u8, - new_size: usize, - return_address: usize, - ) bool { - _ = log2_buf_align; - _ = return_address; - const new_size_aligned = mem.alignForward(new_size, mem.page_size); - - if (builtin.os.tag == .windows) { - const w = os.windows; - if (new_size <= buf_unaligned.len) { - const base_addr = @ptrToInt(buf_unaligned.ptr); - const old_addr_end = base_addr + buf_unaligned.len; - const new_addr_end = mem.alignForward(base_addr + new_size, mem.page_size); - if (old_addr_end > new_addr_end) { - // For shrinking that is not releasing, we will only - // decommit the pages not needed anymore. - w.VirtualFree( - @intToPtr(*anyopaque, new_addr_end), - old_addr_end - new_addr_end, - w.MEM_DECOMMIT, - ); - } - return true; - } - const old_size_aligned = mem.alignForward(buf_unaligned.len, mem.page_size); - if (new_size_aligned <= old_size_aligned) { - return true; - } - return false; - } - - const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size); - if (new_size_aligned == buf_aligned_len) - return true; - - if (new_size_aligned < buf_aligned_len) { - const ptr = @alignCast(mem.page_size, buf_unaligned.ptr + new_size_aligned); - // TODO: if the next_mmap_addr_hint is within the unmapped range, update it - os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]); - return true; - } - - // TODO: call mremap - // TODO: if the next_mmap_addr_hint is within the remapped range, update it - return false; - } - - fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void { - _ = log2_buf_align; - _ = return_address; - - if (builtin.os.tag == .windows) { - os.windows.VirtualFree(slice.ptr, 0, os.windows.MEM_RELEASE); - } else { - const buf_aligned_len = mem.alignForward(slice.len, mem.page_size); - const ptr = @alignCast(mem.page_size, slice.ptr); - os.munmap(ptr[0..buf_aligned_len]); - } - } -}; - -const WasmPageAllocator = struct { - comptime { - if (!builtin.target.isWasm()) { - @compileError("WasmPageAllocator is only available for wasm32 arch"); - } - } - - const vtable = Allocator.VTable{ - .alloc = alloc, - .resize = resize, - .free = free, - }; - - const PageStatus = enum(u1) { - used = 0, - free = 1, - - pub const none_free: u8 = 0; - }; - - const FreeBlock = struct { - data: []u128, - - const Io = std.packed_int_array.PackedIntIo(u1, .Little); - - fn totalPages(self: FreeBlock) usize { - return self.data.len * 128; - } - - fn isInitialized(self: FreeBlock) bool { - return self.data.len > 0; - } - - fn getBit(self: FreeBlock, idx: usize) PageStatus { - const bit_offset = 0; - return @intToEnum(PageStatus, Io.get(mem.sliceAsBytes(self.data), idx, bit_offset)); - } - - fn setBits(self: FreeBlock, start_idx: usize, len: usize, val: PageStatus) void { - const bit_offset = 0; - var i: usize = 0; - while (i < len) : (i += 1) { - Io.set(mem.sliceAsBytes(self.data), start_idx + i, bit_offset, @enumToInt(val)); - } - } - - // Use '0xFFFFFFFF' as a _missing_ sentinel - // This saves ~50 bytes compared to returning a nullable - - // We can guarantee that conventional memory never gets this big, - // and wasm32 would not be able to address this memory (32 GB > usize). - - // Revisit if this is settled: https://github.com/ziglang/zig/issues/3806 - const not_found = std.math.maxInt(usize); - - fn useRecycled(self: FreeBlock, num_pages: usize, log2_align: u8) usize { - @setCold(true); - for (self.data) |segment, i| { - const spills_into_next = @bitCast(i128, segment) < 0; - const has_enough_bits = @popCount(segment) >= num_pages; - - if (!spills_into_next and !has_enough_bits) continue; - - var j: usize = i * 128; - while (j < (i + 1) * 128) : (j += 1) { - var count: usize = 0; - while (j + count < self.totalPages() and self.getBit(j + count) == .free) { - count += 1; - const addr = j * mem.page_size; - if (count >= num_pages and mem.isAlignedLog2(addr, log2_align)) { - self.setBits(j, num_pages, .used); - return j; - } - } - j += count; - } - } - return not_found; - } - - fn recycle(self: FreeBlock, start_idx: usize, len: usize) void { - self.setBits(start_idx, len, .free); - } - }; - - var _conventional_data = [_]u128{0} ** 16; - // Marking `conventional` as const saves ~40 bytes - const conventional = FreeBlock{ .data = &_conventional_data }; - var extended = FreeBlock{ .data = &[_]u128{} }; - - fn extendedOffset() usize { - return conventional.totalPages(); - } - - fn nPages(memsize: usize) usize { - return mem.alignForward(memsize, mem.page_size) / mem.page_size; - } - - fn alloc(_: *anyopaque, len: usize, log2_align: u8, ra: usize) ?[*]u8 { - _ = ra; - if (len > maxInt(usize) - (mem.page_size - 1)) return null; - const page_count = nPages(len); - const page_idx = allocPages(page_count, log2_align) catch return null; - return @intToPtr([*]u8, page_idx * mem.page_size); - } - - fn allocPages(page_count: usize, log2_align: u8) !usize { - { - const idx = conventional.useRecycled(page_count, log2_align); - if (idx != FreeBlock.not_found) { - return idx; - } - } - - const idx = extended.useRecycled(page_count, log2_align); - if (idx != FreeBlock.not_found) { - return idx + extendedOffset(); - } - - const next_page_idx = @wasmMemorySize(0); - const next_page_addr = next_page_idx * mem.page_size; - const aligned_addr = mem.alignForwardLog2(next_page_addr, log2_align); - const drop_page_count = @divExact(aligned_addr - next_page_addr, mem.page_size); - const result = @wasmMemoryGrow(0, @intCast(u32, drop_page_count + page_count)); - if (result <= 0) - return error.OutOfMemory; - assert(result == next_page_idx); - const aligned_page_idx = next_page_idx + drop_page_count; - if (drop_page_count > 0) { - freePages(next_page_idx, aligned_page_idx); - } - return @intCast(usize, aligned_page_idx); - } - - fn freePages(start: usize, end: usize) void { - if (start < extendedOffset()) { - conventional.recycle(start, @min(extendedOffset(), end) - start); - } - if (end > extendedOffset()) { - var new_end = end; - if (!extended.isInitialized()) { - // Steal the last page from the memory currently being recycled - // TODO: would it be better if we use the first page instead? - new_end -= 1; - - extended.data = @intToPtr([*]u128, new_end * mem.page_size)[0 .. mem.page_size / @sizeOf(u128)]; - // Since this is the first page being freed and we consume it, assume *nothing* is free. - mem.set(u128, extended.data, PageStatus.none_free); - } - const clamped_start = @max(extendedOffset(), start); - extended.recycle(clamped_start - extendedOffset(), new_end - clamped_start); - } - } - - fn resize( - _: *anyopaque, - buf: []u8, - log2_buf_align: u8, - new_len: usize, - return_address: usize, - ) bool { - _ = log2_buf_align; - _ = return_address; - const aligned_len = mem.alignForward(buf.len, mem.page_size); - if (new_len > aligned_len) return false; - const current_n = nPages(aligned_len); - const new_n = nPages(new_len); - if (new_n != current_n) { - const base = nPages(@ptrToInt(buf.ptr)); - freePages(base + new_n, base + current_n); - } - return true; - } - - fn free( - _: *anyopaque, - buf: []u8, - log2_buf_align: u8, - return_address: usize, - ) void { - _ = log2_buf_align; - _ = return_address; - const aligned_len = mem.alignForward(buf.len, mem.page_size); - const current_n = nPages(aligned_len); - const base = nPages(@ptrToInt(buf.ptr)); - freePages(base, base + current_n); - } -}; - pub const HeapAllocator = switch (builtin.os.tag) { .windows => struct { heap_handle: ?HeapHandle, @@ -1163,7 +869,10 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void { try testing.expect(slice[60] == 0x34); } -test "heap" { - _ = @import("heap/logging_allocator.zig"); - _ = @import("heap/log_to_writer_allocator.zig"); +test { + _ = LoggingAllocator; + _ = LogToWriterAllocator; + _ = ScopedLoggingAllocator; + _ = ArenaAllocator; + _ = GeneralPurposeAllocator; } diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig new file mode 100644 index 000000000000..2c8146caf3b7 --- /dev/null +++ b/lib/std/heap/PageAllocator.zig @@ -0,0 +1,110 @@ +const std = @import("../std.zig"); +const builtin = @import("builtin"); +const Allocator = std.mem.Allocator; +const mem = std.mem; +const os = std.os; +const maxInt = std.math.maxInt; +const assert = std.debug.assert; + +pub const vtable = Allocator.VTable{ + .alloc = alloc, + .resize = resize, + .free = free, +}; + +fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { + _ = ra; + _ = log2_align; + assert(n > 0); + if (n > maxInt(usize) - (mem.page_size - 1)) return null; + const aligned_len = mem.alignForward(n, mem.page_size); + + if (builtin.os.tag == .windows) { + const w = os.windows; + const addr = w.VirtualAlloc( + null, + aligned_len, + w.MEM_COMMIT | w.MEM_RESERVE, + w.PAGE_READWRITE, + ) catch return null; + return @ptrCast([*]align(mem.page_size) u8, @alignCast(mem.page_size, addr)); + } + + const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .Unordered); + const slice = os.mmap( + hint, + aligned_len, + os.PROT.READ | os.PROT.WRITE, + os.MAP.PRIVATE | os.MAP.ANONYMOUS, + -1, + 0, + ) catch return null; + assert(mem.isAligned(@ptrToInt(slice.ptr), mem.page_size)); + const new_hint = @alignCast(mem.page_size, slice.ptr + aligned_len); + _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .Monotonic, .Monotonic); + return slice.ptr; +} + +fn resize( + _: *anyopaque, + buf_unaligned: []u8, + log2_buf_align: u8, + new_size: usize, + return_address: usize, +) bool { + _ = log2_buf_align; + _ = return_address; + const new_size_aligned = mem.alignForward(new_size, mem.page_size); + + if (builtin.os.tag == .windows) { + const w = os.windows; + if (new_size <= buf_unaligned.len) { + const base_addr = @ptrToInt(buf_unaligned.ptr); + const old_addr_end = base_addr + buf_unaligned.len; + const new_addr_end = mem.alignForward(base_addr + new_size, mem.page_size); + if (old_addr_end > new_addr_end) { + // For shrinking that is not releasing, we will only + // decommit the pages not needed anymore. + w.VirtualFree( + @intToPtr(*anyopaque, new_addr_end), + old_addr_end - new_addr_end, + w.MEM_DECOMMIT, + ); + } + return true; + } + const old_size_aligned = mem.alignForward(buf_unaligned.len, mem.page_size); + if (new_size_aligned <= old_size_aligned) { + return true; + } + return false; + } + + const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size); + if (new_size_aligned == buf_aligned_len) + return true; + + if (new_size_aligned < buf_aligned_len) { + const ptr = @alignCast(mem.page_size, buf_unaligned.ptr + new_size_aligned); + // TODO: if the next_mmap_addr_hint is within the unmapped range, update it + os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]); + return true; + } + + // TODO: call mremap + // TODO: if the next_mmap_addr_hint is within the remapped range, update it + return false; +} + +fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void { + _ = log2_buf_align; + _ = return_address; + + if (builtin.os.tag == .windows) { + os.windows.VirtualFree(slice.ptr, 0, os.windows.MEM_RELEASE); + } else { + const buf_aligned_len = mem.alignForward(slice.len, mem.page_size); + const ptr = @alignCast(mem.page_size, slice.ptr); + os.munmap(ptr[0..buf_aligned_len]); + } +} diff --git a/lib/std/heap/WasmPageAllocator.zig b/lib/std/heap/WasmPageAllocator.zig new file mode 100644 index 000000000000..ef0c5cb1d578 --- /dev/null +++ b/lib/std/heap/WasmPageAllocator.zig @@ -0,0 +1,193 @@ +const std = @import("../std.zig"); +const builtin = @import("builtin"); +const Allocator = std.mem.Allocator; +const mem = std.mem; +const maxInt = std.math.maxInt; +const assert = std.debug.assert; + +comptime { + if (!builtin.target.isWasm()) { + @compileError("WasmPageAllocator is only available for wasm32 arch"); + } +} + +pub const vtable = Allocator.VTable{ + .alloc = alloc, + .resize = resize, + .free = free, +}; + +const PageStatus = enum(u1) { + used = 0, + free = 1, + + pub const none_free: u8 = 0; +}; + +const FreeBlock = struct { + data: []u128, + + const Io = std.packed_int_array.PackedIntIo(u1, .Little); + + fn totalPages(self: FreeBlock) usize { + return self.data.len * 128; + } + + fn isInitialized(self: FreeBlock) bool { + return self.data.len > 0; + } + + fn getBit(self: FreeBlock, idx: usize) PageStatus { + const bit_offset = 0; + return @intToEnum(PageStatus, Io.get(mem.sliceAsBytes(self.data), idx, bit_offset)); + } + + fn setBits(self: FreeBlock, start_idx: usize, len: usize, val: PageStatus) void { + const bit_offset = 0; + var i: usize = 0; + while (i < len) : (i += 1) { + Io.set(mem.sliceAsBytes(self.data), start_idx + i, bit_offset, @enumToInt(val)); + } + } + + // Use '0xFFFFFFFF' as a _missing_ sentinel + // This saves ~50 bytes compared to returning a nullable + + // We can guarantee that conventional memory never gets this big, + // and wasm32 would not be able to address this memory (32 GB > usize). + + // Revisit if this is settled: https://github.com/ziglang/zig/issues/3806 + const not_found = maxInt(usize); + + fn useRecycled(self: FreeBlock, num_pages: usize, log2_align: u8) usize { + @setCold(true); + for (self.data) |segment, i| { + const spills_into_next = @bitCast(i128, segment) < 0; + const has_enough_bits = @popCount(segment) >= num_pages; + + if (!spills_into_next and !has_enough_bits) continue; + + var j: usize = i * 128; + while (j < (i + 1) * 128) : (j += 1) { + var count: usize = 0; + while (j + count < self.totalPages() and self.getBit(j + count) == .free) { + count += 1; + const addr = j * mem.page_size; + if (count >= num_pages and mem.isAlignedLog2(addr, log2_align)) { + self.setBits(j, num_pages, .used); + return j; + } + } + j += count; + } + } + return not_found; + } + + fn recycle(self: FreeBlock, start_idx: usize, len: usize) void { + self.setBits(start_idx, len, .free); + } +}; + +var _conventional_data = [_]u128{0} ** 16; +// Marking `conventional` as const saves ~40 bytes +const conventional = FreeBlock{ .data = &_conventional_data }; +var extended = FreeBlock{ .data = &[_]u128{} }; + +fn extendedOffset() usize { + return conventional.totalPages(); +} + +fn nPages(memsize: usize) usize { + return mem.alignForward(memsize, mem.page_size) / mem.page_size; +} + +fn alloc(_: *anyopaque, len: usize, log2_align: u8, ra: usize) ?[*]u8 { + _ = ra; + if (len > maxInt(usize) - (mem.page_size - 1)) return null; + const page_count = nPages(len); + const page_idx = allocPages(page_count, log2_align) catch return null; + return @intToPtr([*]u8, page_idx * mem.page_size); +} + +fn allocPages(page_count: usize, log2_align: u8) !usize { + { + const idx = conventional.useRecycled(page_count, log2_align); + if (idx != FreeBlock.not_found) { + return idx; + } + } + + const idx = extended.useRecycled(page_count, log2_align); + if (idx != FreeBlock.not_found) { + return idx + extendedOffset(); + } + + const next_page_idx = @wasmMemorySize(0); + const next_page_addr = next_page_idx * mem.page_size; + const aligned_addr = mem.alignForwardLog2(next_page_addr, log2_align); + const drop_page_count = @divExact(aligned_addr - next_page_addr, mem.page_size); + const result = @wasmMemoryGrow(0, @intCast(u32, drop_page_count + page_count)); + if (result <= 0) + return error.OutOfMemory; + assert(result == next_page_idx); + const aligned_page_idx = next_page_idx + drop_page_count; + if (drop_page_count > 0) { + freePages(next_page_idx, aligned_page_idx); + } + return @intCast(usize, aligned_page_idx); +} + +fn freePages(start: usize, end: usize) void { + if (start < extendedOffset()) { + conventional.recycle(start, @min(extendedOffset(), end) - start); + } + if (end > extendedOffset()) { + var new_end = end; + if (!extended.isInitialized()) { + // Steal the last page from the memory currently being recycled + // TODO: would it be better if we use the first page instead? + new_end -= 1; + + extended.data = @intToPtr([*]u128, new_end * mem.page_size)[0 .. mem.page_size / @sizeOf(u128)]; + // Since this is the first page being freed and we consume it, assume *nothing* is free. + mem.set(u128, extended.data, PageStatus.none_free); + } + const clamped_start = @max(extendedOffset(), start); + extended.recycle(clamped_start - extendedOffset(), new_end - clamped_start); + } +} + +fn resize( + _: *anyopaque, + buf: []u8, + log2_buf_align: u8, + new_len: usize, + return_address: usize, +) bool { + _ = log2_buf_align; + _ = return_address; + const aligned_len = mem.alignForward(buf.len, mem.page_size); + if (new_len > aligned_len) return false; + const current_n = nPages(aligned_len); + const new_n = nPages(new_len); + if (new_n != current_n) { + const base = nPages(@ptrToInt(buf.ptr)); + freePages(base + new_n, base + current_n); + } + return true; +} + +fn free( + _: *anyopaque, + buf: []u8, + log2_buf_align: u8, + return_address: usize, +) void { + _ = log2_buf_align; + _ = return_address; + const aligned_len = mem.alignForward(buf.len, mem.page_size); + const current_n = nPages(aligned_len); + const base = nPages(@ptrToInt(buf.ptr)); + freePages(base, base + current_n); +} From 3ea04ed64cd47e65bee3bad1771d349042bb4392 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Nov 2022 18:49:02 -0700 Subject: [PATCH 2/8] introduce std.heap.WasmAllocator fast allocator for WebAssembly eventually this is intended to be merged into `std.heap.GeneralPurposeAllocator` --- lib/std/heap/WasmAllocator.zig | 222 +++++++++++++++++++++++++++++ lib/std/heap/WasmPageAllocator.zig | 9 +- 2 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 lib/std/heap/WasmAllocator.zig diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig new file mode 100644 index 000000000000..408b474a264e --- /dev/null +++ b/lib/std/heap/WasmAllocator.zig @@ -0,0 +1,222 @@ +//! This is intended to be merged into GeneralPurposeAllocator at some point. + +const std = @import("../std.zig"); +const builtin = @import("builtin"); +const Allocator = std.mem.Allocator; +const mem = std.mem; +const assert = std.debug.assert; +const wasm = std.wasm; +const math = std.math; + +comptime { + if (!builtin.target.isWasm()) { + @compileError("WasmPageAllocator is only available for wasm32 arch"); + } +} + +pub const vtable = Allocator.VTable{ + .alloc = alloc, + .resize = resize, + .free = free, +}; + +pub const Error = Allocator.Error; + +const max_usize = math.maxInt(usize); +const bigpage_size = 512 * 1024; +const pages_per_bigpage = bigpage_size / wasm.page_size; +const bigpage_count = max_usize / bigpage_size; + +//// This has a length of 1024 usizes. +//var bigpages_used = [1]usize{0} ** (bigpage_count / @bitSizeOf(usize)); + +/// We have a small size class for all sizes up to 512kb. +const size_class_count = math.log2(bigpage_size); + +const FreeList = struct { + /// Each element is the address of a freed pointer. + ptr: [*]usize, + len: usize, + cap: usize, + + const init: FreeList = .{ + .ptr = undefined, + .len = 0, + .cap = 0, + }; +}; + +var next_addrs = [1]usize{0} ** size_class_count; +var frees = [1]FreeList{FreeList.init} ** size_class_count; +var bigpage_free_list: FreeList = .{ + .ptr = &bigpage_free_buf, + .len = 0, + .cap = bigpage_free_buf.len, +}; +var bigpage_free_buf: [16]usize = undefined; + +fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 { + _ = ctx; + _ = len_align; + _ = ra; + const slot_size = math.ceilPowerOfTwoAssert(usize, @max(len, alignment)); + const class = math.log2(slot_size); + if (class < size_class_count) { + const addr = a: { + const free_list = &frees[class]; + if (free_list.len > 0) { + free_list.len -= 1; + break :a free_list.ptr[free_list.len]; + } + + // Ensure unused capacity in the corresponding free list. + // This prevents memory allocation within free(). + if (free_list.len >= free_list.cap) { + const old_bigpage_count = free_list.cap / bigpage_size; + if (bigpage_free_list.cap - bigpage_free_list.len < old_bigpage_count) { + return error.OutOfMemory; + } + const new_bigpage_count = old_bigpage_count + 1; + const addr = try allocBigPages(new_bigpage_count); + const new_ptr = @intToPtr([*]usize, addr); + const old_ptr = free_list.ptr; + @memcpy( + @ptrCast([*]u8, new_ptr), + @ptrCast([*]u8, old_ptr), + @sizeOf(usize) * free_list.len, + ); + free_list.ptr = new_ptr; + free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize)); + + var i: usize = 0; + while (i < old_bigpage_count) : (i += 1) { + bigpage_free_list.ptr[bigpage_free_list.len] = @ptrToInt(old_ptr) + + i * bigpage_size; + bigpage_free_list.len += 1; + } + } + + const next_addr = next_addrs[class]; + if (next_addr % bigpage_size == 0) { + //std.debug.print("alloc big page len={d} class={d} slot_size={d}\n", .{ + // len, class, slot_size, + //}); + const addr = try allocBigPages(1); + next_addrs[class] = addr + slot_size; + break :a addr; + } else { + //std.debug.print("easy! len={d} class={d} slot_size={d}\n", .{ + // len, class, slot_size, + //}); + next_addrs[class] = next_addr + slot_size; + break :a next_addr; + } + }; + return @intToPtr([*]u8, addr)[0..len]; + } else { + std.debug.panic("big alloc: len={d} align={d} slot_size={d} class={d}", .{ + len, alignment, slot_size, class, + }); + } +} + +fn resize( + ctx: *anyopaque, + buf: []u8, + buf_align: u29, + new_len: usize, + len_align: u29, + return_address: usize, +) ?usize { + _ = ctx; + _ = buf_align; + _ = return_address; + _ = len_align; + _ = new_len; + _ = buf; + @panic("handle resize"); +} + +fn free( + ctx: *anyopaque, + buf: []u8, + buf_align: u29, + return_address: usize, +) void { + _ = ctx; + _ = return_address; + const class_size = @max(buf.len, buf_align); + const class = math.log2(class_size); + if (class < size_class_count) { + const free_list = &frees[class]; + assert(free_list.len < free_list.cap); + free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); + free_list.len += 1; + } else { + std.debug.panic("big free: len={d} align={d}", .{ + buf.len, buf_align, + }); + } +} + +inline fn allocBigPages(n: usize) !usize { + if (n == 1 and bigpage_free_list.len > 0) { + bigpage_free_list.len -= 1; + return bigpage_free_list.ptr[bigpage_free_list.len]; + } + const page_index = @wasmMemoryGrow(0, n * pages_per_bigpage); + if (page_index <= 0) + return error.OutOfMemory; + return @intCast(u32, page_index) * wasm.page_size; +} + +const test_ally = Allocator{ + .ptr = undefined, + .vtable = &vtable, +}; + +test "small allocations - free in same order" { + var list: [513]*u64 = undefined; + + var i: usize = 0; + while (i < 513) : (i += 1) { + const ptr = try test_ally.create(u64); + list[i] = ptr; + } + + for (list) |ptr| { + test_ally.destroy(ptr); + } +} + +test "small allocations - free in reverse order" { + var list: [513]*u64 = undefined; + + var i: usize = 0; + while (i < 513) : (i += 1) { + const ptr = try test_ally.create(u64); + list[i] = ptr; + } + + i = list.len; + while (i > 0) { + i -= 1; + const ptr = list[i]; + test_ally.destroy(ptr); + } +} + +test "large allocations" { + std.debug.print("alloc ptr1\n", .{}); + const ptr1 = try test_ally.alloc(u64, 42768); + std.debug.print("alloc ptr2\n", .{}); + const ptr2 = try test_ally.alloc(u64, 52768); + std.debug.print("free ptr1\n", .{}); + test_ally.free(ptr1); + std.debug.print("alloc ptr3\n", .{}); + const ptr3 = try test_ally.alloc(u64, 62768); + std.debug.print("free ptr3\n", .{}); + test_ally.free(ptr3); + std.debug.print("free ptr2\n", .{}); + test_ally.free(ptr2); +} diff --git a/lib/std/heap/WasmPageAllocator.zig b/lib/std/heap/WasmPageAllocator.zig index ef0c5cb1d578..fb8fc4158100 100644 --- a/lib/std/heap/WasmPageAllocator.zig +++ b/lib/std/heap/WasmPageAllocator.zig @@ -102,7 +102,8 @@ fn nPages(memsize: usize) usize { return mem.alignForward(memsize, mem.page_size) / mem.page_size; } -fn alloc(_: *anyopaque, len: usize, log2_align: u8, ra: usize) ?[*]u8 { +fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, ra: usize) ?[*]u8 { + _ = ctx; _ = ra; if (len > maxInt(usize) - (mem.page_size - 1)) return null; const page_count = nPages(len); @@ -159,12 +160,13 @@ fn freePages(start: usize, end: usize) void { } fn resize( - _: *anyopaque, + ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, return_address: usize, ) bool { + _ = ctx; _ = log2_buf_align; _ = return_address; const aligned_len = mem.alignForward(buf.len, mem.page_size); @@ -179,11 +181,12 @@ fn resize( } fn free( - _: *anyopaque, + ctx: *anyopaque, buf: []u8, log2_buf_align: u8, return_address: usize, ) void { + _ = ctx; _ = log2_buf_align; _ = return_address; const aligned_len = mem.alignForward(buf.len, mem.page_size); From 0c0c70ee82df6e727dd488318e6d44d5010eef79 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Nov 2022 21:08:29 -0700 Subject: [PATCH 3/8] std.heap.WasmAllocator: large allocations --- lib/std/heap/WasmAllocator.zig | 152 ++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 69 deletions(-) diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig index 408b474a264e..6405335fee9e 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/WasmAllocator.zig @@ -23,15 +23,18 @@ pub const vtable = Allocator.VTable{ pub const Error = Allocator.Error; const max_usize = math.maxInt(usize); +const ushift = math.Log2Int(usize); const bigpage_size = 512 * 1024; const pages_per_bigpage = bigpage_size / wasm.page_size; const bigpage_count = max_usize / bigpage_size; -//// This has a length of 1024 usizes. -//var bigpages_used = [1]usize{0} ** (bigpage_count / @bitSizeOf(usize)); - /// We have a small size class for all sizes up to 512kb. const size_class_count = math.log2(bigpage_size); +/// 0 - 1 bigpage +/// 1 - 2 bigpages +/// 2 - 4 bigpages +/// etc. +const big_size_class_count = math.log2(bigpage_count); const FreeList = struct { /// Each element is the address of a freed pointer. @@ -46,20 +49,26 @@ const FreeList = struct { }; }; -var next_addrs = [1]usize{0} ** size_class_count; -var frees = [1]FreeList{FreeList.init} ** size_class_count; -var bigpage_free_list: FreeList = .{ - .ptr = &bigpage_free_buf, - .len = 0, - .cap = bigpage_free_buf.len, +const Bucket = struct { + ptr: usize, + end: usize, + + const init: Bucket = .{ + .ptr = 0, + .end = 0, + }; }; -var bigpage_free_buf: [16]usize = undefined; + +var next_addrs = [1]Bucket{Bucket.init} ** size_class_count; +var frees = [1]FreeList{FreeList.init} ** size_class_count; +var big_frees = [1]FreeList{FreeList.init} ** big_size_class_count; fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 { _ = ctx; _ = len_align; _ = ra; - const slot_size = math.ceilPowerOfTwoAssert(usize, @max(len, alignment)); + const aligned_len = @max(len, alignment); + const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len); const class = math.log2(slot_size); if (class < size_class_count) { const addr = a: { @@ -69,55 +78,30 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) break :a free_list.ptr[free_list.len]; } - // Ensure unused capacity in the corresponding free list. // This prevents memory allocation within free(). - if (free_list.len >= free_list.cap) { - const old_bigpage_count = free_list.cap / bigpage_size; - if (bigpage_free_list.cap - bigpage_free_list.len < old_bigpage_count) { - return error.OutOfMemory; - } - const new_bigpage_count = old_bigpage_count + 1; - const addr = try allocBigPages(new_bigpage_count); - const new_ptr = @intToPtr([*]usize, addr); - const old_ptr = free_list.ptr; - @memcpy( - @ptrCast([*]u8, new_ptr), - @ptrCast([*]u8, old_ptr), - @sizeOf(usize) * free_list.len, - ); - free_list.ptr = new_ptr; - free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize)); - - var i: usize = 0; - while (i < old_bigpage_count) : (i += 1) { - bigpage_free_list.ptr[bigpage_free_list.len] = @ptrToInt(old_ptr) + - i * bigpage_size; - bigpage_free_list.len += 1; - } - } + try ensureFreeListCapacity(free_list); const next_addr = next_addrs[class]; - if (next_addr % bigpage_size == 0) { - //std.debug.print("alloc big page len={d} class={d} slot_size={d}\n", .{ - // len, class, slot_size, - //}); + if (next_addr.ptr == next_addr.end) { const addr = try allocBigPages(1); - next_addrs[class] = addr + slot_size; + //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ + // slot_size, class, addr, + //}); + next_addrs[class] = .{ + .ptr = addr + slot_size, + .end = addr + bigpage_size, + }; break :a addr; } else { - //std.debug.print("easy! len={d} class={d} slot_size={d}\n", .{ - // len, class, slot_size, - //}); - next_addrs[class] = next_addr + slot_size; - break :a next_addr; + next_addrs[class].ptr = next_addr.ptr + slot_size; + break :a next_addr.ptr; } }; return @intToPtr([*]u8, addr)[0..len]; - } else { - std.debug.panic("big alloc: len={d} align={d} slot_size={d} class={d}", .{ - len, alignment, slot_size, class, - }); } + const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size; + const addr = try allocBigPages(bigpages_needed); + return @intToPtr([*]u8, addr)[0..len]; } fn resize( @@ -145,29 +129,65 @@ fn free( ) void { _ = ctx; _ = return_address; - const class_size = @max(buf.len, buf_align); - const class = math.log2(class_size); + const aligned_len = @max(buf.len, buf_align); + const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len); + const class = math.log2(slot_size); if (class < size_class_count) { const free_list = &frees[class]; assert(free_list.len < free_list.cap); free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); free_list.len += 1; } else { - std.debug.panic("big free: len={d} align={d}", .{ - buf.len, buf_align, - }); + const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size; + const big_slot_size = math.ceilPowerOfTwoAssert(usize, bigpages_needed); + const big_class = math.log2(big_slot_size); + const free_list = &big_frees[big_class]; + assert(free_list.len < free_list.cap); + free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); + free_list.len += 1; } } -inline fn allocBigPages(n: usize) !usize { - if (n == 1 and bigpage_free_list.len > 0) { - bigpage_free_list.len -= 1; - return bigpage_free_list.ptr[bigpage_free_list.len]; +fn allocBigPages(n: usize) !usize { + const slot_size = math.ceilPowerOfTwoAssert(usize, n); + const class = math.log2(slot_size); + + const free_list = &big_frees[class]; + if (free_list.len > 0) { + free_list.len -= 1; + return free_list.ptr[free_list.len]; } - const page_index = @wasmMemoryGrow(0, n * pages_per_bigpage); - if (page_index <= 0) - return error.OutOfMemory; - return @intCast(u32, page_index) * wasm.page_size; + + //std.debug.print("ensureFreeListCapacity slot_size={d} big_class={d}\n", .{ + // slot_size, class, + //}); + // This prevents memory allocation within free(). + try ensureFreeListCapacity(free_list); + + const page_index = @wasmMemoryGrow(0, slot_size * pages_per_bigpage); + if (page_index <= 0) return error.OutOfMemory; + const addr = @intCast(u32, page_index) * wasm.page_size; + //std.debug.print("got 0x{x}..0x{x} from memory.grow\n", .{ + // addr, addr + wasm.page_size * slot_size * pages_per_bigpage, + //}); + return addr; +} + +fn ensureFreeListCapacity(free_list: *FreeList) Allocator.Error!void { + if (free_list.len < free_list.cap) return; + const old_bigpage_count = free_list.cap / bigpage_size; + free_list.cap = math.maxInt(usize); // Prevent recursive calls. + const new_bigpage_count = @max(old_bigpage_count * 2, 1); + const addr = try allocBigPages(new_bigpage_count); + //std.debug.print("allocated {d} big pages: 0x{x}\n", .{ new_bigpage_count, addr }); + const new_ptr = @intToPtr([*]usize, addr); + @memcpy( + @ptrCast([*]u8, new_ptr), + @ptrCast([*]u8, free_list.ptr), + @sizeOf(usize) * free_list.len, + ); + free_list.ptr = new_ptr; + free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize)); } const test_ally = Allocator{ @@ -207,16 +227,10 @@ test "small allocations - free in reverse order" { } test "large allocations" { - std.debug.print("alloc ptr1\n", .{}); const ptr1 = try test_ally.alloc(u64, 42768); - std.debug.print("alloc ptr2\n", .{}); const ptr2 = try test_ally.alloc(u64, 52768); - std.debug.print("free ptr1\n", .{}); test_ally.free(ptr1); - std.debug.print("alloc ptr3\n", .{}); const ptr3 = try test_ally.alloc(u64, 62768); - std.debug.print("free ptr3\n", .{}); test_ally.free(ptr3); - std.debug.print("free ptr2\n", .{}); test_ally.free(ptr2); } From d4a1ae474a7d4607ec8ac306963872eb48cfd731 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Nov 2022 21:24:27 -0700 Subject: [PATCH 4/8] std.heap.WasmAllocator: resize in place without force shrinking --- lib/std/heap/WasmAllocator.zig | 67 +++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig index 6405335fee9e..2ba97296f71c 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/WasmAllocator.zig @@ -68,7 +68,7 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) _ = len_align; _ = ra; const aligned_len = @max(len, alignment); - const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len); + const slot_size = math.ceilPowerOfTwo(usize, aligned_len) catch return error.OutOfMemory; const class = math.log2(slot_size); if (class < size_class_count) { const addr = a: { @@ -113,12 +113,28 @@ fn resize( return_address: usize, ) ?usize { _ = ctx; - _ = buf_align; _ = return_address; _ = len_align; - _ = new_len; - _ = buf; - @panic("handle resize"); + // We don't want to move anything from one size class to another. But we can recover bytes + // in between powers of two. + const old_aligned_len = @max(buf.len, buf_align); + const new_aligned_len = @max(new_len, buf_align); + const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_aligned_len); + const old_small_class = math.log2(old_small_slot_size); + if (old_small_class < size_class_count) { + const new_small_slot_size = math.ceilPowerOfTwo(usize, new_aligned_len) catch return null; + //std.debug.print("resize: old_small_slot_size={d} new_small_slot_size={d}\n", .{ + // old_small_slot_size, new_small_slot_size, + //}); + if (old_small_slot_size != new_small_slot_size) return null; + } else { + const old_bigpages_needed = (old_aligned_len + (bigpage_size - 1)) / bigpage_size; + const old_big_slot_size = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed); + const new_bigpages_needed = (new_aligned_len + (bigpage_size - 1)) / bigpage_size; + const new_big_slot_size = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null; + if (old_big_slot_size != new_big_slot_size) return null; + } + return new_len; } fn free( @@ -234,3 +250,44 @@ test "large allocations" { test_ally.free(ptr3); test_ally.free(ptr2); } + +test "very large allocation" { + try std.testing.expectError(error.OutOfMemory, test_ally.alloc(u8, math.maxInt(usize))); +} + +test "realloc" { + var slice = try test_ally.alignedAlloc(u8, @alignOf(u32), 1); + defer test_ally.free(slice); + slice[0] = 0x12; + + // This reallocation should keep its pointer address. + const old_slice = slice; + slice = try test_ally.realloc(slice, 2); + try std.testing.expect(old_slice.ptr == slice.ptr); + try std.testing.expect(slice[0] == 0x12); + slice[1] = 0x34; + + // This requires upgrading to a larger size class + slice = try test_ally.realloc(slice, 17); + try std.testing.expect(slice[0] == 0x12); + try std.testing.expect(slice[1] == 0x34); +} + +test "shrink" { + var slice = try test_ally.alloc(u8, 20); + defer test_ally.free(slice); + + mem.set(u8, slice, 0x11); + + slice = test_ally.shrink(slice, 17); + + for (slice) |b| { + try std.testing.expect(b == 0x11); + } + + slice = test_ally.shrink(slice, 16); + + for (slice) |b| { + try std.testing.expect(b == 0x11); + } +} From 3dcea95ffe97ee93af50bf8906e53c7c7a7ec84e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Nov 2022 21:40:48 -0700 Subject: [PATCH 5/8] std.heap.WasmAllocator: implement resizing --- lib/std/heap/WasmAllocator.zig | 91 +++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 6 deletions(-) diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig index 2ba97296f71c..75e7cb3db5ce 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/WasmAllocator.zig @@ -67,6 +67,7 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) _ = ctx; _ = len_align; _ = ra; + if (alignment > wasm.page_size) return error.OutOfMemory; // calm down const aligned_len = @max(len, alignment); const slot_size = math.ceilPowerOfTwo(usize, aligned_len) catch return error.OutOfMemory; const class = math.log2(slot_size); @@ -110,11 +111,8 @@ fn resize( buf_align: u29, new_len: usize, len_align: u29, - return_address: usize, + ra: usize, ) ?usize { - _ = ctx; - _ = return_address; - _ = len_align; // We don't want to move anything from one size class to another. But we can recover bytes // in between powers of two. const old_aligned_len = @max(buf.len, buf_align); @@ -126,13 +124,29 @@ fn resize( //std.debug.print("resize: old_small_slot_size={d} new_small_slot_size={d}\n", .{ // old_small_slot_size, new_small_slot_size, //}); - if (old_small_slot_size != new_small_slot_size) return null; + if (old_small_slot_size != new_small_slot_size) { + if (new_aligned_len >= old_aligned_len) { + return null; + } + // TODO this panic is a design flaw in the Allocator interface that + // should be addressed. + const new = alloc(ctx, new_len, buf_align, len_align, ra) catch @panic("out of memory"); + @memcpy(new.ptr, buf.ptr, buf.len); + } } else { const old_bigpages_needed = (old_aligned_len + (bigpage_size - 1)) / bigpage_size; const old_big_slot_size = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed); const new_bigpages_needed = (new_aligned_len + (bigpage_size - 1)) / bigpage_size; const new_big_slot_size = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null; - if (old_big_slot_size != new_big_slot_size) return null; + if (old_big_slot_size != new_big_slot_size) { + if (new_aligned_len >= old_aligned_len) { + return null; + } + // TODO this panic is a design flaw in the Allocator interface that + // should be addressed. + const new = alloc(ctx, new_len, buf_align, len_align, ra) catch @panic("out of memory"); + @memcpy(new.ptr, buf.ptr, buf.len); + } } return new_len; } @@ -291,3 +305,68 @@ test "shrink" { try std.testing.expect(b == 0x11); } } + +test "large object - grow" { + var slice1 = try test_ally.alloc(u8, bigpage_size * 2 - 20); + defer test_ally.free(slice1); + + const old = slice1; + slice1 = try test_ally.realloc(slice1, bigpage_size * 2 - 10); + try std.testing.expect(slice1.ptr == old.ptr); + + slice1 = try test_ally.realloc(slice1, bigpage_size * 2); + try std.testing.expect(slice1.ptr == old.ptr); + + slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1); +} + +test "realloc small object to large object" { + var slice = try test_ally.alloc(u8, 70); + defer test_ally.free(slice); + slice[0] = 0x12; + slice[60] = 0x34; + + // This requires upgrading to a large object + const large_object_size = bigpage_size * 2 + 50; + slice = try test_ally.realloc(slice, large_object_size); + try std.testing.expect(slice[0] == 0x12); + try std.testing.expect(slice[60] == 0x34); +} + +test "shrink large object to large object" { + var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50); + defer test_ally.free(slice); + slice[0] = 0x12; + slice[60] = 0x34; + + slice = test_ally.resize(slice, bigpage_size * 2 + 1) orelse return; + try std.testing.expect(slice[0] == 0x12); + try std.testing.expect(slice[60] == 0x34); + + slice = test_ally.shrink(slice, bigpage_size * 2 + 1); + try std.testing.expect(slice[0] == 0x12); + try std.testing.expect(slice[60] == 0x34); + + slice = try test_ally.realloc(slice, bigpage_size * 2); + try std.testing.expect(slice[0] == 0x12); + try std.testing.expect(slice[60] == 0x34); +} + +test "realloc large object to small object" { + var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50); + defer test_ally.free(slice); + slice[0] = 0x12; + slice[16] = 0x34; + + slice = try test_ally.realloc(slice, 19); + try std.testing.expect(slice[0] == 0x12); + try std.testing.expect(slice[16] == 0x34); +} + +test "objects of size 1024 and 2048" { + const slice = try test_ally.alloc(u8, 1025); + const slice2 = try test_ally.alloc(u8, 3000); + + test_ally.free(slice); + test_ally.free(slice2); +} From e2e60f5ff9942902e97aebfdf234bb6a0f821dfe Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 11 Nov 2022 22:48:06 -0700 Subject: [PATCH 6/8] std.heap.WasmAllocator: redo The previous version had a fatal flaw: it did ensureCapacity(1) on the freelist when allocating, but I neglected to consider that you could free() twice in a row. Silly! This strategy allocates an intrusive freelist node with every allocation, big or small. It also does not have the problems with resize because in this case we can push the upper areas of freed stuff into the corresponding freelist. --- lib/std/heap.zig | 42 +----- lib/std/heap/WasmAllocator.zig | 215 ++++++++++++----------------- lib/std/heap/WasmPageAllocator.zig | 39 ++++++ 3 files changed, 136 insertions(+), 160 deletions(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 71933bc18657..ac77db7a798f 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -16,6 +16,7 @@ pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").Log pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator; pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator; +pub const WasmAllocator = @import("heap/WasmAllocator.zig"); pub const WasmPageAllocator = @import("heap/WasmPageAllocator.zig"); pub const PageAllocator = @import("heap/PageAllocator.zig"); @@ -565,43 +566,6 @@ test "raw_c_allocator" { } } -test "WasmPageAllocator internals" { - if (comptime builtin.target.isWasm()) { - const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size; - const initial = try page_allocator.alloc(u8, mem.page_size); - try testing.expect(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite. - - var inplace = try page_allocator.realloc(initial, 1); - try testing.expectEqual(initial.ptr, inplace.ptr); - inplace = try page_allocator.realloc(inplace, 4); - try testing.expectEqual(initial.ptr, inplace.ptr); - page_allocator.free(inplace); - - const reuse = try page_allocator.alloc(u8, 1); - try testing.expectEqual(initial.ptr, reuse.ptr); - page_allocator.free(reuse); - - // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now. - const padding = try page_allocator.alloc(u8, conventional_memsize); - page_allocator.free(padding); - - const extended = try page_allocator.alloc(u8, conventional_memsize); - try testing.expect(@ptrToInt(extended.ptr) >= conventional_memsize); - - const use_small = try page_allocator.alloc(u8, 1); - try testing.expectEqual(initial.ptr, use_small.ptr); - page_allocator.free(use_small); - - inplace = try page_allocator.realloc(extended, 1); - try testing.expectEqual(extended.ptr, inplace.ptr); - page_allocator.free(inplace); - - const reuse_extended = try page_allocator.alloc(u8, conventional_memsize); - try testing.expectEqual(extended.ptr, reuse_extended.ptr); - page_allocator.free(reuse_extended); - } -} - test "PageAllocator" { const allocator = page_allocator; try testAllocator(allocator); @@ -875,4 +839,8 @@ test { _ = ScopedLoggingAllocator; _ = ArenaAllocator; _ = GeneralPurposeAllocator; + if (comptime builtin.target.isWasm()) { + _ = WasmAllocator; + _ = WasmPageAllocator; + } } diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig index 75e7cb3db5ce..e29ebc7f93f5 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/WasmAllocator.zig @@ -24,83 +24,59 @@ pub const Error = Allocator.Error; const max_usize = math.maxInt(usize); const ushift = math.Log2Int(usize); -const bigpage_size = 512 * 1024; +const bigpage_size = 64 * 1024; const pages_per_bigpage = bigpage_size / wasm.page_size; const bigpage_count = max_usize / bigpage_size; -/// We have a small size class for all sizes up to 512kb. -const size_class_count = math.log2(bigpage_size); +/// Because of storing free list pointers, the minimum size class is 3. +const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize))); +const size_class_count = math.log2(bigpage_size) - min_class; /// 0 - 1 bigpage /// 1 - 2 bigpages /// 2 - 4 bigpages /// etc. const big_size_class_count = math.log2(bigpage_count); -const FreeList = struct { - /// Each element is the address of a freed pointer. - ptr: [*]usize, - len: usize, - cap: usize, - - const init: FreeList = .{ - .ptr = undefined, - .len = 0, - .cap = 0, - }; -}; - -const Bucket = struct { - ptr: usize, - end: usize, - - const init: Bucket = .{ - .ptr = 0, - .end = 0, - }; -}; - -var next_addrs = [1]Bucket{Bucket.init} ** size_class_count; -var frees = [1]FreeList{FreeList.init} ** size_class_count; -var big_frees = [1]FreeList{FreeList.init} ** big_size_class_count; +var next_addrs = [1]usize{0} ** size_class_count; +/// For each size class, points to the freed pointer. +var frees = [1]usize{0} ** size_class_count; +/// For each big size class, points to the freed pointer. +var big_frees = [1]usize{0} ** big_size_class_count; fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 { _ = ctx; _ = len_align; _ = ra; if (alignment > wasm.page_size) return error.OutOfMemory; // calm down - const aligned_len = @max(len, alignment); - const slot_size = math.ceilPowerOfTwo(usize, aligned_len) catch return error.OutOfMemory; - const class = math.log2(slot_size); + // Make room for the freelist next pointer. + const actual_len = @max(len +| @sizeOf(usize), alignment); + const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return error.OutOfMemory; + const class = math.log2(slot_size) - min_class; if (class < size_class_count) { const addr = a: { - const free_list = &frees[class]; - if (free_list.len > 0) { - free_list.len -= 1; - break :a free_list.ptr[free_list.len]; + const top_free_ptr = frees[class]; + if (top_free_ptr != 0) { + const node = @intToPtr(*usize, top_free_ptr + (slot_size - @sizeOf(usize))); + frees[class] = node.*; + break :a top_free_ptr; } - // This prevents memory allocation within free(). - try ensureFreeListCapacity(free_list); - const next_addr = next_addrs[class]; - if (next_addr.ptr == next_addr.end) { + if (next_addr % wasm.page_size == 0) { const addr = try allocBigPages(1); //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ // slot_size, class, addr, //}); - next_addrs[class] = .{ - .ptr = addr + slot_size, - .end = addr + bigpage_size, - }; + next_addrs[class] = addr + slot_size; break :a addr; } else { - next_addrs[class].ptr = next_addr.ptr + slot_size; - break :a next_addr.ptr; + next_addrs[class] = next_addr + slot_size; + break :a next_addr; } }; return @intToPtr([*]u8, addr)[0..len]; } - const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size; + const bigpages_needed = bigPagesNeeded(actual_len); const addr = try allocBigPages(bigpages_needed); return @intToPtr([*]u8, addr)[0..len]; } @@ -113,39 +89,49 @@ fn resize( len_align: u29, ra: usize, ) ?usize { + _ = ctx; + _ = len_align; + _ = ra; // We don't want to move anything from one size class to another. But we can recover bytes // in between powers of two. - const old_aligned_len = @max(buf.len, buf_align); - const new_aligned_len = @max(new_len, buf_align); - const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_aligned_len); - const old_small_class = math.log2(old_small_slot_size); + const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align); + const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align); + const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len); + const old_small_class = math.log2(old_small_slot_size) - min_class; if (old_small_class < size_class_count) { - const new_small_slot_size = math.ceilPowerOfTwo(usize, new_aligned_len) catch return null; - //std.debug.print("resize: old_small_slot_size={d} new_small_slot_size={d}\n", .{ - // old_small_slot_size, new_small_slot_size, - //}); - if (old_small_slot_size != new_small_slot_size) { - if (new_aligned_len >= old_aligned_len) { - return null; - } - // TODO this panic is a design flaw in the Allocator interface that - // should be addressed. - const new = alloc(ctx, new_len, buf_align, len_align, ra) catch @panic("out of memory"); - @memcpy(new.ptr, buf.ptr, buf.len); + const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return null; + if (old_small_slot_size == new_small_slot_size) return new_len; + if (new_actual_len >= old_actual_len) return null; + const new_small_class = math.log2(new_small_slot_size) - min_class; + assert(new_small_class < old_small_class); + // Split the small allocation into frees. + var class = old_small_class - 1; + while (true) { + const slot_size = @as(usize, 1) << @intCast(ushift, class + min_class); + const upper_addr = @ptrToInt(buf.ptr) + slot_size; + const node = @intToPtr(*usize, upper_addr + (slot_size - @sizeOf(usize))); + node.* = frees[class]; + frees[class] = upper_addr; + if (class == new_small_class) break; + class -= 1; } } else { - const old_bigpages_needed = (old_aligned_len + (bigpage_size - 1)) / bigpage_size; + const old_bigpages_needed = bigPagesNeeded(old_actual_len); const old_big_slot_size = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed); - const new_bigpages_needed = (new_aligned_len + (bigpage_size - 1)) / bigpage_size; + const new_bigpages_needed = bigPagesNeeded(new_actual_len); const new_big_slot_size = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null; - if (old_big_slot_size != new_big_slot_size) { - if (new_aligned_len >= old_aligned_len) { - return null; - } - // TODO this panic is a design flaw in the Allocator interface that - // should be addressed. - const new = alloc(ctx, new_len, buf_align, len_align, ra) catch @panic("out of memory"); - @memcpy(new.ptr, buf.ptr, buf.len); + if (old_big_slot_size == new_big_slot_size) return new_len; + if (new_actual_len >= old_actual_len) return null; + + const new_small_slot_size = math.ceilPowerOfTwoAssert(usize, new_actual_len); + if (new_small_slot_size < size_class_count) { + const new_small_class = math.log2(new_small_slot_size) - min_class; + // TODO: push the big allocation into the free list + _ = new_small_class; + } else { + const new_big_class = math.log2(new_big_slot_size); + // TODO: push the upper area into the free list + _ = new_big_class; } } return new_len; @@ -159,67 +145,47 @@ fn free( ) void { _ = ctx; _ = return_address; - const aligned_len = @max(buf.len, buf_align); - const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len); - const class = math.log2(slot_size); + const actual_len = @max(buf.len + @sizeOf(usize), buf_align); + const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len); + const class = math.log2(slot_size) - min_class; + const addr = @ptrToInt(buf.ptr); if (class < size_class_count) { - const free_list = &frees[class]; - assert(free_list.len < free_list.cap); - free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); - free_list.len += 1; + const node = @intToPtr(*usize, addr + (slot_size - @sizeOf(usize))); + node.* = frees[class]; + frees[class] = addr; } else { - const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size; - const big_slot_size = math.ceilPowerOfTwoAssert(usize, bigpages_needed); - const big_class = math.log2(big_slot_size); - const free_list = &big_frees[big_class]; - assert(free_list.len < free_list.cap); - free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); - free_list.len += 1; + const bigpages_needed = bigPagesNeeded(actual_len); + const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed); + const big_slot_size_bytes = pow2_pages * bigpage_size; + const node = @intToPtr(*usize, addr + (big_slot_size_bytes - @sizeOf(usize))); + const big_class = math.log2(pow2_pages); + node.* = big_frees[big_class]; + big_frees[big_class] = addr; } } -fn allocBigPages(n: usize) !usize { - const slot_size = math.ceilPowerOfTwoAssert(usize, n); - const class = math.log2(slot_size); +inline fn bigPagesNeeded(byte_count: usize) usize { + return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size; +} - const free_list = &big_frees[class]; - if (free_list.len > 0) { - free_list.len -= 1; - return free_list.ptr[free_list.len]; +fn allocBigPages(n: usize) !usize { + const pow2_pages = math.ceilPowerOfTwoAssert(usize, n); + const slot_size_bytes = pow2_pages * bigpage_size; + const class = math.log2(pow2_pages); + + const top_free_ptr = big_frees[class]; + if (top_free_ptr != 0) { + const node = @intToPtr(*usize, top_free_ptr + (slot_size_bytes - @sizeOf(usize))); + big_frees[class] = node.*; + return top_free_ptr; } - //std.debug.print("ensureFreeListCapacity slot_size={d} big_class={d}\n", .{ - // slot_size, class, - //}); - // This prevents memory allocation within free(). - try ensureFreeListCapacity(free_list); - - const page_index = @wasmMemoryGrow(0, slot_size * pages_per_bigpage); + const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage); if (page_index <= 0) return error.OutOfMemory; const addr = @intCast(u32, page_index) * wasm.page_size; - //std.debug.print("got 0x{x}..0x{x} from memory.grow\n", .{ - // addr, addr + wasm.page_size * slot_size * pages_per_bigpage, - //}); return addr; } -fn ensureFreeListCapacity(free_list: *FreeList) Allocator.Error!void { - if (free_list.len < free_list.cap) return; - const old_bigpage_count = free_list.cap / bigpage_size; - free_list.cap = math.maxInt(usize); // Prevent recursive calls. - const new_bigpage_count = @max(old_bigpage_count * 2, 1); - const addr = try allocBigPages(new_bigpage_count); - //std.debug.print("allocated {d} big pages: 0x{x}\n", .{ new_bigpage_count, addr }); - const new_ptr = @intToPtr([*]usize, addr); - @memcpy( - @ptrCast([*]u8, new_ptr), - @ptrCast([*]u8, free_list.ptr), - @sizeOf(usize) * free_list.len, - ); - free_list.ptr = new_ptr; - free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize)); -} - const test_ally = Allocator{ .ptr = undefined, .vtable = &vtable, @@ -315,8 +281,6 @@ test "large object - grow" { try std.testing.expect(slice1.ptr == old.ptr); slice1 = try test_ally.realloc(slice1, bigpage_size * 2); - try std.testing.expect(slice1.ptr == old.ptr); - slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1); } @@ -370,3 +334,8 @@ test "objects of size 1024 and 2048" { test_ally.free(slice); test_ally.free(slice2); } + +test "standard allocator tests" { + try std.heap.testAllocator(test_ally); + try std.heap.testAllocatorAligned(test_ally); +} diff --git a/lib/std/heap/WasmPageAllocator.zig b/lib/std/heap/WasmPageAllocator.zig index fb8fc4158100..70b93c0508a1 100644 --- a/lib/std/heap/WasmPageAllocator.zig +++ b/lib/std/heap/WasmPageAllocator.zig @@ -1,3 +1,4 @@ +const WasmPageAllocator = @This(); const std = @import("../std.zig"); const builtin = @import("builtin"); const Allocator = std.mem.Allocator; @@ -194,3 +195,41 @@ fn free( const base = nPages(@ptrToInt(buf.ptr)); freePages(base, base + current_n); } + +test "internals" { + const page_allocator = std.heap.page_allocator; + const testing = std.testing; + + const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size; + const initial = try page_allocator.alloc(u8, mem.page_size); + try testing.expect(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite. + + var inplace = try page_allocator.realloc(initial, 1); + try testing.expectEqual(initial.ptr, inplace.ptr); + inplace = try page_allocator.realloc(inplace, 4); + try testing.expectEqual(initial.ptr, inplace.ptr); + page_allocator.free(inplace); + + const reuse = try page_allocator.alloc(u8, 1); + try testing.expectEqual(initial.ptr, reuse.ptr); + page_allocator.free(reuse); + + // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now. + const padding = try page_allocator.alloc(u8, conventional_memsize); + page_allocator.free(padding); + + const ext = try page_allocator.alloc(u8, conventional_memsize); + try testing.expect(@ptrToInt(ext.ptr) >= conventional_memsize); + + const use_small = try page_allocator.alloc(u8, 1); + try testing.expectEqual(initial.ptr, use_small.ptr); + page_allocator.free(use_small); + + inplace = try page_allocator.realloc(ext, 1); + try testing.expectEqual(ext.ptr, inplace.ptr); + page_allocator.free(inplace); + + const reuse_extended = try page_allocator.alloc(u8, conventional_memsize); + try testing.expectEqual(ext.ptr, reuse_extended.ptr); + page_allocator.free(reuse_extended); +} From 931261752d167ea52e0d1e938a0c2bd7fed85327 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 11 Nov 2022 22:59:03 -0700 Subject: [PATCH 7/8] rename a couple variables --- lib/std/heap/WasmAllocator.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig index e29ebc7f93f5..6b74908baff8 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/WasmAllocator.zig @@ -117,10 +117,10 @@ fn resize( } } else { const old_bigpages_needed = bigPagesNeeded(old_actual_len); - const old_big_slot_size = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed); + const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed); const new_bigpages_needed = bigPagesNeeded(new_actual_len); - const new_big_slot_size = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null; - if (old_big_slot_size == new_big_slot_size) return new_len; + const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null; + if (old_big_slot_pages == new_big_slot_pages) return new_len; if (new_actual_len >= old_actual_len) return null; const new_small_slot_size = math.ceilPowerOfTwoAssert(usize, new_actual_len); @@ -129,7 +129,7 @@ fn resize( // TODO: push the big allocation into the free list _ = new_small_class; } else { - const new_big_class = math.log2(new_big_slot_size); + const new_big_class = math.log2(new_big_slot_pages); // TODO: push the upper area into the free list _ = new_big_class; } From 7f063b2c52b5daf55b8b1184502a94d0def4cd22 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 29 Nov 2022 14:05:08 -0700 Subject: [PATCH 8/8] WasmAllocator: simplify thanks to new Allocator interface Now it can refuse to resize when it would disturb the metadata tracking strategy, resulting in smaller code size, a simpler implementation, and less fragmentation. --- lib/std/heap/WasmAllocator.zig | 86 ++++++++++++---------------------- 1 file changed, 31 insertions(+), 55 deletions(-) diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig index 6b74908baff8..3b980ac9d312 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/WasmAllocator.zig @@ -43,14 +43,13 @@ var frees = [1]usize{0} ** size_class_count; /// For each big size class, points to the freed pointer. var big_frees = [1]usize{0} ** big_size_class_count; -fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 { +fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, return_address: usize) ?[*]u8 { _ = ctx; - _ = len_align; - _ = ra; - if (alignment > wasm.page_size) return error.OutOfMemory; // calm down + _ = return_address; // Make room for the freelist next pointer. + const alignment = @as(usize, 1) << @intCast(Allocator.Log2Align, log2_align); const actual_len = @max(len +| @sizeOf(usize), alignment); - const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return error.OutOfMemory; + const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null; const class = math.log2(slot_size) - min_class; if (class < size_class_count) { const addr = a: { @@ -63,7 +62,8 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) const next_addr = next_addrs[class]; if (next_addr % wasm.page_size == 0) { - const addr = try allocBigPages(1); + const addr = allocBigPages(1); + if (addr == 0) return null; //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ // slot_size, class, addr, //}); @@ -74,77 +74,50 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) break :a next_addr; } }; - return @intToPtr([*]u8, addr)[0..len]; + return @intToPtr([*]u8, addr); } const bigpages_needed = bigPagesNeeded(actual_len); - const addr = try allocBigPages(bigpages_needed); - return @intToPtr([*]u8, addr)[0..len]; + const addr = allocBigPages(bigpages_needed); + return @intToPtr([*]u8, addr); } fn resize( ctx: *anyopaque, buf: []u8, - buf_align: u29, + log2_buf_align: u8, new_len: usize, - len_align: u29, - ra: usize, -) ?usize { + return_address: usize, +) bool { _ = ctx; - _ = len_align; - _ = ra; - // We don't want to move anything from one size class to another. But we can recover bytes - // in between powers of two. + _ = return_address; + // We don't want to move anything from one size class to another, but we + // can recover bytes in between powers of two. + const buf_align = @as(usize, 1) << @intCast(Allocator.Log2Align, log2_buf_align); const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align); const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align); const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len); const old_small_class = math.log2(old_small_slot_size) - min_class; if (old_small_class < size_class_count) { - const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return null; - if (old_small_slot_size == new_small_slot_size) return new_len; - if (new_actual_len >= old_actual_len) return null; - const new_small_class = math.log2(new_small_slot_size) - min_class; - assert(new_small_class < old_small_class); - // Split the small allocation into frees. - var class = old_small_class - 1; - while (true) { - const slot_size = @as(usize, 1) << @intCast(ushift, class + min_class); - const upper_addr = @ptrToInt(buf.ptr) + slot_size; - const node = @intToPtr(*usize, upper_addr + (slot_size - @sizeOf(usize))); - node.* = frees[class]; - frees[class] = upper_addr; - if (class == new_small_class) break; - class -= 1; - } + const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false; + return old_small_slot_size == new_small_slot_size; } else { const old_bigpages_needed = bigPagesNeeded(old_actual_len); const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed); const new_bigpages_needed = bigPagesNeeded(new_actual_len); - const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null; - if (old_big_slot_pages == new_big_slot_pages) return new_len; - if (new_actual_len >= old_actual_len) return null; - - const new_small_slot_size = math.ceilPowerOfTwoAssert(usize, new_actual_len); - if (new_small_slot_size < size_class_count) { - const new_small_class = math.log2(new_small_slot_size) - min_class; - // TODO: push the big allocation into the free list - _ = new_small_class; - } else { - const new_big_class = math.log2(new_big_slot_pages); - // TODO: push the upper area into the free list - _ = new_big_class; - } + const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false; + return old_big_slot_pages == new_big_slot_pages; } - return new_len; } fn free( ctx: *anyopaque, buf: []u8, - buf_align: u29, + log2_buf_align: u8, return_address: usize, ) void { _ = ctx; _ = return_address; + const buf_align = @as(usize, 1) << @intCast(Allocator.Log2Align, log2_buf_align); const actual_len = @max(buf.len + @sizeOf(usize), buf_align); const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len); const class = math.log2(slot_size) - min_class; @@ -168,7 +141,7 @@ inline fn bigPagesNeeded(byte_count: usize) usize { return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size; } -fn allocBigPages(n: usize) !usize { +fn allocBigPages(n: usize) usize { const pow2_pages = math.ceilPowerOfTwoAssert(usize, n); const slot_size_bytes = pow2_pages * bigpage_size; const class = math.log2(pow2_pages); @@ -181,7 +154,7 @@ fn allocBigPages(n: usize) !usize { } const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage); - if (page_index <= 0) return error.OutOfMemory; + if (page_index <= 0) return 0; const addr = @intCast(u32, page_index) * wasm.page_size; return addr; } @@ -259,13 +232,15 @@ test "shrink" { mem.set(u8, slice, 0x11); - slice = test_ally.shrink(slice, 17); + try std.testing.expect(test_ally.resize(slice, 17)); + slice = slice[0..17]; for (slice) |b| { try std.testing.expect(b == 0x11); } - slice = test_ally.shrink(slice, 16); + try std.testing.expect(test_ally.resize(slice, 16)); + slice = slice[0..16]; for (slice) |b| { try std.testing.expect(b == 0x11); @@ -303,11 +278,12 @@ test "shrink large object to large object" { slice[0] = 0x12; slice[60] = 0x34; - slice = test_ally.resize(slice, bigpage_size * 2 + 1) orelse return; + try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1)); + slice = slice[0 .. bigpage_size * 2 + 1]; try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); - slice = test_ally.shrink(slice, bigpage_size * 2 + 1); + try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1)); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34);