From 439667be0476f5bf60f3efbb82a3c4d5aae96ee4 Mon Sep 17 00:00:00 2001 From: Archbirdplus Date: Sun, 20 Oct 2024 14:55:57 -0700 Subject: [PATCH 01/39] runtime page size detection heap.zig: define new default page sizes heap.zig: add min/max_page_size and their options lib/std/c: add miscellaneous declarations heap.zig: add pageSize() and its options switch to new page sizes, especially in GPA/stdlib mem.zig: remove page_size --- lib/fuzzer.zig | 2 +- lib/std/Build/Fuzz/WebServer.zig | 2 +- lib/std/Thread.zig | 6 +- lib/std/c.zig | 59 ++- lib/std/c/solaris.zig | 4 - lib/std/crypto/tlcsprng.zig | 5 +- lib/std/debug.zig | 17 +- lib/std/debug/Dwarf.zig | 10 +- lib/std/debug/Info.zig | 1 - lib/std/debug/MemoryAccessor.zig | 9 +- lib/std/debug/SelfInfo.zig | 6 +- lib/std/dynamic_library.zig | 11 +- lib/std/fifo.zig | 2 +- lib/std/heap.zig | 398 ++++++++++++++++++++- lib/std/heap/PageAllocator.zig | 19 +- lib/std/heap/general_purpose_allocator.zig | 129 ++++--- lib/std/heap/sbrk_allocator.zig | 7 +- lib/std/mem.zig | 37 +- lib/std/mem/Allocator.zig | 2 +- lib/std/os/linux/IoUring.zig | 15 +- lib/std/os/linux/tls.zig | 7 +- lib/std/os/plan9.zig | 4 +- lib/std/os/windows/kernel32.zig | 7 +- lib/std/posix.zig | 19 +- lib/std/process.zig | 2 +- lib/std/start.zig | 2 +- lib/std/std.zig | 4 + lib/std/zip.zig | 2 +- src/Package/Fetch.zig | 2 +- 29 files changed, 614 insertions(+), 176 deletions(-) diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 3c00ec7b48cd..eb37e5e2fdb0 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -480,7 +480,7 @@ pub const MemoryMappedList = struct { /// of this ArrayList in accordance with the respective documentation. In /// all cases, "invalidated" means that the memory has been passed to this /// allocator's resize or free function. - items: []align(std.mem.page_size) volatile u8, + items: []align(std.heap.min_page_size) volatile u8, /// How many bytes this list can hold without allocating additional memory. capacity: usize, diff --git a/lib/std/Build/Fuzz/WebServer.zig b/lib/std/Build/Fuzz/WebServer.zig index ac4336476e97..0563d6782ac2 100644 --- a/lib/std/Build/Fuzz/WebServer.zig +++ b/lib/std/Build/Fuzz/WebServer.zig @@ -41,7 +41,7 @@ const fuzzer_arch_os_abi = "wasm32-freestanding"; const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; const CoverageMap = struct { - mapped_memory: []align(std.mem.page_size) const u8, + mapped_memory: []align(std.heap.min_page_size) const u8, coverage: Coverage, source_locations: []Coverage.SourceLocation, /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. diff --git a/lib/std/Thread.zig b/lib/std/Thread.zig index 4e691de98c89..3be80c1641e5 100644 --- a/lib/std/Thread.zig +++ b/lib/std/Thread.zig @@ -769,7 +769,7 @@ const PosixThreadImpl = struct { // Use the same set of parameters used by the libc-less impl. const stack_size = @max(config.stack_size, 16 * 1024); assert(c.pthread_attr_setstacksize(&attr, stack_size) == .SUCCESS); - assert(c.pthread_attr_setguardsize(&attr, std.mem.page_size) == .SUCCESS); + assert(c.pthread_attr_setguardsize(&attr, std.heap.pageSize()) == .SUCCESS); var handle: c.pthread_t = undefined; switch (c.pthread_create( @@ -1155,7 +1155,7 @@ const LinuxThreadImpl = struct { completion: Completion = Completion.init(.running), child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1), parent_tid: i32 = undefined, - mapped: []align(std.mem.page_size) u8, + mapped: []align(std.heap.min_page_size) u8, /// Calls `munmap(mapped.ptr, mapped.len)` then `exit(1)` without touching the stack (which lives in `mapped.ptr`). /// Ported over from musl libc's pthread detached implementation: @@ -1362,7 +1362,7 @@ const LinuxThreadImpl = struct { }; fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl { - const page_size = std.mem.page_size; + const page_size = std.heap.pageSize(); const Args = @TypeOf(args); const Instance = struct { fn_args: Args, diff --git a/lib/std/c.zig b/lib/std/c.zig index fbd0c1d55cbb..df03f8167369 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -3,7 +3,7 @@ const builtin = @import("builtin"); const c = @This(); const maxInt = std.math.maxInt; const assert = std.debug.assert; -const page_size = std.mem.page_size; +const min_page_size = std.heap.min_page_size; const native_abi = builtin.abi; const native_arch = builtin.cpu.arch; const native_os = builtin.os.tag; @@ -2227,6 +2227,39 @@ pub const SC = switch (native_os) { .linux => linux.SC, else => void, }; + +pub const _SC = switch (native_os) { + .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => enum(c_int) { + PAGESIZE = 29, + }, + .dragonfly => enum(c_int) { + PAGESIZE = 47, + }, + .freebsd => enum(c_int) { + PAGESIZE = 47, + }, + .fuchsia => enum(c_int) { + PAGESIZE = 30, + }, + .haiku => enum(c_int) { + PAGESIZE = 27, + }, + .linux => enum(c_int) { + PAGESIZE = 30, + }, + .netbsd => enum(c_int) { + PAGESIZE = 28, + }, + .openbsd => enum(c_int) { + PAGESIZE = 28, + }, + .solaris, .illumos => enum(c_int) { + PAGESIZE = 11, + NPROCESSORS_ONLN = 15, + }, + else => void, +}; + pub const SEEK = switch (native_os) { .linux => linux.SEEK, .emscripten => emscripten.SEEK, @@ -9232,7 +9265,7 @@ pub extern "c" fn getpwnam(name: [*:0]const u8) ?*passwd; pub extern "c" fn getpwuid(uid: uid_t) ?*passwd; pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int; pub extern "c" fn lseek64(fd: fd_t, offset: i64, whence: c_int) i64; -pub extern "c" fn mmap64(addr: ?*align(std.mem.page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *anyopaque; +pub extern "c" fn mmap64(addr: ?*align(min_page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *anyopaque; pub extern "c" fn open64(path: [*:0]const u8, oflag: O, ...) c_int; pub extern "c" fn openat64(fd: c_int, path: [*:0]const u8, oflag: O, ...) c_int; pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: i64) isize; @@ -9324,13 +9357,13 @@ pub extern "c" fn signalfd(fd: fd_t, mask: *const sigset_t, flags: u32) c_int; pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: *const rlimit, old_limit: *rlimit) c_int; pub extern "c" fn mincore( - addr: *align(std.mem.page_size) anyopaque, + addr: *align(min_page_size) anyopaque, length: usize, vec: [*]u8, ) c_int; pub extern "c" fn madvise( - addr: *align(std.mem.page_size) anyopaque, + addr: *align(min_page_size) anyopaque, length: usize, advice: u32, ) c_int; @@ -9428,6 +9461,10 @@ pub const posix_memalign = switch (native_os) { .dragonfly, .netbsd, .freebsd, .solaris, .openbsd, .linux, .macos, .ios, .tvos, .watchos, .visionos => private.posix_memalign, else => {}, }; +pub const sysconf = switch (native_os) { + .solaris => solaris.sysconf, + else => private.sysconf, +}; pub const sf_hdtr = switch (native_os) { .freebsd, .macos, .ios, .tvos, .watchos, .visionos => extern struct { @@ -9469,9 +9506,9 @@ pub extern "c" fn writev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint) i pub extern "c" fn pwritev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint, offset: off_t) isize; pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize; pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize; -pub extern "c" fn mmap(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque; -pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int; -pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int; +pub extern "c" fn mmap(addr: ?*align(min_page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque; +pub extern "c" fn munmap(addr: *align(min_page_size) const anyopaque, len: usize) c_int; +pub extern "c" fn mprotect(addr: *align(min_page_size) anyopaque, len: usize, prot: c_uint) c_int; pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8) c_int; pub extern "c" fn linkat(oldfd: fd_t, oldpath: [*:0]const u8, newfd: fd_t, newpath: [*:0]const u8, flags: c_int) c_int; pub extern "c" fn unlink(path: [*:0]const u8) c_int; @@ -9823,7 +9860,6 @@ pub const SCM = solaris.SCM; pub const SETCONTEXT = solaris.SETCONTEXT; pub const SETUSTACK = solaris.GETUSTACK; pub const SFD = solaris.SFD; -pub const _SC = solaris._SC; pub const cmsghdr = solaris.cmsghdr; pub const ctid_t = solaris.ctid_t; pub const file_obj = solaris.file_obj; @@ -9840,7 +9876,6 @@ pub const priority = solaris.priority; pub const procfs = solaris.procfs; pub const projid_t = solaris.projid_t; pub const signalfd_siginfo = solaris.signalfd_siginfo; -pub const sysconf = solaris.sysconf; pub const taskid_t = solaris.taskid_t; pub const zoneid_t = solaris.zoneid_t; @@ -9997,6 +10032,7 @@ pub const host_t = darwin.host_t; pub const ipc_space_t = darwin.ipc_space_t; pub const ipc_space_port_t = darwin.ipc_space_port_t; pub const kern_return_t = darwin.kern_return_t; +pub const vm_size_t = darwin.vm_size_t; pub const kevent64 = darwin.kevent64; pub const kevent64_s = darwin.kevent64_s; pub const mach_absolute_time = darwin.mach_absolute_time; @@ -10155,7 +10191,7 @@ const private = struct { }; extern "c" fn getrusage(who: c_int, usage: *rusage) c_int; extern "c" fn gettimeofday(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; - extern "c" fn msync(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int; + extern "c" fn msync(addr: *align(min_page_size) const anyopaque, len: usize, flags: c_int) c_int; extern "c" fn nanosleep(rqtp: *const timespec, rmtp: ?*timespec) c_int; extern "c" fn pipe2(fds: *[2]fd_t, flags: O) c_int; extern "c" fn readdir(dir: *DIR) ?*dirent; @@ -10168,6 +10204,7 @@ const private = struct { extern "c" fn socket(domain: c_uint, sock_type: c_uint, protocol: c_uint) c_int; extern "c" fn stat(noalias path: [*:0]const u8, noalias buf: *Stat) c_int; extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int; + extern "c" fn sysconf(sc: c_int) c_long; extern "c" fn pthread_setname_np(thread: pthread_t, name: [*:0]const u8) c_int; extern "c" fn getcontext(ucp: *ucontext_t) c_int; @@ -10202,7 +10239,7 @@ const private = struct { extern "c" fn __getrusage50(who: c_int, usage: *rusage) c_int; extern "c" fn __gettimeofday50(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; extern "c" fn __libc_thr_yield() c_int; - extern "c" fn __msync13(addr: *align(std.mem.page_size) const anyopaque, len: usize, flags: c_int) c_int; + extern "c" fn __msync13(addr: *align(min_page_size) const anyopaque, len: usize, flags: c_int) c_int; extern "c" fn __nanosleep50(rqtp: *const timespec, rmtp: ?*timespec) c_int; extern "c" fn __sigaction14(sig: c_int, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) c_int; extern "c" fn __sigfillset14(set: ?*sigset_t) void; diff --git a/lib/std/c/solaris.zig b/lib/std/c/solaris.zig index c84478e2c0fa..df7dbca16a64 100644 --- a/lib/std/c/solaris.zig +++ b/lib/std/c/solaris.zig @@ -154,10 +154,6 @@ pub const AF_SUN = struct { pub const NOPLM = 0x00000004; }; -pub const _SC = struct { - pub const NPROCESSORS_ONLN = 15; -}; - pub const procfs = struct { pub const misc_header = extern struct { size: u32, diff --git a/lib/std/crypto/tlcsprng.zig b/lib/std/crypto/tlcsprng.zig index bad4df2ea85b..dbe56a390f11 100644 --- a/lib/std/crypto/tlcsprng.zig +++ b/lib/std/crypto/tlcsprng.zig @@ -6,6 +6,7 @@ const std = @import("std"); const builtin = @import("builtin"); const mem = std.mem; +const heap = std.heap; const native_os = builtin.os.tag; const posix = std.posix; @@ -42,7 +43,7 @@ var install_atfork_handler = std.once(struct { } }.do); -threadlocal var wipe_mem: []align(mem.page_size) u8 = &[_]u8{}; +threadlocal var wipe_mem: []align(heap.min_page_size) u8 = &[_]u8{}; fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void { if (os_has_arc4random) { @@ -77,7 +78,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void { } else { // Use a static thread-local buffer. const S = struct { - threadlocal var buf: Context align(mem.page_size) = .{ + threadlocal var buf: Context align(heap.min_page_size) = .{ .init_state = .uninitialized, .rng = undefined, }; diff --git a/lib/std/debug.zig b/lib/std/debug.zig index 643dcf731acc..a3aacf769eda 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -2,6 +2,7 @@ const builtin = @import("builtin"); const std = @import("std.zig"); const math = std.math; const mem = std.mem; +const heap = std.heap; const io = std.io; const posix = std.posix; const fs = std.fs; @@ -1134,7 +1135,7 @@ fn printLineFromFileAnyOs(out_stream: anytype, source_location: SourceLocation) defer f.close(); // TODO fstat and make sure that the file has the correct size - var buf: [mem.page_size]u8 = undefined; + var buf: [4096]u8 = undefined; var amt_read = try f.read(buf[0..]); const line_start = seek: { var current_line_start: usize = 0; @@ -1237,7 +1238,7 @@ test printLineFromFileAnyOs { const overlap = 10; var writer = file.writer(); - try writer.writeByteNTimes('a', mem.page_size - overlap); + try writer.writeByteNTimes('a', heap.min_page_size - overlap); try writer.writeByte('\n'); try writer.writeByteNTimes('a', overlap); @@ -1252,10 +1253,10 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - try writer.writeByteNTimes('a', mem.page_size); + try writer.writeByteNTimes('a', heap.max_page_size); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** mem.page_size) ++ "\n", output.items); + try expectEqualStrings(("a" ** heap.max_page_size) ++ "\n", output.items); output.clearRetainingCapacity(); } { @@ -1265,18 +1266,18 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - try writer.writeByteNTimes('a', 3 * mem.page_size); + try writer.writeByteNTimes('a', 3 * heap.max_page_size); try expectError(error.EndOfFile, printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 })); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** (3 * mem.page_size)) ++ "\n", output.items); + try expectEqualStrings(("a" ** (3 * heap.max_page_size)) ++ "\n", output.items); output.clearRetainingCapacity(); try writer.writeAll("a\na"); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** (3 * mem.page_size)) ++ "a\n", output.items); + try expectEqualStrings(("a" ** (3 * heap.max_page_size)) ++ "a\n", output.items); output.clearRetainingCapacity(); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 }); @@ -1290,7 +1291,7 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - const real_file_start = 3 * mem.page_size; + const real_file_start = 3 * heap.min_page_size; try writer.writeByteNTimes('\n', real_file_start); try writer.writeAll("abc\ndef"); diff --git a/lib/std/debug/Dwarf.zig b/lib/std/debug/Dwarf.zig index 3469deaae927..d4cd6748985b 100644 --- a/lib/std/debug/Dwarf.zig +++ b/lib/std/debug/Dwarf.zig @@ -2120,8 +2120,8 @@ fn pcRelBase(field_ptr: usize, pc_rel_offset: i64) !usize { pub const ElfModule = struct { base_address: usize, dwarf: Dwarf, - mapped_memory: []align(std.mem.page_size) const u8, - external_mapped_memory: ?[]align(std.mem.page_size) const u8, + mapped_memory: []align(std.heap.min_page_size) const u8, + external_mapped_memory: ?[]align(std.heap.min_page_size) const u8, pub fn deinit(self: *@This(), allocator: Allocator) void { self.dwarf.deinit(allocator); @@ -2167,11 +2167,11 @@ pub const ElfModule = struct { /// sections from an external file. pub fn load( gpa: Allocator, - mapped_mem: []align(std.mem.page_size) const u8, + mapped_mem: []align(std.heap.min_page_size) const u8, build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(std.mem.page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, elf_filename: ?[]const u8, ) LoadError!Dwarf.ElfModule { if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo; @@ -2423,7 +2423,7 @@ pub const ElfModule = struct { build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(std.mem.page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, ) LoadError!Dwarf.ElfModule { const elf_file = elf_file_path.root_dir.handle.openFile(elf_file_path.sub_path, .{}) catch |err| switch (err) { error.FileNotFound => return missing(), diff --git a/lib/std/debug/Info.zig b/lib/std/debug/Info.zig index 0a07d9ba15a2..c809547f7376 100644 --- a/lib/std/debug/Info.zig +++ b/lib/std/debug/Info.zig @@ -10,7 +10,6 @@ const std = @import("../std.zig"); const Allocator = std.mem.Allocator; const Path = std.Build.Cache.Path; const Dwarf = std.debug.Dwarf; -const page_size = std.mem.page_size; const assert = std.debug.assert; const Coverage = std.debug.Coverage; const SourceLocation = std.debug.Coverage.SourceLocation; diff --git a/lib/std/debug/MemoryAccessor.zig b/lib/std/debug/MemoryAccessor.zig index 5f57ad585312..8c9c93b52eab 100644 --- a/lib/std/debug/MemoryAccessor.zig +++ b/lib/std/debug/MemoryAccessor.zig @@ -7,7 +7,7 @@ const native_os = builtin.os.tag; const std = @import("../std.zig"); const posix = std.posix; const File = std.fs.File; -const page_size = std.mem.page_size; +const min_page_size = std.heap.min_page_size; const MemoryAccessor = @This(); @@ -93,9 +93,10 @@ pub fn isValidMemory(address: usize) bool { // We are unable to determine validity of memory for freestanding targets if (native_os == .freestanding or native_os == .other or native_os == .uefi) return true; - const aligned_address = address & ~@as(usize, @intCast((page_size - 1))); + const page_size = std.heap.pageSize(); + const aligned_address = address & ~(page_size - 1); if (aligned_address == 0) return false; - const aligned_memory = @as([*]align(page_size) u8, @ptrFromInt(aligned_address))[0..page_size]; + const aligned_memory = @as([*]align(min_page_size) u8, @ptrFromInt(aligned_address))[0..page_size]; if (native_os == .windows) { const windows = std.os.windows; @@ -104,7 +105,7 @@ pub fn isValidMemory(address: usize) bool { // The only error this function can throw is ERROR_INVALID_PARAMETER. // supply an address that invalid i'll be thrown. - const rc = windows.VirtualQuery(aligned_memory, &memory_info, aligned_memory.len) catch { + const rc = windows.VirtualQuery(@ptrCast(aligned_memory), &memory_info, aligned_memory.len) catch { return false; }; diff --git a/lib/std/debug/SelfInfo.zig b/lib/std/debug/SelfInfo.zig index a2cea70d3745..ee676ecf3e51 100644 --- a/lib/std/debug/SelfInfo.zig +++ b/lib/std/debug/SelfInfo.zig @@ -504,7 +504,7 @@ pub const Module = switch (native_os) { .macos, .ios, .watchos, .tvos, .visionos => struct { base_address: usize, vmaddr_slide: usize, - mapped_memory: []align(mem.page_size) const u8, + mapped_memory: []align(std.heap.min_page_size) const u8, symbols: []const MachoSymbol, strings: [:0]const u8, ofiles: OFileTable, @@ -1046,7 +1046,7 @@ pub fn readElfDebugInfo( build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(mem.page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, ) !Dwarf.ElfModule { nosuspend { const elf_file = (if (elf_filename) |filename| blk: { @@ -1088,7 +1088,7 @@ const MachoSymbol = struct { /// Takes ownership of file, even on error. /// TODO it's weird to take ownership even on error, rework this code. -fn mapWholeFile(file: File) ![]align(mem.page_size) const u8 { +fn mapWholeFile(file: File) ![]align(std.heap.min_page_size) const u8 { nosuspend { defer file.close(); diff --git a/lib/std/dynamic_library.zig b/lib/std/dynamic_library.zig index 8f07db68dab2..38511f7f292d 100644 --- a/lib/std/dynamic_library.zig +++ b/lib/std/dynamic_library.zig @@ -1,6 +1,7 @@ const std = @import("std.zig"); const builtin = @import("builtin"); const mem = std.mem; +const heap = std.heap; const testing = std.testing; const elf = std.elf; const windows = std.os.windows; @@ -143,7 +144,7 @@ pub const ElfDynLib = struct { hashtab: [*]posix.Elf_Symndx, versym: ?[*]elf.Versym, verdef: ?*elf.Verdef, - memory: []align(mem.page_size) u8, + memory: []align(heap.min_page_size) u8, pub const Error = ElfDynLibError; @@ -223,7 +224,7 @@ pub const ElfDynLib = struct { // corresponding to the actual LOAD sections. const file_bytes = try posix.mmap( null, - mem.alignForward(usize, size, mem.page_size), + mem.alignForward(usize, size, heap.pageSize()), posix.PROT.READ, .{ .TYPE = .PRIVATE }, fd, @@ -284,10 +285,10 @@ pub const ElfDynLib = struct { elf.PT_LOAD => { // The VirtAddr may not be page-aligned; in such case there will be // extra nonsense mapped before/after the VirtAddr,MemSiz - const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, mem.page_size) - 1); + const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, heap.pageSize()) - 1); const extra_bytes = (base + ph.p_vaddr) - aligned_addr; - const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, mem.page_size); - const ptr = @as([*]align(mem.page_size) u8, @ptrFromInt(aligned_addr)); + const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, heap.pageSize()); + const ptr = @as([*]align(heap.min_page_size) u8, @ptrFromInt(aligned_addr)); const prot = elfToMmapProt(ph.p_flags); if ((ph.p_flags & elf.PF_W) == 0) { // If it does not need write access, it can be mapped from the fd. diff --git a/lib/std/fifo.zig b/lib/std/fifo.zig index b07e870f0460..fa5357cd6c6c 100644 --- a/lib/std/fifo.zig +++ b/lib/std/fifo.zig @@ -91,7 +91,7 @@ pub fn LinearFifo( mem.copyForwards(T, self.buf[0..self.count], self.buf[self.head..][0..self.count]); self.head = 0; } else { - var tmp: [mem.page_size / 2 / @sizeOf(T)]T = undefined; + var tmp: [4096 / 2 / @sizeOf(T)]T = undefined; while (self.head != 0) { const n = @min(self.head, tmp.len); diff --git a/lib/std/heap.zig b/lib/std/heap.zig index df72786f0fd0..7351c6fa09ec 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -8,6 +8,376 @@ const c = std.c; const Allocator = std.mem.Allocator; const windows = std.os.windows; +const default_min_page_size: ?usize = switch (builtin.os.tag) { + .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { + .x86_64 => 4 << 10, + .aarch64 => 16 << 10, + else => null, + }, + .windows => switch (builtin.cpu.arch) { + // -- + .x86, .x86_64 => 4 << 10, + // SuperH => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + // DEC Alpha => 8 << 10, + // Itanium => 8 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + else => null, + }, + .wasi => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 + .uefi => 4 << 10, + .freebsd => switch (builtin.cpu.arch) { + // FreeBSD/sys/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + else => null, + }, + .netbsd => switch (builtin.cpu.arch) { + // NetBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .riscv32, .riscv64 => 4 << 10, + // Sun-2 + .m68k => 2 << 10, + else => null, + }, + .dragonfly => switch (builtin.cpu.arch) { + .x86, .x86_64 => 4 << 10, + else => null, + }, + .openbsd => switch (builtin.cpu.arch) { + // OpenBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + else => null, + }, + .solaris, .illumos => switch (builtin.cpu.arch) { + // src/uts/*/sys/machparam.h + .x86, .x86_64 => 4 << 10, + .sparc, .sparc64 => 8 << 10, + else => null, + }, + .fuchsia => switch (builtin.cpu.arch) { + // fuchsia/kernel/arch/*/include/arch/defines.h + .x86_64 => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .riscv64 => 4 << 10, + else => null, + }, + // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 + .serenity => 4 << 10, + .haiku => switch (builtin.cpu.arch) { + // haiku/headers/posix/arch/*/limits.h + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + else => null, + }, + .hurd => switch (builtin.cpu.arch) { + // gnumach/*/include/mach/*/vm_param.h + .x86, .x86_64 => 4 << 10, + .aarch64 => null, + else => null, + }, + .plan9 => switch (builtin.cpu.arch) { + // 9front/sys/src/9/*/mem.h + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + .sparc => 4 << 10, + else => null, + }, + .ps3 => switch (builtin.cpu.arch) { + // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html + .powerpc64 => 1 << 20, // 1 MiB + else => null, + }, + .ps4 => switch (builtin.cpu.arch) { + // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 + .x86, .x86_64 => 4 << 10, + else => null, + }, + .ps5 => switch (builtin.cpu.arch) { + // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 + .x86, .x86_64 => 16 << 10, + else => null, + }, + // system/lib/libc/musl/arch/emscripten/bits/limits.h + .emscripten => 64 << 10, + .linux => switch (builtin.cpu.arch) { + // Linux/arch/*/Kconfig + .arc => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .csky => 4 << 10, + .hexagon => 4 << 10, + .loongarch32, .loongarch64 => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + .s390x => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + .xtensa => 4 << 10, + else => null, + }, + .freestanding => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + else => null, +}; + +const default_max_page_size: ?usize = switch (builtin.os.tag) { + .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { + .x86_64 => 4 << 10, + .aarch64 => 16 << 10, + else => null, + }, + .windows => switch (builtin.cpu.arch) { + // -- + .x86, .x86_64 => 4 << 10, + // SuperH => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + // DEC Alpha => 8 << 10, + // Itanium => 8 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + else => null, + }, + .wasi => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 + .uefi => 4 << 10, + .freebsd => switch (builtin.cpu.arch) { + // FreeBSD/sys/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + else => null, + }, + .netbsd => switch (builtin.cpu.arch) { + // NetBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .mips, .mipsel, .mips64, .mips64el => 16 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 16 << 10, + .sparc => 8 << 10, + .sparc64 => 8 << 10, + .riscv32, .riscv64 => 4 << 10, + .m68k => 8 << 10, + else => null, + }, + .dragonfly => switch (builtin.cpu.arch) { + .x86, .x86_64 => 4 << 10, + else => null, + }, + .openbsd => switch (builtin.cpu.arch) { + // OpenBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + .mips64, .mips64el => 16 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + else => null, + }, + .solaris, .illumos => switch (builtin.cpu.arch) { + // src/uts/*/sys/machparam.h + .x86, .x86_64 => 4 << 10, + .sparc, .sparc64 => 8 << 10, + else => null, + }, + .fuchsia => switch (builtin.cpu.arch) { + // fuchsia/kernel/arch/*/include/arch/defines.h + .x86_64 => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .riscv64 => 4 << 10, + else => null, + }, + // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 + .serenity => 4 << 10, + .haiku => switch (builtin.cpu.arch) { + // haiku/headers/posix/arch/*/limits.h + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + else => null, + }, + .hurd => switch (builtin.cpu.arch) { + // gnumach/*/include/mach/*/vm_param.h + .x86, .x86_64 => 4 << 10, + .aarch64 => null, + else => null, + }, + .plan9 => switch (builtin.cpu.arch) { + // 9front/sys/src/9/*/mem.h + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .mips, .mipsel, .mips64, .mips64el => 16 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + .sparc => 4 << 10, + else => null, + }, + .ps3 => switch (builtin.cpu.arch) { + // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html + .powerpc64 => 1 << 20, // 1 MiB + else => null, + }, + .ps4 => switch (builtin.cpu.arch) { + // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 + .x86, .x86_64 => 4 << 10, + else => null, + }, + .ps5 => switch (builtin.cpu.arch) { + // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 + .x86, .x86_64 => 16 << 10, + else => null, + }, + // system/lib/libc/musl/arch/emscripten/bits/limits.h + .emscripten => 64 << 10, + .linux => switch (builtin.cpu.arch) { + // Linux/arch/*/Kconfig + .arc => 16 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .csky => 4 << 10, + .hexagon => 256 << 10, + .loongarch32, .loongarch64 => 64 << 10, + .m68k => 8 << 10, + .mips, .mipsel, .mips64, .mips64el => 64 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 256 << 10, + .riscv32, .riscv64 => 4 << 10, + .s390x => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + .xtensa => 4 << 10, + else => null, + }, + .freestanding => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + else => null, +}; + +/// The compile-time minimum page size that the target might have. +/// All pointers from `mmap` or `VirtualAlloc` are aligned to at least `min_page_size`, but their +/// actual alignment may be much bigger. +/// This value can be overridden via `std.options.min_page_size`. +/// On many systems, the actual page size can only be determined at runtime with `pageSize()`. +pub const min_page_size: usize = std.options.min_page_size orelse (default_min_page_size orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) + @compileError("freestanding/other explicitly has no min_page_size. One can be provided with std.options.min_page_size") +else + @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has no min_page_size. One can be provided with std.options.min_page_size")); + +/// The compile-time maximum page size that the target might have. +/// Targeting a system with a larger page size may require overriding `std.options.max_page_size`, +/// as well as using the linker arugment `-z max-page-size=`. +/// The actual page size can only be determined at runtime with `pageSize()`. +pub const max_page_size: usize = std.options.max_page_size orelse (default_max_page_size orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) + @compileError("freestanding/other explicitly has no max_page_size. One can be provided with std.options.max_page_size") +else + @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has no max_page_size. One can be provided with std.options.max_page_size")); + +/// Returns the system page size. +/// If the page size is comptime-known, `pageSize()` returns it directly. +/// Otherwise, `pageSize()` defers to `std.options.queryPageSizeFn()`. +pub fn pageSize() usize { + if (min_page_size == max_page_size) { + return min_page_size; + } + return std.options.queryPageSizeFn(); +} + +// A cache used by `defaultQueryPageSize()` to avoid repeating syscalls. +var page_size_cache = std.atomic.Value(usize).init(0); + +// The default implementation in `std.options.queryPageSizeFn`. +// The first time it is called, it asserts that the page size is within the comptime bounds. +pub fn defaultQueryPageSize() usize { + var size = page_size_cache.load(.unordered); + if (size > 0) return size; + size = switch (builtin.os.tag) { + .linux => if (builtin.link_libc) @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) else std.os.linux.getauxval(std.elf.AT_PAGESZ), + .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => blk: { + const task_port = std.c.mach_task_self(); + // mach_task_self may fail "if there are any resource failures or other errors". + if (task_port == std.c.TASK_NULL) + break :blk 0; + var info_count = std.c.TASK_VM_INFO_COUNT; + var vm_info: std.c.task_vm_info_data_t = undefined; + vm_info.page_size = 0; + _ = std.c.task_info( + task_port, + std.c.TASK_VM_INFO, + @as(std.c.task_info_t, @ptrCast(&vm_info)), + &info_count, + ); + assert(vm_info.page_size != 0); + break :blk @as(usize, @intCast(vm_info.page_size)); + }, + .windows => blk: { + var info: std.os.windows.SYSTEM_INFO = undefined; + std.os.windows.kernel32.GetSystemInfo(&info); + break :blk info.dwPageSize; + }, + else => if (builtin.link_libc) + if (std.c._SC != void and @hasDecl(std.c._SC, "PAGESIZE")) + @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) + else + @compileError("missing _SC.PAGESIZE declaration for " ++ @tagName(builtin.os.tag) ++ "-" ++ @tagName(builtin.os.tag)) + else if (builtin.os.tag == .freestanding or builtin.os.tag == .other) + @compileError("pageSize on freestanding/other is not supported with the default std.options.queryPageSizeFn") + else + @compileError("pageSize on " ++ @tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " is not supported without linking libc, using the default implementation"), + }; + + assert(size >= min_page_size); + assert(size <= max_page_size); + page_size_cache.store(size, .unordered); + + return size; +} + pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator; pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator; pub const ScopedLoggingAllocator = @import("heap/logging_allocator.zig").ScopedLoggingAllocator; @@ -29,7 +399,7 @@ pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra; pub const MemoryPoolOptions = memory_pool.Options; /// TODO Utilize this on Windows. -pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null; +pub var next_mmap_addr_hint: ?[*]align(min_page_size) u8 = null; const CAllocator = struct { comptime { @@ -256,7 +626,7 @@ pub const wasm_allocator: Allocator = .{ /// Verifies that the adjusted length will still map to the full length pub fn alignPageAllocLen(full_len: usize, len: usize) usize { const aligned_len = mem.alignAllocLen(full_len, len); - assert(mem.alignForward(usize, aligned_len, mem.page_size) == full_len); + assert(mem.alignForward(usize, aligned_len, pageSize()) == full_len); return aligned_len; } @@ -615,13 +985,13 @@ test "PageAllocator" { } if (builtin.os.tag == .windows) { - const slice = try allocator.alignedAlloc(u8, mem.page_size, 128); + const slice = try allocator.alignedAlloc(u8, min_page_size, 128); slice[0] = 0x12; slice[127] = 0x34; allocator.free(slice); } { - var buf = try allocator.alloc(u8, mem.page_size + 1); + var buf = try allocator.alloc(u8, pageSize() + 1); defer allocator.free(buf); buf = try allocator.realloc(buf, 1); // shrink past the page boundary } @@ -824,7 +1194,7 @@ pub fn testAllocatorLargeAlignment(base_allocator: mem.Allocator) !void { var validationAllocator = mem.validationWrap(base_allocator); const allocator = validationAllocator.allocator(); - const large_align: usize = mem.page_size / 2; + const large_align: usize = min_page_size / 2; var align_mask: usize = undefined; align_mask = @shlWithOverflow(~@as(usize, 0), @as(Allocator.Log2Align, @ctz(large_align)))[0]; @@ -857,7 +1227,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void { var fib = FixedBufferAllocator.init(&debug_buffer); const debug_allocator = fib.allocator(); - const alloc_size = mem.page_size * 2 + 50; + const alloc_size = pageSize() * 2 + 50; var slice = try allocator.alignedAlloc(u8, 16, alloc_size); defer allocator.free(slice); @@ -866,7 +1236,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void { // which is 16 pages, hence the 32. This test may require to increase // the size of the allocations feeding the `allocator` parameter if they // fail, because of this high over-alignment we want to have. - while (@intFromPtr(slice.ptr) == mem.alignForward(usize, @intFromPtr(slice.ptr), mem.page_size * 32)) { + while (@intFromPtr(slice.ptr) == mem.alignForward(usize, @intFromPtr(slice.ptr), pageSize() * 32)) { try stuff_to_free.append(slice); slice = try allocator.alignedAlloc(u8, 16, alloc_size); } @@ -881,6 +1251,20 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void { try testing.expect(slice[60] == 0x34); } +test "pageSize() smoke test" { + const size = std.heap.pageSize(); + // Check that pageSize is a power of 2. + std.debug.assert(size & (size - 1) == 0); +} + +test "defaultQueryPageSize() smoke test" { + // queryPageSize() does not always get called by pageSize() + if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; + const size = defaultQueryPageSize(); + // Check that pageSize is a power of 2. + std.debug.assert(size & (size - 1) == 0); +} + test { _ = LoggingAllocator; _ = LogToWriterAllocator; diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index 4188c255285c..1e9058717eaf 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -2,6 +2,7 @@ const std = @import("../std.zig"); const builtin = @import("builtin"); const Allocator = std.mem.Allocator; const mem = std.mem; +const heap = std.heap; const maxInt = std.math.maxInt; const assert = std.debug.assert; const native_os = builtin.os.tag; @@ -18,7 +19,7 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { _ = ra; _ = log2_align; assert(n > 0); - if (n > maxInt(usize) - (mem.page_size - 1)) return null; + if (n > maxInt(usize) - (heap.pageSize() - 1)) return null; if (native_os == .windows) { const addr = windows.VirtualAlloc( @@ -34,7 +35,7 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { return @ptrCast(addr); } - const aligned_len = mem.alignForward(usize, n, mem.page_size); + const aligned_len = mem.alignForward(usize, n, heap.pageSize()); const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered); const slice = posix.mmap( hint, @@ -44,8 +45,8 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { -1, 0, ) catch return null; - assert(mem.isAligned(@intFromPtr(slice.ptr), mem.page_size)); - const new_hint: [*]align(mem.page_size) u8 = @alignCast(slice.ptr + aligned_len); + assert(mem.isAligned(@intFromPtr(slice.ptr), heap.pageSize())); + const new_hint: [*]align(heap.min_page_size) u8 = @alignCast(slice.ptr + aligned_len); _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic); return slice.ptr; } @@ -59,13 +60,13 @@ fn resize( ) bool { _ = log2_buf_align; _ = return_address; - const new_size_aligned = mem.alignForward(usize, new_size, mem.page_size); + const new_size_aligned = mem.alignForward(usize, new_size, heap.pageSize()); if (native_os == .windows) { if (new_size <= buf_unaligned.len) { const base_addr = @intFromPtr(buf_unaligned.ptr); const old_addr_end = base_addr + buf_unaligned.len; - const new_addr_end = mem.alignForward(usize, base_addr + new_size, mem.page_size); + const new_addr_end = mem.alignForward(usize, base_addr + new_size, heap.pageSize()); if (old_addr_end > new_addr_end) { // For shrinking that is not releasing, we will only // decommit the pages not needed anymore. @@ -77,14 +78,14 @@ fn resize( } return true; } - const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, mem.page_size); + const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, heap.pageSize()); if (new_size_aligned <= old_size_aligned) { return true; } return false; } - const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, mem.page_size); + const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, heap.pageSize()); if (new_size_aligned == buf_aligned_len) return true; @@ -107,7 +108,7 @@ fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) v if (native_os == .windows) { windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE); } else { - const buf_aligned_len = mem.alignForward(usize, slice.len, mem.page_size); + const buf_aligned_len = mem.alignForward(usize, slice.len, heap.pageSize()); posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len])); } } diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index c23f8dcd7983..bfb824d4aa97 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -48,7 +48,7 @@ //! //! ## Basic Design: //! -//! Small allocations are divided into buckets: +//! Small allocations are divided into buckets. For a max page size of 4K: //! //! ``` //! index obj_size @@ -75,6 +75,9 @@ //! BucketHeader, followed by "used bits", and two stack traces for each slot //! (allocation trace and free trace). //! +//! The buckets array contains buckets for every size class below `max_page_size`. +//! At runtime, only size classes below `pageSize()` will actually be used for allocations. +//! //! The "used bits" are 1 bit per slot representing whether the slot is used. //! Allocations use the data to iterate to find a free slot. Frees assert that the //! corresponding bit is 1 and set it to 0. @@ -99,11 +102,13 @@ const math = std.math; const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; -const page_size = std.mem.page_size; +const min_page_size = std.heap.min_page_size; +const max_page_size = std.heap.max_page_size; +const pageSize = std.heap.pageSize; const StackTrace = std.builtin.StackTrace; /// Integer type for pointing to slots in a small allocation -const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); +const SlotIndex = std.meta.Int(.unsigned, math.log2(max_page_size) + 1); const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6; const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0; @@ -157,6 +162,9 @@ pub const Config = struct { pub const Check = enum { ok, leak }; +var used_small_bucket_count_cache = std.atomic.Value(usize).init(0); +var largest_used_bucket_object_size_cache = std.atomic.Value(usize).init(0); + /// Default initialization of this struct is deprecated; use `.init` instead. pub fn GeneralPurposeAllocator(comptime config: Config) type { return struct { @@ -206,9 +214,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { pub const Error = mem.Allocator.Error; - const small_bucket_count = math.log2(page_size); + const small_bucket_count = math.log2(max_page_size); const largest_bucket_object_size = 1 << (small_bucket_count - 1); const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); + fn used_small_bucket_count() usize { + const cached = used_small_bucket_count_cache.load(.monotonic); + if (cached != 0) { + return cached; + } + const val = math.log2(pageSize()); + used_small_bucket_count_cache.store(val, .monotonic); + return val; + } + fn largest_used_bucket_object_size() usize { + const cached = largest_used_bucket_object_size_cache.load(.monotonic); + if (cached != 0) { + return cached; + } + const val = @as(usize, 1) << @truncate(used_small_bucket_count() - 1); + largest_used_bucket_object_size_cache.store(val, .monotonic); + return val; + } const bucketCompare = struct { fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order { @@ -261,7 +287,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation const BucketHeader = struct { - page: [*]align(page_size) u8, + page: [*]align(min_page_size) u8, alloc_cursor: SlotIndex, used_count: SlotIndex, @@ -273,14 +299,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (!config.safety) @compileError("requested size is only stored when safety is enabled"); const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(size_class); const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); - const slot_count = @divExact(page_size, size_class); + const slot_count = @divExact(pageSize(), size_class); return sizes[0..slot_count]; } fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 { if (!config.safety) @compileError("requested size is only stored when safety is enabled"); const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class); - const slot_count = @divExact(page_size, size_class); + const slot_count = @divExact(pageSize(), size_class); return aligns_ptr[0..slot_count]; } @@ -312,7 +338,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { /// Only valid for buckets within `empty_buckets`, and relies on the `alloc_cursor` /// of empty buckets being set to `slot_count` when they are added to `empty_buckets` fn emptyBucketSizeClass(bucket: *BucketHeader) usize { - return @divExact(page_size, bucket.alloc_cursor); + return @divExact(pageSize(), bucket.alloc_cursor); } }; @@ -355,13 +381,13 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn bucketAlignsStart(size_class: usize) usize { if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); - const slot_count = @divExact(page_size, size_class); + const slot_count = @divExact(pageSize(), size_class); return bucketRequestedSizesStart(size_class) + (@sizeOf(LargestSizeClassInt) * slot_count); } fn bucketStackFramesStart(size_class: usize) usize { const unaligned_start = if (config.safety) blk: { - const slot_count = @divExact(page_size, size_class); + const slot_count = @divExact(pageSize(), size_class); break :blk bucketAlignsStart(size_class) + slot_count; } else @sizeOf(BucketHeader) + usedBitsCount(size_class); return mem.alignForward( @@ -372,12 +398,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn bucketSize(size_class: usize) usize { - const slot_count = @divExact(page_size, size_class); + const slot_count = @divExact(pageSize(), size_class); return bucketStackFramesStart(size_class) + one_trace_size * traces_per_slot * slot_count; } fn usedBitsCount(size_class: usize) usize { - const slot_count = @divExact(page_size, size_class); + const slot_count = @divExact(pageSize(), size_class); if (slot_count < 8) return 1; return @divExact(slot_count, 8); } @@ -416,7 +442,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { pub fn detectLeaks(self: *Self) bool { var leaks = false; - for (&self.buckets, 0..) |*buckets, bucket_i| { + for (0..used_small_bucket_count()) |bucket_i| { + const buckets = &self.buckets[bucket_i]; if (buckets.root == null) continue; const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i)); const used_bits_count = usedBitsCount(size_class); @@ -464,7 +491,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { var bucket = node.key; if (config.never_unmap) { // free page that was intentionally leaked by never_unmap - self.backing_allocator.free(bucket.page[0..page_size]); + self.backing_allocator.free(bucket.page[0..pageSize()]); } // alloc_cursor was set to slot count when bucket added to empty_buckets self.freeBucket(bucket, bucket.emptyBucketSizeClass()); @@ -531,7 +558,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error!Slot { const bucket_index = math.log2(size_class); var buckets = &self.buckets[bucket_index]; - const slot_count = @divExact(page_size, size_class); + const slot_count = @divExact(pageSize(), size_class); if (self.cur_buckets[bucket_index] == null or self.cur_buckets[bucket_index].?.alloc_cursor == slot_count) { const new_bucket = try self.createBucket(size_class); errdefer self.freeBucket(new_bucket, size_class); @@ -564,7 +591,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { addr: usize, current_bucket: ?*BucketHeader, ) ?*BucketHeader { - const search_page: [*]align(page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, page_size)); + const search_page: [*]align(min_page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, pageSize())); if (current_bucket != null and current_bucket.?.page == search_page) { return current_bucket; } @@ -729,14 +756,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { assert(old_mem.len != 0); const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align); - if (aligned_size > largest_bucket_object_size) { + if (aligned_size > largest_used_bucket_object_size()) { return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr); } const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size); var bucket_index = math.log2(size_class_hint); var size_class: usize = size_class_hint; - const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { + const bucket = while (bucket_index < used_small_bucket_count()) : (bucket_index += 1) { if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| { break bucket; } @@ -847,7 +874,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { assert(old_mem.len != 0); const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align); - if (aligned_size > largest_bucket_object_size) { + if (aligned_size > largest_used_bucket_object_size()) { self.freeLarge(old_mem, log2_old_align, ret_addr); return; } @@ -855,7 +882,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { var bucket_index = math.log2(size_class_hint); var size_class: usize = size_class_hint; - const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { + const bucket = while (bucket_index < used_small_bucket_count()) : (bucket_index += 1) { if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| { break bucket; } @@ -944,14 +971,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { self.cur_buckets[bucket_index] = null; } if (!config.never_unmap) { - self.backing_allocator.free(bucket.page[0..page_size]); + self.backing_allocator.free(bucket.page[0..pageSize()]); } if (!config.retain_metadata) { self.freeBucket(bucket, size_class); self.bucket_node_pool.destroy(node); } else { // move alloc_cursor to end so we can tell size_class later - const slot_count = @divExact(page_size, size_class); + const slot_count = @divExact(pageSize(), size_class); bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count)); var empty_entry = self.empty_buckets.getEntryFor(node.key); empty_entry.set(node); @@ -992,7 +1019,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { ret_addr: usize, ) Allocator.Error![*]u8 { const new_aligned_size = @max(len, @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align))); - if (new_aligned_size > largest_bucket_object_size) { + if (new_aligned_size > largest_used_bucket_object_size()) { try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1); const ptr = self.backing_allocator.rawAlloc(len, log2_ptr_align, ret_addr) orelse return error.OutOfMemory; @@ -1035,7 +1062,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader { - const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size); + const page = try self.backing_allocator.alignedAlloc(u8, min_page_size, pageSize()); errdefer self.backing_allocator.free(page); const bucket_size = bucketSize(size_class); @@ -1179,17 +1206,17 @@ test "large object - grow" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice1 = try allocator.alloc(u8, page_size * 2 - 20); + var slice1 = try allocator.alloc(u8, pageSize() * 2 - 20); defer allocator.free(slice1); const old = slice1; - slice1 = try allocator.realloc(slice1, page_size * 2 - 10); + slice1 = try allocator.realloc(slice1, pageSize() * 2 - 10); try std.testing.expect(slice1.ptr == old.ptr); - slice1 = try allocator.realloc(slice1, page_size * 2); + slice1 = try allocator.realloc(slice1, pageSize() * 2); try std.testing.expect(slice1.ptr == old.ptr); - slice1 = try allocator.realloc(slice1, page_size * 2 + 1); + slice1 = try allocator.realloc(slice1, pageSize() * 2 + 1); } test "realloc small object to large object" { @@ -1203,7 +1230,7 @@ test "realloc small object to large object" { slice[60] = 0x34; // This requires upgrading to a large object - const large_object_size = page_size * 2 + 50; + const large_object_size = pageSize() * 2 + 50; slice = try allocator.realloc(slice, large_object_size); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); @@ -1214,22 +1241,22 @@ test "shrink large object to large object" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, page_size * 2 + 50); + var slice = try allocator.alloc(u8, pageSize() * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[60] = 0x34; - if (!allocator.resize(slice, page_size * 2 + 1)) return; - slice = slice.ptr[0 .. page_size * 2 + 1]; + if (!allocator.resize(slice, pageSize() * 2 + 1)) return; + slice = slice.ptr[0 .. pageSize() * 2 + 1]; try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); - try std.testing.expect(allocator.resize(slice, page_size * 2 + 1)); - slice = slice[0 .. page_size * 2 + 1]; + try std.testing.expect(allocator.resize(slice, pageSize() * 2 + 1)); + slice = slice[0 .. pageSize() * 2 + 1]; try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); - slice = try allocator.realloc(slice, page_size * 2); + slice = try allocator.realloc(slice, pageSize() * 2); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); } @@ -1245,13 +1272,13 @@ test "shrink large object to large object with larger alignment" { var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); const debug_allocator = fba.allocator(); - const alloc_size = page_size * 2 + 50; + const alloc_size = pageSize() * 2 + 50; var slice = try allocator.alignedAlloc(u8, 16, alloc_size); defer allocator.free(slice); const big_alignment: usize = switch (builtin.os.tag) { - .windows => page_size * 32, // Windows aligns to 64K. - else => page_size * 2, + .windows => pageSize() * 32, // Windows aligns to 64K. + else => pageSize() * 2, }; // This loop allocates until we find a page that is not aligned to the big // alignment. Then we shrink the allocation after the loop, but increase the @@ -1277,7 +1304,7 @@ test "realloc large object to small object" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, page_size * 2 + 50); + var slice = try allocator.alloc(u8, pageSize() * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[16] = 0x34; @@ -1319,18 +1346,18 @@ test "realloc large object to larger alignment" { var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); const debug_allocator = fba.allocator(); - var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); + var slice = try allocator.alignedAlloc(u8, 16, pageSize() * 2 + 50); defer allocator.free(slice); const big_alignment: usize = switch (builtin.os.tag) { - .windows => page_size * 32, // Windows aligns to 64K. - else => page_size * 2, + .windows => pageSize() * 32, // Windows aligns to 64K. + else => pageSize() * 2, }; // This loop allocates until we find a page that is not aligned to the big alignment. var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { try stuff_to_free.append(slice); - slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); + slice = try allocator.alignedAlloc(u8, 16, pageSize() * 2 + 50); } while (stuff_to_free.popOrNull()) |item| { allocator.free(item); @@ -1338,15 +1365,15 @@ test "realloc large object to larger alignment" { slice[0] = 0x12; slice[16] = 0x34; - slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100); + slice = try allocator.reallocAdvanced(slice, 32, pageSize() * 2 + 100); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); - slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25); + slice = try allocator.reallocAdvanced(slice, 32, pageSize() * 2 + 25); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); - slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100); + slice = try allocator.reallocAdvanced(slice, big_alignment, pageSize() * 2 + 100); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); } @@ -1362,7 +1389,7 @@ test "large object shrinks to small but allocation fails during shrink" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, page_size * 2 + 50); + var slice = try allocator.alloc(u8, pageSize() * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[3] = 0x34; @@ -1433,7 +1460,7 @@ test "double frees" { try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null) != null); // detect a large allocation double free - const large = try allocator.alloc(u8, 2 * page_size); + const large = try allocator.alloc(u8, 2 * pageSize()); try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr))); try std.testing.expectEqual(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.bytes, large); allocator.free(large); @@ -1442,7 +1469,7 @@ test "double frees" { const normal_small = try allocator.alloc(u8, size_class); defer allocator.free(normal_small); - const normal_large = try allocator.alloc(u8, 2 * page_size); + const normal_large = try allocator.alloc(u8, 2 * pageSize()); defer allocator.free(normal_large); // check that flushing retained metadata doesn't disturb live allocations @@ -1475,8 +1502,8 @@ test "bug 9995 fix, large allocs count requested size not backing size" { var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; const allocator = gpa.allocator(); - var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); - try std.testing.expect(gpa.total_requested_bytes == page_size + 1); + var buf = try allocator.alignedAlloc(u8, 1, pageSize() + 1); + try std.testing.expect(gpa.total_requested_bytes == pageSize() + 1); buf = try allocator.realloc(buf, 1); try std.testing.expect(gpa.total_requested_bytes == 1); buf = try allocator.realloc(buf, 2); diff --git a/lib/std/heap/sbrk_allocator.zig b/lib/std/heap/sbrk_allocator.zig index 08933fed5257..b39f8ac344f5 100644 --- a/lib/std/heap/sbrk_allocator.zig +++ b/lib/std/heap/sbrk_allocator.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const math = std.math; const Allocator = std.mem.Allocator; const mem = std.mem; +const heap = std.heap; const assert = std.debug.assert; pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type { @@ -18,7 +19,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type { const max_usize = math.maxInt(usize); const ushift = math.Log2Int(usize); const bigpage_size = 64 * 1024; - const pages_per_bigpage = bigpage_size / mem.page_size; + const pages_per_bigpage = bigpage_size / heap.pageSize(); const bigpage_count = max_usize / bigpage_size; /// Because of storing free list pointers, the minimum size class is 3. @@ -58,7 +59,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type { } const next_addr = next_addrs[class]; - if (next_addr % mem.page_size == 0) { + if (next_addr % heap.pageSize == 0) { const addr = allocBigPages(1); if (addr == 0) return null; //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ @@ -153,7 +154,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type { big_frees[class] = node.*; return top_free_ptr; } - return sbrk(pow2_pages * pages_per_bigpage * mem.page_size); + return sbrk(pow2_pages * pages_per_bigpage * heap.pageSize()); } }; } diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 9e3f4f3936fe..a0ed952e7ba1 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -8,26 +8,6 @@ const testing = std.testing; const Endian = std.builtin.Endian; const native_endian = builtin.cpu.arch.endian(); -/// Compile time known minimum page size. -/// https://github.com/ziglang/zig/issues/4082 -pub const page_size = switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 * 1024, - .aarch64 => switch (builtin.os.tag) { - .macos, .ios, .watchos, .tvos, .visionos => 16 * 1024, - else => 4 * 1024, - }, - .sparc64 => 8 * 1024, - .loongarch32, .loongarch64 => switch (builtin.os.tag) { - // Linux default KConfig value is 16KiB - .linux => 16 * 1024, - // FIXME: - // There is no other OS supported yet. Use the same value - // as Linux for now. - else => 16 * 1024, - }, - else => 4 * 1024, -}; - /// The standard library currently thoroughly depends on byte size /// being 8 bits. (see the use of u8 throughout allocation code as /// the "byte" type.) Code which depends on this can reference this @@ -1072,12 +1052,13 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co const Block = @Vector(block_len, T); const mask: Block = @splat(sentinel); - comptime std.debug.assert(std.mem.page_size % block_size == 0); + comptime std.debug.assert(std.heap.max_page_size % @sizeOf(Block) == 0); + std.debug.assert(std.heap.pageSize() % @sizeOf(Block) == 0); // First block may be unaligned const start_addr = @intFromPtr(&p[i]); - const offset_in_page = start_addr & (std.mem.page_size - 1); - if (offset_in_page <= std.mem.page_size - block_size) { + const offset_in_page = start_addr & (std.heap.pageSize() - 1); + if (offset_in_page <= std.heap.pageSize() - @sizeOf(Block)) { // Will not read past the end of a page, full block. const block: Block = p[i..][0..block_len].*; const matches = block == mask; @@ -1125,18 +1106,18 @@ test "indexOfSentinel vector paths" { const block_len = std.simd.suggestVectorLength(T) orelse continue; // Allocate three pages so we guarantee a page-crossing address with a full page after - const memory = try allocator.alloc(T, 3 * std.mem.page_size / @sizeOf(T)); + const memory = try allocator.alloc(T, 3 * std.heap.pageSize() / @sizeOf(T)); defer allocator.free(memory); @memset(memory, 0xaa); // Find starting page-alignment = 0 var start: usize = 0; const start_addr = @intFromPtr(&memory); - start += (std.mem.alignForward(usize, start_addr, std.mem.page_size) - start_addr) / @sizeOf(T); - try testing.expect(start < std.mem.page_size / @sizeOf(T)); + start += (std.mem.alignForward(usize, start_addr, std.heap.pageSize()) - start_addr) / @sizeOf(T); + try testing.expect(start < std.heap.pageSize() / @sizeOf(T)); // Validate all sub-block alignments - const search_len = std.mem.page_size / @sizeOf(T); + const search_len = std.heap.pageSize() / @sizeOf(T); memory[start + search_len] = 0; for (0..block_len) |offset| { try testing.expectEqual(search_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start + offset]))); @@ -1144,7 +1125,7 @@ test "indexOfSentinel vector paths" { memory[start + search_len] = 0xaa; // Validate page boundary crossing - const start_page_boundary = start + (std.mem.page_size / @sizeOf(T)); + const start_page_boundary = start + (std.heap.pageSize() / @sizeOf(T)); memory[start_page_boundary + block_len] = 0; for (0..block_len) |offset| { try testing.expectEqual(2 * block_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start_page_boundary - block_len + offset]))); diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index bc3ef028b913..38fb9a5b96b2 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -218,7 +218,7 @@ fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: // The Zig Allocator interface is not intended to solve alignments beyond // the minimum OS page size. For these use cases, the caller must use OS // APIs directly. - comptime assert(alignment <= mem.page_size); + if (!@inComptime() and alignment > std.heap.pageSize()) @panic("Alignment must be smaller than page size."); if (byte_count == 0) { const ptr = comptime std.mem.alignBackward(usize, math.maxInt(usize), alignment); diff --git a/lib/std/os/linux/IoUring.zig b/lib/std/os/linux/IoUring.zig index 731877e5aef1..d01fc6e0ec11 100644 --- a/lib/std/os/linux/IoUring.zig +++ b/lib/std/os/linux/IoUring.zig @@ -3,6 +3,7 @@ const std = @import("std"); const builtin = @import("builtin"); const assert = std.debug.assert; const mem = std.mem; +const heap = std.heap; const net = std.net; const posix = std.posix; const linux = std.os.linux; @@ -1341,8 +1342,8 @@ pub const SubmissionQueue = struct { dropped: *u32, array: []u32, sqes: []linux.io_uring_sqe, - mmap: []align(mem.page_size) u8, - mmap_sqes: []align(mem.page_size) u8, + mmap: []align(heap.min_page_size) u8, + mmap_sqes: []align(heap.min_page_size) u8, // We use `sqe_head` and `sqe_tail` in the same way as liburing: // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`. @@ -1460,7 +1461,7 @@ pub const BufferGroup = struct { /// Pointer to the memory shared by the kernel. /// `buffers_count` of `io_uring_buf` structures are shared by the kernel. /// First `io_uring_buf` is overlaid by `io_uring_buf_ring` struct. - br: *align(mem.page_size) linux.io_uring_buf_ring, + br: *align(heap.min_page_size) linux.io_uring_buf_ring, /// Contiguous block of memory of size (buffers_count * buffer_size). buffers: []u8, /// Size of each buffer in buffers. @@ -1555,7 +1556,7 @@ pub const BufferGroup = struct { /// `fd` is IO_Uring.fd for which the provided buffer ring is being registered. /// `entries` is the number of entries requested in the buffer ring, must be power of 2. /// `group_id` is the chosen buffer group ID, unique in IO_Uring. -pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(mem.page_size) linux.io_uring_buf_ring { +pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(heap.min_page_size) linux.io_uring_buf_ring { if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange; if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo; @@ -1571,7 +1572,7 @@ pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(mem.p errdefer posix.munmap(mmap); assert(mmap.len == mmap_size); - const br: *align(mem.page_size) linux.io_uring_buf_ring = @ptrCast(mmap.ptr); + const br: *align(heap.min_page_size) linux.io_uring_buf_ring = @ptrCast(mmap.ptr); try register_buf_ring(fd, @intFromPtr(br), entries, group_id); return br; } @@ -1613,9 +1614,9 @@ fn handle_register_buf_ring_result(res: usize) !void { } // Unregisters a previously registered shared buffer ring, returned from io_uring_setup_buf_ring. -pub fn free_buf_ring(fd: posix.fd_t, br: *align(mem.page_size) linux.io_uring_buf_ring, entries: u32, group_id: u16) void { +pub fn free_buf_ring(fd: posix.fd_t, br: *align(heap.min_page_size) linux.io_uring_buf_ring, entries: u32, group_id: u16) void { unregister_buf_ring(fd, group_id) catch {}; - var mmap: []align(mem.page_size) u8 = undefined; + var mmap: []align(heap.min_page_size) u8 = undefined; mmap.ptr = @ptrCast(br); mmap.len = entries * @sizeOf(linux.io_uring_buf); posix.munmap(mmap); diff --git a/lib/std/os/linux/tls.zig b/lib/std/os/linux/tls.zig index d1292e86dd03..7917fe9d1bb4 100644 --- a/lib/std/os/linux/tls.zig +++ b/lib/std/os/linux/tls.zig @@ -11,6 +11,7 @@ const std = @import("std"); const mem = std.mem; +const heap = std.heap; const elf = std.elf; const math = std.math; const assert = std.debug.assert; @@ -490,7 +491,7 @@ pub fn prepareArea(area: []u8) usize { // and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I // think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is // equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead. -var main_thread_area_buffer: [0x2100]u8 align(mem.page_size) = undefined; +var main_thread_area_buffer: [0x2100]u8 align(heap.min_page_size) = undefined; /// Computes the layout of the static TLS area, allocates the area, initializes all of its fields, /// and assigns the architecture-specific value to the TP register. @@ -503,7 +504,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void { const area = blk: { // Fast path for the common case where the TLS data is really small, avoid an allocation and // use our local buffer. - if (area_desc.alignment <= mem.page_size and area_desc.size <= main_thread_area_buffer.len) { + if (area_desc.alignment <= heap.min_page_size and area_desc.size <= main_thread_area_buffer.len) { break :blk main_thread_area_buffer[0..area_desc.size]; } @@ -517,7 +518,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void { ); if (@as(isize, @bitCast(begin_addr)) < 0) @trap(); - const area_ptr: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr); + const area_ptr: [*]align(heap.min_page_size) u8 = @ptrFromInt(begin_addr); // Make sure the slice is correctly aligned. const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment); diff --git a/lib/std/os/plan9.zig b/lib/std/os/plan9.zig index 90f45ef7fb48..1882eda47696 100644 --- a/lib/std/os/plan9.zig +++ b/lib/std/os/plan9.zig @@ -367,8 +367,8 @@ pub fn sbrk(n: usize) usize { bloc = @intFromPtr(&ExecData.end); bloc_max = @intFromPtr(&ExecData.end); } - const bl = std.mem.alignForward(usize, bloc, std.mem.page_size); - const n_aligned = std.mem.alignForward(usize, n, std.mem.page_size); + const bl = std.mem.alignForward(usize, bloc, std.heap.pageSize()); + const n_aligned = std.mem.alignForward(usize, n, std.heap.pageSize()); if (bl + n_aligned > bloc_max) { // we need to allocate if (brk_(bl + n_aligned) < 0) return 0; diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index f2da7957a802..a8ff5fb6d32b 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -42,6 +42,7 @@ const WCHAR = windows.WCHAR; const WIN32_FIND_DATAW = windows.WIN32_FIND_DATAW; const Win32Error = windows.Win32Error; const WORD = windows.WORD; +const SYSTEM_INFO = windows.SYSTEM_INFO; // I/O - Filesystem @@ -667,6 +668,6 @@ pub extern "kernel32" fn SetLastError( // TODO: // Wrapper around KUSER_SHARED_DATA.SystemTime. // Much better to use NtQuerySystemTime or NtQuerySystemTimePrecise for guaranteed 0.1ns precision. -pub extern "kernel32" fn GetSystemTimeAsFileTime( - lpSystemTimeAsFileTime: *FILETIME, -) callconv(.winapi) void; +pub extern "kernel32" fn GetSystemTimeAsFileTime(lpSystemTimeAsFileTime: *FILETIME) callconv(.winapi) void; + +pub extern "kernel32" fn GetSystemInfo(lpSystemInfo: *SYSTEM_INFO) callconv(.winapi) void; diff --git a/lib/std/posix.zig b/lib/std/posix.zig index c9e67f7d388c..46cab28e78e8 100644 --- a/lib/std/posix.zig +++ b/lib/std/posix.zig @@ -18,6 +18,7 @@ const builtin = @import("builtin"); const root = @import("root"); const std = @import("std.zig"); const mem = std.mem; +const heap = std.heap; const fs = std.fs; const max_path_bytes = fs.max_path_bytes; const maxInt = std.math.maxInt; @@ -4694,7 +4695,7 @@ pub const MProtectError = error{ OutOfMemory, } || UnexpectedError; -pub fn mprotect(memory: []align(mem.page_size) u8, protection: u32) MProtectError!void { +pub fn mprotect(memory: []align(heap.min_page_size) u8, protection: u32) MProtectError!void { if (native_os == .windows) { const win_prot: windows.DWORD = switch (@as(u3, @truncate(protection))) { 0b000 => windows.PAGE_NOACCESS, @@ -4759,21 +4760,21 @@ pub const MMapError = error{ /// * SIGSEGV - Attempted write into a region mapped as read-only. /// * SIGBUS - Attempted access to a portion of the buffer that does not correspond to the file pub fn mmap( - ptr: ?[*]align(mem.page_size) u8, + ptr: ?[*]align(heap.min_page_size) u8, length: usize, prot: u32, flags: system.MAP, fd: fd_t, offset: u64, -) MMapError![]align(mem.page_size) u8 { +) MMapError![]align(heap.min_page_size) u8 { const mmap_sym = if (lfs64_abi) system.mmap64 else system.mmap; const rc = mmap_sym(ptr, length, prot, @bitCast(flags), fd, @bitCast(offset)); const err: E = if (builtin.link_libc) blk: { - if (rc != std.c.MAP_FAILED) return @as([*]align(mem.page_size) u8, @ptrCast(@alignCast(rc)))[0..length]; + if (rc != std.c.MAP_FAILED) return @as([*]align(heap.min_page_size) u8, @ptrCast(@alignCast(rc)))[0..length]; break :blk @enumFromInt(system._errno().*); } else blk: { const err = errno(rc); - if (err == .SUCCESS) return @as([*]align(mem.page_size) u8, @ptrFromInt(rc))[0..length]; + if (err == .SUCCESS) return @as([*]align(heap.min_page_size) u8, @ptrFromInt(rc))[0..length]; break :blk err; }; switch (err) { @@ -4799,7 +4800,7 @@ pub fn mmap( /// Zig's munmap function does not, for two reasons: /// * It violates the Zig principle that resource deallocation must succeed. /// * The Windows function, VirtualFree, has this restriction. -pub fn munmap(memory: []align(mem.page_size) const u8) void { +pub fn munmap(memory: []align(heap.min_page_size) const u8) void { switch (errno(system.munmap(memory.ptr, memory.len))) { .SUCCESS => return, .INVAL => unreachable, // Invalid parameters. @@ -4813,7 +4814,7 @@ pub const MSyncError = error{ PermissionDenied, } || UnexpectedError; -pub fn msync(memory: []align(mem.page_size) u8, flags: i32) MSyncError!void { +pub fn msync(memory: []align(heap.min_page_size) u8, flags: i32) MSyncError!void { switch (errno(system.msync(memory.ptr, memory.len, flags))) { .SUCCESS => return, .PERM => return error.PermissionDenied, @@ -7135,7 +7136,7 @@ pub const MincoreError = error{ } || UnexpectedError; /// Determine whether pages are resident in memory. -pub fn mincore(ptr: [*]align(mem.page_size) u8, length: usize, vec: [*]u8) MincoreError!void { +pub fn mincore(ptr: [*]align(heap.min_page_size) u8, length: usize, vec: [*]u8) MincoreError!void { return switch (errno(system.mincore(ptr, length, vec))) { .SUCCESS => {}, .AGAIN => error.SystemResources, @@ -7181,7 +7182,7 @@ pub const MadviseError = error{ /// Give advice about use of memory. /// This syscall is optional and is sometimes configured to be disabled. -pub fn madvise(ptr: [*]align(mem.page_size) u8, length: usize, advice: u32) MadviseError!void { +pub fn madvise(ptr: [*]align(heap.min_page_size) u8, length: usize, advice: u32) MadviseError!void { switch (errno(system.madvise(ptr, length, advice))) { .SUCCESS => return, .PERM => return error.PermissionDenied, diff --git a/lib/std/process.zig b/lib/std/process.zig index bc798e68e06f..0e10128e0e22 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -1560,7 +1560,7 @@ pub fn posixGetUserInfo(name: []const u8) !UserInfo { ReadGroupId, }; - var buf: [std.mem.page_size]u8 = undefined; + var buf: [std.heap.min_page_size]u8 = undefined; var name_index: usize = 0; var state = State.Start; var uid: posix.uid_t = 0; diff --git a/lib/std/start.zig b/lib/std/start.zig index 8b0d6183cfea..0163f0054e0f 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -576,7 +576,7 @@ fn expandStackSize(phdrs: []elf.Phdr) void { switch (phdr.p_type) { elf.PT_GNU_STACK => { if (phdr.p_memsz == 0) break; - assert(phdr.p_memsz % std.mem.page_size == 0); + assert(phdr.p_memsz % std.heap.pageSize() == 0); // Silently fail if we are unable to get limits. const limits = std.posix.getrlimit(.STACK) catch break; diff --git a/lib/std/std.zig b/lib/std/std.zig index 5c997aebaf2e..43b163bb2daa 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -119,6 +119,10 @@ pub const Options = struct { args: anytype, ) void = log.defaultLog, + min_page_size: ?usize = null, + max_page_size: ?usize = null, + queryPageSizeFn: fn () usize = heap.defaultQueryPageSize, + fmt_max_depth: usize = fmt.default_max_depth, cryptoRandomSeed: fn (buffer: []u8) void = @import("crypto/tlcsprng.zig").defaultRandomSeed, diff --git a/lib/std/zip.zig b/lib/std/zip.zig index 0717694c4994..c149584fd561 100644 --- a/lib/std/zip.zig +++ b/lib/std/zip.zig @@ -162,7 +162,7 @@ pub fn decompress( var total_uncompressed: u64 = 0; switch (method) { .store => { - var buf: [std.mem.page_size]u8 = undefined; + var buf: [4096]u8 = undefined; while (true) { const len = try reader.read(&buf); if (len == 0) break; diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 4d14b23d9dce..e8e0840f09bf 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1249,7 +1249,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { .{@errorName(err)}, )); defer zip_file.close(); - var buf: [std.mem.page_size]u8 = undefined; + var buf: [std.heap.min_page_size]u8 = undefined; while (true) { const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( "read zip stream failed: {s}", From 284de7d957037c8a7032bd6e2a95bd5f55b73666 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Jan 2025 14:16:25 -0800 Subject: [PATCH 02/39] adjust runtime page size APIs * fix merge conflicts * rename the declarations * reword documentation * extract FixedBufferAllocator to separate file * take advantage of locals * remove the assertion about max alignment in Allocator API, leaving it Allocator implementation defined * fix non-inline function call in start logic The GeneralPurposeAllocator implementation is totally broken because it uses global state but I didn't address that in this commit. --- lib/fuzzer.zig | 2 +- lib/std/Build/Fuzz/WebServer.zig | 2 +- lib/std/Thread.zig | 2 +- lib/std/c.zig | 20 +- lib/std/crypto/tlcsprng.zig | 11 +- lib/std/debug.zig | 15 +- lib/std/debug/Dwarf.zig | 10 +- lib/std/debug/MemoryAccessor.zig | 4 +- lib/std/debug/SelfInfo.zig | 6 +- lib/std/dynamic_library.zig | 13 +- lib/std/heap.zig | 953 ++++++++------------- lib/std/heap/FixedBufferAllocator.zig | 218 +++++ lib/std/heap/PageAllocator.zig | 25 +- lib/std/heap/general_purpose_allocator.zig | 16 +- lib/std/mem.zig | 22 +- lib/std/mem/Allocator.zig | 11 +- lib/std/os/linux/IoUring.zig | 16 +- lib/std/os/linux/tls.zig | 20 +- lib/std/posix.zig | 20 +- lib/std/process.zig | 2 +- lib/std/start.zig | 2 +- lib/std/std.zig | 9 +- src/Package/Fetch.zig | 2 +- 23 files changed, 703 insertions(+), 698 deletions(-) create mode 100644 lib/std/heap/FixedBufferAllocator.zig diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index eb37e5e2fdb0..026b7cbc159e 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -480,7 +480,7 @@ pub const MemoryMappedList = struct { /// of this ArrayList in accordance with the respective documentation. In /// all cases, "invalidated" means that the memory has been passed to this /// allocator's resize or free function. - items: []align(std.heap.min_page_size) volatile u8, + items: []align(std.heap.page_size_min) volatile u8, /// How many bytes this list can hold without allocating additional memory. capacity: usize, diff --git a/lib/std/Build/Fuzz/WebServer.zig b/lib/std/Build/Fuzz/WebServer.zig index 0563d6782ac2..87cd7a1a1deb 100644 --- a/lib/std/Build/Fuzz/WebServer.zig +++ b/lib/std/Build/Fuzz/WebServer.zig @@ -41,7 +41,7 @@ const fuzzer_arch_os_abi = "wasm32-freestanding"; const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; const CoverageMap = struct { - mapped_memory: []align(std.heap.min_page_size) const u8, + mapped_memory: []align(std.heap.page_size_min) const u8, coverage: Coverage, source_locations: []Coverage.SourceLocation, /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. diff --git a/lib/std/Thread.zig b/lib/std/Thread.zig index 3be80c1641e5..6dcb95618497 100644 --- a/lib/std/Thread.zig +++ b/lib/std/Thread.zig @@ -1155,7 +1155,7 @@ const LinuxThreadImpl = struct { completion: Completion = Completion.init(.running), child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1), parent_tid: i32 = undefined, - mapped: []align(std.heap.min_page_size) u8, + mapped: []align(std.heap.page_size_min) u8, /// Calls `munmap(mapped.ptr, mapped.len)` then `exit(1)` without touching the stack (which lives in `mapped.ptr`). /// Ported over from musl libc's pthread detached implementation: diff --git a/lib/std/c.zig b/lib/std/c.zig index df03f8167369..6e1e9beb9f43 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -3,7 +3,7 @@ const builtin = @import("builtin"); const c = @This(); const maxInt = std.math.maxInt; const assert = std.debug.assert; -const min_page_size = std.heap.min_page_size; +const page_size = std.heap.page_size_min; const native_abi = builtin.abi; const native_arch = builtin.cpu.arch; const native_os = builtin.os.tag; @@ -2229,7 +2229,7 @@ pub const SC = switch (native_os) { }; pub const _SC = switch (native_os) { - .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => enum(c_int) { + .driverkit, .ios, .macos, .tvos, .visionos, .watchos => enum(c_int) { PAGESIZE = 29, }, .dragonfly => enum(c_int) { @@ -9265,7 +9265,7 @@ pub extern "c" fn getpwnam(name: [*:0]const u8) ?*passwd; pub extern "c" fn getpwuid(uid: uid_t) ?*passwd; pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int; pub extern "c" fn lseek64(fd: fd_t, offset: i64, whence: c_int) i64; -pub extern "c" fn mmap64(addr: ?*align(min_page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *anyopaque; +pub extern "c" fn mmap64(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *anyopaque; pub extern "c" fn open64(path: [*:0]const u8, oflag: O, ...) c_int; pub extern "c" fn openat64(fd: c_int, path: [*:0]const u8, oflag: O, ...) c_int; pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: i64) isize; @@ -9357,13 +9357,13 @@ pub extern "c" fn signalfd(fd: fd_t, mask: *const sigset_t, flags: u32) c_int; pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: *const rlimit, old_limit: *rlimit) c_int; pub extern "c" fn mincore( - addr: *align(min_page_size) anyopaque, + addr: *align(page_size) anyopaque, length: usize, vec: [*]u8, ) c_int; pub extern "c" fn madvise( - addr: *align(min_page_size) anyopaque, + addr: *align(page_size) anyopaque, length: usize, advice: u32, ) c_int; @@ -9506,9 +9506,9 @@ pub extern "c" fn writev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint) i pub extern "c" fn pwritev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint, offset: off_t) isize; pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize; pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize; -pub extern "c" fn mmap(addr: ?*align(min_page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque; -pub extern "c" fn munmap(addr: *align(min_page_size) const anyopaque, len: usize) c_int; -pub extern "c" fn mprotect(addr: *align(min_page_size) anyopaque, len: usize, prot: c_uint) c_int; +pub extern "c" fn mmap(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque; +pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int; +pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int; pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8) c_int; pub extern "c" fn linkat(oldfd: fd_t, oldpath: [*:0]const u8, newfd: fd_t, newpath: [*:0]const u8, flags: c_int) c_int; pub extern "c" fn unlink(path: [*:0]const u8) c_int; @@ -10191,7 +10191,7 @@ const private = struct { }; extern "c" fn getrusage(who: c_int, usage: *rusage) c_int; extern "c" fn gettimeofday(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; - extern "c" fn msync(addr: *align(min_page_size) const anyopaque, len: usize, flags: c_int) c_int; + extern "c" fn msync(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int; extern "c" fn nanosleep(rqtp: *const timespec, rmtp: ?*timespec) c_int; extern "c" fn pipe2(fds: *[2]fd_t, flags: O) c_int; extern "c" fn readdir(dir: *DIR) ?*dirent; @@ -10239,7 +10239,7 @@ const private = struct { extern "c" fn __getrusage50(who: c_int, usage: *rusage) c_int; extern "c" fn __gettimeofday50(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; extern "c" fn __libc_thr_yield() c_int; - extern "c" fn __msync13(addr: *align(min_page_size) const anyopaque, len: usize, flags: c_int) c_int; + extern "c" fn __msync13(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int; extern "c" fn __nanosleep50(rqtp: *const timespec, rmtp: ?*timespec) c_int; extern "c" fn __sigaction14(sig: c_int, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) c_int; extern "c" fn __sigfillset14(set: ?*sigset_t) void; diff --git a/lib/std/crypto/tlcsprng.zig b/lib/std/crypto/tlcsprng.zig index dbe56a390f11..a1d9beb9b509 100644 --- a/lib/std/crypto/tlcsprng.zig +++ b/lib/std/crypto/tlcsprng.zig @@ -6,7 +6,6 @@ const std = @import("std"); const builtin = @import("builtin"); const mem = std.mem; -const heap = std.heap; const native_os = builtin.os.tag; const posix = std.posix; @@ -43,7 +42,7 @@ var install_atfork_handler = std.once(struct { } }.do); -threadlocal var wipe_mem: []align(heap.min_page_size) u8 = &[_]u8{}; +threadlocal var wipe_mem: []align(std.heap.page_size_min) u8 = &[_]u8{}; fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void { if (os_has_arc4random) { @@ -78,7 +77,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void { } else { // Use a static thread-local buffer. const S = struct { - threadlocal var buf: Context align(heap.min_page_size) = .{ + threadlocal var buf: Context align(std.heap.page_size_min) = .{ .init_state = .uninitialized, .rng = undefined, }; @@ -86,7 +85,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void { wipe_mem = mem.asBytes(&S.buf); } } - const ctx = @as(*Context, @ptrCast(wipe_mem.ptr)); + const ctx: *Context = @ptrCast(wipe_mem.ptr); switch (ctx.init_state) { .uninitialized => { @@ -142,7 +141,7 @@ fn childAtForkHandler() callconv(.c) void { } fn fillWithCsprng(buffer: []u8) void { - const ctx = @as(*Context, @ptrCast(wipe_mem.ptr)); + const ctx: *Context = @ptrCast(wipe_mem.ptr); return ctx.rng.fill(buffer); } @@ -158,7 +157,7 @@ fn initAndFill(buffer: []u8) void { // the `std.options.cryptoRandomSeed` function is provided. std.options.cryptoRandomSeed(&seed); - const ctx = @as(*Context, @ptrCast(wipe_mem.ptr)); + const ctx: *Context = @ptrCast(wipe_mem.ptr); ctx.rng = Rng.init(seed); std.crypto.secureZero(u8, &seed); diff --git a/lib/std/debug.zig b/lib/std/debug.zig index a3aacf769eda..9deca6de4987 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -2,7 +2,6 @@ const builtin = @import("builtin"); const std = @import("std.zig"); const math = std.math; const mem = std.mem; -const heap = std.heap; const io = std.io; const posix = std.posix; const fs = std.fs; @@ -1238,7 +1237,7 @@ test printLineFromFileAnyOs { const overlap = 10; var writer = file.writer(); - try writer.writeByteNTimes('a', heap.min_page_size - overlap); + try writer.writeByteNTimes('a', std.heap.page_size_min - overlap); try writer.writeByte('\n'); try writer.writeByteNTimes('a', overlap); @@ -1253,10 +1252,10 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - try writer.writeByteNTimes('a', heap.max_page_size); + try writer.writeByteNTimes('a', std.heap.page_size_max); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** heap.max_page_size) ++ "\n", output.items); + try expectEqualStrings(("a" ** std.heap.page_size_max) ++ "\n", output.items); output.clearRetainingCapacity(); } { @@ -1266,18 +1265,18 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - try writer.writeByteNTimes('a', 3 * heap.max_page_size); + try writer.writeByteNTimes('a', 3 * std.heap.page_size_max); try expectError(error.EndOfFile, printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 })); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** (3 * heap.max_page_size)) ++ "\n", output.items); + try expectEqualStrings(("a" ** (3 * std.heap.page_size_max)) ++ "\n", output.items); output.clearRetainingCapacity(); try writer.writeAll("a\na"); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** (3 * heap.max_page_size)) ++ "a\n", output.items); + try expectEqualStrings(("a" ** (3 * std.heap.page_size_max)) ++ "a\n", output.items); output.clearRetainingCapacity(); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 }); @@ -1291,7 +1290,7 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - const real_file_start = 3 * heap.min_page_size; + const real_file_start = 3 * std.heap.page_size_min; try writer.writeByteNTimes('\n', real_file_start); try writer.writeAll("abc\ndef"); diff --git a/lib/std/debug/Dwarf.zig b/lib/std/debug/Dwarf.zig index d4cd6748985b..b72ddcac4765 100644 --- a/lib/std/debug/Dwarf.zig +++ b/lib/std/debug/Dwarf.zig @@ -2120,8 +2120,8 @@ fn pcRelBase(field_ptr: usize, pc_rel_offset: i64) !usize { pub const ElfModule = struct { base_address: usize, dwarf: Dwarf, - mapped_memory: []align(std.heap.min_page_size) const u8, - external_mapped_memory: ?[]align(std.heap.min_page_size) const u8, + mapped_memory: []align(std.heap.page_size_min) const u8, + external_mapped_memory: ?[]align(std.heap.page_size_min) const u8, pub fn deinit(self: *@This(), allocator: Allocator) void { self.dwarf.deinit(allocator); @@ -2167,11 +2167,11 @@ pub const ElfModule = struct { /// sections from an external file. pub fn load( gpa: Allocator, - mapped_mem: []align(std.heap.min_page_size) const u8, + mapped_mem: []align(std.heap.page_size_min) const u8, build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8, elf_filename: ?[]const u8, ) LoadError!Dwarf.ElfModule { if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo; @@ -2423,7 +2423,7 @@ pub const ElfModule = struct { build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8, ) LoadError!Dwarf.ElfModule { const elf_file = elf_file_path.root_dir.handle.openFile(elf_file_path.sub_path, .{}) catch |err| switch (err) { error.FileNotFound => return missing(), diff --git a/lib/std/debug/MemoryAccessor.zig b/lib/std/debug/MemoryAccessor.zig index 8c9c93b52eab..7857656554e5 100644 --- a/lib/std/debug/MemoryAccessor.zig +++ b/lib/std/debug/MemoryAccessor.zig @@ -7,7 +7,7 @@ const native_os = builtin.os.tag; const std = @import("../std.zig"); const posix = std.posix; const File = std.fs.File; -const min_page_size = std.heap.min_page_size; +const page_size_min = std.heap.page_size_min; const MemoryAccessor = @This(); @@ -96,7 +96,7 @@ pub fn isValidMemory(address: usize) bool { const page_size = std.heap.pageSize(); const aligned_address = address & ~(page_size - 1); if (aligned_address == 0) return false; - const aligned_memory = @as([*]align(min_page_size) u8, @ptrFromInt(aligned_address))[0..page_size]; + const aligned_memory = @as([*]align(page_size_min) u8, @ptrFromInt(aligned_address))[0..page_size]; if (native_os == .windows) { const windows = std.os.windows; diff --git a/lib/std/debug/SelfInfo.zig b/lib/std/debug/SelfInfo.zig index ee676ecf3e51..b51a8f18d206 100644 --- a/lib/std/debug/SelfInfo.zig +++ b/lib/std/debug/SelfInfo.zig @@ -504,7 +504,7 @@ pub const Module = switch (native_os) { .macos, .ios, .watchos, .tvos, .visionos => struct { base_address: usize, vmaddr_slide: usize, - mapped_memory: []align(std.heap.min_page_size) const u8, + mapped_memory: []align(std.heap.page_size_min) const u8, symbols: []const MachoSymbol, strings: [:0]const u8, ofiles: OFileTable, @@ -1046,7 +1046,7 @@ pub fn readElfDebugInfo( build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8, ) !Dwarf.ElfModule { nosuspend { const elf_file = (if (elf_filename) |filename| blk: { @@ -1088,7 +1088,7 @@ const MachoSymbol = struct { /// Takes ownership of file, even on error. /// TODO it's weird to take ownership even on error, rework this code. -fn mapWholeFile(file: File) ![]align(std.heap.min_page_size) const u8 { +fn mapWholeFile(file: File) ![]align(std.heap.page_size_min) const u8 { nosuspend { defer file.close(); diff --git a/lib/std/dynamic_library.zig b/lib/std/dynamic_library.zig index 38511f7f292d..b31fa5ea4d0b 100644 --- a/lib/std/dynamic_library.zig +++ b/lib/std/dynamic_library.zig @@ -1,7 +1,6 @@ const std = @import("std.zig"); const builtin = @import("builtin"); const mem = std.mem; -const heap = std.heap; const testing = std.testing; const elf = std.elf; const windows = std.os.windows; @@ -144,7 +143,7 @@ pub const ElfDynLib = struct { hashtab: [*]posix.Elf_Symndx, versym: ?[*]elf.Versym, verdef: ?*elf.Verdef, - memory: []align(heap.min_page_size) u8, + memory: []align(std.heap.page_size_min) u8, pub const Error = ElfDynLibError; @@ -220,11 +219,13 @@ pub const ElfDynLib = struct { const stat = try file.stat(); const size = std.math.cast(usize, stat.size) orelse return error.FileTooBig; + const page_size = std.heap.pageSize(); + // This one is to read the ELF info. We do more mmapping later // corresponding to the actual LOAD sections. const file_bytes = try posix.mmap( null, - mem.alignForward(usize, size, heap.pageSize()), + mem.alignForward(usize, size, page_size), posix.PROT.READ, .{ .TYPE = .PRIVATE }, fd, @@ -285,10 +286,10 @@ pub const ElfDynLib = struct { elf.PT_LOAD => { // The VirtAddr may not be page-aligned; in such case there will be // extra nonsense mapped before/after the VirtAddr,MemSiz - const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, heap.pageSize()) - 1); + const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, page_size) - 1); const extra_bytes = (base + ph.p_vaddr) - aligned_addr; - const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, heap.pageSize()); - const ptr = @as([*]align(heap.min_page_size) u8, @ptrFromInt(aligned_addr)); + const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, page_size); + const ptr = @as([*]align(std.heap.page_size_min) u8, @ptrFromInt(aligned_addr)); const prot = elfToMmapProt(ph.p_flags); if ((ph.p_flags & elf.PF_W) == 0) { // If it does not need write access, it can be mapped from the fd. diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 7351c6fa09ec..746fc74bd0c0 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -8,337 +8,79 @@ const c = std.c; const Allocator = std.mem.Allocator; const windows = std.os.windows; -const default_min_page_size: ?usize = switch (builtin.os.tag) { - .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { - .x86_64 => 4 << 10, - .aarch64 => 16 << 10, - else => null, - }, - .windows => switch (builtin.cpu.arch) { - // -- - .x86, .x86_64 => 4 << 10, - // SuperH => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, - // DEC Alpha => 8 << 10, - // Itanium => 8 << 10, - .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, - else => null, - }, - .wasi => switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 << 10, - else => null, - }, - // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 - .uefi => 4 << 10, - .freebsd => switch (builtin.cpu.arch) { - // FreeBSD/sys/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv32, .riscv64 => 4 << 10, - else => null, - }, - .netbsd => switch (builtin.cpu.arch) { - // NetBSD/sys/arch/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .sparc => 4 << 10, - .sparc64 => 8 << 10, - .riscv32, .riscv64 => 4 << 10, - // Sun-2 - .m68k => 2 << 10, - else => null, - }, - .dragonfly => switch (builtin.cpu.arch) { - .x86, .x86_64 => 4 << 10, - else => null, - }, - .openbsd => switch (builtin.cpu.arch) { - // OpenBSD/sys/arch/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, - .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv64 => 4 << 10, - .sparc64 => 8 << 10, - else => null, - }, - .solaris, .illumos => switch (builtin.cpu.arch) { - // src/uts/*/sys/machparam.h - .x86, .x86_64 => 4 << 10, - .sparc, .sparc64 => 8 << 10, - else => null, - }, - .fuchsia => switch (builtin.cpu.arch) { - // fuchsia/kernel/arch/*/include/arch/defines.h - .x86_64 => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .riscv64 => 4 << 10, - else => null, - }, - // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 - .serenity => 4 << 10, - .haiku => switch (builtin.cpu.arch) { - // haiku/headers/posix/arch/*/limits.h - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .m68k => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv64 => 4 << 10, - .sparc64 => 8 << 10, - .x86, .x86_64 => 4 << 10, - else => null, - }, - .hurd => switch (builtin.cpu.arch) { - // gnumach/*/include/mach/*/vm_param.h - .x86, .x86_64 => 4 << 10, - .aarch64 => null, - else => null, - }, - .plan9 => switch (builtin.cpu.arch) { - // 9front/sys/src/9/*/mem.h - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, - .sparc => 4 << 10, - else => null, - }, - .ps3 => switch (builtin.cpu.arch) { - // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html - .powerpc64 => 1 << 20, // 1 MiB - else => null, - }, - .ps4 => switch (builtin.cpu.arch) { - // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 - .x86, .x86_64 => 4 << 10, - else => null, - }, - .ps5 => switch (builtin.cpu.arch) { - // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 - .x86, .x86_64 => 16 << 10, - else => null, - }, - // system/lib/libc/musl/arch/emscripten/bits/limits.h - .emscripten => 64 << 10, - .linux => switch (builtin.cpu.arch) { - // Linux/arch/*/Kconfig - .arc => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .csky => 4 << 10, - .hexagon => 4 << 10, - .loongarch32, .loongarch64 => 4 << 10, - .m68k => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv32, .riscv64 => 4 << 10, - .s390x => 4 << 10, - .sparc => 4 << 10, - .sparc64 => 8 << 10, - .x86, .x86_64 => 4 << 10, - .xtensa => 4 << 10, - else => null, - }, - .freestanding => switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 << 10, - else => null, - }, - else => null, -}; +pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator; +pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator; +pub const ScopedLoggingAllocator = @import("heap/logging_allocator.zig").ScopedLoggingAllocator; +pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").LogToWriterAllocator; +pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator; +pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; +pub const GeneralPurposeAllocatorConfig = @import("heap/general_purpose_allocator.zig").Config; +pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator; +pub const Check = @import("heap/general_purpose_allocator.zig").Check; +pub const WasmAllocator = @import("heap/WasmAllocator.zig"); +pub const PageAllocator = @import("heap/PageAllocator.zig"); +pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); +pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; +pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); -const default_max_page_size: ?usize = switch (builtin.os.tag) { - .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { - .x86_64 => 4 << 10, - .aarch64 => 16 << 10, - else => null, - }, - .windows => switch (builtin.cpu.arch) { - // -- - .x86, .x86_64 => 4 << 10, - // SuperH => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, - // DEC Alpha => 8 << 10, - // Itanium => 8 << 10, - .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, - else => null, - }, - .wasi => switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 << 10, - else => null, - }, - // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 - .uefi => 4 << 10, - .freebsd => switch (builtin.cpu.arch) { - // FreeBSD/sys/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv32, .riscv64 => 4 << 10, - else => null, - }, - .netbsd => switch (builtin.cpu.arch) { - // NetBSD/sys/arch/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 64 << 10, - .mips, .mipsel, .mips64, .mips64el => 16 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 16 << 10, - .sparc => 8 << 10, - .sparc64 => 8 << 10, - .riscv32, .riscv64 => 4 << 10, - .m68k => 8 << 10, - else => null, - }, - .dragonfly => switch (builtin.cpu.arch) { - .x86, .x86_64 => 4 << 10, - else => null, - }, - .openbsd => switch (builtin.cpu.arch) { - // OpenBSD/sys/arch/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, - .mips64, .mips64el => 16 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv64 => 4 << 10, - .sparc64 => 8 << 10, - else => null, - }, - .solaris, .illumos => switch (builtin.cpu.arch) { - // src/uts/*/sys/machparam.h - .x86, .x86_64 => 4 << 10, - .sparc, .sparc64 => 8 << 10, - else => null, - }, - .fuchsia => switch (builtin.cpu.arch) { - // fuchsia/kernel/arch/*/include/arch/defines.h - .x86_64 => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .riscv64 => 4 << 10, - else => null, - }, - // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 - .serenity => 4 << 10, - .haiku => switch (builtin.cpu.arch) { - // haiku/headers/posix/arch/*/limits.h - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .m68k => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv64 => 4 << 10, - .sparc64 => 8 << 10, - .x86, .x86_64 => 4 << 10, - else => null, - }, - .hurd => switch (builtin.cpu.arch) { - // gnumach/*/include/mach/*/vm_param.h - .x86, .x86_64 => 4 << 10, - .aarch64 => null, - else => null, - }, - .plan9 => switch (builtin.cpu.arch) { - // 9front/sys/src/9/*/mem.h - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 64 << 10, - .mips, .mipsel, .mips64, .mips64el => 16 << 10, - .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, - .sparc => 4 << 10, - else => null, - }, - .ps3 => switch (builtin.cpu.arch) { - // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html - .powerpc64 => 1 << 20, // 1 MiB - else => null, - }, - .ps4 => switch (builtin.cpu.arch) { - // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 - .x86, .x86_64 => 4 << 10, - else => null, - }, - .ps5 => switch (builtin.cpu.arch) { - // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 - .x86, .x86_64 => 16 << 10, - else => null, - }, - // system/lib/libc/musl/arch/emscripten/bits/limits.h - .emscripten => 64 << 10, - .linux => switch (builtin.cpu.arch) { - // Linux/arch/*/Kconfig - .arc => 16 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 64 << 10, - .csky => 4 << 10, - .hexagon => 256 << 10, - .loongarch32, .loongarch64 => 64 << 10, - .m68k => 8 << 10, - .mips, .mipsel, .mips64, .mips64el => 64 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 256 << 10, - .riscv32, .riscv64 => 4 << 10, - .s390x => 4 << 10, - .sparc => 4 << 10, - .sparc64 => 8 << 10, - .x86, .x86_64 => 4 << 10, - .xtensa => 4 << 10, - else => null, - }, - .freestanding => switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 << 10, - else => null, - }, - else => null, -}; +const memory_pool = @import("heap/memory_pool.zig"); +pub const MemoryPool = memory_pool.MemoryPool; +pub const MemoryPoolAligned = memory_pool.MemoryPoolAligned; +pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra; +pub const MemoryPoolOptions = memory_pool.Options; + +/// TODO Utilize this on Windows. +pub var next_mmap_addr_hint: ?[*]align(page_size_min) u8 = null; -/// The compile-time minimum page size that the target might have. -/// All pointers from `mmap` or `VirtualAlloc` are aligned to at least `min_page_size`, but their -/// actual alignment may be much bigger. -/// This value can be overridden via `std.options.min_page_size`. -/// On many systems, the actual page size can only be determined at runtime with `pageSize()`. -pub const min_page_size: usize = std.options.min_page_size orelse (default_min_page_size orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) - @compileError("freestanding/other explicitly has no min_page_size. One can be provided with std.options.min_page_size") +/// comptime-known minimum page size of the target. +/// +/// All pointers from `mmap` or `VirtualAlloc` are aligned to at least +/// `page_size_min`, but their actual alignment may be bigger. +/// +/// This value can be overridden via `std.options.page_size_min`. +/// +/// On many systems, the actual page size can only be determined at runtime +/// with `pageSize`. +pub const page_size_min: usize = std.options.page_size_min orelse (page_size_min_default orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) + @compileError("freestanding/other page_size_min must provided with std.options.page_size_min") else - @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has no min_page_size. One can be provided with std.options.min_page_size")); - -/// The compile-time maximum page size that the target might have. -/// Targeting a system with a larger page size may require overriding `std.options.max_page_size`, -/// as well as using the linker arugment `-z max-page-size=`. -/// The actual page size can only be determined at runtime with `pageSize()`. -pub const max_page_size: usize = std.options.max_page_size orelse (default_max_page_size orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) - @compileError("freestanding/other explicitly has no max_page_size. One can be provided with std.options.max_page_size") + @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has unknown page_size_min; populate std.options.page_size_min")); + +/// comptime-known maximum page size of the target. +/// +/// Targeting a system with a larger page size may require overriding +/// `std.options.page_size_max`, as well as providing a corresponding linker +/// option. +/// +/// The actual page size can only be determined at runtime with `pageSize`. +pub const page_size_max: usize = std.options.page_size_max orelse (page_size_max_default orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) + @compileError("freestanding/other page_size_max must provided with std.options.page_size_max") else - @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has no max_page_size. One can be provided with std.options.max_page_size")); - -/// Returns the system page size. -/// If the page size is comptime-known, `pageSize()` returns it directly. -/// Otherwise, `pageSize()` defers to `std.options.queryPageSizeFn()`. -pub fn pageSize() usize { - if (min_page_size == max_page_size) { - return min_page_size; - } - return std.options.queryPageSizeFn(); + @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has unknown page_size_max; populate std.options.page_size_max")); + +/// If the page size is comptime-known, return value is comptime. +/// Otherwise, calls `std.options.queryPageSize` which by default queries the +/// host operating system at runtime. +pub inline fn pageSize() usize { + if (page_size_min == page_size_max) return page_size_min; + return std.options.queryPageSize(); } -// A cache used by `defaultQueryPageSize()` to avoid repeating syscalls. -var page_size_cache = std.atomic.Value(usize).init(0); +test pageSize { + assert(std.math.isPowerOfTwo(pageSize())); +} -// The default implementation in `std.options.queryPageSizeFn`. -// The first time it is called, it asserts that the page size is within the comptime bounds. +/// The default implementation of `std.options.queryPageSize`. +/// Asserts that the page size is within `page_size_min` and `page_size_max` pub fn defaultQueryPageSize() usize { - var size = page_size_cache.load(.unordered); + const global = struct { + var cached_result: std.atomic.Value(usize) = .init(0); + }; + var size = global.cached_result.load(.unordered); if (size > 0) return size; size = switch (builtin.os.tag) { .linux => if (builtin.link_libc) @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) else std.os.linux.getauxval(std.elf.AT_PAGESZ), - .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => blk: { + .driverkit, .ios, .macos, .tvos, .visionos, .watchos => blk: { const task_port = std.c.mach_task_self(); // mach_task_self may fail "if there are any resource failures or other errors". if (task_port == std.c.TASK_NULL) @@ -353,7 +95,7 @@ pub fn defaultQueryPageSize() usize { &info_count, ); assert(vm_info.page_size != 0); - break :blk @as(usize, @intCast(vm_info.page_size)); + break :blk @intCast(vm_info.page_size); }, .windows => blk: { var info: std.os.windows.SYSTEM_INFO = undefined; @@ -361,45 +103,24 @@ pub fn defaultQueryPageSize() usize { break :blk info.dwPageSize; }, else => if (builtin.link_libc) - if (std.c._SC != void and @hasDecl(std.c._SC, "PAGESIZE")) - @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) - else - @compileError("missing _SC.PAGESIZE declaration for " ++ @tagName(builtin.os.tag) ++ "-" ++ @tagName(builtin.os.tag)) + @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) else if (builtin.os.tag == .freestanding or builtin.os.tag == .other) - @compileError("pageSize on freestanding/other is not supported with the default std.options.queryPageSizeFn") + @compileError("unsupported target: freestanding/other") else @compileError("pageSize on " ++ @tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " is not supported without linking libc, using the default implementation"), }; - assert(size >= min_page_size); - assert(size <= max_page_size); - page_size_cache.store(size, .unordered); + assert(size >= page_size_min); + assert(size <= page_size_max); + global.cached_result.store(size, .unordered); return size; } -pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator; -pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator; -pub const ScopedLoggingAllocator = @import("heap/logging_allocator.zig").ScopedLoggingAllocator; -pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").LogToWriterAllocator; -pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator; -pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; -pub const GeneralPurposeAllocatorConfig = @import("heap/general_purpose_allocator.zig").Config; -pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator; -pub const Check = @import("heap/general_purpose_allocator.zig").Check; -pub const WasmAllocator = @import("heap/WasmAllocator.zig"); -pub const PageAllocator = @import("heap/PageAllocator.zig"); -pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); -pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; - -const memory_pool = @import("heap/memory_pool.zig"); -pub const MemoryPool = memory_pool.MemoryPool; -pub const MemoryPoolAligned = memory_pool.MemoryPoolAligned; -pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra; -pub const MemoryPoolOptions = memory_pool.Options; - -/// TODO Utilize this on Windows. -pub var next_mmap_addr_hint: ?[*]align(min_page_size) u8 = null; +test defaultQueryPageSize { + if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; + assert(std.math.isPowerOfTwo(defaultQueryPageSize())); +} const CAllocator = struct { comptime { @@ -623,13 +344,6 @@ pub const wasm_allocator: Allocator = .{ .vtable = &WasmAllocator.vtable, }; -/// Verifies that the adjusted length will still map to the full length -pub fn alignPageAllocLen(full_len: usize, len: usize) usize { - const aligned_len = mem.alignAllocLen(full_len, len); - assert(mem.alignForward(usize, aligned_len, pageSize()) == full_len); - return aligned_len; -} - pub const HeapAllocator = switch (builtin.os.tag) { .windows => struct { heap_handle: ?HeapHandle, @@ -730,145 +444,6 @@ pub const HeapAllocator = switch (builtin.os.tag) { else => @compileError("Unsupported OS"), }; -fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool { - return @intFromPtr(ptr) >= @intFromPtr(container.ptr) and - @intFromPtr(ptr) < (@intFromPtr(container.ptr) + container.len); -} - -fn sliceContainsSlice(container: []u8, slice: []u8) bool { - return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and - (@intFromPtr(slice.ptr) + slice.len) <= (@intFromPtr(container.ptr) + container.len); -} - -pub const FixedBufferAllocator = struct { - end_index: usize, - buffer: []u8, - - pub fn init(buffer: []u8) FixedBufferAllocator { - return FixedBufferAllocator{ - .buffer = buffer, - .end_index = 0, - }; - } - - /// *WARNING* using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe - pub fn allocator(self: *FixedBufferAllocator) Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .free = free, - }, - }; - } - - /// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator` - /// *WARNING* using this at the same time as the interface returned by `allocator` is not thread safe - pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = threadSafeAlloc, - .resize = Allocator.noResize, - .free = Allocator.noFree, - }, - }; - } - - pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool { - return sliceContainsPtr(self.buffer, ptr); - } - - pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool { - return sliceContainsSlice(self.buffer, slice); - } - - /// NOTE: this will not work in all cases, if the last allocation had an adjusted_index - /// then we won't be able to determine what the last allocation was. This is because - /// the alignForward operation done in alloc is not reversible. - pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool { - return buf.ptr + buf.len == self.buffer.ptr + self.end_index; - } - - fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { - const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = ra; - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); - const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null; - const adjusted_index = self.end_index + adjust_off; - const new_end_index = adjusted_index + n; - if (new_end_index > self.buffer.len) return null; - self.end_index = new_end_index; - return self.buffer.ptr + adjusted_index; - } - - fn resize( - ctx: *anyopaque, - buf: []u8, - log2_buf_align: u8, - new_size: usize, - return_address: usize, - ) bool { - const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = log2_buf_align; - _ = return_address; - assert(@inComptime() or self.ownsSlice(buf)); - - if (!self.isLastAllocation(buf)) { - if (new_size > buf.len) return false; - return true; - } - - if (new_size <= buf.len) { - const sub = buf.len - new_size; - self.end_index -= sub; - return true; - } - - const add = new_size - buf.len; - if (add + self.end_index > self.buffer.len) return false; - - self.end_index += add; - return true; - } - - fn free( - ctx: *anyopaque, - buf: []u8, - log2_buf_align: u8, - return_address: usize, - ) void { - const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = log2_buf_align; - _ = return_address; - assert(@inComptime() or self.ownsSlice(buf)); - - if (self.isLastAllocation(buf)) { - self.end_index -= buf.len; - } - } - - fn threadSafeAlloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { - const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = ra; - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); - var end_index = @atomicLoad(usize, &self.end_index, .seq_cst); - while (true) { - const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null; - const adjusted_index = end_index + adjust_off; - const new_end_index = adjusted_index + n; - if (new_end_index > self.buffer.len) return null; - end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse - return self.buffer[adjusted_index..new_end_index].ptr; - } - } - - pub fn reset(self: *FixedBufferAllocator) void { - self.end_index = 0; - } -}; - /// Returns a `StackFallbackAllocator` allocating using either a /// `FixedBufferAllocator` on an array of size `size` and falling back to /// `fallback_allocator` if that fails. @@ -975,7 +550,7 @@ test "raw_c_allocator" { } } -test "PageAllocator" { +test PageAllocator { const allocator = page_allocator; try testAllocator(allocator); try testAllocatorAligned(allocator); @@ -985,7 +560,7 @@ test "PageAllocator" { } if (builtin.os.tag == .windows) { - const slice = try allocator.alignedAlloc(u8, min_page_size, 128); + const slice = try allocator.alignedAlloc(u8, page_size_min, 128); slice[0] = 0x12; slice[127] = 0x34; allocator.free(slice); @@ -997,7 +572,7 @@ test "PageAllocator" { } } -test "HeapAllocator" { +test HeapAllocator { if (builtin.os.tag == .windows) { // https://github.com/ziglang/zig/issues/13702 if (builtin.cpu.arch == .aarch64) return error.SkipZigTest; @@ -1013,7 +588,7 @@ test "HeapAllocator" { } } -test "ArenaAllocator" { +test ArenaAllocator { var arena_allocator = ArenaAllocator.init(page_allocator); defer arena_allocator.deinit(); const allocator = arena_allocator.allocator(); @@ -1024,38 +599,6 @@ test "ArenaAllocator" { try testAllocatorAlignedShrink(allocator); } -var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined; -test "FixedBufferAllocator" { - var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..])); - const allocator = fixed_buffer_allocator.allocator(); - - try testAllocator(allocator); - try testAllocatorAligned(allocator); - try testAllocatorLargeAlignment(allocator); - try testAllocatorAlignedShrink(allocator); -} - -test "FixedBufferAllocator.reset" { - var buf: [8]u8 align(@alignOf(u64)) = undefined; - var fba = FixedBufferAllocator.init(buf[0..]); - const allocator = fba.allocator(); - - const X = 0xeeeeeeeeeeeeeeee; - const Y = 0xffffffffffffffff; - - const x = try allocator.create(u64); - x.* = X; - try testing.expectError(error.OutOfMemory, allocator.create(u64)); - - fba.reset(); - const y = try allocator.create(u64); - y.* = Y; - - // we expect Y to have overwritten X. - try testing.expect(x.* == y.*); - try testing.expect(y.* == Y); -} - test "StackFallbackAllocator" { { var stack_allocator = stackFallback(4096, std.testing.allocator); @@ -1075,46 +618,6 @@ test "StackFallbackAllocator" { } } -test "FixedBufferAllocator Reuse memory on realloc" { - var small_fixed_buffer: [10]u8 = undefined; - // check if we re-use the memory - { - var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]); - const allocator = fixed_buffer_allocator.allocator(); - - const slice0 = try allocator.alloc(u8, 5); - try testing.expect(slice0.len == 5); - const slice1 = try allocator.realloc(slice0, 10); - try testing.expect(slice1.ptr == slice0.ptr); - try testing.expect(slice1.len == 10); - try testing.expectError(error.OutOfMemory, allocator.realloc(slice1, 11)); - } - // check that we don't re-use the memory if it's not the most recent block - { - var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]); - const allocator = fixed_buffer_allocator.allocator(); - - var slice0 = try allocator.alloc(u8, 2); - slice0[0] = 1; - slice0[1] = 2; - const slice1 = try allocator.alloc(u8, 2); - const slice2 = try allocator.realloc(slice0, 4); - try testing.expect(slice0.ptr != slice2.ptr); - try testing.expect(slice1.ptr != slice2.ptr); - try testing.expect(slice2[0] == 1); - try testing.expect(slice2[1] == 2); - } -} - -test "Thread safe FixedBufferAllocator" { - var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); - - try testAllocator(fixed_buffer_allocator.threadSafeAllocator()); - try testAllocatorAligned(fixed_buffer_allocator.threadSafeAllocator()); - try testAllocatorLargeAlignment(fixed_buffer_allocator.threadSafeAllocator()); - try testAllocatorAlignedShrink(fixed_buffer_allocator.threadSafeAllocator()); -} - /// This one should not try alignments that exceed what C malloc can handle. pub fn testAllocator(base_allocator: mem.Allocator) !void { var validationAllocator = mem.validationWrap(base_allocator); @@ -1194,7 +697,7 @@ pub fn testAllocatorLargeAlignment(base_allocator: mem.Allocator) !void { var validationAllocator = mem.validationWrap(base_allocator); const allocator = validationAllocator.allocator(); - const large_align: usize = min_page_size / 2; + const large_align: usize = page_size_min / 2; var align_mask: usize = undefined; align_mask = @shlWithOverflow(~@as(usize, 0), @as(Allocator.Log2Align, @ctz(large_align)))[0]; @@ -1251,19 +754,296 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void { try testing.expect(slice[60] == 0x34); } -test "pageSize() smoke test" { - const size = std.heap.pageSize(); - // Check that pageSize is a power of 2. - std.debug.assert(size & (size - 1) == 0); -} +const page_size_min_default: ?usize = switch (builtin.os.tag) { + .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { + .x86_64 => 4 << 10, + .aarch64 => 16 << 10, + else => null, + }, + .windows => switch (builtin.cpu.arch) { + // -- + .x86, .x86_64 => 4 << 10, + // SuperH => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + // DEC Alpha => 8 << 10, + // Itanium => 8 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + else => null, + }, + .wasi => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 + .uefi => 4 << 10, + .freebsd => switch (builtin.cpu.arch) { + // FreeBSD/sys/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + else => null, + }, + .netbsd => switch (builtin.cpu.arch) { + // NetBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .riscv32, .riscv64 => 4 << 10, + // Sun-2 + .m68k => 2 << 10, + else => null, + }, + .dragonfly => switch (builtin.cpu.arch) { + .x86, .x86_64 => 4 << 10, + else => null, + }, + .openbsd => switch (builtin.cpu.arch) { + // OpenBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + else => null, + }, + .solaris, .illumos => switch (builtin.cpu.arch) { + // src/uts/*/sys/machparam.h + .x86, .x86_64 => 4 << 10, + .sparc, .sparc64 => 8 << 10, + else => null, + }, + .fuchsia => switch (builtin.cpu.arch) { + // fuchsia/kernel/arch/*/include/arch/defines.h + .x86_64 => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .riscv64 => 4 << 10, + else => null, + }, + // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 + .serenity => 4 << 10, + .haiku => switch (builtin.cpu.arch) { + // haiku/headers/posix/arch/*/limits.h + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + else => null, + }, + .hurd => switch (builtin.cpu.arch) { + // gnumach/*/include/mach/*/vm_param.h + .x86, .x86_64 => 4 << 10, + .aarch64 => null, + else => null, + }, + .plan9 => switch (builtin.cpu.arch) { + // 9front/sys/src/9/*/mem.h + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + .sparc => 4 << 10, + else => null, + }, + .ps3 => switch (builtin.cpu.arch) { + // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html + .powerpc64 => 1 << 20, // 1 MiB + else => null, + }, + .ps4 => switch (builtin.cpu.arch) { + // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 + .x86, .x86_64 => 4 << 10, + else => null, + }, + .ps5 => switch (builtin.cpu.arch) { + // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 + .x86, .x86_64 => 16 << 10, + else => null, + }, + // system/lib/libc/musl/arch/emscripten/bits/limits.h + .emscripten => 64 << 10, + .linux => switch (builtin.cpu.arch) { + // Linux/arch/*/Kconfig + .arc => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .csky => 4 << 10, + .hexagon => 4 << 10, + .loongarch32, .loongarch64 => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + .s390x => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + .xtensa => 4 << 10, + else => null, + }, + .freestanding => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + else => null, +}; -test "defaultQueryPageSize() smoke test" { - // queryPageSize() does not always get called by pageSize() - if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; - const size = defaultQueryPageSize(); - // Check that pageSize is a power of 2. - std.debug.assert(size & (size - 1) == 0); -} +const page_size_max_default: ?usize = switch (builtin.os.tag) { + .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { + .x86_64 => 4 << 10, + .aarch64 => 16 << 10, + else => null, + }, + .windows => switch (builtin.cpu.arch) { + // -- + .x86, .x86_64 => 4 << 10, + // SuperH => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + // DEC Alpha => 8 << 10, + // Itanium => 8 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + else => null, + }, + .wasi => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 + .uefi => 4 << 10, + .freebsd => switch (builtin.cpu.arch) { + // FreeBSD/sys/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + else => null, + }, + .netbsd => switch (builtin.cpu.arch) { + // NetBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .mips, .mipsel, .mips64, .mips64el => 16 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 16 << 10, + .sparc => 8 << 10, + .sparc64 => 8 << 10, + .riscv32, .riscv64 => 4 << 10, + .m68k => 8 << 10, + else => null, + }, + .dragonfly => switch (builtin.cpu.arch) { + .x86, .x86_64 => 4 << 10, + else => null, + }, + .openbsd => switch (builtin.cpu.arch) { + // OpenBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + .mips64, .mips64el => 16 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + else => null, + }, + .solaris, .illumos => switch (builtin.cpu.arch) { + // src/uts/*/sys/machparam.h + .x86, .x86_64 => 4 << 10, + .sparc, .sparc64 => 8 << 10, + else => null, + }, + .fuchsia => switch (builtin.cpu.arch) { + // fuchsia/kernel/arch/*/include/arch/defines.h + .x86_64 => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .riscv64 => 4 << 10, + else => null, + }, + // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 + .serenity => 4 << 10, + .haiku => switch (builtin.cpu.arch) { + // haiku/headers/posix/arch/*/limits.h + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + else => null, + }, + .hurd => switch (builtin.cpu.arch) { + // gnumach/*/include/mach/*/vm_param.h + .x86, .x86_64 => 4 << 10, + .aarch64 => null, + else => null, + }, + .plan9 => switch (builtin.cpu.arch) { + // 9front/sys/src/9/*/mem.h + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .mips, .mipsel, .mips64, .mips64el => 16 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + .sparc => 4 << 10, + else => null, + }, + .ps3 => switch (builtin.cpu.arch) { + // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html + .powerpc64 => 1 << 20, // 1 MiB + else => null, + }, + .ps4 => switch (builtin.cpu.arch) { + // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 + .x86, .x86_64 => 4 << 10, + else => null, + }, + .ps5 => switch (builtin.cpu.arch) { + // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 + .x86, .x86_64 => 16 << 10, + else => null, + }, + // system/lib/libc/musl/arch/emscripten/bits/limits.h + .emscripten => 64 << 10, + .linux => switch (builtin.cpu.arch) { + // Linux/arch/*/Kconfig + .arc => 16 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .csky => 4 << 10, + .hexagon => 256 << 10, + .loongarch32, .loongarch64 => 64 << 10, + .m68k => 8 << 10, + .mips, .mipsel, .mips64, .mips64el => 64 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 256 << 10, + .riscv32, .riscv64 => 4 << 10, + .s390x => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + .xtensa => 4 << 10, + else => null, + }, + .freestanding => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + else => null, +}; test { _ = LoggingAllocator; @@ -1272,6 +1052,7 @@ test { _ = @import("heap/memory_pool.zig"); _ = ArenaAllocator; _ = GeneralPurposeAllocator; + _ = FixedBufferAllocator; if (builtin.target.isWasm()) { _ = WasmAllocator; } diff --git a/lib/std/heap/FixedBufferAllocator.zig b/lib/std/heap/FixedBufferAllocator.zig new file mode 100644 index 000000000000..5995dfe154aa --- /dev/null +++ b/lib/std/heap/FixedBufferAllocator.zig @@ -0,0 +1,218 @@ +const std = @import("../std.zig"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const mem = std.mem; + +const FixedBufferAllocator = @This(); + +end_index: usize, +buffer: []u8, + +pub fn init(buffer: []u8) FixedBufferAllocator { + return FixedBufferAllocator{ + .buffer = buffer, + .end_index = 0, + }; +} + +/// Using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe. +pub fn allocator(self: *FixedBufferAllocator) Allocator { + return .{ + .ptr = self, + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .free = free, + }, + }; +} + +/// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator` +/// +/// Using this at the same time as the interface returned by `allocator` is not thread safe. +pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator { + return .{ + .ptr = self, + .vtable = &.{ + .alloc = threadSafeAlloc, + .resize = Allocator.noResize, + .free = Allocator.noFree, + }, + }; +} + +pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool { + return sliceContainsPtr(self.buffer, ptr); +} + +pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool { + return sliceContainsSlice(self.buffer, slice); +} + +/// This has false negatives when the last allocation had an +/// adjusted_index. In such case we won't be able to determine what the +/// last allocation was because the alignForward operation done in alloc is +/// not reversible. +pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool { + return buf.ptr + buf.len == self.buffer.ptr + self.end_index; +} + +pub fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { + const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = ra; + const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); + const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null; + const adjusted_index = self.end_index + adjust_off; + const new_end_index = adjusted_index + n; + if (new_end_index > self.buffer.len) return null; + self.end_index = new_end_index; + return self.buffer.ptr + adjusted_index; +} + +pub fn resize( + ctx: *anyopaque, + buf: []u8, + log2_buf_align: u8, + new_size: usize, + return_address: usize, +) bool { + const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = log2_buf_align; + _ = return_address; + assert(@inComptime() or self.ownsSlice(buf)); + + if (!self.isLastAllocation(buf)) { + if (new_size > buf.len) return false; + return true; + } + + if (new_size <= buf.len) { + const sub = buf.len - new_size; + self.end_index -= sub; + return true; + } + + const add = new_size - buf.len; + if (add + self.end_index > self.buffer.len) return false; + + self.end_index += add; + return true; +} + +pub fn free( + ctx: *anyopaque, + buf: []u8, + log2_buf_align: u8, + return_address: usize, +) void { + const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = log2_buf_align; + _ = return_address; + assert(@inComptime() or self.ownsSlice(buf)); + + if (self.isLastAllocation(buf)) { + self.end_index -= buf.len; + } +} + +fn threadSafeAlloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { + const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = ra; + const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); + var end_index = @atomicLoad(usize, &self.end_index, .seq_cst); + while (true) { + const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null; + const adjusted_index = end_index + adjust_off; + const new_end_index = adjusted_index + n; + if (new_end_index > self.buffer.len) return null; + end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse + return self.buffer[adjusted_index..new_end_index].ptr; + } +} + +pub fn reset(self: *FixedBufferAllocator) void { + self.end_index = 0; +} + +fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool { + return @intFromPtr(ptr) >= @intFromPtr(container.ptr) and + @intFromPtr(ptr) < (@intFromPtr(container.ptr) + container.len); +} + +fn sliceContainsSlice(container: []u8, slice: []u8) bool { + return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and + (@intFromPtr(slice.ptr) + slice.len) <= (@intFromPtr(container.ptr) + container.len); +} + +var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined; + +test FixedBufferAllocator { + var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..])); + const a = fixed_buffer_allocator.allocator(); + + try std.heap.testAllocator(a); + try std.heap.testAllocatorAligned(a); + try std.heap.testAllocatorLargeAlignment(a); + try std.heap.testAllocatorAlignedShrink(a); +} + +test reset { + var buf: [8]u8 align(@alignOf(u64)) = undefined; + var fba = FixedBufferAllocator.init(buf[0..]); + const a = fba.allocator(); + + const X = 0xeeeeeeeeeeeeeeee; + const Y = 0xffffffffffffffff; + + const x = try a.create(u64); + x.* = X; + try std.testing.expectError(error.OutOfMemory, a.create(u64)); + + fba.reset(); + const y = try a.create(u64); + y.* = Y; + + // we expect Y to have overwritten X. + try std.testing.expect(x.* == y.*); + try std.testing.expect(y.* == Y); +} + +test "reuse memory on realloc" { + var small_fixed_buffer: [10]u8 = undefined; + // check if we re-use the memory + { + var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]); + const a = fixed_buffer_allocator.allocator(); + + const slice0 = try a.alloc(u8, 5); + try std.testing.expect(slice0.len == 5); + const slice1 = try a.realloc(slice0, 10); + try std.testing.expect(slice1.ptr == slice0.ptr); + try std.testing.expect(slice1.len == 10); + try std.testing.expectError(error.OutOfMemory, a.realloc(slice1, 11)); + } + // check that we don't re-use the memory if it's not the most recent block + { + var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]); + const a = fixed_buffer_allocator.allocator(); + + var slice0 = try a.alloc(u8, 2); + slice0[0] = 1; + slice0[1] = 2; + const slice1 = try a.alloc(u8, 2); + const slice2 = try a.realloc(slice0, 4); + try std.testing.expect(slice0.ptr != slice2.ptr); + try std.testing.expect(slice1.ptr != slice2.ptr); + try std.testing.expect(slice2[0] == 1); + try std.testing.expect(slice2[1] == 2); + } +} + +test "thread safe version" { + var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); + + try std.heap.testAllocator(fixed_buffer_allocator.threadSafeAllocator()); + try std.heap.testAllocatorAligned(fixed_buffer_allocator.threadSafeAllocator()); + try std.heap.testAllocatorLargeAlignment(fixed_buffer_allocator.threadSafeAllocator()); + try std.heap.testAllocatorAlignedShrink(fixed_buffer_allocator.threadSafeAllocator()); +} diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index 1e9058717eaf..c261dcec43bf 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -2,14 +2,14 @@ const std = @import("../std.zig"); const builtin = @import("builtin"); const Allocator = std.mem.Allocator; const mem = std.mem; -const heap = std.heap; const maxInt = std.math.maxInt; const assert = std.debug.assert; const native_os = builtin.os.tag; const windows = std.os.windows; const posix = std.posix; +const page_size_min = std.heap.page_size_min; -pub const vtable = Allocator.VTable{ +pub const vtable: Allocator.VTable = .{ .alloc = alloc, .resize = resize, .free = free, @@ -19,7 +19,6 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { _ = ra; _ = log2_align; assert(n > 0); - if (n > maxInt(usize) - (heap.pageSize() - 1)) return null; if (native_os == .windows) { const addr = windows.VirtualAlloc( @@ -35,7 +34,10 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { return @ptrCast(addr); } - const aligned_len = mem.alignForward(usize, n, heap.pageSize()); + const page_size = std.heap.pageSize(); + if (n >= maxInt(usize) - page_size) return null; + + const aligned_len = mem.alignForward(usize, n, page_size); const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered); const slice = posix.mmap( hint, @@ -45,8 +47,8 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { -1, 0, ) catch return null; - assert(mem.isAligned(@intFromPtr(slice.ptr), heap.pageSize())); - const new_hint: [*]align(heap.min_page_size) u8 = @alignCast(slice.ptr + aligned_len); + assert(mem.isAligned(@intFromPtr(slice.ptr), page_size_min)); + const new_hint: [*]align(std.heap.page_size_min) u8 = @alignCast(slice.ptr + aligned_len); _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic); return slice.ptr; } @@ -60,13 +62,14 @@ fn resize( ) bool { _ = log2_buf_align; _ = return_address; - const new_size_aligned = mem.alignForward(usize, new_size, heap.pageSize()); + const page_size = std.heap.pageSize(); + const new_size_aligned = mem.alignForward(usize, new_size, page_size); if (native_os == .windows) { if (new_size <= buf_unaligned.len) { const base_addr = @intFromPtr(buf_unaligned.ptr); const old_addr_end = base_addr + buf_unaligned.len; - const new_addr_end = mem.alignForward(usize, base_addr + new_size, heap.pageSize()); + const new_addr_end = mem.alignForward(usize, base_addr + new_size, page_size); if (old_addr_end > new_addr_end) { // For shrinking that is not releasing, we will only // decommit the pages not needed anymore. @@ -78,14 +81,14 @@ fn resize( } return true; } - const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, heap.pageSize()); + const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, page_size); if (new_size_aligned <= old_size_aligned) { return true; } return false; } - const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, heap.pageSize()); + const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, page_size); if (new_size_aligned == buf_aligned_len) return true; @@ -108,7 +111,7 @@ fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) v if (native_os == .windows) { windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE); } else { - const buf_aligned_len = mem.alignForward(usize, slice.len, heap.pageSize()); + const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize()); posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len])); } } diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index bfb824d4aa97..4a03dfecb88a 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -75,7 +75,7 @@ //! BucketHeader, followed by "used bits", and two stack traces for each slot //! (allocation trace and free trace). //! -//! The buckets array contains buckets for every size class below `max_page_size`. +//! The buckets array contains buckets for every size class below `page_size_max`. //! At runtime, only size classes below `pageSize()` will actually be used for allocations. //! //! The "used bits" are 1 bit per slot representing whether the slot is used. @@ -102,13 +102,13 @@ const math = std.math; const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; -const min_page_size = std.heap.min_page_size; -const max_page_size = std.heap.max_page_size; +const page_size_min = std.heap.page_size_min; +const page_size_max = std.heap.page_size_max; const pageSize = std.heap.pageSize; const StackTrace = std.builtin.StackTrace; /// Integer type for pointing to slots in a small allocation -const SlotIndex = std.meta.Int(.unsigned, math.log2(max_page_size) + 1); +const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size_max) + 1); const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6; const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0; @@ -214,7 +214,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { pub const Error = mem.Allocator.Error; - const small_bucket_count = math.log2(max_page_size); + const small_bucket_count = math.log2(page_size_max); const largest_bucket_object_size = 1 << (small_bucket_count - 1); const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); fn used_small_bucket_count() usize { @@ -287,7 +287,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation const BucketHeader = struct { - page: [*]align(min_page_size) u8, + page: [*]align(page_size_min) u8, alloc_cursor: SlotIndex, used_count: SlotIndex, @@ -591,7 +591,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { addr: usize, current_bucket: ?*BucketHeader, ) ?*BucketHeader { - const search_page: [*]align(min_page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, pageSize())); + const search_page: [*]align(page_size_min) u8 = @ptrFromInt(mem.alignBackward(usize, addr, pageSize())); if (current_bucket != null and current_bucket.?.page == search_page) { return current_bucket; } @@ -1062,7 +1062,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader { - const page = try self.backing_allocator.alignedAlloc(u8, min_page_size, pageSize()); + const page = try self.backing_allocator.alignedAlloc(u8, page_size_min, pageSize()); errdefer self.backing_allocator.free(page); const bucket_size = bucketSize(size_class); diff --git a/lib/std/mem.zig b/lib/std/mem.zig index a0ed952e7ba1..f0531bab1ea5 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -1048,17 +1048,18 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co // as we don't read into a new page. This should be the case for most architectures // which use paged memory, however should be confirmed before adding a new arch below. .aarch64, .x86, .x86_64 => if (std.simd.suggestVectorLength(T)) |block_len| { + const page_size = std.heap.pageSize(); const block_size = @sizeOf(T) * block_len; const Block = @Vector(block_len, T); const mask: Block = @splat(sentinel); - comptime std.debug.assert(std.heap.max_page_size % @sizeOf(Block) == 0); - std.debug.assert(std.heap.pageSize() % @sizeOf(Block) == 0); + comptime assert(std.heap.page_size_max % @sizeOf(Block) == 0); + assert(page_size % @sizeOf(Block) == 0); // First block may be unaligned const start_addr = @intFromPtr(&p[i]); - const offset_in_page = start_addr & (std.heap.pageSize() - 1); - if (offset_in_page <= std.heap.pageSize() - @sizeOf(Block)) { + const offset_in_page = start_addr & (page_size - 1); + if (offset_in_page <= page_size - @sizeOf(Block)) { // Will not read past the end of a page, full block. const block: Block = p[i..][0..block_len].*; const matches = block == mask; @@ -1078,7 +1079,7 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co } } - std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size)); + assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size)); while (true) { const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len])); const matches = block.* == mask; @@ -1101,23 +1102,24 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co test "indexOfSentinel vector paths" { const Types = [_]type{ u8, u16, u32, u64 }; const allocator = std.testing.allocator; + const page_size = std.heap.pageSize(); inline for (Types) |T| { const block_len = std.simd.suggestVectorLength(T) orelse continue; // Allocate three pages so we guarantee a page-crossing address with a full page after - const memory = try allocator.alloc(T, 3 * std.heap.pageSize() / @sizeOf(T)); + const memory = try allocator.alloc(T, 3 * page_size / @sizeOf(T)); defer allocator.free(memory); @memset(memory, 0xaa); // Find starting page-alignment = 0 var start: usize = 0; const start_addr = @intFromPtr(&memory); - start += (std.mem.alignForward(usize, start_addr, std.heap.pageSize()) - start_addr) / @sizeOf(T); - try testing.expect(start < std.heap.pageSize() / @sizeOf(T)); + start += (std.mem.alignForward(usize, start_addr, page_size) - start_addr) / @sizeOf(T); + try testing.expect(start < page_size / @sizeOf(T)); // Validate all sub-block alignments - const search_len = std.heap.pageSize() / @sizeOf(T); + const search_len = page_size / @sizeOf(T); memory[start + search_len] = 0; for (0..block_len) |offset| { try testing.expectEqual(search_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start + offset]))); @@ -1125,7 +1127,7 @@ test "indexOfSentinel vector paths" { memory[start + search_len] = 0xaa; // Validate page boundary crossing - const start_page_boundary = start + (std.heap.pageSize() / @sizeOf(T)); + const start_page_boundary = start + (page_size / @sizeOf(T)); memory[start_page_boundary + block_len] = 0; for (0..block_len) |offset| { try testing.expectEqual(2 * block_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start_page_boundary - block_len + offset]))); diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index 38fb9a5b96b2..7f3334f1d2e6 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -18,11 +18,15 @@ ptr: *anyopaque, vtable: *const VTable, pub const VTable = struct { - /// Attempt to allocate exactly `len` bytes aligned to `1 << ptr_align`. + /// Allocate exactly `len` bytes aligned to `1 << ptr_align`, or return `null` + /// indicating the allocation failed. /// /// `ret_addr` is optionally provided as the first return address of the /// allocation call stack. If the value is `0` it means no return address /// has been provided. + /// + /// The returned slice of memory must have been `@memset` to `undefined` + /// by the allocator implementation. alloc: *const fn (ctx: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8, /// Attempt to expand or shrink memory in place. `buf.len` must equal the @@ -215,11 +219,6 @@ fn allocWithSizeAndAlignment(self: Allocator, comptime size: usize, comptime ali } fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: usize, return_address: usize) Error![*]align(alignment) u8 { - // The Zig Allocator interface is not intended to solve alignments beyond - // the minimum OS page size. For these use cases, the caller must use OS - // APIs directly. - if (!@inComptime() and alignment > std.heap.pageSize()) @panic("Alignment must be smaller than page size."); - if (byte_count == 0) { const ptr = comptime std.mem.alignBackward(usize, math.maxInt(usize), alignment); return @as([*]align(alignment) u8, @ptrFromInt(ptr)); diff --git a/lib/std/os/linux/IoUring.zig b/lib/std/os/linux/IoUring.zig index d01fc6e0ec11..2e43709075aa 100644 --- a/lib/std/os/linux/IoUring.zig +++ b/lib/std/os/linux/IoUring.zig @@ -3,12 +3,12 @@ const std = @import("std"); const builtin = @import("builtin"); const assert = std.debug.assert; const mem = std.mem; -const heap = std.heap; const net = std.net; const posix = std.posix; const linux = std.os.linux; const testing = std.testing; const is_linux = builtin.os.tag == .linux; +const page_size_min = std.heap.page_size_min; fd: posix.fd_t = -1, sq: SubmissionQueue, @@ -1342,8 +1342,8 @@ pub const SubmissionQueue = struct { dropped: *u32, array: []u32, sqes: []linux.io_uring_sqe, - mmap: []align(heap.min_page_size) u8, - mmap_sqes: []align(heap.min_page_size) u8, + mmap: []align(page_size_min) u8, + mmap_sqes: []align(page_size_min) u8, // We use `sqe_head` and `sqe_tail` in the same way as liburing: // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`. @@ -1461,7 +1461,7 @@ pub const BufferGroup = struct { /// Pointer to the memory shared by the kernel. /// `buffers_count` of `io_uring_buf` structures are shared by the kernel. /// First `io_uring_buf` is overlaid by `io_uring_buf_ring` struct. - br: *align(heap.min_page_size) linux.io_uring_buf_ring, + br: *align(page_size_min) linux.io_uring_buf_ring, /// Contiguous block of memory of size (buffers_count * buffer_size). buffers: []u8, /// Size of each buffer in buffers. @@ -1556,7 +1556,7 @@ pub const BufferGroup = struct { /// `fd` is IO_Uring.fd for which the provided buffer ring is being registered. /// `entries` is the number of entries requested in the buffer ring, must be power of 2. /// `group_id` is the chosen buffer group ID, unique in IO_Uring. -pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(heap.min_page_size) linux.io_uring_buf_ring { +pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(page_size_min) linux.io_uring_buf_ring { if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange; if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo; @@ -1572,7 +1572,7 @@ pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(heap. errdefer posix.munmap(mmap); assert(mmap.len == mmap_size); - const br: *align(heap.min_page_size) linux.io_uring_buf_ring = @ptrCast(mmap.ptr); + const br: *align(page_size_min) linux.io_uring_buf_ring = @ptrCast(mmap.ptr); try register_buf_ring(fd, @intFromPtr(br), entries, group_id); return br; } @@ -1614,9 +1614,9 @@ fn handle_register_buf_ring_result(res: usize) !void { } // Unregisters a previously registered shared buffer ring, returned from io_uring_setup_buf_ring. -pub fn free_buf_ring(fd: posix.fd_t, br: *align(heap.min_page_size) linux.io_uring_buf_ring, entries: u32, group_id: u16) void { +pub fn free_buf_ring(fd: posix.fd_t, br: *align(page_size_min) linux.io_uring_buf_ring, entries: u32, group_id: u16) void { unregister_buf_ring(fd, group_id) catch {}; - var mmap: []align(heap.min_page_size) u8 = undefined; + var mmap: []align(page_size_min) u8 = undefined; mmap.ptr = @ptrCast(br); mmap.len = entries * @sizeOf(linux.io_uring_buf); posix.munmap(mmap); diff --git a/lib/std/os/linux/tls.zig b/lib/std/os/linux/tls.zig index 7917fe9d1bb4..3180b04d2bb9 100644 --- a/lib/std/os/linux/tls.zig +++ b/lib/std/os/linux/tls.zig @@ -11,13 +11,13 @@ const std = @import("std"); const mem = std.mem; -const heap = std.heap; const elf = std.elf; const math = std.math; const assert = std.debug.assert; const native_arch = @import("builtin").cpu.arch; const linux = std.os.linux; const posix = std.posix; +const page_size_min = std.heap.page_size_min; /// Represents an ELF TLS variant. /// @@ -485,13 +485,13 @@ pub fn prepareArea(area: []u8) usize { }; } -// The main motivation for the size chosen here is that this is how much ends up being requested for -// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up -// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned -// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I -// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is -// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead. -var main_thread_area_buffer: [0x2100]u8 align(heap.min_page_size) = undefined; +/// The main motivation for the size chosen here is that this is how much ends up being requested for +/// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up +/// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned +/// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I +/// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is +/// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead. +var main_thread_area_buffer: [0x2100]u8 align(page_size_min) = undefined; /// Computes the layout of the static TLS area, allocates the area, initializes all of its fields, /// and assigns the architecture-specific value to the TP register. @@ -504,7 +504,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void { const area = blk: { // Fast path for the common case where the TLS data is really small, avoid an allocation and // use our local buffer. - if (area_desc.alignment <= heap.min_page_size and area_desc.size <= main_thread_area_buffer.len) { + if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) { break :blk main_thread_area_buffer[0..area_desc.size]; } @@ -518,7 +518,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void { ); if (@as(isize, @bitCast(begin_addr)) < 0) @trap(); - const area_ptr: [*]align(heap.min_page_size) u8 = @ptrFromInt(begin_addr); + const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr); // Make sure the slice is correctly aligned. const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment); diff --git a/lib/std/posix.zig b/lib/std/posix.zig index 46cab28e78e8..5b36c9b13978 100644 --- a/lib/std/posix.zig +++ b/lib/std/posix.zig @@ -18,13 +18,13 @@ const builtin = @import("builtin"); const root = @import("root"); const std = @import("std.zig"); const mem = std.mem; -const heap = std.heap; const fs = std.fs; const max_path_bytes = fs.max_path_bytes; const maxInt = std.math.maxInt; const cast = std.math.cast; const assert = std.debug.assert; const native_os = builtin.os.tag; +const page_size_min = std.heap.page_size_min; test { _ = @import("posix/test.zig"); @@ -4695,7 +4695,7 @@ pub const MProtectError = error{ OutOfMemory, } || UnexpectedError; -pub fn mprotect(memory: []align(heap.min_page_size) u8, protection: u32) MProtectError!void { +pub fn mprotect(memory: []align(page_size_min) u8, protection: u32) MProtectError!void { if (native_os == .windows) { const win_prot: windows.DWORD = switch (@as(u3, @truncate(protection))) { 0b000 => windows.PAGE_NOACCESS, @@ -4760,21 +4760,21 @@ pub const MMapError = error{ /// * SIGSEGV - Attempted write into a region mapped as read-only. /// * SIGBUS - Attempted access to a portion of the buffer that does not correspond to the file pub fn mmap( - ptr: ?[*]align(heap.min_page_size) u8, + ptr: ?[*]align(page_size_min) u8, length: usize, prot: u32, flags: system.MAP, fd: fd_t, offset: u64, -) MMapError![]align(heap.min_page_size) u8 { +) MMapError![]align(page_size_min) u8 { const mmap_sym = if (lfs64_abi) system.mmap64 else system.mmap; const rc = mmap_sym(ptr, length, prot, @bitCast(flags), fd, @bitCast(offset)); const err: E = if (builtin.link_libc) blk: { - if (rc != std.c.MAP_FAILED) return @as([*]align(heap.min_page_size) u8, @ptrCast(@alignCast(rc)))[0..length]; + if (rc != std.c.MAP_FAILED) return @as([*]align(page_size_min) u8, @ptrCast(@alignCast(rc)))[0..length]; break :blk @enumFromInt(system._errno().*); } else blk: { const err = errno(rc); - if (err == .SUCCESS) return @as([*]align(heap.min_page_size) u8, @ptrFromInt(rc))[0..length]; + if (err == .SUCCESS) return @as([*]align(page_size_min) u8, @ptrFromInt(rc))[0..length]; break :blk err; }; switch (err) { @@ -4800,7 +4800,7 @@ pub fn mmap( /// Zig's munmap function does not, for two reasons: /// * It violates the Zig principle that resource deallocation must succeed. /// * The Windows function, VirtualFree, has this restriction. -pub fn munmap(memory: []align(heap.min_page_size) const u8) void { +pub fn munmap(memory: []align(page_size_min) const u8) void { switch (errno(system.munmap(memory.ptr, memory.len))) { .SUCCESS => return, .INVAL => unreachable, // Invalid parameters. @@ -4814,7 +4814,7 @@ pub const MSyncError = error{ PermissionDenied, } || UnexpectedError; -pub fn msync(memory: []align(heap.min_page_size) u8, flags: i32) MSyncError!void { +pub fn msync(memory: []align(page_size_min) u8, flags: i32) MSyncError!void { switch (errno(system.msync(memory.ptr, memory.len, flags))) { .SUCCESS => return, .PERM => return error.PermissionDenied, @@ -7136,7 +7136,7 @@ pub const MincoreError = error{ } || UnexpectedError; /// Determine whether pages are resident in memory. -pub fn mincore(ptr: [*]align(heap.min_page_size) u8, length: usize, vec: [*]u8) MincoreError!void { +pub fn mincore(ptr: [*]align(page_size_min) u8, length: usize, vec: [*]u8) MincoreError!void { return switch (errno(system.mincore(ptr, length, vec))) { .SUCCESS => {}, .AGAIN => error.SystemResources, @@ -7182,7 +7182,7 @@ pub const MadviseError = error{ /// Give advice about use of memory. /// This syscall is optional and is sometimes configured to be disabled. -pub fn madvise(ptr: [*]align(heap.min_page_size) u8, length: usize, advice: u32) MadviseError!void { +pub fn madvise(ptr: [*]align(page_size_min) u8, length: usize, advice: u32) MadviseError!void { switch (errno(system.madvise(ptr, length, advice))) { .SUCCESS => return, .PERM => return error.PermissionDenied, diff --git a/lib/std/process.zig b/lib/std/process.zig index 0e10128e0e22..dd08e88af24f 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -1560,7 +1560,7 @@ pub fn posixGetUserInfo(name: []const u8) !UserInfo { ReadGroupId, }; - var buf: [std.heap.min_page_size]u8 = undefined; + var buf: [std.heap.page_size_min]u8 = undefined; var name_index: usize = 0; var state = State.Start; var uid: posix.uid_t = 0; diff --git a/lib/std/start.zig b/lib/std/start.zig index 0163f0054e0f..a91df357006d 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -576,7 +576,7 @@ fn expandStackSize(phdrs: []elf.Phdr) void { switch (phdr.p_type) { elf.PT_GNU_STACK => { if (phdr.p_memsz == 0) break; - assert(phdr.p_memsz % std.heap.pageSize() == 0); + assert(phdr.p_memsz % std.heap.page_size_min == 0); // Silently fail if we are unable to get limits. const limits = std.posix.getrlimit(.STACK) catch break; diff --git a/lib/std/std.zig b/lib/std/std.zig index 43b163bb2daa..558710015c09 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -119,9 +119,12 @@ pub const Options = struct { args: anytype, ) void = log.defaultLog, - min_page_size: ?usize = null, - max_page_size: ?usize = null, - queryPageSizeFn: fn () usize = heap.defaultQueryPageSize, + /// Overrides `std.heap.page_size_min`. + page_size_min: ?usize = null, + /// Overrides `std.heap.page_size_max`. + page_size_max: ?usize = null, + /// Overrides default implementation for determining OS page size at runtime. + queryPageSize: fn () usize = heap.defaultQueryPageSize, fmt_max_depth: usize = fmt.default_max_depth, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index e8e0840f09bf..9b119161de59 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1249,7 +1249,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { .{@errorName(err)}, )); defer zip_file.close(); - var buf: [std.heap.min_page_size]u8 = undefined; + var buf: [4096]u8 = undefined; while (true) { const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( "read zip stream failed: {s}", From 95a0474dc6bf26a7a86bea4c93e06b3ae0cd37cc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Jan 2025 21:48:05 -0800 Subject: [PATCH 03/39] revert GPA to before this branch --- lib/std/heap/general_purpose_allocator.zig | 129 ++++++++------------- 1 file changed, 51 insertions(+), 78 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 4a03dfecb88a..c23f8dcd7983 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -48,7 +48,7 @@ //! //! ## Basic Design: //! -//! Small allocations are divided into buckets. For a max page size of 4K: +//! Small allocations are divided into buckets: //! //! ``` //! index obj_size @@ -75,9 +75,6 @@ //! BucketHeader, followed by "used bits", and two stack traces for each slot //! (allocation trace and free trace). //! -//! The buckets array contains buckets for every size class below `page_size_max`. -//! At runtime, only size classes below `pageSize()` will actually be used for allocations. -//! //! The "used bits" are 1 bit per slot representing whether the slot is used. //! Allocations use the data to iterate to find a free slot. Frees assert that the //! corresponding bit is 1 and set it to 0. @@ -102,13 +99,11 @@ const math = std.math; const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; -const page_size_min = std.heap.page_size_min; -const page_size_max = std.heap.page_size_max; -const pageSize = std.heap.pageSize; +const page_size = std.mem.page_size; const StackTrace = std.builtin.StackTrace; /// Integer type for pointing to slots in a small allocation -const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size_max) + 1); +const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6; const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0; @@ -162,9 +157,6 @@ pub const Config = struct { pub const Check = enum { ok, leak }; -var used_small_bucket_count_cache = std.atomic.Value(usize).init(0); -var largest_used_bucket_object_size_cache = std.atomic.Value(usize).init(0); - /// Default initialization of this struct is deprecated; use `.init` instead. pub fn GeneralPurposeAllocator(comptime config: Config) type { return struct { @@ -214,27 +206,9 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { pub const Error = mem.Allocator.Error; - const small_bucket_count = math.log2(page_size_max); + const small_bucket_count = math.log2(page_size); const largest_bucket_object_size = 1 << (small_bucket_count - 1); const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); - fn used_small_bucket_count() usize { - const cached = used_small_bucket_count_cache.load(.monotonic); - if (cached != 0) { - return cached; - } - const val = math.log2(pageSize()); - used_small_bucket_count_cache.store(val, .monotonic); - return val; - } - fn largest_used_bucket_object_size() usize { - const cached = largest_used_bucket_object_size_cache.load(.monotonic); - if (cached != 0) { - return cached; - } - const val = @as(usize, 1) << @truncate(used_small_bucket_count() - 1); - largest_used_bucket_object_size_cache.store(val, .monotonic); - return val; - } const bucketCompare = struct { fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order { @@ -287,7 +261,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation const BucketHeader = struct { - page: [*]align(page_size_min) u8, + page: [*]align(page_size) u8, alloc_cursor: SlotIndex, used_count: SlotIndex, @@ -299,14 +273,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (!config.safety) @compileError("requested size is only stored when safety is enabled"); const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(size_class); const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); - const slot_count = @divExact(pageSize(), size_class); + const slot_count = @divExact(page_size, size_class); return sizes[0..slot_count]; } fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 { if (!config.safety) @compileError("requested size is only stored when safety is enabled"); const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class); - const slot_count = @divExact(pageSize(), size_class); + const slot_count = @divExact(page_size, size_class); return aligns_ptr[0..slot_count]; } @@ -338,7 +312,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { /// Only valid for buckets within `empty_buckets`, and relies on the `alloc_cursor` /// of empty buckets being set to `slot_count` when they are added to `empty_buckets` fn emptyBucketSizeClass(bucket: *BucketHeader) usize { - return @divExact(pageSize(), bucket.alloc_cursor); + return @divExact(page_size, bucket.alloc_cursor); } }; @@ -381,13 +355,13 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn bucketAlignsStart(size_class: usize) usize { if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); - const slot_count = @divExact(pageSize(), size_class); + const slot_count = @divExact(page_size, size_class); return bucketRequestedSizesStart(size_class) + (@sizeOf(LargestSizeClassInt) * slot_count); } fn bucketStackFramesStart(size_class: usize) usize { const unaligned_start = if (config.safety) blk: { - const slot_count = @divExact(pageSize(), size_class); + const slot_count = @divExact(page_size, size_class); break :blk bucketAlignsStart(size_class) + slot_count; } else @sizeOf(BucketHeader) + usedBitsCount(size_class); return mem.alignForward( @@ -398,12 +372,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn bucketSize(size_class: usize) usize { - const slot_count = @divExact(pageSize(), size_class); + const slot_count = @divExact(page_size, size_class); return bucketStackFramesStart(size_class) + one_trace_size * traces_per_slot * slot_count; } fn usedBitsCount(size_class: usize) usize { - const slot_count = @divExact(pageSize(), size_class); + const slot_count = @divExact(page_size, size_class); if (slot_count < 8) return 1; return @divExact(slot_count, 8); } @@ -442,8 +416,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { pub fn detectLeaks(self: *Self) bool { var leaks = false; - for (0..used_small_bucket_count()) |bucket_i| { - const buckets = &self.buckets[bucket_i]; + for (&self.buckets, 0..) |*buckets, bucket_i| { if (buckets.root == null) continue; const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i)); const used_bits_count = usedBitsCount(size_class); @@ -491,7 +464,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { var bucket = node.key; if (config.never_unmap) { // free page that was intentionally leaked by never_unmap - self.backing_allocator.free(bucket.page[0..pageSize()]); + self.backing_allocator.free(bucket.page[0..page_size]); } // alloc_cursor was set to slot count when bucket added to empty_buckets self.freeBucket(bucket, bucket.emptyBucketSizeClass()); @@ -558,7 +531,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error!Slot { const bucket_index = math.log2(size_class); var buckets = &self.buckets[bucket_index]; - const slot_count = @divExact(pageSize(), size_class); + const slot_count = @divExact(page_size, size_class); if (self.cur_buckets[bucket_index] == null or self.cur_buckets[bucket_index].?.alloc_cursor == slot_count) { const new_bucket = try self.createBucket(size_class); errdefer self.freeBucket(new_bucket, size_class); @@ -591,7 +564,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { addr: usize, current_bucket: ?*BucketHeader, ) ?*BucketHeader { - const search_page: [*]align(page_size_min) u8 = @ptrFromInt(mem.alignBackward(usize, addr, pageSize())); + const search_page: [*]align(page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, page_size)); if (current_bucket != null and current_bucket.?.page == search_page) { return current_bucket; } @@ -756,14 +729,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { assert(old_mem.len != 0); const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align); - if (aligned_size > largest_used_bucket_object_size()) { + if (aligned_size > largest_bucket_object_size) { return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr); } const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size); var bucket_index = math.log2(size_class_hint); var size_class: usize = size_class_hint; - const bucket = while (bucket_index < used_small_bucket_count()) : (bucket_index += 1) { + const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| { break bucket; } @@ -874,7 +847,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { assert(old_mem.len != 0); const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align); - if (aligned_size > largest_used_bucket_object_size()) { + if (aligned_size > largest_bucket_object_size) { self.freeLarge(old_mem, log2_old_align, ret_addr); return; } @@ -882,7 +855,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { var bucket_index = math.log2(size_class_hint); var size_class: usize = size_class_hint; - const bucket = while (bucket_index < used_small_bucket_count()) : (bucket_index += 1) { + const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| { break bucket; } @@ -971,14 +944,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { self.cur_buckets[bucket_index] = null; } if (!config.never_unmap) { - self.backing_allocator.free(bucket.page[0..pageSize()]); + self.backing_allocator.free(bucket.page[0..page_size]); } if (!config.retain_metadata) { self.freeBucket(bucket, size_class); self.bucket_node_pool.destroy(node); } else { // move alloc_cursor to end so we can tell size_class later - const slot_count = @divExact(pageSize(), size_class); + const slot_count = @divExact(page_size, size_class); bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count)); var empty_entry = self.empty_buckets.getEntryFor(node.key); empty_entry.set(node); @@ -1019,7 +992,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { ret_addr: usize, ) Allocator.Error![*]u8 { const new_aligned_size = @max(len, @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align))); - if (new_aligned_size > largest_used_bucket_object_size()) { + if (new_aligned_size > largest_bucket_object_size) { try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1); const ptr = self.backing_allocator.rawAlloc(len, log2_ptr_align, ret_addr) orelse return error.OutOfMemory; @@ -1062,7 +1035,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader { - const page = try self.backing_allocator.alignedAlloc(u8, page_size_min, pageSize()); + const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size); errdefer self.backing_allocator.free(page); const bucket_size = bucketSize(size_class); @@ -1206,17 +1179,17 @@ test "large object - grow" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice1 = try allocator.alloc(u8, pageSize() * 2 - 20); + var slice1 = try allocator.alloc(u8, page_size * 2 - 20); defer allocator.free(slice1); const old = slice1; - slice1 = try allocator.realloc(slice1, pageSize() * 2 - 10); + slice1 = try allocator.realloc(slice1, page_size * 2 - 10); try std.testing.expect(slice1.ptr == old.ptr); - slice1 = try allocator.realloc(slice1, pageSize() * 2); + slice1 = try allocator.realloc(slice1, page_size * 2); try std.testing.expect(slice1.ptr == old.ptr); - slice1 = try allocator.realloc(slice1, pageSize() * 2 + 1); + slice1 = try allocator.realloc(slice1, page_size * 2 + 1); } test "realloc small object to large object" { @@ -1230,7 +1203,7 @@ test "realloc small object to large object" { slice[60] = 0x34; // This requires upgrading to a large object - const large_object_size = pageSize() * 2 + 50; + const large_object_size = page_size * 2 + 50; slice = try allocator.realloc(slice, large_object_size); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); @@ -1241,22 +1214,22 @@ test "shrink large object to large object" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, pageSize() * 2 + 50); + var slice = try allocator.alloc(u8, page_size * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[60] = 0x34; - if (!allocator.resize(slice, pageSize() * 2 + 1)) return; - slice = slice.ptr[0 .. pageSize() * 2 + 1]; + if (!allocator.resize(slice, page_size * 2 + 1)) return; + slice = slice.ptr[0 .. page_size * 2 + 1]; try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); - try std.testing.expect(allocator.resize(slice, pageSize() * 2 + 1)); - slice = slice[0 .. pageSize() * 2 + 1]; + try std.testing.expect(allocator.resize(slice, page_size * 2 + 1)); + slice = slice[0 .. page_size * 2 + 1]; try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); - slice = try allocator.realloc(slice, pageSize() * 2); + slice = try allocator.realloc(slice, page_size * 2); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); } @@ -1272,13 +1245,13 @@ test "shrink large object to large object with larger alignment" { var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); const debug_allocator = fba.allocator(); - const alloc_size = pageSize() * 2 + 50; + const alloc_size = page_size * 2 + 50; var slice = try allocator.alignedAlloc(u8, 16, alloc_size); defer allocator.free(slice); const big_alignment: usize = switch (builtin.os.tag) { - .windows => pageSize() * 32, // Windows aligns to 64K. - else => pageSize() * 2, + .windows => page_size * 32, // Windows aligns to 64K. + else => page_size * 2, }; // This loop allocates until we find a page that is not aligned to the big // alignment. Then we shrink the allocation after the loop, but increase the @@ -1304,7 +1277,7 @@ test "realloc large object to small object" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, pageSize() * 2 + 50); + var slice = try allocator.alloc(u8, page_size * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[16] = 0x34; @@ -1346,18 +1319,18 @@ test "realloc large object to larger alignment" { var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); const debug_allocator = fba.allocator(); - var slice = try allocator.alignedAlloc(u8, 16, pageSize() * 2 + 50); + var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); defer allocator.free(slice); const big_alignment: usize = switch (builtin.os.tag) { - .windows => pageSize() * 32, // Windows aligns to 64K. - else => pageSize() * 2, + .windows => page_size * 32, // Windows aligns to 64K. + else => page_size * 2, }; // This loop allocates until we find a page that is not aligned to the big alignment. var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { try stuff_to_free.append(slice); - slice = try allocator.alignedAlloc(u8, 16, pageSize() * 2 + 50); + slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); } while (stuff_to_free.popOrNull()) |item| { allocator.free(item); @@ -1365,15 +1338,15 @@ test "realloc large object to larger alignment" { slice[0] = 0x12; slice[16] = 0x34; - slice = try allocator.reallocAdvanced(slice, 32, pageSize() * 2 + 100); + slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); - slice = try allocator.reallocAdvanced(slice, 32, pageSize() * 2 + 25); + slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); - slice = try allocator.reallocAdvanced(slice, big_alignment, pageSize() * 2 + 100); + slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); } @@ -1389,7 +1362,7 @@ test "large object shrinks to small but allocation fails during shrink" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, pageSize() * 2 + 50); + var slice = try allocator.alloc(u8, page_size * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[3] = 0x34; @@ -1460,7 +1433,7 @@ test "double frees" { try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null) != null); // detect a large allocation double free - const large = try allocator.alloc(u8, 2 * pageSize()); + const large = try allocator.alloc(u8, 2 * page_size); try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr))); try std.testing.expectEqual(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.bytes, large); allocator.free(large); @@ -1469,7 +1442,7 @@ test "double frees" { const normal_small = try allocator.alloc(u8, size_class); defer allocator.free(normal_small); - const normal_large = try allocator.alloc(u8, 2 * pageSize()); + const normal_large = try allocator.alloc(u8, 2 * page_size); defer allocator.free(normal_large); // check that flushing retained metadata doesn't disturb live allocations @@ -1502,8 +1475,8 @@ test "bug 9995 fix, large allocs count requested size not backing size" { var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; const allocator = gpa.allocator(); - var buf = try allocator.alignedAlloc(u8, 1, pageSize() + 1); - try std.testing.expect(gpa.total_requested_bytes == pageSize() + 1); + var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); + try std.testing.expect(gpa.total_requested_bytes == page_size + 1); buf = try allocator.realloc(buf, 1); try std.testing.expect(gpa.total_requested_bytes == 1); buf = try allocator.realloc(buf, 2); From 4913de3c88d61637490bb450690d769b324508b5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Jan 2025 22:47:29 -0800 Subject: [PATCH 04/39] GeneralPurposeAllocator: minimal fix This keeps the implementation matching master branch, however, introduces a compile error that applications can work around by explicitly setting page_size_max and page_size_min to match their computer's settings, in the case that those values are not already equal. I plan to rework this allocator in a follow-up enhancement with the goal of reducing total active memory mappings. --- lib/std/heap/general_purpose_allocator.zig | 6 +++--- lib/std/mem/Allocator.zig | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index c23f8dcd7983..bf3f9df76dd2 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -99,7 +99,7 @@ const math = std.math; const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; -const page_size = std.mem.page_size; +const page_size = std.heap.pageSize(); // TODO: allow this to be runtime known const StackTrace = std.builtin.StackTrace; /// Integer type for pointing to slots in a small allocation @@ -1040,8 +1040,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const bucket_size = bucketSize(size_class); const bucket_bytes = try self.backing_allocator.alignedAlloc(u8, @alignOf(BucketHeader), bucket_size); - const ptr = @as(*BucketHeader, @ptrCast(bucket_bytes.ptr)); - ptr.* = BucketHeader{ + const ptr: *BucketHeader = @ptrCast(bucket_bytes.ptr); + ptr.* = .{ .page = page.ptr, .alloc_cursor = 0, .used_count = 0, diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index 7f3334f1d2e6..277758501bb3 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -227,7 +227,7 @@ fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: const byte_ptr = self.rawAlloc(byte_count, log2a(alignment), return_address) orelse return Error.OutOfMemory; // TODO: https://github.com/ziglang/zig/issues/4298 @memset(byte_ptr[0..byte_count], undefined); - return @as([*]align(alignment) u8, @alignCast(byte_ptr)); + return @alignCast(byte_ptr); } /// Requests to modify the size of an allocation. It is guaranteed to not move From 5c638845390fed5cf1c504ac013a4f75c3b7c690 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Jan 2025 18:21:19 -0800 Subject: [PATCH 05/39] add std.mem.Alignment API --- lib/std/mem.zig | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/lib/std/mem.zig b/lib/std/mem.zig index f0531bab1ea5..c357c768d768 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -18,6 +18,60 @@ pub const byte_size_in_bits = 8; pub const Allocator = @import("mem/Allocator.zig"); +/// Stored as a power-of-two. +pub const Alignment = enum(math.Log2Int(usize)) { + @"1" = 0, + @"2" = 1, + @"4" = 2, + @"8" = 3, + @"16" = 4, + @"32" = 5, + @"64" = 6, + _, + + pub fn toByteUnits(a: Alignment) usize { + return @as(usize, 1) << @intFromEnum(a); + } + + pub fn fromByteUnits(n: usize) Alignment { + assert(std.math.isPowerOfTwo(n)); + return @enumFromInt(@ctz(n)); + } + + pub fn order(lhs: Alignment, rhs: Alignment) std.math.Order { + return std.math.order(@intFromEnum(lhs), @intFromEnum(rhs)); + } + + pub fn compare(lhs: Alignment, op: std.math.CompareOperator, rhs: Alignment) bool { + return std.math.compare(@intFromEnum(lhs), op, @intFromEnum(rhs)); + } + + pub fn max(lhs: Alignment, rhs: Alignment) Alignment { + return @enumFromInt(@max(@intFromEnum(lhs), @intFromEnum(rhs))); + } + + pub fn min(lhs: Alignment, rhs: Alignment) Alignment { + return @enumFromInt(@min(@intFromEnum(lhs), @intFromEnum(rhs))); + } + + /// Return next address with this alignment. + pub fn forward(a: Alignment, address: usize) usize { + const x = (@as(usize, 1) << @intFromEnum(a)) - 1; + return (address + x) & ~x; + } + + /// Return previous address with this alignment. + pub fn backward(a: Alignment, address: usize) usize { + const x = (@as(usize, 1) << @intFromEnum(a)) - 1; + return address & ~x; + } + + /// Return whether address is aligned to this amount. + pub fn check(a: Alignment, address: usize) bool { + return @ctz(address) >= @intFromEnum(a); + } +}; + /// Detects and asserts if the std.mem.Allocator interface is violated by the caller /// or the allocator. pub fn ValidationAllocator(comptime T: type) type { From 91f41bdc7095635b752af0ca3aee0a41ce864f8c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Jan 2025 18:21:54 -0800 Subject: [PATCH 06/39] std.heap.PageAllocator: restore high alignment functionality This allocator now supports alignments greater than page size, with the same implementation as it used before. This is a partial revert of ceb0a632cfd6a4eada6bd27bf6a3754e95dcac86. It looks like VirtualAlloc2 has better solutions to this problem, including features such as MEM_RESERVE_PLACEHOLDER and MEM_LARGE_PAGES. This possibility can be investigated as a follow-up task. --- lib/std/heap/PageAllocator.zig | 84 +++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index c261dcec43bf..732760b98fff 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -15,51 +15,100 @@ pub const vtable: Allocator.VTable = .{ .free = free, }; -fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { +fn alloc(context: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { + const requested_alignment: mem.Alignment = @enumFromInt(log2_align); + _ = context; _ = ra; - _ = log2_align; assert(n > 0); + const page_size = std.heap.pageSize(); + if (n >= maxInt(usize) - page_size) return null; + const alignment_bytes = requested_alignment.toByteUnits(); + if (native_os == .windows) { + // According to official documentation, VirtualAlloc aligns to page + // boundary, however, empirically it reserves pages on a 64K boundary. + // Since it is very likely the requested alignment will be honored, + // this logic first tries a call with exactly the size requested, + // before falling back to the loop below. + // https://devblogs.microsoft.com/oldnewthing/?p=42223 const addr = windows.VirtualAlloc( null, - // VirtualAlloc will round the length to a multiple of page size. - // VirtualAlloc docs: If the lpAddress parameter is NULL, this value is rounded up to the next page boundary + // "If the lpAddress parameter is NULL, this value is rounded up to + // the next page boundary". n, - windows.MEM_COMMIT | windows.MEM_RESERVE, windows.PAGE_READWRITE, ) catch return null; - return @ptrCast(addr); - } - const page_size = std.heap.pageSize(); - if (n >= maxInt(usize) - page_size) return null; + if (mem.isAligned(@intFromPtr(addr), alignment_bytes)) + return @ptrCast(addr); + + // Fallback: reserve a range of memory large enough to find a + // sufficiently aligned address, then free the entire range and + // immediately allocate the desired subset. Another thread may have won + // the race to map the target range, in which case a retry is needed. + windows.VirtualFree(addr, 0, windows.MEM_RELEASE); + + const overalloc_len = n + alignment_bytes - page_size; + const aligned_len = mem.alignForward(usize, n, page_size); + + while (true) { + const reserved_addr = windows.VirtualAlloc( + null, + overalloc_len, + windows.MEM_RESERVE, + windows.PAGE_NOACCESS, + ) catch return null; + const aligned_addr = mem.alignForward(usize, @intFromPtr(reserved_addr), alignment_bytes); + windows.VirtualFree(reserved_addr, 0, windows.MEM_RELEASE); + const ptr = windows.VirtualAlloc( + @ptrFromInt(aligned_addr), + aligned_len, + windows.MEM_COMMIT | windows.MEM_RESERVE, + windows.PAGE_READWRITE, + ) catch continue; + return @ptrCast(ptr); + } + } const aligned_len = mem.alignForward(usize, n, page_size); + const max_drop_len = alignment_bytes - @min(alignment_bytes, page_size); + const overalloc_len = if (max_drop_len <= aligned_len - n) + aligned_len + else + mem.alignForward(usize, aligned_len + max_drop_len, page_size); const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered); const slice = posix.mmap( hint, - aligned_len, + overalloc_len, posix.PROT.READ | posix.PROT.WRITE, .{ .TYPE = .PRIVATE, .ANONYMOUS = true }, -1, 0, ) catch return null; - assert(mem.isAligned(@intFromPtr(slice.ptr), page_size_min)); - const new_hint: [*]align(std.heap.page_size_min) u8 = @alignCast(slice.ptr + aligned_len); + const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes) orelse return null; + // Unmap the extra bytes that were only requested in order to guarantee + // that the range of memory we were provided had a proper alignment in it + // somewhere. The extra bytes could be at the beginning, or end, or both. + const drop_len = result_ptr - slice.ptr; + if (drop_len != 0) posix.munmap(slice[0..drop_len]); + const remaining_len = overalloc_len - drop_len; + if (remaining_len > aligned_len) posix.munmap(@alignCast(result_ptr[aligned_len..remaining_len])); + const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + aligned_len); _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic); - return slice.ptr; + return result_ptr; } fn resize( - _: *anyopaque, + context: *anyopaque, buf_unaligned: []u8, log2_buf_align: u8, new_size: usize, return_address: usize, ) bool { + _ = context; _ = log2_buf_align; _ = return_address; const page_size = std.heap.pageSize(); @@ -71,8 +120,8 @@ fn resize( const old_addr_end = base_addr + buf_unaligned.len; const new_addr_end = mem.alignForward(usize, base_addr + new_size, page_size); if (old_addr_end > new_addr_end) { - // For shrinking that is not releasing, we will only - // decommit the pages not needed anymore. + // For shrinking that is not releasing, we will only decommit + // the pages not needed anymore. windows.VirtualFree( @as(*anyopaque, @ptrFromInt(new_addr_end)), old_addr_end - new_addr_end, @@ -104,7 +153,8 @@ fn resize( return false; } -fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void { +fn free(context: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void { + _ = context; _ = log2_buf_align; _ = return_address; From b23662feeb0ecea67eefbcfe941e609c5a8ca842 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Jan 2025 18:24:29 -0800 Subject: [PATCH 07/39] std.heap.WasmAllocator: use `@splat` syntax preferred over array multiplication where possible. --- lib/std/heap/WasmAllocator.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig index fea6ae5f529e..2bdfdd84eac6 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/WasmAllocator.zig @@ -40,11 +40,11 @@ const size_class_count = math.log2(bigpage_size) - min_class; /// etc. const big_size_class_count = math.log2(bigpage_count); -var next_addrs = [1]usize{0} ** size_class_count; +var next_addrs: [size_class_count]usize = @splat(0); /// For each size class, points to the freed pointer. -var frees = [1]usize{0} ** size_class_count; +var frees: [size_class_count]usize = @splat(0); /// For each big size class, points to the freed pointer. -var big_frees = [1]usize{0} ** big_size_class_count; +var big_frees: [big_size_class_count]usize = @splat(0); fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, return_address: usize) ?[*]u8 { _ = ctx; @@ -160,7 +160,7 @@ fn allocBigPages(n: usize) usize { return @as(usize, @intCast(page_index)) * wasm.page_size; } -const test_ally = Allocator{ +const test_ally: Allocator = .{ .ptr = undefined, .vtable = &vtable, }; From dd2fa4f75d3d2b1214fde22081f0b88850d1b55d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Jan 2025 18:25:01 -0800 Subject: [PATCH 08/39] std.heap.GeneralPurposeAllocator: runtime-known page size no longer causes compilation failure. This also addresses the problem of high map count causing OOM by choosing a page size of 2MiB for most targets when the page_size_max is smaller than this number. --- lib/std/heap/general_purpose_allocator.zig | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index bf3f9df76dd2..d57253686029 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -99,7 +99,11 @@ const math = std.math; const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; -const page_size = std.heap.pageSize(); // TODO: allow this to be runtime known +const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { + .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. + .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. + else => 2 * 1024 * 1024, // Avoids too many active mappings when `page_size_max` is low. +}); const StackTrace = std.builtin.StackTrace; /// Integer type for pointing to slots in a small allocation From 7eeef5fb2b9dc78679f4091e2a8173d07968b3e5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 19:55:09 -0800 Subject: [PATCH 09/39] std.mem.Allocator: introduce `remap` function to the interface This one changes the size of an allocation, allowing it to be relocated. However, the implementation will still return `null` if it would be equivalent to new = alloc memcpy(new, old) free(old) Mainly this prepares for taking advantage of `mremap` which I thought would be a bigger deal but apparently is only available on Linux. Still, we should use it on Linux. --- lib/std/array_list.zig | 37 ++-- lib/std/heap/FixedBufferAllocator.zig | 30 ++- lib/std/heap/PageAllocator.zig | 99 ++++++---- lib/std/heap/arena_allocator.zig | 42 ++-- lib/std/heap/general_purpose_allocator.zig | 144 ++++++++------ lib/std/mem/Allocator.zig | 214 ++++++++++++++------- lib/std/testing/failing_allocator.zig | 40 +++- 7 files changed, 389 insertions(+), 217 deletions(-) diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index 5eb527e74204..55b6e3cc401c 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -105,21 +105,19 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type { return result; } - /// The caller owns the returned memory. Empties this ArrayList, - /// Its capacity is cleared, making deinit() safe but unnecessary to call. + /// The caller owns the returned memory. Empties this ArrayList. + /// Its capacity is cleared, making `deinit` safe but unnecessary to call. pub fn toOwnedSlice(self: *Self) Allocator.Error!Slice { const allocator = self.allocator; const old_memory = self.allocatedSlice(); - if (allocator.resize(old_memory, self.items.len)) { - const result = self.items; + if (allocator.remap(old_memory, self.items.len)) |new_items| { self.* = init(allocator); - return result; + return new_items; } const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len); @memcpy(new_memory, self.items); - @memset(self.items, undefined); self.clearAndFree(); return new_memory; } @@ -185,8 +183,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type { // extra capacity. const new_capacity = growCapacity(self.capacity, new_len); const old_memory = self.allocatedSlice(); - if (self.allocator.resize(old_memory, new_capacity)) { - self.capacity = new_capacity; + if (self.allocator.remap(old_memory, new_capacity)) |new_memory| { + self.items.ptr = new_memory.ptr; + self.capacity = new_memory.len; return addManyAtAssumeCapacity(self, index, count); } @@ -468,8 +467,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type { // the allocator implementation would pointlessly copy our // extra capacity. const old_memory = self.allocatedSlice(); - if (self.allocator.resize(old_memory, new_capacity)) { - self.capacity = new_capacity; + if (self.allocator.remap(old_memory, new_capacity)) |new_memory| { + self.items.ptr = new_memory.ptr; + self.capacity = new_memory.len; } else { const new_memory = try self.allocator.alignedAlloc(T, alignment, new_capacity); @memcpy(new_memory[0..self.items.len], self.items); @@ -707,15 +707,13 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ /// Its capacity is cleared, making deinit() safe but unnecessary to call. pub fn toOwnedSlice(self: *Self, allocator: Allocator) Allocator.Error!Slice { const old_memory = self.allocatedSlice(); - if (allocator.resize(old_memory, self.items.len)) { - const result = self.items; + if (allocator.remap(old_memory, self.items.len)) |new_items| { self.* = .empty; - return result; + return new_items; } const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len); @memcpy(new_memory, self.items); - @memset(self.items, undefined); self.clearAndFree(allocator); return new_memory; } @@ -1031,9 +1029,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ } const old_memory = self.allocatedSlice(); - if (allocator.resize(old_memory, new_len)) { - self.capacity = new_len; - self.items.len = new_len; + if (allocator.remap(old_memory, new_len)) |new_items| { + self.capacity = new_items.len; + self.items = new_items; return; } @@ -1099,8 +1097,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ // the allocator implementation would pointlessly copy our // extra capacity. const old_memory = self.allocatedSlice(); - if (allocator.resize(old_memory, new_capacity)) { - self.capacity = new_capacity; + if (allocator.remap(old_memory, new_capacity)) |new_memory| { + self.items.ptr = new_memory.ptr; + self.capacity = new_memory.len; } else { const new_memory = try allocator.alignedAlloc(T, alignment, new_capacity); @memcpy(new_memory[0..self.items.len], self.items); diff --git a/lib/std/heap/FixedBufferAllocator.zig b/lib/std/heap/FixedBufferAllocator.zig index 5995dfe154aa..0951dd3bcc88 100644 --- a/lib/std/heap/FixedBufferAllocator.zig +++ b/lib/std/heap/FixedBufferAllocator.zig @@ -9,7 +9,7 @@ end_index: usize, buffer: []u8, pub fn init(buffer: []u8) FixedBufferAllocator { - return FixedBufferAllocator{ + return .{ .buffer = buffer, .end_index = 0, }; @@ -22,6 +22,7 @@ pub fn allocator(self: *FixedBufferAllocator) Allocator { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; @@ -36,6 +37,7 @@ pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator { .vtable = &.{ .alloc = threadSafeAlloc, .resize = Allocator.noResize, + .remap = Allocator.noRemap, .free = Allocator.noFree, }, }; @@ -57,10 +59,10 @@ pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool { return buf.ptr + buf.len == self.buffer.ptr + self.end_index; } -pub fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { +pub fn alloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); _ = ra; - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); + const ptr_align = alignment.toByteUnits(); const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null; const adjusted_index = self.end_index + adjust_off; const new_end_index = adjusted_index + n; @@ -72,12 +74,12 @@ pub fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { pub fn resize( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, new_size: usize, return_address: usize, ) bool { const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = log2_buf_align; + _ = alignment; _ = return_address; assert(@inComptime() or self.ownsSlice(buf)); @@ -99,14 +101,24 @@ pub fn resize( return true; } +pub fn remap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, +) ?[*]u8 { + return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null; +} + pub fn free( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, return_address: usize, ) void { const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = log2_buf_align; + _ = alignment; _ = return_address; assert(@inComptime() or self.ownsSlice(buf)); @@ -115,10 +127,10 @@ pub fn free( } } -fn threadSafeAlloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { +fn threadSafeAlloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); _ = ra; - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); + const ptr_align = alignment.toByteUnits(); var end_index = @atomicLoad(usize, &self.end_index, .seq_cst); while (true) { const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null; diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index 732760b98fff..59b0c6015d18 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -12,18 +12,18 @@ const page_size_min = std.heap.page_size_min; pub const vtable: Allocator.VTable = .{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }; -fn alloc(context: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { - const requested_alignment: mem.Alignment = @enumFromInt(log2_align); +fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { _ = context; _ = ra; assert(n > 0); const page_size = std.heap.pageSize(); if (n >= maxInt(usize) - page_size) return null; - const alignment_bytes = requested_alignment.toByteUnits(); + const alignment_bytes = alignment.toByteUnits(); if (native_os == .windows) { // According to official documentation, VirtualAlloc aligns to page @@ -103,22 +103,52 @@ fn alloc(context: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { fn resize( context: *anyopaque, - buf_unaligned: []u8, - log2_buf_align: u8, - new_size: usize, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, return_address: usize, ) bool { _ = context; - _ = log2_buf_align; + _ = alignment; _ = return_address; + return realloc(memory, new_len, false) != null; +} + +pub fn remap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, +) ?[*]u8 { + _ = context; + _ = alignment; + _ = return_address; + return realloc(memory, new_len, true); +} + +fn free(context: *anyopaque, slice: []u8, alignment: mem.Alignment, return_address: usize) void { + _ = context; + _ = alignment; + _ = return_address; + + if (native_os == .windows) { + windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE); + } else { + const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize()); + posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len])); + } +} + +fn realloc(memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { const page_size = std.heap.pageSize(); - const new_size_aligned = mem.alignForward(usize, new_size, page_size); + const new_size_aligned = mem.alignForward(usize, new_len, page_size); if (native_os == .windows) { - if (new_size <= buf_unaligned.len) { - const base_addr = @intFromPtr(buf_unaligned.ptr); - const old_addr_end = base_addr + buf_unaligned.len; - const new_addr_end = mem.alignForward(usize, base_addr + new_size, page_size); + if (new_len <= memory.len) { + const base_addr = @intFromPtr(memory.ptr); + const old_addr_end = base_addr + memory.len; + const new_addr_end = mem.alignForward(usize, base_addr + new_len, page_size); if (old_addr_end > new_addr_end) { // For shrinking that is not releasing, we will only decommit // the pages not needed anymore. @@ -128,40 +158,31 @@ fn resize( windows.MEM_DECOMMIT, ); } - return true; + return memory.ptr; } - const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, page_size); + const old_size_aligned = mem.alignForward(usize, memory.len, page_size); if (new_size_aligned <= old_size_aligned) { - return true; + return memory.ptr; } - return false; + return null; } - const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, page_size); - if (new_size_aligned == buf_aligned_len) - return true; + const page_aligned_len = mem.alignForward(usize, memory.len, page_size); + if (new_size_aligned == page_aligned_len) + return memory.ptr; - if (new_size_aligned < buf_aligned_len) { - const ptr = buf_unaligned.ptr + new_size_aligned; - // TODO: if the next_mmap_addr_hint is within the unmapped range, update it - posix.munmap(@alignCast(ptr[0 .. buf_aligned_len - new_size_aligned])); - return true; + const mremap_available = false; // native_os == .linux; + if (mremap_available) { + // TODO: if the next_mmap_addr_hint is within the remapped range, update it + return posix.mremap(memory, new_len, .{ .MAYMOVE = may_move }, null) catch return null; } - // TODO: call mremap - // TODO: if the next_mmap_addr_hint is within the remapped range, update it - return false; -} - -fn free(context: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void { - _ = context; - _ = log2_buf_align; - _ = return_address; - - if (native_os == .windows) { - windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE); - } else { - const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize()); - posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len])); + if (new_size_aligned < page_aligned_len) { + const ptr = memory.ptr + new_size_aligned; + // TODO: if the next_mmap_addr_hint is within the unmapped range, update it + posix.munmap(@alignCast(ptr[0 .. page_aligned_len - new_size_aligned])); + return memory.ptr; } + + return null; } diff --git a/lib/std/heap/arena_allocator.zig b/lib/std/heap/arena_allocator.zig index 3cff6b439fbc..15d904447912 100644 --- a/lib/std/heap/arena_allocator.zig +++ b/lib/std/heap/arena_allocator.zig @@ -29,12 +29,14 @@ pub const ArenaAllocator = struct { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; } const BufNode = std.SinglyLinkedList(usize).Node; + const BufNode_alignment: mem.Alignment = .fromByteUnits(@alignOf(BufNode)); pub fn init(child_allocator: Allocator) ArenaAllocator { return (State{}).promote(child_allocator); @@ -47,9 +49,8 @@ pub const ArenaAllocator = struct { while (it) |node| { // this has to occur before the free because the free frees node const next_it = node.next; - const align_bits = std.math.log2_int(usize, @alignOf(BufNode)); const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data]; - self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress()); + self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress()); it = next_it; } } @@ -120,7 +121,6 @@ pub const ArenaAllocator = struct { return true; } const total_size = requested_capacity + @sizeOf(BufNode); - const align_bits = std.math.log2_int(usize, @alignOf(BufNode)); // Free all nodes except for the last one var it = self.state.buffer_list.first; const maybe_first_node = while (it) |node| { @@ -129,7 +129,7 @@ pub const ArenaAllocator = struct { if (next_it == null) break node; const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data]; - self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress()); + self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress()); it = next_it; } else null; std.debug.assert(maybe_first_node == null or maybe_first_node.?.next == null); @@ -141,16 +141,16 @@ pub const ArenaAllocator = struct { if (first_node.data == total_size) return true; const first_alloc_buf = @as([*]u8, @ptrCast(first_node))[0..first_node.data]; - if (self.child_allocator.rawResize(first_alloc_buf, align_bits, total_size, @returnAddress())) { + if (self.child_allocator.rawResize(first_alloc_buf, BufNode_alignment, total_size, @returnAddress())) { // successful resize first_node.data = total_size; } else { // manual realloc - const new_ptr = self.child_allocator.rawAlloc(total_size, align_bits, @returnAddress()) orelse { + const new_ptr = self.child_allocator.rawAlloc(total_size, BufNode_alignment, @returnAddress()) orelse { // we failed to preheat the arena properly, signal this to the user. return false; }; - self.child_allocator.rawFree(first_alloc_buf, align_bits, @returnAddress()); + self.child_allocator.rawFree(first_alloc_buf, BufNode_alignment, @returnAddress()); const node: *BufNode = @ptrCast(@alignCast(new_ptr)); node.* = .{ .data = total_size }; self.state.buffer_list.first = node; @@ -163,8 +163,7 @@ pub const ArenaAllocator = struct { const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16); const big_enough_len = prev_len + actual_min_size; const len = big_enough_len + big_enough_len / 2; - const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode)); - const ptr = self.child_allocator.rawAlloc(len, log2_align, @returnAddress()) orelse + const ptr = self.child_allocator.rawAlloc(len, BufNode_alignment, @returnAddress()) orelse return null; const buf_node: *BufNode = @ptrCast(@alignCast(ptr)); buf_node.* = .{ .data = len }; @@ -173,11 +172,11 @@ pub const ArenaAllocator = struct { return buf_node; } - fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { + fn alloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { const self: *ArenaAllocator = @ptrCast(@alignCast(ctx)); _ = ra; - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); + const ptr_align = alignment.toByteUnits(); var cur_node = if (self.state.buffer_list.first) |first_node| first_node else @@ -197,8 +196,7 @@ pub const ArenaAllocator = struct { } const bigger_buf_size = @sizeOf(BufNode) + new_end_index; - const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode)); - if (self.child_allocator.rawResize(cur_alloc_buf, log2_align, bigger_buf_size, @returnAddress())) { + if (self.child_allocator.rawResize(cur_alloc_buf, BufNode_alignment, bigger_buf_size, @returnAddress())) { cur_node.data = bigger_buf_size; } else { // Allocate a new node if that's not possible @@ -207,9 +205,9 @@ pub const ArenaAllocator = struct { } } - fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool { + fn resize(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) bool { const self: *ArenaAllocator = @ptrCast(@alignCast(ctx)); - _ = log2_buf_align; + _ = alignment; _ = ret_addr; const cur_node = self.state.buffer_list.first orelse return false; @@ -231,8 +229,18 @@ pub const ArenaAllocator = struct { } } - fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void { - _ = log2_buf_align; + fn remap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, + ) ?[*]u8 { + return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null; + } + + fn free(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, ret_addr: usize) void { + _ = alignment; _ = ret_addr; const self: *ArenaAllocator = @ptrCast(@alignCast(ctx)); diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index d57253686029..d1e4cda8335c 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -226,7 +226,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { requested_size: if (config.enable_memory_limit) usize else void, stack_addresses: [trace_n][stack_n]usize, freed: if (config.retain_metadata) bool else void, - log2_ptr_align: if (config.never_unmap and config.retain_metadata) u8 else void, + alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void, const trace_n = if (config.retain_metadata) traces_per_slot else 1; @@ -281,11 +281,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { return sizes[0..slot_count]; } - fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 { + fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []mem.Alignment { if (!config.safety) @compileError("requested size is only stored when safety is enabled"); const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class); const slot_count = @divExact(page_size, size_class); - return aligns_ptr[0..slot_count]; + return @ptrCast(aligns_ptr[0..slot_count]); } fn stackTracePtr( @@ -326,6 +326,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; @@ -455,7 +456,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { var it = self.large_allocations.iterator(); while (it.next()) |large| { if (large.value_ptr.freed) { - self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.log2_ptr_align, @returnAddress()); + self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress()); } } } @@ -583,10 +584,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn resizeLarge( self: *Self, old_mem: []u8, - log2_old_align: u8, + alignment: mem.Alignment, new_size: usize, ret_addr: usize, - ) bool { + may_move: bool, + ) ?[*]u8 { const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { if (config.safety) { @panic("Invalid free"); @@ -628,30 +630,37 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (config.enable_memory_limit) { const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size; if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { - return false; + return null; } self.total_requested_bytes = new_req_bytes; } - if (!self.backing_allocator.rawResize(old_mem, log2_old_align, new_size, ret_addr)) { + const opt_resized_ptr = if (may_move) + self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr) + else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) + old_mem.ptr + else + null; + + const resized_ptr = opt_resized_ptr orelse { if (config.enable_memory_limit) { self.total_requested_bytes = prev_req_bytes; } - return false; - } + return null; + }; if (config.enable_memory_limit) { entry.value_ptr.requested_size = new_size; } if (config.verbose_log) { - log.info("large resize {d} bytes at {*} to {d}", .{ - old_mem.len, old_mem.ptr, new_size, + log.info("large resize {d} bytes at {*} to {d} at {*}", .{ + old_mem.len, old_mem.ptr, new_size, resized_ptr, }); } - entry.value_ptr.bytes = old_mem.ptr[0..new_size]; + entry.value_ptr.bytes = resized_ptr[0..new_size]; entry.value_ptr.captureStackTrace(ret_addr, .alloc); - return true; + return resized_ptr; } /// This function assumes the object is in the large object storage regardless @@ -659,7 +668,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn freeLarge( self: *Self, old_mem: []u8, - log2_old_align: u8, + alignment: mem.Alignment, ret_addr: usize, ) void { const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { @@ -695,7 +704,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } if (!config.never_unmap) { - self.backing_allocator.rawFree(old_mem, log2_old_align, ret_addr); + self.backing_allocator.rawFree(old_mem, alignment, ret_addr); } if (config.enable_memory_limit) { @@ -719,22 +728,42 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn resize( - ctx: *anyopaque, + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, + ) bool { + return realloc(context, memory, alignment, new_len, return_address, false) != null; + } + + fn remap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, + ) ?[*]u8 { + return realloc(context, memory, alignment, new_len, return_address, true); + } + + fn realloc( + context: *anyopaque, old_mem: []u8, - log2_old_align_u8: u8, - new_size: usize, + alignment: mem.Alignment, + new_len: usize, ret_addr: usize, - ) bool { - const self: *Self = @ptrCast(@alignCast(ctx)); - const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8)); + may_move: bool, + ) ?[*]u8 { + const self: *Self = @ptrCast(@alignCast(context)); self.mutex.lock(); defer self.mutex.unlock(); assert(old_mem.len != 0); - const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align); + const aligned_size = @max(old_mem.len, alignment.toByteUnits()); if (aligned_size > largest_bucket_object_size) { - return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr); + return self.resizeLarge(old_mem, alignment, new_len, ret_addr, may_move); } const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size); @@ -758,7 +787,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } } } - return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr); + return self.resizeLarge(old_mem, alignment, new_len, ret_addr, may_move); }; const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page); const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class)); @@ -779,8 +808,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (config.safety) { const requested_size = bucket.requestedSizes(size_class)[slot_index]; if (requested_size == 0) @panic("Invalid free"); - const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index]; - if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) { + const slot_alignment = bucket.log2PtrAligns(size_class)[slot_index]; + if (old_mem.len != requested_size or alignment != slot_alignment) { var addresses: [stack_n]usize = [1]usize{0} ** stack_n; var free_stack_trace = StackTrace{ .instruction_addresses = &addresses, @@ -795,10 +824,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { free_stack_trace, }); } - if (log2_old_align != log2_ptr_align) { + if (alignment != slot_alignment) { log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{ - @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)), - @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)), + slot_alignment.toByteUnits(), + alignment.toByteUnits(), bucketStackTrace(bucket, size_class, slot_index, .alloc), free_stack_trace, }); @@ -807,52 +836,51 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } const prev_req_bytes = self.total_requested_bytes; if (config.enable_memory_limit) { - const new_req_bytes = prev_req_bytes + new_size - old_mem.len; + const new_req_bytes = prev_req_bytes + new_len - old_mem.len; if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { - return false; + return null; } self.total_requested_bytes = new_req_bytes; } - const new_aligned_size = @max(new_size, @as(usize, 1) << log2_old_align); + const new_aligned_size = @max(new_len, alignment.toByteUnits()); const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size); if (new_size_class <= size_class) { - if (old_mem.len > new_size) { - @memset(old_mem[new_size..], undefined); + if (old_mem.len > new_len) { + @memset(old_mem[new_len..], undefined); } if (config.verbose_log) { log.info("small resize {d} bytes at {*} to {d}", .{ - old_mem.len, old_mem.ptr, new_size, + old_mem.len, old_mem.ptr, new_len, }); } if (config.safety) { - bucket.requestedSizes(size_class)[slot_index] = @intCast(new_size); + bucket.requestedSizes(size_class)[slot_index] = @intCast(new_len); } - return true; + return old_mem.ptr; } if (config.enable_memory_limit) { self.total_requested_bytes = prev_req_bytes; } - return false; + return null; } fn free( ctx: *anyopaque, old_mem: []u8, - log2_old_align_u8: u8, + alignment: mem.Alignment, ret_addr: usize, ) void { const self: *Self = @ptrCast(@alignCast(ctx)); - const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8)); self.mutex.lock(); defer self.mutex.unlock(); assert(old_mem.len != 0); - const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align); + const aligned_size = @max(old_mem.len, alignment.toByteUnits()); if (aligned_size > largest_bucket_object_size) { - self.freeLarge(old_mem, log2_old_align, ret_addr); + self.freeLarge(old_mem, alignment, ret_addr); return; } const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size); @@ -877,7 +905,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } } } - self.freeLarge(old_mem, log2_old_align, ret_addr); + self.freeLarge(old_mem, alignment, ret_addr); return; }; const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page); @@ -900,8 +928,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (config.safety) { const requested_size = bucket.requestedSizes(size_class)[slot_index]; if (requested_size == 0) @panic("Invalid free"); - const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index]; - if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) { + const slot_alignment = bucket.log2PtrAligns(size_class)[slot_index]; + if (old_mem.len != requested_size or alignment != slot_alignment) { var addresses: [stack_n]usize = [1]usize{0} ** stack_n; var free_stack_trace = StackTrace{ .instruction_addresses = &addresses, @@ -916,10 +944,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { free_stack_trace, }); } - if (log2_old_align != log2_ptr_align) { + if (alignment != slot_alignment) { log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ - @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)), - @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)), + slot_alignment.toByteUnits(), + alignment.toByteUnits(), bucketStackTrace(bucket, size_class, slot_index, .alloc), free_stack_trace, }); @@ -981,24 +1009,24 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { return true; } - fn alloc(ctx: *anyopaque, len: usize, log2_ptr_align: u8, ret_addr: usize) ?[*]u8 { + fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 { const self: *Self = @ptrCast(@alignCast(ctx)); self.mutex.lock(); defer self.mutex.unlock(); if (!self.isAllocationAllowed(len)) return null; - return allocInner(self, len, @as(Allocator.Log2Align, @intCast(log2_ptr_align)), ret_addr) catch return null; + return allocInner(self, len, alignment, ret_addr) catch return null; } fn allocInner( self: *Self, len: usize, - log2_ptr_align: Allocator.Log2Align, + alignment: mem.Alignment, ret_addr: usize, ) Allocator.Error![*]u8 { - const new_aligned_size = @max(len, @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align))); + const new_aligned_size = @max(len, alignment.toByteUnits()); if (new_aligned_size > largest_bucket_object_size) { try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1); - const ptr = self.backing_allocator.rawAlloc(len, log2_ptr_align, ret_addr) orelse + const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse return error.OutOfMemory; const slice = ptr[0..len]; @@ -1016,7 +1044,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (config.retain_metadata) { gop.value_ptr.freed = false; if (config.never_unmap) { - gop.value_ptr.log2_ptr_align = log2_ptr_align; + gop.value_ptr.alignment = alignment; } } @@ -1030,7 +1058,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const slot = try self.allocSlot(new_size_class, ret_addr); if (config.safety) { slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len); - slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = log2_ptr_align; + slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = alignment; } if (config.verbose_log) { log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr }); @@ -1150,7 +1178,7 @@ test "realloc" { } test "shrink" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa: GeneralPurposeAllocator(test_config) = .{}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1214,7 +1242,7 @@ test "realloc small object to large object" { } test "shrink large object to large object" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa: GeneralPurposeAllocator(test_config) = .{}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index 277758501bb3..7c0e59906c43 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -6,19 +6,21 @@ const math = std.math; const mem = std.mem; const Allocator = @This(); const builtin = @import("builtin"); +const Alignment = std.mem.Alignment; pub const Error = error{OutOfMemory}; pub const Log2Align = math.Log2Int(usize); /// The type erased pointer to the allocator implementation. -/// Any comparison of this field may result in illegal behavior, since it may be set to -/// `undefined` in cases where the allocator implementation does not have any associated -/// state. +/// +/// Any comparison of this field may result in illegal behavior, since it may +/// be set to `undefined` in cases where the allocator implementation does not +/// have any associated state. ptr: *anyopaque, vtable: *const VTable, pub const VTable = struct { - /// Allocate exactly `len` bytes aligned to `1 << ptr_align`, or return `null` + /// Allocate exactly `len` bytes aligned to `alignment`, or return `null` /// indicating the allocation failed. /// /// `ret_addr` is optionally provided as the first return address of the @@ -27,12 +29,14 @@ pub const VTable = struct { /// /// The returned slice of memory must have been `@memset` to `undefined` /// by the allocator implementation. - alloc: *const fn (ctx: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8, + alloc: *const fn (*anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8, - /// Attempt to expand or shrink memory in place. `buf.len` must equal the - /// length requested from the most recent successful call to `alloc` or - /// `resize`. `buf_align` must equal the same value that was passed as the - /// `ptr_align` parameter to the original `alloc` call. + /// Attempt to expand or shrink memory in place. + /// + /// `memory.len` must equal the length requested from the most recent + /// successful call to `alloc` or `resize`. `alignment` must equal the same + /// value that was passed as the `alignment` parameter to the original + /// `alloc` call. /// /// A result of `true` indicates the resize was successful and the /// allocation now has the same address but a size of `new_len`. `false` @@ -44,72 +48,114 @@ pub const VTable = struct { /// `ret_addr` is optionally provided as the first return address of the /// allocation call stack. If the value is `0` it means no return address /// has been provided. - resize: *const fn (ctx: *anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: usize) bool, + resize: *const fn (*anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool, - /// Free and invalidate a buffer. + /// Attempt to expand or shrink memory, allowing relocation. + /// + /// `memory.len` must equal the length requested from the most recent + /// successful call to `alloc` or `resize`. `alignment` must equal the same + /// value that was passed as the `alignment` parameter to the original + /// `alloc` call. + /// + /// A non-`null` return value indicates the resize was successful. The + /// allocation may have same address, or may have been relocated. In either + /// case, the allocation now has size of `new_len`. A `null` return value + /// indicates that the resize would be equivalent to allocating new memory, + /// copying the bytes from the old memory, and then freeing the old memory. + /// In such case, it is more efficient for the caller to perform the copy. /// - /// `buf.len` must equal the most recent length returned by `alloc` or + /// `new_len` must be greater than zero. + /// + /// `ret_addr` is optionally provided as the first return address of the + /// allocation call stack. If the value is `0` it means no return address + /// has been provided. + remap: *const fn (*anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8, + + /// Free and invalidate a region of memory. + /// + /// `memory.len` must equal the most recent length returned by `alloc` or /// given to a successful `resize` call. /// - /// `buf_align` must equal the same value that was passed as the - /// `ptr_align` parameter to the original `alloc` call. + /// `alignment` must equal the same value that was passed as the + /// `alignment` parameter to the original `alloc` call. /// /// `ret_addr` is optionally provided as the first return address of the /// allocation call stack. If the value is `0` it means no return address /// has been provided. - free: *const fn (ctx: *anyopaque, buf: []u8, buf_align: u8, ret_addr: usize) void, + free: *const fn (*anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void, }; pub fn noResize( self: *anyopaque, - buf: []u8, - log2_buf_align: u8, + memory: []u8, + alignment: Alignment, new_len: usize, ret_addr: usize, ) bool { _ = self; - _ = buf; - _ = log2_buf_align; + _ = memory; + _ = alignment; _ = new_len; _ = ret_addr; return false; } +pub fn noRemap( + self: *anyopaque, + memory: []u8, + alignment: Alignment, + new_len: usize, + ret_addr: usize, +) ?[*]u8 { + _ = self; + _ = memory; + _ = alignment; + _ = new_len; + _ = ret_addr; + return null; +} + pub fn noFree( self: *anyopaque, - buf: []u8, - log2_buf_align: u8, + memory: []u8, + alignment: Alignment, ret_addr: usize, ) void { _ = self; - _ = buf; - _ = log2_buf_align; + _ = memory; + _ = alignment; _ = ret_addr; } /// This function is not intended to be called except from within the /// implementation of an Allocator -pub inline fn rawAlloc(self: Allocator, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 { - return self.vtable.alloc(self.ptr, len, ptr_align, ret_addr); +pub inline fn rawAlloc(a: Allocator, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 { + return a.vtable.alloc(a.ptr, len, alignment, ret_addr); } /// This function is not intended to be called except from within the -/// implementation of an Allocator -pub inline fn rawResize(self: Allocator, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool { - return self.vtable.resize(self.ptr, buf, log2_buf_align, new_len, ret_addr); +/// implementation of an Allocator. +pub inline fn rawResize(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { + return a.vtable.resize(a.ptr, memory, alignment, new_len, ret_addr); +} + +/// This function is not intended to be called except from within the +/// implementation of an Allocator. +pub inline fn rawRemap(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + return a.vtable.remap(a.ptr, memory, alignment, new_len, ret_addr); } /// This function is not intended to be called except from within the /// implementation of an Allocator -pub inline fn rawFree(self: Allocator, buf: []u8, log2_buf_align: u8, ret_addr: usize) void { - return self.vtable.free(self.ptr, buf, log2_buf_align, ret_addr); +pub inline fn rawFree(a: Allocator, memory: []u8, alignment: Alignment, ret_addr: usize) void { + return a.vtable.free(a.ptr, memory, alignment, ret_addr); } /// Returns a pointer to undefined memory. /// Call `destroy` with the result to free the memory. -pub fn create(self: Allocator, comptime T: type) Error!*T { +pub fn create(a: Allocator, comptime T: type) Error!*T { if (@sizeOf(T) == 0) return @as(*T, @ptrFromInt(math.maxInt(usize))); - const ptr: *T = @ptrCast(try self.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress())); + const ptr: *T = @ptrCast(try a.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress())); return ptr; } @@ -121,7 +167,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void { const T = info.child; if (@sizeOf(T) == 0) return; const non_const_ptr = @as([*]u8, @ptrCast(@constCast(ptr))); - self.rawFree(non_const_ptr[0..@sizeOf(T)], log2a(info.alignment), @returnAddress()); + self.rawFree(non_const_ptr[0..@sizeOf(T)], .fromByteUnits(info.alignment), @returnAddress()); } /// Allocates an array of `n` items of type `T` and sets all the @@ -224,36 +270,88 @@ fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: return @as([*]align(alignment) u8, @ptrFromInt(ptr)); } - const byte_ptr = self.rawAlloc(byte_count, log2a(alignment), return_address) orelse return Error.OutOfMemory; + const byte_ptr = self.rawAlloc(byte_count, .fromByteUnits(alignment), return_address) orelse return Error.OutOfMemory; // TODO: https://github.com/ziglang/zig/issues/4298 @memset(byte_ptr[0..byte_count], undefined); return @alignCast(byte_ptr); } -/// Requests to modify the size of an allocation. It is guaranteed to not move -/// the pointer, however the allocator implementation may refuse the resize -/// request by returning `false`. -pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) bool { - const Slice = @typeInfo(@TypeOf(old_mem)).pointer; +/// Request to modify the size of an allocation. +/// +/// It is guaranteed to not move the pointer, however the allocator +/// implementation may refuse the resize request by returning `false`. +/// +/// `allocation` may be an empty slice, in which case a new allocation is made. +/// +/// `new_len` may be zero, in which case the allocation is freed. +pub fn resize(self: Allocator, allocation: anytype, new_len: usize) bool { + const Slice = @typeInfo(@TypeOf(allocation)).pointer; const T = Slice.child; - if (new_n == 0) { - self.free(old_mem); + const alignment = Slice.alignment; + if (new_len == 0) { + self.free(allocation); return true; } - if (old_mem.len == 0) { + if (allocation.len == 0) { return false; } - const old_byte_slice = mem.sliceAsBytes(old_mem); + const old_memory = mem.sliceAsBytes(allocation); + // I would like to use saturating multiplication here, but LLVM cannot lower it + // on WebAssembly: https://github.com/ziglang/zig/issues/9660 + //const new_len_bytes = new_len *| @sizeOf(T); + const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return false; + return self.rawResize(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress()); +} + +/// Request to modify the size of an allocation, allowing relocation. +/// +/// A non-`null` return value indicates the resize was successful. The +/// allocation may have same address, or may have been relocated. In either +/// case, the allocation now has size of `new_len`. A `null` return value +/// indicates that the resize would be equivalent to allocating new memory, +/// copying the bytes from the old memory, and then freeing the old memory. +/// In such case, it is more efficient for the caller to perform those +/// operations. +/// +/// `allocation` may be an empty slice, in which case a new allocation is made. +/// +/// `new_len` may be zero, in which case the allocation is freed. +pub fn remap(self: Allocator, allocation: anytype, new_len: usize) t: { + const Slice = @typeInfo(@TypeOf(allocation)).pointer; + break :t ?[]align(Slice.alignment) Slice.child; +} { + const Slice = @typeInfo(@TypeOf(allocation)).pointer; + const T = Slice.child; + const alignment = Slice.alignment; + if (new_len == 0) { + self.free(allocation); + return allocation[0..0]; + } + if (allocation.len == 0) { + return null; + } + const old_memory = mem.sliceAsBytes(allocation); // I would like to use saturating multiplication here, but LLVM cannot lower it // on WebAssembly: https://github.com/ziglang/zig/issues/9660 - //const new_byte_count = new_n *| @sizeOf(T); - const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return false; - return self.rawResize(old_byte_slice, log2a(Slice.alignment), new_byte_count, @returnAddress()); + //const new_len_bytes = new_len *| @sizeOf(T); + const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return null; + const new_ptr = self.rawRemap(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress()) orelse return null; + const new_memory: []align(alignment) u8 = @alignCast(new_ptr[0..new_len_bytes]); + return mem.bytesAsSlice(T, new_memory); } /// This function requests a new byte size for an existing allocation, which /// can be larger, smaller, or the same size as the old memory allocation. +/// /// If `new_n` is 0, this is the same as `free` and it always succeeds. +/// +/// `old_mem` may have length zero, which makes a new allocation. +/// +/// This function only fails on out-of-memory conditions, unlike: +/// * `remap` which returns `null` when the `Allocator` implementation cannot +/// do the realloc more efficiently than the caller +/// * `resize` which returns `false` when the `Allocator` implementation cannot +/// change the size without relocating the allocation. pub fn realloc(self: Allocator, old_mem: anytype, new_n: usize) t: { const Slice = @typeInfo(@TypeOf(old_mem)).pointer; break :t Error![]align(Slice.alignment) Slice.child; @@ -284,18 +382,18 @@ pub fn reallocAdvanced( const old_byte_slice = mem.sliceAsBytes(old_mem); const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory; // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure - if (self.rawResize(old_byte_slice, log2a(Slice.alignment), byte_count, return_address)) { - const new_bytes: []align(Slice.alignment) u8 = @alignCast(old_byte_slice.ptr[0..byte_count]); + if (self.rawRemap(old_byte_slice, .fromByteUnits(Slice.alignment), byte_count, return_address)) |p| { + const new_bytes: []align(Slice.alignment) u8 = @alignCast(p[0..byte_count]); return mem.bytesAsSlice(T, new_bytes); } - const new_mem = self.rawAlloc(byte_count, log2a(Slice.alignment), return_address) orelse + const new_mem = self.rawAlloc(byte_count, .fromByteUnits(Slice.alignment), return_address) orelse return error.OutOfMemory; const copy_len = @min(byte_count, old_byte_slice.len); @memcpy(new_mem[0..copy_len], old_byte_slice[0..copy_len]); // TODO https://github.com/ziglang/zig/issues/4298 @memset(old_byte_slice, undefined); - self.rawFree(old_byte_slice, log2a(Slice.alignment), return_address); + self.rawFree(old_byte_slice, .fromByteUnits(Slice.alignment), return_address); const new_bytes: []align(Slice.alignment) u8 = @alignCast(new_mem[0..byte_count]); return mem.bytesAsSlice(T, new_bytes); @@ -312,7 +410,7 @@ pub fn free(self: Allocator, memory: anytype) void { const non_const_ptr = @constCast(bytes.ptr); // TODO: https://github.com/ziglang/zig/issues/4298 @memset(non_const_ptr[0..bytes_len], undefined); - self.rawFree(non_const_ptr[0..bytes_len], log2a(Slice.alignment), @returnAddress()); + self.rawFree(non_const_ptr[0..bytes_len], .fromByteUnits(Slice.alignment), @returnAddress()); } /// Copies `m` to newly allocated memory. Caller owns the memory. @@ -329,17 +427,3 @@ pub fn dupeZ(allocator: Allocator, comptime T: type, m: []const T) Error![:0]T { new_buf[m.len] = 0; return new_buf[0..m.len :0]; } - -/// TODO replace callsites with `@log2` after this proposal is implemented: -/// https://github.com/ziglang/zig/issues/13642 -inline fn log2a(x: anytype) switch (@typeInfo(@TypeOf(x))) { - .int => math.Log2Int(@TypeOf(x)), - .comptime_int => comptime_int, - else => @compileError("int please"), -} { - switch (@typeInfo(@TypeOf(x))) { - .int => return math.log2_int(@TypeOf(x), x), - .comptime_int => return math.log2(x), - else => @compileError("bad"), - } -} diff --git a/lib/std/testing/failing_allocator.zig b/lib/std/testing/failing_allocator.zig index 3a83f313bf09..e8b70eaf218e 100644 --- a/lib/std/testing/failing_allocator.zig +++ b/lib/std/testing/failing_allocator.zig @@ -62,6 +62,7 @@ pub const FailingAllocator = struct { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; @@ -70,7 +71,7 @@ pub const FailingAllocator = struct { fn alloc( ctx: *anyopaque, len: usize, - log2_ptr_align: u8, + alignment: mem.Alignment, return_address: usize, ) ?[*]u8 { const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); @@ -86,7 +87,7 @@ pub const FailingAllocator = struct { } return null; } - const result = self.internal_allocator.rawAlloc(len, log2_ptr_align, return_address) orelse + const result = self.internal_allocator.rawAlloc(len, alignment, return_address) orelse return null; self.allocated_bytes += len; self.allocations += 1; @@ -96,33 +97,52 @@ pub const FailingAllocator = struct { fn resize( ctx: *anyopaque, - old_mem: []u8, - log2_old_align: u8, + memory: []u8, + alignment: mem.Alignment, new_len: usize, ra: usize, ) bool { const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); if (self.resize_index == self.resize_fail_index) return false; - if (!self.internal_allocator.rawResize(old_mem, log2_old_align, new_len, ra)) + if (!self.internal_allocator.rawResize(memory, alignment, new_len, ra)) return false; - if (new_len < old_mem.len) { - self.freed_bytes += old_mem.len - new_len; + if (new_len < memory.len) { + self.freed_bytes += memory.len - new_len; } else { - self.allocated_bytes += new_len - old_mem.len; + self.allocated_bytes += new_len - memory.len; } self.resize_index += 1; return true; } + fn remap( + ctx: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + ra: usize, + ) ?[*]u8 { + const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); + if (self.resize_index == self.resize_fail_index) return null; + const new_ptr = self.internal_allocator.rawRemap(memory, alignment, new_len, ra) orelse return null; + if (new_len < memory.len) { + self.freed_bytes += memory.len - new_len; + } else { + self.allocated_bytes += new_len - memory.len; + } + self.resize_index += 1; + return new_ptr; + } + fn free( ctx: *anyopaque, old_mem: []u8, - log2_old_align: u8, + alignment: mem.Alignment, ra: usize, ) void { const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); - self.internal_allocator.rawFree(old_mem, log2_old_align, ra); + self.internal_allocator.rawFree(old_mem, alignment, ra); self.deallocations += 1; self.freed_bytes += old_mem.len; } From a0b2a18648ec4c465f3e19c05338b554c0742dfb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 20:00:01 -0800 Subject: [PATCH 10/39] std.testing.FailingAllocator: flatten namespace --- lib/std/testing.zig | 7 +- lib/std/testing/FailingAllocator.zig | 161 +++++++++++++++++++++++++ lib/std/testing/failing_allocator.zig | 162 -------------------------- 3 files changed, 164 insertions(+), 166 deletions(-) create mode 100644 lib/std/testing/FailingAllocator.zig delete mode 100644 lib/std/testing/failing_allocator.zig diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 1c109c48e21f..3387f258b802 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -7,7 +7,9 @@ const math = std.math; /// Initialized on startup. Read-only after that. pub var random_seed: u32 = 0; -pub const FailingAllocator = @import("testing/failing_allocator.zig").FailingAllocator; +pub const FailingAllocator = @import("testing/FailingAllocator.zig"); +pub const failing_allocator = failing_allocator_instance.allocator(); +pub var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.allocator(), .{ .fail_index = 0 }); /// This should only be used in temporary test programs. pub const allocator = allocator_instance.allocator(); @@ -17,9 +19,6 @@ pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{}) = b: { break :b .init; }; -pub const failing_allocator = failing_allocator_instance.allocator(); -pub var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.allocator(), .{ .fail_index = 0 }); - pub var base_allocator_instance = std.heap.FixedBufferAllocator.init(""); /// TODO https://github.com/ziglang/zig/issues/5738 diff --git a/lib/std/testing/FailingAllocator.zig b/lib/std/testing/FailingAllocator.zig new file mode 100644 index 000000000000..c7767ae02fec --- /dev/null +++ b/lib/std/testing/FailingAllocator.zig @@ -0,0 +1,161 @@ +//! Allocator that fails after N allocations, useful for making sure out of +//! memory conditions are handled correctly. +//! +//! To use this, first initialize it and get an allocator with +//! +//! `const failing_allocator = &FailingAllocator.init(, +//! ).allocator;` +//! +//! Then use `failing_allocator` anywhere you would have used a +//! different allocator. +const std = @import("../std.zig"); +const mem = std.mem; +const FailingAllocator = @This(); + +alloc_index: usize, +resize_index: usize, +internal_allocator: mem.Allocator, +allocated_bytes: usize, +freed_bytes: usize, +allocations: usize, +deallocations: usize, +stack_addresses: [num_stack_frames]usize, +has_induced_failure: bool, +fail_index: usize, +resize_fail_index: usize, + +const num_stack_frames = if (std.debug.sys_can_stack_trace) 16 else 0; + +pub const Config = struct { + /// The number of successful allocations you can expect from this allocator. + /// The next allocation will fail. For example, with `fail_index` equal to + /// 2, the following test will pass: + /// + /// var a = try failing_alloc.create(i32); + /// var b = try failing_alloc.create(i32); + /// testing.expectError(error.OutOfMemory, failing_alloc.create(i32)); + fail_index: usize = std.math.maxInt(usize), + + /// Number of successful resizes to expect from this allocator. The next resize will fail. + resize_fail_index: usize = std.math.maxInt(usize), +}; + +pub fn init(internal_allocator: mem.Allocator, config: Config) FailingAllocator { + return FailingAllocator{ + .internal_allocator = internal_allocator, + .alloc_index = 0, + .resize_index = 0, + .allocated_bytes = 0, + .freed_bytes = 0, + .allocations = 0, + .deallocations = 0, + .stack_addresses = undefined, + .has_induced_failure = false, + .fail_index = config.fail_index, + .resize_fail_index = config.resize_fail_index, + }; +} + +pub fn allocator(self: *FailingAllocator) mem.Allocator { + return .{ + .ptr = self, + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }, + }; +} + +fn alloc( + ctx: *anyopaque, + len: usize, + alignment: mem.Alignment, + return_address: usize, +) ?[*]u8 { + const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); + if (self.alloc_index == self.fail_index) { + if (!self.has_induced_failure) { + @memset(&self.stack_addresses, 0); + var stack_trace = std.builtin.StackTrace{ + .instruction_addresses = &self.stack_addresses, + .index = 0, + }; + std.debug.captureStackTrace(return_address, &stack_trace); + self.has_induced_failure = true; + } + return null; + } + const result = self.internal_allocator.rawAlloc(len, alignment, return_address) orelse + return null; + self.allocated_bytes += len; + self.allocations += 1; + self.alloc_index += 1; + return result; +} + +fn resize( + ctx: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + ra: usize, +) bool { + const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); + if (self.resize_index == self.resize_fail_index) + return false; + if (!self.internal_allocator.rawResize(memory, alignment, new_len, ra)) + return false; + if (new_len < memory.len) { + self.freed_bytes += memory.len - new_len; + } else { + self.allocated_bytes += new_len - memory.len; + } + self.resize_index += 1; + return true; +} + +fn remap( + ctx: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + ra: usize, +) ?[*]u8 { + const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); + if (self.resize_index == self.resize_fail_index) return null; + const new_ptr = self.internal_allocator.rawRemap(memory, alignment, new_len, ra) orelse return null; + if (new_len < memory.len) { + self.freed_bytes += memory.len - new_len; + } else { + self.allocated_bytes += new_len - memory.len; + } + self.resize_index += 1; + return new_ptr; +} + +fn free( + ctx: *anyopaque, + old_mem: []u8, + alignment: mem.Alignment, + ra: usize, +) void { + const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); + self.internal_allocator.rawFree(old_mem, alignment, ra); + self.deallocations += 1; + self.freed_bytes += old_mem.len; +} + +/// Only valid once `has_induced_failure == true` +pub fn getStackTrace(self: *FailingAllocator) std.builtin.StackTrace { + std.debug.assert(self.has_induced_failure); + var len: usize = 0; + while (len < self.stack_addresses.len and self.stack_addresses[len] != 0) { + len += 1; + } + return .{ + .instruction_addresses = &self.stack_addresses, + .index = len, + }; +} diff --git a/lib/std/testing/failing_allocator.zig b/lib/std/testing/failing_allocator.zig deleted file mode 100644 index e8b70eaf218e..000000000000 --- a/lib/std/testing/failing_allocator.zig +++ /dev/null @@ -1,162 +0,0 @@ -const std = @import("../std.zig"); -const mem = std.mem; - -pub const Config = struct { - /// The number of successful allocations you can expect from this allocator. - /// The next allocation will fail. For example, with `fail_index` equal to - /// 2, the following test will pass: - /// - /// var a = try failing_alloc.create(i32); - /// var b = try failing_alloc.create(i32); - /// testing.expectError(error.OutOfMemory, failing_alloc.create(i32)); - fail_index: usize = std.math.maxInt(usize), - - /// Number of successful resizes to expect from this allocator. The next resize will fail. - resize_fail_index: usize = std.math.maxInt(usize), -}; - -/// Allocator that fails after N allocations, useful for making sure out of -/// memory conditions are handled correctly. -/// -/// To use this, first initialize it and get an allocator with -/// -/// `const failing_allocator = &FailingAllocator.init(, -/// ).allocator;` -/// -/// Then use `failing_allocator` anywhere you would have used a -/// different allocator. -pub const FailingAllocator = struct { - alloc_index: usize, - resize_index: usize, - internal_allocator: mem.Allocator, - allocated_bytes: usize, - freed_bytes: usize, - allocations: usize, - deallocations: usize, - stack_addresses: [num_stack_frames]usize, - has_induced_failure: bool, - fail_index: usize, - resize_fail_index: usize, - - const num_stack_frames = if (std.debug.sys_can_stack_trace) 16 else 0; - - pub fn init(internal_allocator: mem.Allocator, config: Config) FailingAllocator { - return FailingAllocator{ - .internal_allocator = internal_allocator, - .alloc_index = 0, - .resize_index = 0, - .allocated_bytes = 0, - .freed_bytes = 0, - .allocations = 0, - .deallocations = 0, - .stack_addresses = undefined, - .has_induced_failure = false, - .fail_index = config.fail_index, - .resize_fail_index = config.resize_fail_index, - }; - } - - pub fn allocator(self: *FailingAllocator) mem.Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - fn alloc( - ctx: *anyopaque, - len: usize, - alignment: mem.Alignment, - return_address: usize, - ) ?[*]u8 { - const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); - if (self.alloc_index == self.fail_index) { - if (!self.has_induced_failure) { - @memset(&self.stack_addresses, 0); - var stack_trace = std.builtin.StackTrace{ - .instruction_addresses = &self.stack_addresses, - .index = 0, - }; - std.debug.captureStackTrace(return_address, &stack_trace); - self.has_induced_failure = true; - } - return null; - } - const result = self.internal_allocator.rawAlloc(len, alignment, return_address) orelse - return null; - self.allocated_bytes += len; - self.allocations += 1; - self.alloc_index += 1; - return result; - } - - fn resize( - ctx: *anyopaque, - memory: []u8, - alignment: mem.Alignment, - new_len: usize, - ra: usize, - ) bool { - const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); - if (self.resize_index == self.resize_fail_index) - return false; - if (!self.internal_allocator.rawResize(memory, alignment, new_len, ra)) - return false; - if (new_len < memory.len) { - self.freed_bytes += memory.len - new_len; - } else { - self.allocated_bytes += new_len - memory.len; - } - self.resize_index += 1; - return true; - } - - fn remap( - ctx: *anyopaque, - memory: []u8, - alignment: mem.Alignment, - new_len: usize, - ra: usize, - ) ?[*]u8 { - const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); - if (self.resize_index == self.resize_fail_index) return null; - const new_ptr = self.internal_allocator.rawRemap(memory, alignment, new_len, ra) orelse return null; - if (new_len < memory.len) { - self.freed_bytes += memory.len - new_len; - } else { - self.allocated_bytes += new_len - memory.len; - } - self.resize_index += 1; - return new_ptr; - } - - fn free( - ctx: *anyopaque, - old_mem: []u8, - alignment: mem.Alignment, - ra: usize, - ) void { - const self: *FailingAllocator = @ptrCast(@alignCast(ctx)); - self.internal_allocator.rawFree(old_mem, alignment, ra); - self.deallocations += 1; - self.freed_bytes += old_mem.len; - } - - /// Only valid once `has_induced_failure == true` - pub fn getStackTrace(self: *FailingAllocator) std.builtin.StackTrace { - std.debug.assert(self.has_induced_failure); - var len: usize = 0; - while (len < self.stack_addresses.len and self.stack_addresses[len] != 0) { - len += 1; - } - return .{ - .instruction_addresses = &self.stack_addresses, - .index = len, - }; - } -}; From a4d4e086c59b702c34218105764d4ec491ecc566 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 20:40:57 -0800 Subject: [PATCH 11/39] introduce std.posix.mremap and use it in std.heap.page_allocator --- lib/std/heap/PageAllocator.zig | 10 ++++++---- lib/std/os/linux.zig | 18 +++++++++++++++++ lib/std/posix.zig | 35 ++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index 59b0c6015d18..c0c0c146956d 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -140,7 +140,8 @@ fn free(context: *anyopaque, slice: []u8, alignment: mem.Alignment, return_addre } } -fn realloc(memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { +fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { + const memory: []align(std.heap.page_size_min) u8 = @alignCast(uncasted_memory); const page_size = std.heap.pageSize(); const new_size_aligned = mem.alignForward(usize, new_len, page_size); @@ -153,7 +154,7 @@ fn realloc(memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { // For shrinking that is not releasing, we will only decommit // the pages not needed anymore. windows.VirtualFree( - @as(*anyopaque, @ptrFromInt(new_addr_end)), + @ptrFromInt(new_addr_end), old_addr_end - new_addr_end, windows.MEM_DECOMMIT, ); @@ -171,10 +172,11 @@ fn realloc(memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { if (new_size_aligned == page_aligned_len) return memory.ptr; - const mremap_available = false; // native_os == .linux; + const mremap_available = native_os == .linux; if (mremap_available) { // TODO: if the next_mmap_addr_hint is within the remapped range, update it - return posix.mremap(memory, new_len, .{ .MAYMOVE = may_move }, null) catch return null; + const new_memory = posix.mremap(memory.ptr, memory.len, new_len, .{ .MAYMOVE = may_move }, null) catch return null; + return new_memory.ptr; } if (new_size_aligned < page_aligned_len) { diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 365fb9f05f6b..6e64d022cb4e 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -305,6 +305,13 @@ pub const MAP = switch (native_arch) { else => @compileError("missing std.os.linux.MAP constants for this architecture"), }; +pub const MREMAP = packed struct(u32) { + MAYMOVE: bool = false, + FIXED: bool = false, + DONTUNMAP: bool = false, + _: u29 = 0, +}; + pub const O = switch (native_arch) { .x86_64 => packed struct(u32) { ACCMODE: ACCMODE = .RDONLY, @@ -934,6 +941,17 @@ pub fn mprotect(address: [*]const u8, length: usize, protection: usize) usize { return syscall3(.mprotect, @intFromPtr(address), length, protection); } +pub fn mremap(old_addr: ?[*]const u8, old_len: usize, new_len: usize, flags: MREMAP, new_addr: ?[*]const u8) usize { + return syscall5( + .mremap, + @intFromPtr(old_addr), + old_len, + new_len, + @as(u32, @bitCast(flags)), + @intFromPtr(new_addr), + ); +} + pub const MSF = struct { pub const ASYNC = 1; pub const INVALIDATE = 2; diff --git a/lib/std/posix.zig b/lib/std/posix.zig index 5b36c9b13978..a1409705bf84 100644 --- a/lib/std/posix.zig +++ b/lib/std/posix.zig @@ -83,6 +83,7 @@ pub const MAP = system.MAP; pub const MAX_ADDR_LEN = system.MAX_ADDR_LEN; pub const MFD = system.MFD; pub const MMAP2_UNIT = system.MMAP2_UNIT; +pub const MREMAP = system.MREMAP; pub const MSF = system.MSF; pub const MSG = system.MSG; pub const NAME_MAX = system.NAME_MAX; @@ -4809,6 +4810,40 @@ pub fn munmap(memory: []align(page_size_min) const u8) void { } } +pub const MRemapError = error{ + LockedMemoryLimitExceeded, + /// Either a bug in the calling code, or the operating system abused the + /// EINVAL error code. + InvalidSyscallParameters, + OutOfMemory, +} || UnexpectedError; + +pub fn mremap( + old_address: ?[*]align(page_size_min) u8, + old_len: usize, + new_len: usize, + flags: system.MREMAP, + new_address: ?[*]align(page_size_min) u8, +) MRemapError![]align(page_size_min) u8 { + const rc = system.mremap(old_address, old_len, new_len, flags, new_address); + const err: E = if (builtin.link_libc) blk: { + if (rc != std.c.MAP_FAILED) return @as([*]align(page_size_min) u8, @ptrCast(@alignCast(rc)))[0..new_len]; + break :blk @enumFromInt(system._errno().*); + } else blk: { + const err = errno(rc); + if (err == .SUCCESS) return @as([*]align(page_size_min) u8, @ptrFromInt(rc))[0..new_len]; + break :blk err; + }; + switch (err) { + .SUCCESS => unreachable, + .AGAIN => return error.LockedMemoryLimitExceeded, + .INVAL => return error.InvalidSyscallParameters, + .NOMEM => return error.OutOfMemory, + .FAULT => unreachable, + else => return unexpectedErrno(err), + } +} + pub const MSyncError = error{ UnmappedMemory, PermissionDenied, From 0d8166be3f7e1cc2a2200956155ed5a8792614b8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 21:03:27 -0800 Subject: [PATCH 12/39] std: update to new Allocator API --- lib/std/heap.zig | 82 ++++++++++++++---------- lib/std/heap/log_to_writer_allocator.zig | 39 +++++++++-- lib/std/mem.zig | 50 +++++++-------- 3 files changed, 105 insertions(+), 66 deletions(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 746fc74bd0c0..2abe00c7a07c 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -147,12 +147,12 @@ const CAllocator = struct { return @as(*[*]u8, @ptrFromInt(@intFromPtr(ptr) - @sizeOf(usize))); } - fn alignedAlloc(len: usize, log2_align: u8) ?[*]u8 { - const alignment = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_align)); + fn alignedAlloc(len: usize, alignment: mem.Alignment) ?[*]u8 { + const alignment_bytes = alignment.toByteUnits(); if (supports_posix_memalign) { // The posix_memalign only accepts alignment values that are a // multiple of the pointer size - const eff_alignment = @max(alignment, @sizeOf(usize)); + const eff_alignment = @max(alignment_bytes, @sizeOf(usize)); var aligned_ptr: ?*anyopaque = undefined; if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0) @@ -164,9 +164,9 @@ const CAllocator = struct { // Thin wrapper around regular malloc, overallocate to account for // alignment padding and store the original malloc()'ed pointer before // the aligned address. - const unaligned_ptr = @as([*]u8, @ptrCast(c.malloc(len + alignment - 1 + @sizeOf(usize)) orelse return null)); + const unaligned_ptr = @as([*]u8, @ptrCast(c.malloc(len + alignment_bytes - 1 + @sizeOf(usize)) orelse return null)); const unaligned_addr = @intFromPtr(unaligned_ptr); - const aligned_addr = mem.alignForward(usize, unaligned_addr + @sizeOf(usize), alignment); + const aligned_addr = mem.alignForward(usize, unaligned_addr + @sizeOf(usize), alignment_bytes); const aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr); getHeader(aligned_ptr).* = unaligned_ptr; @@ -195,22 +195,22 @@ const CAllocator = struct { fn alloc( _: *anyopaque, len: usize, - log2_align: u8, + alignment: mem.Alignment, return_address: usize, ) ?[*]u8 { _ = return_address; assert(len > 0); - return alignedAlloc(len, log2_align); + return alignedAlloc(len, alignment); } fn resize( _: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, new_len: usize, return_address: usize, ) bool { - _ = log2_buf_align; + _ = alignment; _ = return_address; if (new_len <= buf.len) { return true; @@ -227,10 +227,10 @@ const CAllocator = struct { fn free( _: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, return_address: usize, ) void { - _ = log2_buf_align; + _ = alignment; _ = return_address; alignedFree(buf.ptr); } @@ -267,28 +267,28 @@ const raw_c_allocator_vtable = Allocator.VTable{ fn rawCAlloc( _: *anyopaque, len: usize, - log2_ptr_align: u8, + alignment: mem.Alignment, ret_addr: usize, ) ?[*]u8 { _ = ret_addr; - assert(log2_ptr_align <= comptime std.math.log2_int(usize, @alignOf(std.c.max_align_t))); + assert(alignment.order(.le, comptime .fromByteUnits(@alignOf(std.c.max_align_t)))); // Note that this pointer cannot be aligncasted to max_align_t because if // len is < max_align_t then the alignment can be smaller. For example, if // max_align_t is 16, but the user requests 8 bytes, there is no built-in // type in C that is size 8 and has 16 byte alignment, so the alignment may // be 8 bytes rather than 16. Similarly if only 1 byte is requested, malloc // is allowed to return a 1-byte aligned pointer. - return @as(?[*]u8, @ptrCast(c.malloc(len))); + return @ptrCast(c.malloc(len)); } fn rawCResize( _: *anyopaque, buf: []u8, - log2_old_align: u8, + alignment: mem.Alignment, new_len: usize, ret_addr: usize, ) bool { - _ = log2_old_align; + _ = alignment; _ = ret_addr; if (new_len <= buf.len) @@ -305,10 +305,10 @@ fn rawCResize( fn rawCFree( _: *anyopaque, buf: []u8, - log2_old_align: u8, + alignment: mem.Alignment, ret_addr: usize, ) void { - _ = log2_old_align; + _ = alignment; _ = ret_addr; c.free(buf.ptr); } @@ -380,13 +380,13 @@ pub const HeapAllocator = switch (builtin.os.tag) { fn alloc( ctx: *anyopaque, n: usize, - log2_ptr_align: u8, + alignment: mem.Alignment, return_address: usize, ) ?[*]u8 { _ = return_address; const self: *HeapAllocator = @ptrCast(@alignCast(ctx)); - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); + const ptr_align = alignment.toByteUnits(); const amt = n + ptr_align - 1 + @sizeOf(usize); const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, .seq_cst); const heap_handle = optional_heap_handle orelse blk: { @@ -407,11 +407,11 @@ pub const HeapAllocator = switch (builtin.os.tag) { fn resize( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, new_size: usize, return_address: usize, ) bool { - _ = log2_buf_align; + _ = alignment; _ = return_address; const self: *HeapAllocator = @ptrCast(@alignCast(ctx)); @@ -432,10 +432,10 @@ pub const HeapAllocator = switch (builtin.os.tag) { fn free( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, return_address: usize, ) void { - _ = log2_buf_align; + _ = alignment; _ = return_address; const self: *HeapAllocator = @ptrCast(@alignCast(ctx)); windows.HeapFree(self.heap_handle.?, 0, @as(*anyopaque, @ptrFromInt(getRecordPtr(buf).*))); @@ -482,6 +482,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; @@ -496,40 +497,55 @@ pub fn StackFallbackAllocator(comptime size: usize) type { fn alloc( ctx: *anyopaque, len: usize, - log2_ptr_align: u8, + alignment: mem.Alignment, ra: usize, ) ?[*]u8 { const self: *Self = @ptrCast(@alignCast(ctx)); - return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, log2_ptr_align, ra) orelse - return self.fallback_allocator.rawAlloc(len, log2_ptr_align, ra); + return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, alignment, ra) orelse + return self.fallback_allocator.rawAlloc(len, alignment, ra); } fn resize( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, new_len: usize, ra: usize, ) bool { const self: *Self = @ptrCast(@alignCast(ctx)); if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) { - return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, log2_buf_align, new_len, ra); + return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, alignment, new_len, ra); } else { - return self.fallback_allocator.rawResize(buf, log2_buf_align, new_len, ra); + return self.fallback_allocator.rawResize(buf, alignment, new_len, ra); + } + } + + fn remap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, + ) ?[*]u8 { + const self: *Self = @ptrCast(@alignCast(context)); + if (self.fixed_buffer_allocator.ownsPtr(memory.ptr)) { + return FixedBufferAllocator.remap(&self.fixed_buffer_allocator, memory, alignment, new_len, return_address); + } else { + return self.fallback_allocator.rawRemap(memory, alignment, new_len, return_address); } } fn free( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, ra: usize, ) void { const self: *Self = @ptrCast(@alignCast(ctx)); if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) { - return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, log2_buf_align, ra); + return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, alignment, ra); } else { - return self.fallback_allocator.rawFree(buf, log2_buf_align, ra); + return self.fallback_allocator.rawFree(buf, alignment, ra); } } }; diff --git a/lib/std/heap/log_to_writer_allocator.zig b/lib/std/heap/log_to_writer_allocator.zig index b5c86c9bebbf..239a580f9bae 100644 --- a/lib/std/heap/log_to_writer_allocator.zig +++ b/lib/std/heap/log_to_writer_allocator.zig @@ -23,6 +23,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; @@ -31,12 +32,12 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { fn alloc( ctx: *anyopaque, len: usize, - log2_ptr_align: u8, + alignment: std.mem.Alignment, ra: usize, ) ?[*]u8 { const self: *Self = @ptrCast(@alignCast(ctx)); self.writer.print("alloc : {}", .{len}) catch {}; - const result = self.parent_allocator.rawAlloc(len, log2_ptr_align, ra); + const result = self.parent_allocator.rawAlloc(len, alignment, ra); if (result != null) { self.writer.print(" success!\n", .{}) catch {}; } else { @@ -48,7 +49,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { fn resize( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: std.mem.Alignment, new_len: usize, ra: usize, ) bool { @@ -59,7 +60,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {}; } - if (self.parent_allocator.rawResize(buf, log2_buf_align, new_len, ra)) { + if (self.parent_allocator.rawResize(buf, alignment, new_len, ra)) { if (new_len > buf.len) { self.writer.print(" success!\n", .{}) catch {}; } @@ -71,15 +72,41 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { return false; } + fn remap( + ctx: *anyopaque, + buf: []u8, + alignment: std.mem.Alignment, + new_len: usize, + ra: usize, + ) ?[*]u8 { + const self: *Self = @ptrCast(@alignCast(ctx)); + if (new_len <= buf.len) { + self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {}; + } else { + self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {}; + } + + if (self.parent_allocator.rawRemap(buf, alignment, new_len, ra)) |new_memory| { + if (new_len > buf.len) { + self.writer.print(" success!\n", .{}) catch {}; + } + return new_memory; + } + + std.debug.assert(new_len > buf.len); + self.writer.print(" failure!\n", .{}) catch {}; + return null; + } + fn free( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: std.mem.Alignment, ra: usize, ) void { const self: *Self = @ptrCast(@alignCast(ctx)); self.writer.print("free : {}\n", .{buf.len}) catch {}; - self.parent_allocator.rawFree(buf, log2_buf_align, ra); + self.parent_allocator.rawFree(buf, alignment, ra); } }; } diff --git a/lib/std/mem.zig b/lib/std/mem.zig index c357c768d768..0f56762c2782 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -92,6 +92,7 @@ pub fn ValidationAllocator(comptime T: type) type { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; @@ -105,41 +106,54 @@ pub fn ValidationAllocator(comptime T: type) type { pub fn alloc( ctx: *anyopaque, n: usize, - log2_ptr_align: u8, + alignment: mem.Alignment, ret_addr: usize, ) ?[*]u8 { assert(n > 0); const self: *Self = @ptrCast(@alignCast(ctx)); const underlying = self.getUnderlyingAllocatorPtr(); - const result = underlying.rawAlloc(n, log2_ptr_align, ret_addr) orelse + const result = underlying.rawAlloc(n, alignment, ret_addr) orelse return null; - assert(mem.isAlignedLog2(@intFromPtr(result), log2_ptr_align)); + assert(alignment.check(@intFromPtr(result))); return result; } pub fn resize( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: Alignment, new_len: usize, ret_addr: usize, ) bool { const self: *Self = @ptrCast(@alignCast(ctx)); assert(buf.len > 0); const underlying = self.getUnderlyingAllocatorPtr(); - return underlying.rawResize(buf, log2_buf_align, new_len, ret_addr); + return underlying.rawResize(buf, alignment, new_len, ret_addr); + } + + pub fn remap( + ctx: *anyopaque, + buf: []u8, + alignment: Alignment, + new_len: usize, + ret_addr: usize, + ) ?[*]u8 { + const self: *Self = @ptrCast(@alignCast(ctx)); + assert(buf.len > 0); + const underlying = self.getUnderlyingAllocatorPtr(); + return underlying.rawRemap(buf, alignment, new_len, ret_addr); } pub fn free( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: Alignment, ret_addr: usize, ) void { const self: *Self = @ptrCast(@alignCast(ctx)); assert(buf.len > 0); const underlying = self.getUnderlyingAllocatorPtr(); - underlying.rawFree(buf, log2_buf_align, ret_addr); + underlying.rawFree(buf, alignment, ret_addr); } pub fn reset(self: *Self) void { @@ -167,27 +181,9 @@ pub fn alignAllocLen(full_len: usize, alloc_len: usize, len_align: u29) usize { return adjusted; } -const fail_allocator = Allocator{ - .ptr = undefined, - .vtable = &failAllocator_vtable, -}; - -const failAllocator_vtable = Allocator.VTable{ - .alloc = failAllocatorAlloc, - .resize = Allocator.noResize, - .free = Allocator.noFree, -}; - -fn failAllocatorAlloc(_: *anyopaque, n: usize, log2_alignment: u8, ra: usize) ?[*]u8 { - _ = n; - _ = log2_alignment; - _ = ra; - return null; -} - test "Allocator basics" { - try testing.expectError(error.OutOfMemory, fail_allocator.alloc(u8, 1)); - try testing.expectError(error.OutOfMemory, fail_allocator.allocSentinel(u8, 1, 0)); + try testing.expectError(error.OutOfMemory, testing.failing_allocator.alloc(u8, 1)); + try testing.expectError(error.OutOfMemory, testing.failing_allocator.allocSentinel(u8, 1, 0)); } test "Allocator.resize" { From f1717777a2ce12905f86e75a11ff6388332d6926 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 21:07:49 -0800 Subject: [PATCH 13/39] std.heap: delete LoggingAllocator and friends I don't think these belong in std, at least not in their current form. If someone wants to add these back I'd like to review the patch before it lands. Reverts 629e2e784495dd8ac91493fa7bb11e1772698e42 --- lib/std/heap.zig | 8 -- lib/std/heap/log_to_writer_allocator.zig | 145 ----------------------- lib/std/heap/logging_allocator.zig | 133 --------------------- test/compare_output.zig | 43 ------- 4 files changed, 329 deletions(-) delete mode 100644 lib/std/heap/log_to_writer_allocator.zig delete mode 100644 lib/std/heap/logging_allocator.zig diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 2abe00c7a07c..bec7c9149ff7 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -8,11 +8,6 @@ const c = std.c; const Allocator = std.mem.Allocator; const windows = std.os.windows; -pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator; -pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator; -pub const ScopedLoggingAllocator = @import("heap/logging_allocator.zig").ScopedLoggingAllocator; -pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").LogToWriterAllocator; -pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator; pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; pub const GeneralPurposeAllocatorConfig = @import("heap/general_purpose_allocator.zig").Config; pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator; @@ -1062,9 +1057,6 @@ const page_size_max_default: ?usize = switch (builtin.os.tag) { }; test { - _ = LoggingAllocator; - _ = LogToWriterAllocator; - _ = ScopedLoggingAllocator; _ = @import("heap/memory_pool.zig"); _ = ArenaAllocator; _ = GeneralPurposeAllocator; diff --git a/lib/std/heap/log_to_writer_allocator.zig b/lib/std/heap/log_to_writer_allocator.zig deleted file mode 100644 index 239a580f9bae..000000000000 --- a/lib/std/heap/log_to_writer_allocator.zig +++ /dev/null @@ -1,145 +0,0 @@ -const std = @import("../std.zig"); -const Allocator = std.mem.Allocator; - -/// This allocator is used in front of another allocator and logs to the provided writer -/// on every call to the allocator. Writer errors are ignored. -pub fn LogToWriterAllocator(comptime Writer: type) type { - return struct { - parent_allocator: Allocator, - writer: Writer, - - const Self = @This(); - - pub fn init(parent_allocator: Allocator, writer: Writer) Self { - return Self{ - .parent_allocator = parent_allocator, - .writer = writer, - }; - } - - pub fn allocator(self: *Self) Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - fn alloc( - ctx: *anyopaque, - len: usize, - alignment: std.mem.Alignment, - ra: usize, - ) ?[*]u8 { - const self: *Self = @ptrCast(@alignCast(ctx)); - self.writer.print("alloc : {}", .{len}) catch {}; - const result = self.parent_allocator.rawAlloc(len, alignment, ra); - if (result != null) { - self.writer.print(" success!\n", .{}) catch {}; - } else { - self.writer.print(" failure!\n", .{}) catch {}; - } - return result; - } - - fn resize( - ctx: *anyopaque, - buf: []u8, - alignment: std.mem.Alignment, - new_len: usize, - ra: usize, - ) bool { - const self: *Self = @ptrCast(@alignCast(ctx)); - if (new_len <= buf.len) { - self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {}; - } else { - self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {}; - } - - if (self.parent_allocator.rawResize(buf, alignment, new_len, ra)) { - if (new_len > buf.len) { - self.writer.print(" success!\n", .{}) catch {}; - } - return true; - } - - std.debug.assert(new_len > buf.len); - self.writer.print(" failure!\n", .{}) catch {}; - return false; - } - - fn remap( - ctx: *anyopaque, - buf: []u8, - alignment: std.mem.Alignment, - new_len: usize, - ra: usize, - ) ?[*]u8 { - const self: *Self = @ptrCast(@alignCast(ctx)); - if (new_len <= buf.len) { - self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {}; - } else { - self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {}; - } - - if (self.parent_allocator.rawRemap(buf, alignment, new_len, ra)) |new_memory| { - if (new_len > buf.len) { - self.writer.print(" success!\n", .{}) catch {}; - } - return new_memory; - } - - std.debug.assert(new_len > buf.len); - self.writer.print(" failure!\n", .{}) catch {}; - return null; - } - - fn free( - ctx: *anyopaque, - buf: []u8, - alignment: std.mem.Alignment, - ra: usize, - ) void { - const self: *Self = @ptrCast(@alignCast(ctx)); - self.writer.print("free : {}\n", .{buf.len}) catch {}; - self.parent_allocator.rawFree(buf, alignment, ra); - } - }; -} - -/// This allocator is used in front of another allocator and logs to the provided writer -/// on every call to the allocator. Writer errors are ignored. -pub fn logToWriterAllocator( - parent_allocator: Allocator, - writer: anytype, -) LogToWriterAllocator(@TypeOf(writer)) { - return LogToWriterAllocator(@TypeOf(writer)).init(parent_allocator, writer); -} - -test "LogToWriterAllocator" { - var log_buf: [255]u8 = undefined; - var fbs = std.io.fixedBufferStream(&log_buf); - - var allocator_buf: [10]u8 = undefined; - var fixedBufferAllocator = std.mem.validationWrap(std.heap.FixedBufferAllocator.init(&allocator_buf)); - var allocator_state = logToWriterAllocator(fixedBufferAllocator.allocator(), fbs.writer()); - const allocator = allocator_state.allocator(); - - var a = try allocator.alloc(u8, 10); - try std.testing.expect(allocator.resize(a, 5)); - a = a[0..5]; - try std.testing.expect(!allocator.resize(a, 20)); - allocator.free(a); - - try std.testing.expectEqualSlices(u8, - \\alloc : 10 success! - \\shrink: 10 to 5 - \\expand: 5 to 20 failure! - \\free : 5 - \\ - , fbs.getWritten()); -} diff --git a/lib/std/heap/logging_allocator.zig b/lib/std/heap/logging_allocator.zig deleted file mode 100644 index 706f2ac544e3..000000000000 --- a/lib/std/heap/logging_allocator.zig +++ /dev/null @@ -1,133 +0,0 @@ -const std = @import("../std.zig"); -const Allocator = std.mem.Allocator; - -/// This allocator is used in front of another allocator and logs to `std.log` -/// on every call to the allocator. -/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator` -pub fn LoggingAllocator( - comptime success_log_level: std.log.Level, - comptime failure_log_level: std.log.Level, -) type { - return ScopedLoggingAllocator(.default, success_log_level, failure_log_level); -} - -/// This allocator is used in front of another allocator and logs to `std.log` -/// with the given scope on every call to the allocator. -/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator` -pub fn ScopedLoggingAllocator( - comptime scope: @Type(.enum_literal), - comptime success_log_level: std.log.Level, - comptime failure_log_level: std.log.Level, -) type { - const log = std.log.scoped(scope); - - return struct { - parent_allocator: Allocator, - - const Self = @This(); - - pub fn init(parent_allocator: Allocator) Self { - return .{ - .parent_allocator = parent_allocator, - }; - } - - pub fn allocator(self: *Self) Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .free = free, - }, - }; - } - - // This function is required as the `std.log.log` function is not public - inline fn logHelper(comptime log_level: std.log.Level, comptime format: []const u8, args: anytype) void { - switch (log_level) { - .err => log.err(format, args), - .warn => log.warn(format, args), - .info => log.info(format, args), - .debug => log.debug(format, args), - } - } - - fn alloc( - ctx: *anyopaque, - len: usize, - log2_ptr_align: u8, - ra: usize, - ) ?[*]u8 { - const self: *Self = @ptrCast(@alignCast(ctx)); - const result = self.parent_allocator.rawAlloc(len, log2_ptr_align, ra); - if (result != null) { - logHelper( - success_log_level, - "alloc - success - len: {}, ptr_align: {}", - .{ len, log2_ptr_align }, - ); - } else { - logHelper( - failure_log_level, - "alloc - failure: OutOfMemory - len: {}, ptr_align: {}", - .{ len, log2_ptr_align }, - ); - } - return result; - } - - fn resize( - ctx: *anyopaque, - buf: []u8, - log2_buf_align: u8, - new_len: usize, - ra: usize, - ) bool { - const self: *Self = @ptrCast(@alignCast(ctx)); - if (self.parent_allocator.rawResize(buf, log2_buf_align, new_len, ra)) { - if (new_len <= buf.len) { - logHelper( - success_log_level, - "shrink - success - {} to {}, buf_align: {}", - .{ buf.len, new_len, log2_buf_align }, - ); - } else { - logHelper( - success_log_level, - "expand - success - {} to {}, buf_align: {}", - .{ buf.len, new_len, log2_buf_align }, - ); - } - - return true; - } - - std.debug.assert(new_len > buf.len); - logHelper( - failure_log_level, - "expand - failure - {} to {}, buf_align: {}", - .{ buf.len, new_len, log2_buf_align }, - ); - return false; - } - - fn free( - ctx: *anyopaque, - buf: []u8, - log2_buf_align: u8, - ra: usize, - ) void { - const self: *Self = @ptrCast(@alignCast(ctx)); - self.parent_allocator.rawFree(buf, log2_buf_align, ra); - logHelper(success_log_level, "free - len: {}", .{buf.len}); - } - }; -} - -/// This allocator is used in front of another allocator and logs to `std.log` -/// on every call to the allocator. -/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator` -pub fn loggingAllocator(parent_allocator: Allocator) LoggingAllocator(.debug, .err) { - return LoggingAllocator(.debug, .err).init(parent_allocator); -} diff --git a/test/compare_output.zig b/test/compare_output.zig index e766641cde8a..3603163a2d3e 100644 --- a/test/compare_output.zig +++ b/test/compare_output.zig @@ -493,49 +493,6 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ ); - // It is required to override the log function in order to print to stdout instead of stderr - cases.add("std.heap.LoggingAllocator logs to std.log", - \\const std = @import("std"); - \\ - \\pub const std_options: std.Options = .{ - \\ .log_level = .debug, - \\ .logFn = log, - \\}; - \\ - \\pub fn main() !void { - \\ var allocator_buf: [10]u8 = undefined; - \\ const fba = std.heap.FixedBufferAllocator.init(&allocator_buf); - \\ var fba_wrapped = std.mem.validationWrap(fba); - \\ var logging_allocator = std.heap.loggingAllocator(fba_wrapped.allocator()); - \\ const allocator = logging_allocator.allocator(); - \\ - \\ var a = try allocator.alloc(u8, 10); - \\ try std.testing.expect(allocator.resize(a, 5)); - \\ a = a[0..5]; - \\ try std.testing.expect(a.len == 5); - \\ try std.testing.expect(!allocator.resize(a, 20)); - \\ allocator.free(a); - \\} - \\ - \\pub fn log( - \\ comptime level: std.log.Level, - \\ comptime scope: @TypeOf(.EnumLiteral), - \\ comptime format: []const u8, - \\ args: anytype, - \\) void { - \\ const level_txt = comptime level.asText(); - \\ const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; - \\ const stdout = std.io.getStdOut().writer(); - \\ nosuspend stdout.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch return; - \\} - , - \\debug: alloc - success - len: 10, ptr_align: 0 - \\debug: shrink - success - 10 to 5, buf_align: 0 - \\error: expand - failure - 5 to 20, buf_align: 0 - \\debug: free - len: 5 - \\ - ); - cases.add("valid carriage return example", "const io = @import(\"std\").io;\r\n" ++ // Testing CRLF line endings are valid "\r\n" ++ "pub \r fn main() void {\r\n" ++ // Testing isolated carriage return as whitespace is valid From becd16859dbc9d6357099f660408b54d00aa18ef Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 21:20:52 -0800 Subject: [PATCH 14/39] std.hash_map: placeholder for doc comments --- lib/std/hash_map.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig index 270cd1b27309..1cb4bfe010a3 100644 --- a/lib/std/hash_map.zig +++ b/lib/std/hash_map.zig @@ -413,10 +413,15 @@ pub fn HashMap( /// If there is an `Entry` with a matching key, it is deleted from /// the hash map, and this function returns true. Otherwise this /// function returns false. + /// + /// TODO: answer the question in these doc comments, does this + /// increase the unused capacity by one? pub fn remove(self: *Self, key: K) bool { return self.unmanaged.removeContext(key, self.ctx); } + /// TODO: answer the question in these doc comments, does this + /// increase the unused capacity by one? pub fn removeAdapted(self: *Self, key: anytype, ctx: anytype) bool { return self.unmanaged.removeAdapted(key, ctx); } @@ -424,6 +429,9 @@ pub fn HashMap( /// Delete the entry with key pointed to by key_ptr from the hash map. /// key_ptr is assumed to be a valid pointer to a key that is present /// in the hash map. + /// + /// TODO: answer the question in these doc comments, does this + /// increase the unused capacity by one? pub fn removeByPtr(self: *Self, key_ptr: *K) void { self.unmanaged.removeByPtr(key_ptr); } @@ -1225,14 +1233,23 @@ pub fn HashMapUnmanaged( /// If there is an `Entry` with a matching key, it is deleted from /// the hash map, and this function returns true. Otherwise this /// function returns false. + /// + /// TODO: answer the question in these doc comments, does this + /// increase the unused capacity by one? pub fn remove(self: *Self, key: K) bool { if (@sizeOf(Context) != 0) @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call removeContext instead."); return self.removeContext(key, undefined); } + + /// TODO: answer the question in these doc comments, does this + /// increase the unused capacity by one? pub fn removeContext(self: *Self, key: K, ctx: Context) bool { return self.removeAdapted(key, ctx); } + + /// TODO: answer the question in these doc comments, does this + /// increase the unused capacity by one? pub fn removeAdapted(self: *Self, key: anytype, ctx: anytype) bool { if (self.getIndex(key, ctx)) |idx| { self.removeByIndex(idx); @@ -1245,6 +1262,9 @@ pub fn HashMapUnmanaged( /// Delete the entry with key pointed to by key_ptr from the hash map. /// key_ptr is assumed to be a valid pointer to a key that is present /// in the hash map. + /// + /// TODO: answer the question in these doc comments, does this + /// increase the unused capacity by one? pub fn removeByPtr(self: *Self, key_ptr: *K) void { // TODO: replace with pointer subtraction once supported by zig // if @sizeOf(K) == 0 then there is at most one item in the hash From 601f632c274cee4e8b9780c436be0771d856ade0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 21:38:08 -0800 Subject: [PATCH 15/39] std.heap.GeneralPurposeAllocator: fix large alloc accounting when mremap relocates an allocation --- lib/std/heap/general_purpose_allocator.zig | 33 +++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index d1e4cda8335c..32de2fad6303 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -608,7 +608,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { var addresses: [stack_n]usize = [1]usize{0} ** stack_n; - var free_stack_trace = StackTrace{ + var free_stack_trace: StackTrace = .{ .instruction_addresses = &addresses, .index = 0, }; @@ -635,9 +635,15 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { self.total_requested_bytes = new_req_bytes; } - const opt_resized_ptr = if (may_move) - self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr) - else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) + const opt_resized_ptr = if (may_move) b: { + // So that if the allocation moves, we can memcpy the + // `LargeAlloc` value directly from old to new location. + // It's also not clear to me whether removing one item from std + // lib hash map guarantees that unused capacity increases by + // one. + self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; + break :b self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr); + } else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) old_mem.ptr else null; @@ -660,6 +666,25 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } entry.value_ptr.bytes = resized_ptr[0..new_size]; entry.value_ptr.captureStackTrace(ret_addr, .alloc); + + // Update the key of the hash map if the memory was relocated. + if (resized_ptr != old_mem.ptr) { + const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(resized_ptr)); + if (config.retain_metadata and !config.never_unmap) { + // Backing allocator may be reusing memory that we're retaining metadata for + assert(!gop.found_existing or gop.value_ptr.freed); + } else { + assert(!gop.found_existing); // This would mean the kernel double-mapped pages. + } + gop.value_ptr.* = entry.value_ptr.*; + if (!config.retain_metadata) { + self.large_allocations.removeByPtr(entry.key_ptr); + } else { + entry.value_ptr.freed = true; + entry.value_ptr.captureStackTrace(ret_addr, .free); + } + } + return resized_ptr; } From 2c5113f6d1ee9d5041d9d79cfbe13e6bea2aff56 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 21:41:12 -0800 Subject: [PATCH 16/39] std.os.linux.mmap: remove logic that does not belong here --- lib/std/os/linux.zig | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 6e64d022cb4e..118340869434 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -899,10 +899,6 @@ pub fn umount2(special: [*:0]const u8, flags: u32) usize { pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: MAP, fd: i32, offset: i64) usize { if (@hasField(SYS, "mmap2")) { - // Make sure the offset is also specified in multiples of page size - if ((offset & (MMAP2_UNIT - 1)) != 0) - return @bitCast(-@as(isize, @intFromEnum(E.INVAL))); - return syscall6( .mmap2, @intFromPtr(address), From 36e9b0f0261a5421f22dbf17f513a8c2f685c5d3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Feb 2025 22:34:29 -0800 Subject: [PATCH 17/39] std.mem.Allocator: keep the undefined memset Reversal on the decision: the Allocator interface is the correct place for the memset to undefined because it allows Allocator implementations to bypass the interface and use a backing allocator directly, skipping the performance penalty of memsetting the entire allocation, which may be very large, as well as having valuable zeroes on them. closes #4298 --- lib/std/heap/general_purpose_allocator.zig | 35 ++++++++++++++-------- lib/std/mem/Allocator.zig | 11 +++---- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 32de2fad6303..cd230e48d928 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -157,6 +157,9 @@ pub const Config = struct { /// Enables emitting info messages with the size and address of every allocation. verbose_log: bool = false, + + /// Tell whether the backing allocator returns already-zeroed memory. + backing_allocator_zeroes: bool = true, }; pub const Check = enum { ok, leak }; @@ -179,7 +182,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const Self = @This(); - /// The initial state of a `GeneralPurposeAllocator`, containing no allocations and backed by the system page allocator. + /// The initial state of a `GeneralPurposeAllocator`, containing no + /// allocations and backed by the system page allocator. pub const init: Self = .{ .backing_allocator = std.heap.page_allocator, .buckets = [1]Buckets{.{}} ** small_bucket_count, @@ -508,7 +512,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void { if (stack_n == 0) return; @memset(addresses, 0); - var stack_trace = StackTrace{ + var stack_trace: StackTrace = .{ .instruction_addresses = addresses, .index = 0, }; @@ -1092,22 +1096,29 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader { - const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size); - errdefer self.backing_allocator.free(page); + const alignment: mem.Alignment = .fromByteUnits(page_size); + const page = self.backing_allocator.rawAlloc(page_size, alignment, @returnAddress()) orelse + return error.OutOfMemory; + errdefer self.backing_allocator.rawFree(page[0..page_size], alignment, @returnAddress()); const bucket_size = bucketSize(size_class); - const bucket_bytes = try self.backing_allocator.alignedAlloc(u8, @alignOf(BucketHeader), bucket_size); - const ptr: *BucketHeader = @ptrCast(bucket_bytes.ptr); + const header_align: mem.Alignment = .fromByteUnits(@alignOf(BucketHeader)); + const ptr: *BucketHeader = @alignCast(@ptrCast(self.backing_allocator.rawAlloc( + bucket_size, + header_align, + @returnAddress(), + ) orelse return error.OutOfMemory)); ptr.* = .{ - .page = page.ptr, + .page = @alignCast(page), .alloc_cursor = 0, .used_count = 0, }; - // Set the used bits to all zeroes - @memset(@as([*]u8, @as(*[1]u8, ptr.usedBits(0)))[0..usedBitsCount(size_class)], 0); - if (config.safety) { - // Set the requested sizes to zeroes - @memset(mem.sliceAsBytes(ptr.requestedSizes(size_class)), 0); + if (!config.backing_allocator_zeroes) { + @memset(@as([*]u8, @as(*[1]u8, ptr.usedBits(0)))[0..usedBitsCount(size_class)], 0); + if (config.safety) { + // Set the requested sizes to zeroes + @memset(mem.sliceAsBytes(ptr.requestedSizes(size_class)), 0); + } } return ptr; } diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index 7c0e59906c43..1a0108f3be38 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -128,25 +128,25 @@ pub fn noFree( } /// This function is not intended to be called except from within the -/// implementation of an Allocator +/// implementation of an `Allocator`. pub inline fn rawAlloc(a: Allocator, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 { return a.vtable.alloc(a.ptr, len, alignment, ret_addr); } /// This function is not intended to be called except from within the -/// implementation of an Allocator. +/// implementation of an `Allocator`. pub inline fn rawResize(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { return a.vtable.resize(a.ptr, memory, alignment, new_len, ret_addr); } /// This function is not intended to be called except from within the -/// implementation of an Allocator. +/// implementation of an `Allocator`. pub inline fn rawRemap(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { return a.vtable.remap(a.ptr, memory, alignment, new_len, ret_addr); } /// This function is not intended to be called except from within the -/// implementation of an Allocator +/// implementation of an `Allocator`. pub inline fn rawFree(a: Allocator, memory: []u8, alignment: Alignment, ret_addr: usize) void { return a.vtable.free(a.ptr, memory, alignment, ret_addr); } @@ -271,7 +271,6 @@ fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: } const byte_ptr = self.rawAlloc(byte_count, .fromByteUnits(alignment), return_address) orelse return Error.OutOfMemory; - // TODO: https://github.com/ziglang/zig/issues/4298 @memset(byte_ptr[0..byte_count], undefined); return @alignCast(byte_ptr); } @@ -391,7 +390,6 @@ pub fn reallocAdvanced( return error.OutOfMemory; const copy_len = @min(byte_count, old_byte_slice.len); @memcpy(new_mem[0..copy_len], old_byte_slice[0..copy_len]); - // TODO https://github.com/ziglang/zig/issues/4298 @memset(old_byte_slice, undefined); self.rawFree(old_byte_slice, .fromByteUnits(Slice.alignment), return_address); @@ -408,7 +406,6 @@ pub fn free(self: Allocator, memory: anytype) void { const bytes_len = bytes.len + if (Slice.sentinel() != null) @sizeOf(Slice.child) else 0; if (bytes_len == 0) return; const non_const_ptr = @constCast(bytes.ptr); - // TODO: https://github.com/ziglang/zig/issues/4298 @memset(non_const_ptr[0..bytes_len], undefined); self.rawFree(non_const_ptr[0..bytes_len], .fromByteUnits(Slice.alignment), @returnAddress()); } From 1a6d87d699a9e0867f3a50566dc72b3e2347d8d1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Feb 2025 23:01:24 -0800 Subject: [PATCH 18/39] std.heap.ThreadSafeAllocator: update to new Allocator API --- lib/std/heap.zig | 1 + lib/std/heap/ThreadSafeAllocator.zig | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index bec7c9149ff7..156fbdeab478 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -1061,6 +1061,7 @@ test { _ = ArenaAllocator; _ = GeneralPurposeAllocator; _ = FixedBufferAllocator; + _ = ThreadSafeAllocator; if (builtin.target.isWasm()) { _ = WasmAllocator; } diff --git a/lib/std/heap/ThreadSafeAllocator.zig b/lib/std/heap/ThreadSafeAllocator.zig index 12bb095b30fd..dc8bf89017b7 100644 --- a/lib/std/heap/ThreadSafeAllocator.zig +++ b/lib/std/heap/ThreadSafeAllocator.zig @@ -9,35 +9,45 @@ pub fn allocator(self: *ThreadSafeAllocator) Allocator { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; } -fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { +fn alloc(ctx: *anyopaque, n: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 { const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx)); self.mutex.lock(); defer self.mutex.unlock(); - return self.child_allocator.rawAlloc(n, log2_ptr_align, ra); + return self.child_allocator.rawAlloc(n, alignment, ra); } -fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool { +fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx)); self.mutex.lock(); defer self.mutex.unlock(); - return self.child_allocator.rawResize(buf, log2_buf_align, new_len, ret_addr); + return self.child_allocator.rawResize(buf, alignment, new_len, ret_addr); } -fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void { +fn remap(context: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, return_address: usize) ?[*]u8 { + const self: *ThreadSafeAllocator = @ptrCast(@alignCast(context)); + + self.mutex.lock(); + defer self.mutex.unlock(); + + return self.child_allocator.rawRemap(memory, alignment, new_len, return_address); +} + +fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx)); self.mutex.lock(); defer self.mutex.unlock(); - return self.child_allocator.rawFree(buf, log2_buf_align, ret_addr); + return self.child_allocator.rawFree(buf, alignment, ret_addr); } const std = @import("../std.zig"); From 8ff7481e8245f916966973e2c830bc3fb5e4c748 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Feb 2025 23:05:06 -0800 Subject: [PATCH 19/39] std.heap.GeneralPurposeAllocator: inline small allocation metadata Put the small allocation metadata directly into the (large) pages allocated. --- lib/std/heap/general_purpose_allocator.zig | 792 +++++++-------------- 1 file changed, 261 insertions(+), 531 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index cd230e48d928..1fdb09cb413e 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -99,12 +99,14 @@ const math = std.math; const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; +const StackTrace = std.builtin.StackTrace; + const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. else => 2 * 1024 * 1024, // Avoids too many active mappings when `page_size_max` is low. }); -const StackTrace = std.builtin.StackTrace; +const page_align: mem.Alignment = .fromByteUnits(page_size); /// Integer type for pointing to slots in a small allocation const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); @@ -168,29 +170,23 @@ pub const Check = enum { ok, leak }; pub fn GeneralPurposeAllocator(comptime config: Config) type { return struct { backing_allocator: Allocator = std.heap.page_allocator, - buckets: [small_bucket_count]Buckets = [1]Buckets{Buckets{}} ** small_bucket_count, - cur_buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, - large_allocations: LargeAllocTable = .{}, - empty_buckets: if (config.retain_metadata) Buckets else void = - if (config.retain_metadata) Buckets{} else {}, - bucket_node_pool: std.heap.MemoryPool(Buckets.Node) = std.heap.MemoryPool(Buckets.Node).init(std.heap.page_allocator), - + /// Tracks the active bucket, which is the one that has free slots in it. + buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, + large_allocations: LargeAllocTable = .empty, total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, - mutex: @TypeOf(mutex_init) = mutex_init, const Self = @This(); - /// The initial state of a `GeneralPurposeAllocator`, containing no - /// allocations and backed by the system page allocator. - pub const init: Self = .{ - .backing_allocator = std.heap.page_allocator, - .buckets = [1]Buckets{.{}} ** small_bucket_count, - .cur_buckets = [1]?*BucketHeader{null} ** small_bucket_count, - .large_allocations = .{}, - .empty_buckets = if (config.retain_metadata) .{} else {}, - .bucket_node_pool = .init(std.heap.page_allocator), + pub const init: Self = .{}; + + /// These can be derived from size_class_index but the calculation is nontrivial. + const slot_counts: [small_bucket_count]SlotIndex = init: { + @setEvalBranchQuota(10000); + var result: [small_bucket_count]SlotIndex = undefined; + for (&result, 0..) |*elem, i| elem.* = calculateSlotCount(i); + break :init result; }; const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {}; @@ -204,8 +200,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { DummyMutex{}; const DummyMutex = struct { - fn lock(_: *DummyMutex) void {} - fn unlock(_: *DummyMutex) void {} + inline fn lock(_: *DummyMutex) void {} + inline fn unlock(_: *DummyMutex) void {} }; const stack_n = config.stack_trace_frames; @@ -223,7 +219,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page)); } }.compare; - const Buckets = std.Treap(*BucketHeader, bucketCompare); const LargeAlloc = struct { bytes: []u8, @@ -259,46 +254,50 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { }; const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); - // Bucket: In memory, in order: - // * BucketHeader - // * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots - // -- below only exists when config.safety is true -- - // * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot - // * log2_ptr_aligns: [N]u8 // 1 byte for every slot - // -- above only exists when config.safety is true -- - // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation - + /// Bucket: In memory, in order: + /// * BucketHeader + /// * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots + /// -- below only exists when config.safety is true -- + /// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot + /// * log2_ptr_aligns: [N]u8 // 1 byte for every slot + /// -- above only exists when config.safety is true -- + /// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation const BucketHeader = struct { - page: [*]align(page_size) u8, - alloc_cursor: SlotIndex, - used_count: SlotIndex, + allocated_count: SlotIndex, + freed_count: SlotIndex, + prev: ?*BucketHeader, + + fn fromPage(page_addr: usize, slot_count: usize) *BucketHeader { + const unaligned = page_addr + page_size - bucketSize(slot_count); + return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1)); + } + // TODO use usize instead of u8 fn usedBits(bucket: *BucketHeader, index: usize) *u8 { - return @as(*u8, @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index)); + // TODO avoid ptr to int + return @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index); } - fn requestedSizes(bucket: *BucketHeader, size_class: usize) []LargestSizeClassInt { + fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt { if (!config.safety) @compileError("requested size is only stored when safety is enabled"); - const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(size_class); + const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(slot_count); const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); - const slot_count = @divExact(page_size, size_class); return sizes[0..slot_count]; } - fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []mem.Alignment { + fn log2PtrAligns(bucket: *BucketHeader, slot_count: usize) []mem.Alignment { if (!config.safety) @compileError("requested size is only stored when safety is enabled"); - const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class); - const slot_count = @divExact(page_size, size_class); + const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(slot_count); return @ptrCast(aligns_ptr[0..slot_count]); } fn stackTracePtr( bucket: *BucketHeader, - size_class: usize, + slot_count: usize, slot_index: SlotIndex, trace_kind: TraceKind, ) *[stack_n]usize { - const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(size_class); + const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(slot_count); const addr = start_ptr + one_trace_size * traces_per_slot * slot_index + @intFromEnum(trace_kind) * @as(usize, one_trace_size); return @ptrCast(@alignCast(addr)); @@ -307,21 +306,15 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn captureStackTrace( bucket: *BucketHeader, ret_addr: usize, - size_class: usize, + slot_count: usize, slot_index: SlotIndex, trace_kind: TraceKind, ) void { // Initialize them to 0. When determining the count we must look // for non zero addresses. - const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind); + const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); collectStackTrace(ret_addr, stack_addresses); } - - /// Only valid for buckets within `empty_buckets`, and relies on the `alloc_cursor` - /// of empty buckets being set to `slot_count` when they are added to `empty_buckets` - fn emptyBucketSizeClass(bucket: *BucketHeader) usize { - return @divExact(page_size, bucket.alloc_cursor); - } }; pub fn allocator(self: *Self) Allocator { @@ -338,64 +331,76 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn bucketStackTrace( bucket: *BucketHeader, - size_class: usize, + slot_count: usize, slot_index: SlotIndex, trace_kind: TraceKind, ) StackTrace { - const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind); + const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); var len: usize = 0; while (len < stack_n and stack_addresses[len] != 0) { len += 1; } - return StackTrace{ + return .{ .instruction_addresses = stack_addresses, .index = len, }; } - fn bucketRequestedSizesStart(size_class: usize) usize { + fn bucketRequestedSizesStart(slot_count: usize) usize { if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); return mem.alignForward( usize, - @sizeOf(BucketHeader) + usedBitsCount(size_class), + @sizeOf(BucketHeader) + usedBitsCount(slot_count), @alignOf(LargestSizeClassInt), ); } - fn bucketAlignsStart(size_class: usize) usize { + fn bucketAlignsStart(slot_count: usize) usize { if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); - const slot_count = @divExact(page_size, size_class); - return bucketRequestedSizesStart(size_class) + (@sizeOf(LargestSizeClassInt) * slot_count); + return bucketRequestedSizesStart(slot_count) + (@sizeOf(LargestSizeClassInt) * slot_count); } - fn bucketStackFramesStart(size_class: usize) usize { - const unaligned_start = if (config.safety) blk: { - const slot_count = @divExact(page_size, size_class); - break :blk bucketAlignsStart(size_class) + slot_count; - } else @sizeOf(BucketHeader) + usedBitsCount(size_class); - return mem.alignForward( - usize, - unaligned_start, - @alignOf(usize), - ); + fn bucketStackFramesStart(slot_count: usize) usize { + const unaligned_start = if (config.safety) + bucketAlignsStart(slot_count) + slot_count + else + @sizeOf(BucketHeader) + usedBitsCount(slot_count); + return mem.alignForward(usize, unaligned_start, @alignOf(usize)); } - fn bucketSize(size_class: usize) usize { - const slot_count = @divExact(page_size, size_class); - return bucketStackFramesStart(size_class) + one_trace_size * traces_per_slot * slot_count; + fn bucketSize(slot_count: usize) usize { + return bucketStackFramesStart(slot_count) + one_trace_size * traces_per_slot * slot_count; } - fn usedBitsCount(size_class: usize) usize { - const slot_count = @divExact(page_size, size_class); - if (slot_count < 8) return 1; - return @divExact(slot_count, 8); + /// This is executed only at compile-time to prepopulate a lookup table. + fn calculateSlotCount(size_class_index: usize) SlotIndex { + const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + var lower: usize = 8; + var upper: usize = (page_size - bucketSize(lower)) / size_class; + while (upper > lower) { + const proposed: usize = lower + (upper - lower) / 2; + if (proposed == lower) return lower; + const slots_end = proposed * size_class; + const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); + const bucket_size = bucketSize(proposed); + const end = header_begin + bucket_size; + if (end > page_size) { + upper = proposed - 1; + } else { + lower = proposed; + } + } + return lower; } - fn detectLeaksInBucket( - bucket: *BucketHeader, - size_class: usize, - used_bits_count: usize, - ) bool { + fn usedBitsCount(slot_count: usize) usize { + assert(slot_count >= 8); + return (slot_count + 7) / 8; + } + + fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool { + const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + const slot_count = slot_counts[size_class_index]; var leaks = false; var used_bits_byte: usize = 0; while (used_bits_byte < used_bits_count) : (used_bits_byte += 1) { @@ -405,12 +410,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { while (true) : (bit_index += 1) { const is_used = @as(u1, @truncate(used_byte >> bit_index)) != 0; if (is_used) { - const slot_index = @as(SlotIndex, @intCast(used_bits_byte * 8 + bit_index)); - const stack_trace = bucketStackTrace(bucket, size_class, slot_index, .alloc); - const addr = bucket.page + slot_index * size_class; - log.err("memory address 0x{x} leaked: {}", .{ - @intFromPtr(addr), stack_trace, - }); + const slot_index: SlotIndex = @intCast(used_bits_byte * 8 + bit_index); + const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc); + const page_addr = @intFromPtr(bucket) & ~(page_size - 1); + const addr = page_addr + slot_index * size_class; + log.err("memory address 0x{x} leaked: {}", .{ addr, stack_trace }); leaks = true; } if (bit_index == math.maxInt(u3)) @@ -425,16 +429,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { pub fn detectLeaks(self: *Self) bool { var leaks = false; - for (&self.buckets, 0..) |*buckets, bucket_i| { - if (buckets.root == null) continue; - const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i)); - const used_bits_count = usedBitsCount(size_class); - var it = buckets.inorderIterator(); - while (it.next()) |node| { - const bucket = node.key; - leaks = detectLeaksInBucket(bucket, size_class, used_bits_count) or leaks; + for (self.buckets, 0..) |init_optional_bucket, size_class_index| { + var optional_bucket = init_optional_bucket; + const slot_count = slot_counts[size_class_index]; + const used_bits_count = usedBitsCount(slot_count); + while (optional_bucket) |bucket| { + leaks = detectLeaksInBucket(bucket, size_class_index, used_bits_count) or leaks; + optional_bucket = bucket.prev; } } + var it = self.large_allocations.valueIterator(); while (it.next()) |large_alloc| { if (config.retain_metadata and large_alloc.freed) continue; @@ -447,46 +451,21 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { return leaks; } - fn freeBucket(self: *Self, bucket: *BucketHeader, size_class: usize) void { - const bucket_size = bucketSize(size_class); - const bucket_slice = @as([*]align(@alignOf(BucketHeader)) u8, @ptrCast(bucket))[0..bucket_size]; - self.backing_allocator.free(bucket_slice); - } - fn freeRetainedMetadata(self: *Self) void { - if (config.retain_metadata) { - if (config.never_unmap) { - // free large allocations that were intentionally leaked by never_unmap - var it = self.large_allocations.iterator(); - while (it.next()) |large| { - if (large.value_ptr.freed) { - self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress()); - } - } - } - // free retained metadata for small allocations - while (self.empty_buckets.getMin()) |node| { - // remove the node from the tree before destroying it - var entry = self.empty_buckets.getEntryForExisting(node); - entry.set(null); - - var bucket = node.key; - if (config.never_unmap) { - // free page that was intentionally leaked by never_unmap - self.backing_allocator.free(bucket.page[0..page_size]); + comptime assert(config.retain_metadata); + if (config.never_unmap) { + // free large allocations that were intentionally leaked by never_unmap + var it = self.large_allocations.iterator(); + while (it.next()) |large| { + if (large.value_ptr.freed) { + self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress()); } - // alloc_cursor was set to slot count when bucket added to empty_buckets - self.freeBucket(bucket, bucket.emptyBucketSizeClass()); - self.bucket_node_pool.destroy(node); } - self.empty_buckets.root = null; } } pub fn flushRetainedMetadata(self: *Self) void { - if (!config.retain_metadata) { - @compileError("'flushRetainedMetadata' requires 'config.retain_metadata = true'"); - } + comptime assert(config.retain_metadata); self.freeRetainedMetadata(); // also remove entries from large_allocations var it = self.large_allocations.iterator(); @@ -500,13 +479,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { /// Returns `Check.leak` if there were leaks; `Check.ok` otherwise. pub fn deinit(self: *Self) Check { const leaks = if (config.safety) self.detectLeaks() else false; - if (config.retain_metadata) { - self.freeRetainedMetadata(); - } + if (config.retain_metadata) self.freeRetainedMetadata(); self.large_allocations.deinit(self.backing_allocator); - self.bucket_node_pool.deinit(); self.* = undefined; - return @as(Check, @enumFromInt(@intFromBool(leaks))); + return if (leaks) .leak else .ok; } fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void { @@ -520,8 +496,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void { - var addresses: [stack_n]usize = [1]usize{0} ** stack_n; - var second_free_stack_trace = StackTrace{ + var addresses: [stack_n]usize = @splat(0); + var second_free_stack_trace: StackTrace = .{ .instruction_addresses = &addresses, .index = 0, }; @@ -531,58 +507,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { }); } - const Slot = struct { - bucket: *BucketHeader, - slot_index: usize, - ptr: [*]u8, - }; - - fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error!Slot { - const bucket_index = math.log2(size_class); - var buckets = &self.buckets[bucket_index]; - const slot_count = @divExact(page_size, size_class); - if (self.cur_buckets[bucket_index] == null or self.cur_buckets[bucket_index].?.alloc_cursor == slot_count) { - const new_bucket = try self.createBucket(size_class); - errdefer self.freeBucket(new_bucket, size_class); - const node = try self.bucket_node_pool.create(); - node.key = new_bucket; - var entry = buckets.getEntryFor(new_bucket); - std.debug.assert(entry.node == null); - entry.set(node); - self.cur_buckets[bucket_index] = node.key; - } - const bucket = self.cur_buckets[bucket_index].?; - - const slot_index = bucket.alloc_cursor; - bucket.alloc_cursor += 1; - - const used_bits_byte = bucket.usedBits(slot_index / 8); - const used_bit_index: u3 = @as(u3, @intCast(slot_index % 8)); // TODO cast should be unnecessary - used_bits_byte.* |= (@as(u8, 1) << used_bit_index); - bucket.used_count += 1; - bucket.captureStackTrace(trace_addr, size_class, slot_index, .alloc); - return .{ - .bucket = bucket, - .slot_index = slot_index, - .ptr = bucket.page + slot_index * size_class, - }; - } - - fn searchBucket( - buckets: *Buckets, - addr: usize, - current_bucket: ?*BucketHeader, - ) ?*BucketHeader { - const search_page: [*]align(page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, page_size)); - if (current_bucket != null and current_bucket.?.page == search_page) { - return current_bucket; - } - var search_header: BucketHeader = undefined; - search_header.page = search_page; - const entry = buckets.getEntryFor(&search_header); - return if (entry.node) |node| node.key else null; - } - /// This function assumes the object is in the large object storage regardless /// of the parameters. fn resizeLarge( @@ -752,201 +676,177 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } } - pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void { - self.requested_memory_limit = limit; - } - - fn resize( - context: *anyopaque, - memory: []u8, - alignment: mem.Alignment, - new_len: usize, - return_address: usize, - ) bool { - return realloc(context, memory, alignment, new_len, return_address, false) != null; - } - - fn remap( - context: *anyopaque, - memory: []u8, - alignment: mem.Alignment, - new_len: usize, - return_address: usize, - ) ?[*]u8 { - return realloc(context, memory, alignment, new_len, return_address, true); - } - - fn realloc( - context: *anyopaque, - old_mem: []u8, - alignment: mem.Alignment, - new_len: usize, - ret_addr: usize, - may_move: bool, - ) ?[*]u8 { + fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 { const self: *Self = @ptrCast(@alignCast(context)); self.mutex.lock(); defer self.mutex.unlock(); - assert(old_mem.len != 0); - - const aligned_size = @max(old_mem.len, alignment.toByteUnits()); - if (aligned_size > largest_bucket_object_size) { - return self.resizeLarge(old_mem, alignment, new_len, ret_addr, may_move); + if (config.enable_memory_limit) { + const new_req_bytes = self.total_requested_bytes + len; + if (new_req_bytes > self.requested_memory_limit) return null; + self.total_requested_bytes = new_req_bytes; } - const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size); - var bucket_index = math.log2(size_class_hint); - var size_class: usize = size_class_hint; - const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { - if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| { - break bucket; + const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment)); + if (size_class_index >= self.buckets.len) { + @branchHint(.unlikely); + self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; + const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse return null; + const slice = ptr[0..len]; + + const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr)); + if (config.retain_metadata and !config.never_unmap) { + // Backing allocator may be reusing memory that we're retaining metadata for + assert(!gop.found_existing or gop.value_ptr.freed); + } else { + assert(!gop.found_existing); // This would mean the kernel double-mapped pages. } - size_class *= 2; - } else blk: { + gop.value_ptr.bytes = slice; + if (config.enable_memory_limit) + gop.value_ptr.requested_size = len; + gop.value_ptr.captureStackTrace(ret_addr, .alloc); if (config.retain_metadata) { - if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { - // object not in active buckets or a large allocation, so search empty buckets - if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) |bucket| { - size_class = bucket.emptyBucketSizeClass(); - // bucket is empty so is_used below will always be false and we exit there - break :blk bucket; - } else { - @panic("Invalid free"); - } + gop.value_ptr.freed = false; + if (config.never_unmap) { + gop.value_ptr.alignment = alignment; } } - return self.resizeLarge(old_mem, alignment, new_len, ret_addr, may_move); - }; - const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page); - const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class)); - const used_byte_index = slot_index / 8; - const used_bit_index = @as(u3, @intCast(slot_index % 8)); - const used_byte = bucket.usedBits(used_byte_index); - const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; - if (!is_used) { - if (config.safety) { - reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free)); - @panic("Unrecoverable double free"); - } else { - unreachable; + + if (config.verbose_log) { + log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr }); } + return slice.ptr; } - // Definitely an in-use small alloc now. - if (config.safety) { - const requested_size = bucket.requestedSizes(size_class)[slot_index]; - if (requested_size == 0) @panic("Invalid free"); - const slot_alignment = bucket.log2PtrAligns(size_class)[slot_index]; - if (old_mem.len != requested_size or alignment != slot_alignment) { - var addresses: [stack_n]usize = [1]usize{0} ** stack_n; - var free_stack_trace = StackTrace{ - .instruction_addresses = &addresses, - .index = 0, - }; - std.debug.captureStackTrace(ret_addr, &free_stack_trace); - if (old_mem.len != requested_size) { - log.err("Allocation size {d} bytes does not match resize size {d}. Allocation: {} Resize: {}", .{ - requested_size, - old_mem.len, - bucketStackTrace(bucket, size_class, slot_index, .alloc), - free_stack_trace, - }); + const slot_count = slot_counts[size_class_index]; + + if (self.buckets[size_class_index]) |bucket| { + @branchHint(.likely); + const slot_index = bucket.allocated_count; + if (slot_index < slot_count) { + @branchHint(.likely); + bucket.allocated_count = slot_index + 1; + const used_bits_byte = bucket.usedBits(slot_index / 8); + const used_bit_index: u3 = @intCast(slot_index % 8); + used_bits_byte.* |= (@as(u8, 1) << used_bit_index); + const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + if (config.stack_trace_frames > 0) { + bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc); } - if (alignment != slot_alignment) { - log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{ - slot_alignment.toByteUnits(), - alignment.toByteUnits(), - bucketStackTrace(bucket, size_class, slot_index, .alloc), - free_stack_trace, - }); + if (config.safety) { + bucket.requestedSizes(slot_count)[slot_index] = @intCast(len); + bucket.log2PtrAligns(slot_count)[slot_index] = alignment; } + const page_addr = @intFromPtr(bucket) & ~(page_size - 1); + const addr = page_addr + slot_index * size_class; + if (config.verbose_log) { + log.info("small alloc {d} bytes at 0x{x}", .{ len, addr }); + } + return @ptrFromInt(addr); } } - const prev_req_bytes = self.total_requested_bytes; - if (config.enable_memory_limit) { - const new_req_bytes = prev_req_bytes + new_len - old_mem.len; - if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { - return null; - } - self.total_requested_bytes = new_req_bytes; + + const page = self.backing_allocator.rawAlloc(page_size, page_align, @returnAddress()) orelse + return null; + const bucket: *BucketHeader = .fromPage(@intFromPtr(page), slot_count); + bucket.* = .{ + .allocated_count = 1, + .freed_count = 0, + .prev = self.buckets[size_class_index], + }; + self.buckets[size_class_index] = bucket; + + if (!config.backing_allocator_zeroes) { + @memset(@as([*]u8, @as(*[1]u8, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0); + if (config.safety) @memset(bucket.requestedSizes(slot_count), 0); } - const new_aligned_size = @max(new_len, alignment.toByteUnits()); - const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size); - if (new_size_class <= size_class) { - if (old_mem.len > new_len) { - @memset(old_mem[new_len..], undefined); - } - if (config.verbose_log) { - log.info("small resize {d} bytes at {*} to {d}", .{ - old_mem.len, old_mem.ptr, new_len, - }); - } - if (config.safety) { - bucket.requestedSizes(size_class)[slot_index] = @intCast(new_len); - } - return old_mem.ptr; + bucket.usedBits(0).* = 0b1; + + if (config.stack_trace_frames > 0) { + bucket.captureStackTrace(ret_addr, slot_count, 0, .alloc); } - if (config.enable_memory_limit) { - self.total_requested_bytes = prev_req_bytes; + if (config.safety) { + bucket.requestedSizes(slot_count)[0] = @intCast(len); + bucket.log2PtrAligns(slot_count)[0] = alignment; } + + if (config.verbose_log) { + log.info("small alloc {d} bytes at 0x{x}", .{ len, @intFromPtr(page) }); + } + + return page; + } + + fn resize( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, + ) bool { + _ = context; + _ = memory; + _ = alignment; + _ = new_len; + _ = return_address; + return false; + } + + fn remap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, + ) ?[*]u8 { + _ = context; + _ = memory; + _ = alignment; + _ = new_len; + _ = return_address; return null; } fn free( - ctx: *anyopaque, - old_mem: []u8, + context: *anyopaque, + old_memory: []u8, alignment: mem.Alignment, - ret_addr: usize, + return_address: usize, ) void { - const self: *Self = @ptrCast(@alignCast(ctx)); + const self: *Self = @ptrCast(@alignCast(context)); self.mutex.lock(); defer self.mutex.unlock(); - assert(old_mem.len != 0); + assert(old_memory.len != 0); - const aligned_size = @max(old_mem.len, alignment.toByteUnits()); - if (aligned_size > largest_bucket_object_size) { - self.freeLarge(old_mem, alignment, ret_addr); + const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment)); + if (size_class_index >= self.buckets.len) { + @branchHint(.unlikely); + self.freeLarge(old_memory, alignment, return_address); return; } - const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size); - var bucket_index = math.log2(size_class_hint); - var size_class: usize = size_class_hint; - const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { - if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| { - break bucket; - } - size_class *= 2; - } else blk: { - if (config.retain_metadata) { - if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { - // object not in active buckets or a large allocation, so search empty buckets - if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) |bucket| { - size_class = bucket.emptyBucketSizeClass(); - // bucket is empty so is_used below will always be false and we exit there - break :blk bucket; - } else { - @panic("Invalid free"); - } - } - } - self.freeLarge(old_mem, alignment, ret_addr); - return; - }; - const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page); - const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class)); + const slot_count = slot_counts[size_class_index]; + const freed_addr = @intFromPtr(old_memory.ptr); + const page_addr = freed_addr & ~(page_size - 1); + const bucket: *BucketHeader = .fromPage(page_addr, slot_count); + const page_offset = freed_addr - page_addr; + const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + const slot_index: SlotIndex = @intCast(page_offset / size_class); const used_byte_index = slot_index / 8; - const used_bit_index = @as(u3, @intCast(slot_index % 8)); + const used_bit_index: u3 = @intCast(slot_index % 8); const used_byte = bucket.usedBits(used_byte_index); const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; if (!is_used) { if (config.safety) { - reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free)); - // Recoverable if this is a free. + reportDoubleFree( + return_address, + bucketStackTrace(bucket, slot_count, slot_index, .alloc), + bucketStackTrace(bucket, slot_count, slot_index, .free), + ); + // Recoverable since this is a free. return; } else { unreachable; @@ -955,21 +855,21 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { // Definitely an in-use small alloc now. if (config.safety) { - const requested_size = bucket.requestedSizes(size_class)[slot_index]; + const requested_size = bucket.requestedSizes(slot_count)[slot_index]; if (requested_size == 0) @panic("Invalid free"); - const slot_alignment = bucket.log2PtrAligns(size_class)[slot_index]; - if (old_mem.len != requested_size or alignment != slot_alignment) { + const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; + if (old_memory.len != requested_size or alignment != slot_alignment) { var addresses: [stack_n]usize = [1]usize{0} ** stack_n; - var free_stack_trace = StackTrace{ + var free_stack_trace: StackTrace = .{ .instruction_addresses = &addresses, .index = 0, }; - std.debug.captureStackTrace(ret_addr, &free_stack_trace); - if (old_mem.len != requested_size) { + std.debug.captureStackTrace(return_address, &free_stack_trace); + if (old_memory.len != requested_size) { log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ requested_size, - old_mem.len, - bucketStackTrace(bucket, size_class, slot_index, .alloc), + old_memory.len, + bucketStackTrace(bucket, slot_count, slot_index, .alloc), free_stack_trace, }); } @@ -977,7 +877,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ slot_alignment.toByteUnits(), alignment.toByteUnits(), - bucketStackTrace(bucket, size_class, slot_index, .alloc), + bucketStackTrace(bucket, slot_count, slot_index, .alloc), free_stack_trace, }); } @@ -985,142 +885,29 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } if (config.enable_memory_limit) { - self.total_requested_bytes -= old_mem.len; + self.total_requested_bytes -= old_memory.len; } // Capture stack trace to be the "first free", in case a double free happens. - bucket.captureStackTrace(ret_addr, size_class, slot_index, .free); + bucket.captureStackTrace(return_address, slot_count, slot_index, .free); used_byte.* &= ~(@as(u8, 1) << used_bit_index); - bucket.used_count -= 1; if (config.safety) { - bucket.requestedSizes(size_class)[slot_index] = 0; + bucket.requestedSizes(slot_count)[slot_index] = 0; } - if (bucket.used_count == 0) { - var entry = self.buckets[bucket_index].getEntryFor(bucket); - // save the node for destruction/insertion into in empty_buckets - const node = entry.node.?; - entry.set(null); - if (self.cur_buckets[bucket_index] == bucket) { - self.cur_buckets[bucket_index] = null; + bucket.freed_count += 1; + if (bucket.freed_count == bucket.allocated_count) { + if (self.buckets[size_class_index] == bucket) { + self.buckets[size_class_index] = null; } if (!config.never_unmap) { - self.backing_allocator.free(bucket.page[0..page_size]); - } - if (!config.retain_metadata) { - self.freeBucket(bucket, size_class); - self.bucket_node_pool.destroy(node); - } else { - // move alloc_cursor to end so we can tell size_class later - const slot_count = @divExact(page_size, size_class); - bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count)); - var empty_entry = self.empty_buckets.getEntryFor(node.key); - empty_entry.set(node); - } - } else { - @memset(old_mem, undefined); - } - if (config.verbose_log) { - log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); - } - } - - // Returns true if an allocation of `size` bytes is within the specified - // limits if enable_memory_limit is true - fn isAllocationAllowed(self: *Self, size: usize) bool { - if (config.enable_memory_limit) { - const new_req_bytes = self.total_requested_bytes + size; - if (new_req_bytes > self.requested_memory_limit) - return false; - self.total_requested_bytes = new_req_bytes; - } - - return true; - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 { - const self: *Self = @ptrCast(@alignCast(ctx)); - self.mutex.lock(); - defer self.mutex.unlock(); - if (!self.isAllocationAllowed(len)) return null; - return allocInner(self, len, alignment, ret_addr) catch return null; - } - - fn allocInner( - self: *Self, - len: usize, - alignment: mem.Alignment, - ret_addr: usize, - ) Allocator.Error![*]u8 { - const new_aligned_size = @max(len, alignment.toByteUnits()); - if (new_aligned_size > largest_bucket_object_size) { - try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1); - const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse - return error.OutOfMemory; - const slice = ptr[0..len]; - - const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr)); - if (config.retain_metadata and !config.never_unmap) { - // Backing allocator may be reusing memory that we're retaining metadata for - assert(!gop.found_existing or gop.value_ptr.freed); - } else { - assert(!gop.found_existing); // This would mean the kernel double-mapped pages. + const page: [*]align(page_size) u8 = @ptrFromInt(page_addr); + self.backing_allocator.rawFree(page[0..page_size], page_align, @returnAddress()); } - gop.value_ptr.bytes = slice; - if (config.enable_memory_limit) - gop.value_ptr.requested_size = len; - gop.value_ptr.captureStackTrace(ret_addr, .alloc); - if (config.retain_metadata) { - gop.value_ptr.freed = false; - if (config.never_unmap) { - gop.value_ptr.alignment = alignment; - } - } - - if (config.verbose_log) { - log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr }); - } - return slice.ptr; - } - - const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size); - const slot = try self.allocSlot(new_size_class, ret_addr); - if (config.safety) { - slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len); - slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = alignment; } if (config.verbose_log) { - log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr }); - } - return slot.ptr; - } - - fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader { - const alignment: mem.Alignment = .fromByteUnits(page_size); - const page = self.backing_allocator.rawAlloc(page_size, alignment, @returnAddress()) orelse - return error.OutOfMemory; - errdefer self.backing_allocator.rawFree(page[0..page_size], alignment, @returnAddress()); - - const bucket_size = bucketSize(size_class); - const header_align: mem.Alignment = .fromByteUnits(@alignOf(BucketHeader)); - const ptr: *BucketHeader = @alignCast(@ptrCast(self.backing_allocator.rawAlloc( - bucket_size, - header_align, - @returnAddress(), - ) orelse return error.OutOfMemory)); - ptr.* = .{ - .page = @alignCast(page), - .alloc_cursor = 0, - .used_count = 0, - }; - if (!config.backing_allocator_zeroes) { - @memset(@as([*]u8, @as(*[1]u8, ptr.usedBits(0)))[0..usedBitsCount(size_class)], 0); - if (config.safety) { - // Set the requested sizes to zeroes - @memset(mem.sliceAsBytes(ptr.requestedSizes(size_class)), 0); - } + log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); } - return ptr; } }; } @@ -1460,7 +1247,7 @@ test "setting a memory cap" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - gpa.setRequestedMemoryLimit(1010); + gpa.requested_memory_limit = 1010; const small = try allocator.create(i32); try std.testing.expect(gpa.total_requested_bytes == 4); @@ -1481,63 +1268,6 @@ test "setting a memory cap" { allocator.free(exact); } -test "double frees" { - // use a GPA to back a GPA to check for leaks of the latter's metadata - var backing_gpa = GeneralPurposeAllocator(.{ .safety = true }){}; - defer std.testing.expect(backing_gpa.deinit() == .ok) catch @panic("leak"); - - const GPA = GeneralPurposeAllocator(.{ .safety = true, .never_unmap = true, .retain_metadata = true }); - var gpa = GPA{ .backing_allocator = backing_gpa.allocator() }; - defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); - const allocator = gpa.allocator(); - - // detect a small allocation double free, even though bucket is emptied - const index: usize = 6; - const size_class: usize = @as(usize, 1) << 6; - const small = try allocator.alloc(u8, size_class); - try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) != null); - allocator.free(small); - try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) == null); - try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null) != null); - - // detect a large allocation double free - const large = try allocator.alloc(u8, 2 * page_size); - try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr))); - try std.testing.expectEqual(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.bytes, large); - allocator.free(large); - try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr))); - try std.testing.expect(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.freed); - - const normal_small = try allocator.alloc(u8, size_class); - defer allocator.free(normal_small); - const normal_large = try allocator.alloc(u8, 2 * page_size); - defer allocator.free(normal_large); - - // check that flushing retained metadata doesn't disturb live allocations - gpa.flushRetainedMetadata(); - try std.testing.expect(gpa.empty_buckets.root == null); - try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr), gpa.cur_buckets[index]) != null); - try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr))); - try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr))); -} - -test "empty bucket size class" { - const GPA = GeneralPurposeAllocator(.{ .safety = true, .never_unmap = true, .retain_metadata = true }); - var gpa = GPA{}; - defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); - const allocator = gpa.allocator(); - - // allocate and free to create an empty bucket - const size_class: usize = @as(usize, 1) << 6; - const small = try allocator.alloc(u8, size_class); - allocator.free(small); - - // the metadata tracking system relies on alloc_cursor of empty buckets - // being set to the slot count so that we can get back the size class. - const empty_bucket = GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null).?; - try std.testing.expect(empty_bucket.emptyBucketSizeClass() == size_class); -} - test "bug 9995 fix, large allocs count requested size not backing size" { // with AtLeast, buffer likely to be larger than requested, especially when shrinking var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; From 0e0f0c9625c5517856883e506f572395d5fac58d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Feb 2025 23:12:55 -0800 Subject: [PATCH 20/39] std.heap.GeneralPurposeAllocator: check canary in free --- lib/std/heap/general_purpose_allocator.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 1fdb09cb413e..accdb114576b 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -162,6 +162,10 @@ pub const Config = struct { /// Tell whether the backing allocator returns already-zeroed memory. backing_allocator_zeroes: bool = true, + + /// Magic value that distinguishes allocations owned by this allocator from + /// other regions of memory. + canary: usize = @truncate(0x9232a6ff85dff10f), }; pub const Check = enum { ok, leak }; @@ -266,6 +270,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { allocated_count: SlotIndex, freed_count: SlotIndex, prev: ?*BucketHeader, + canary: usize = config.canary, fn fromPage(page_addr: usize, slot_count: usize) *BucketHeader { const unaligned = page_addr + page_size - bucketSize(slot_count); @@ -832,6 +837,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const freed_addr = @intFromPtr(old_memory.ptr); const page_addr = freed_addr & ~(page_size - 1); const bucket: *BucketHeader = .fromPage(page_addr, slot_count); + if (bucket.canary != config.canary) @panic("Invalid free"); const page_offset = freed_addr - page_addr; const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); const slot_index: SlotIndex = @intCast(page_offset / size_class); From 82b5a1d313754037ee03d7cd20ee9bc8a64a1ba2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 00:18:43 -0800 Subject: [PATCH 21/39] std.heap.GeneralPurposeAllocator: implement resize and remap --- lib/std/heap/general_purpose_allocator.zig | 164 ++++++++++++++++----- 1 file changed, 131 insertions(+), 33 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index accdb114576b..2464db11989e 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -163,6 +163,10 @@ pub const Config = struct { /// Tell whether the backing allocator returns already-zeroed memory. backing_allocator_zeroes: bool = true, + /// When resizing an allocation, refresh the stack trace with the resize + /// callsite. Comes with a performance penalty. + resize_stack_traces: bool = false, + /// Magic value that distinguishes allocations owned by this allocator from /// other regions of memory. canary: usize = @truncate(0x9232a6ff85dff10f), @@ -554,6 +558,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { }); } + // If this would move the allocation into a small size class, + // refuse the request, because it would require creating small + // allocation metadata. + const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment)); + if (new_size_class_index < self.buckets.len) return null; + // Do memory limit accounting with requested sizes rather than what // backing_allocator returns because if we want to return // error.OutOfMemory, we have to leave allocation untouched, and @@ -598,7 +608,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { }); } entry.value_ptr.bytes = resized_ptr[0..new_size]; - entry.value_ptr.captureStackTrace(ret_addr, .alloc); + if (config.resize_stack_traces) + entry.value_ptr.captureStackTrace(ret_addr, .alloc); // Update the key of the hash map if the memory was relocated. if (resized_ptr != old_mem.ptr) { @@ -791,12 +802,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { new_len: usize, return_address: usize, ) bool { - _ = context; - _ = memory; - _ = alignment; - _ = new_len; - _ = return_address; - return false; + const self: *Self = @ptrCast(@alignCast(context)); + self.mutex.lock(); + defer self.mutex.unlock(); + + const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); + if (size_class_index >= self.buckets.len) { + return self.resizeLarge(memory, alignment, new_len, return_address, false) != null; + } else { + return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index); + } } fn remap( @@ -806,12 +821,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { new_len: usize, return_address: usize, ) ?[*]u8 { - _ = context; - _ = memory; - _ = alignment; - _ = new_len; - _ = return_address; - return null; + const self: *Self = @ptrCast(@alignCast(context)); + self.mutex.lock(); + defer self.mutex.unlock(); + + const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); + if (size_class_index >= self.buckets.len) { + return self.resizeLarge(memory, alignment, new_len, return_address, true); + } else { + return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null; + } } fn free( @@ -894,8 +913,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { self.total_requested_bytes -= old_memory.len; } - // Capture stack trace to be the "first free", in case a double free happens. - bucket.captureStackTrace(return_address, slot_count, slot_index, .free); + if (config.stack_trace_frames > 0) { + // Capture stack trace to be the "first free", in case a double free happens. + bucket.captureStackTrace(return_address, slot_count, slot_index, .free); + } used_byte.* &= ~(@as(u8, 1) << used_bit_index); if (config.safety) { @@ -915,6 +936,91 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); } } + + fn resizeSmall( + self: *Self, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, + size_class_index: usize, + ) bool { + const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment)); + if (!config.safety) return new_size_class_index == size_class_index; + const slot_count = slot_counts[size_class_index]; + const memory_addr = @intFromPtr(memory.ptr); + const page_addr = memory_addr & ~(page_size - 1); + const bucket: *BucketHeader = .fromPage(page_addr, slot_count); + if (bucket.canary != config.canary) @panic("Invalid free"); + const page_offset = memory_addr - page_addr; + const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + const slot_index: SlotIndex = @intCast(page_offset / size_class); + const used_byte_index = slot_index / 8; + const used_bit_index: u3 = @intCast(slot_index % 8); + const used_byte = bucket.usedBits(used_byte_index); + const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; + if (!is_used) { + reportDoubleFree( + return_address, + bucketStackTrace(bucket, slot_count, slot_index, .alloc), + bucketStackTrace(bucket, slot_count, slot_index, .free), + ); + // Recoverable since this is a free. + return false; + } + + // Definitely an in-use small alloc now. + const requested_size = bucket.requestedSizes(slot_count)[slot_index]; + if (requested_size == 0) @panic("Invalid free"); + const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; + if (memory.len != requested_size or alignment != slot_alignment) { + var addresses: [stack_n]usize = [1]usize{0} ** stack_n; + var free_stack_trace: StackTrace = .{ + .instruction_addresses = &addresses, + .index = 0, + }; + std.debug.captureStackTrace(return_address, &free_stack_trace); + if (memory.len != requested_size) { + log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ + requested_size, + memory.len, + bucketStackTrace(bucket, slot_count, slot_index, .alloc), + free_stack_trace, + }); + } + if (alignment != slot_alignment) { + log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ + slot_alignment.toByteUnits(), + alignment.toByteUnits(), + bucketStackTrace(bucket, slot_count, slot_index, .alloc), + free_stack_trace, + }); + } + } + + if (new_size_class_index != size_class_index) return false; + + const prev_req_bytes = self.total_requested_bytes; + if (config.enable_memory_limit) { + const new_req_bytes = prev_req_bytes - memory.len + new_len; + if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { + return false; + } + self.total_requested_bytes = new_req_bytes; + } + + if (memory.len > new_len) @memset(memory[new_len..], undefined); + if (config.verbose_log) + log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len }); + + if (config.safety) + bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len); + + if (config.resize_stack_traces) + bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc); + + return true; + } }; } @@ -1023,12 +1129,8 @@ test "shrink" { try std.testing.expect(b == 0x11); } - try std.testing.expect(allocator.resize(slice, 16)); - slice = slice[0..16]; - - for (slice) |b| { - try std.testing.expect(b == 0x11); - } + // Does not cross size class boundaries when shrinking. + try std.testing.expect(!allocator.resize(slice, 16)); } test "large object - grow" { @@ -1212,14 +1314,14 @@ test "realloc large object to larger alignment" { try std.testing.expect(slice[16] == 0x34); } -test "large object shrinks to small but allocation fails during shrink" { +test "large object rejects shrinking to small" { if (builtin.target.isWasm()) { // Not expected to pass on targets that do not have memory mapping. return error.SkipZigTest; } var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); - var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = failing_allocator.allocator() }; + var gpa: GeneralPurposeAllocator(.{}) = .{ .backing_allocator = failing_allocator.allocator() }; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1228,10 +1330,7 @@ test "large object shrinks to small but allocation fails during shrink" { slice[0] = 0x12; slice[3] = 0x34; - // Next allocation will fail in the backing allocator of the GeneralPurposeAllocator - - try std.testing.expect(allocator.resize(slice, 4)); - slice = slice[0..4]; + try std.testing.expect(!allocator.resize(slice, 4)); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[3] == 0x34); } @@ -1274,17 +1373,16 @@ test "setting a memory cap" { allocator.free(exact); } -test "bug 9995 fix, large allocs count requested size not backing size" { - // with AtLeast, buffer likely to be larger than requested, especially when shrinking - var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; +test "large allocations count requested size not backing size" { + var gpa: GeneralPurposeAllocator(.{ .enable_memory_limit = true }) = .{}; const allocator = gpa.allocator(); var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); - try std.testing.expect(gpa.total_requested_bytes == page_size + 1); + try std.testing.expectEqual(page_size + 1, gpa.total_requested_bytes); buf = try allocator.realloc(buf, 1); - try std.testing.expect(gpa.total_requested_bytes == 1); + try std.testing.expectEqual(1, gpa.total_requested_bytes); buf = try allocator.realloc(buf, 2); - try std.testing.expect(gpa.total_requested_bytes == 2); + try std.testing.expectEqual(2, gpa.total_requested_bytes); } test "retain metadata and never unmap" { From 00b723dc9dec6a85e2018c9875aeae32ea331ad7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 01:04:24 -0800 Subject: [PATCH 22/39] std.heap.WasmAllocator: update to new Allocator API --- lib/std/heap/WasmAllocator.zig | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/WasmAllocator.zig index 2bdfdd84eac6..e30ac5ab0127 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/WasmAllocator.zig @@ -20,6 +20,7 @@ comptime { pub const vtable: Allocator.VTable = .{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }; @@ -46,12 +47,11 @@ var frees: [size_class_count]usize = @splat(0); /// For each big size class, points to the freed pointer. var big_frees: [big_size_class_count]usize = @splat(0); -fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, return_address: usize) ?[*]u8 { +fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 { _ = ctx; _ = return_address; // Make room for the freelist next pointer. - const alignment = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_align)); - const actual_len = @max(len +| @sizeOf(usize), alignment); + const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits()); const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null; const class = math.log2(slot_size) - min_class; if (class < size_class_count) { @@ -86,7 +86,7 @@ fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, return_address: usize) ?[* fn resize( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, new_len: usize, return_address: usize, ) bool { @@ -94,7 +94,7 @@ fn resize( _ = return_address; // We don't want to move anything from one size class to another, but we // can recover bytes in between powers of two. - const buf_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_buf_align)); + const buf_align = alignment.toByteUnits(); const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align); const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align); const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len); @@ -111,15 +111,25 @@ fn resize( } } +fn remap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, +) ?[*]u8 { + return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null; +} + fn free( ctx: *anyopaque, buf: []u8, - log2_buf_align: u8, + alignment: mem.Alignment, return_address: usize, ) void { _ = ctx; _ = return_address; - const buf_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_buf_align)); + const buf_align = alignment.toByteUnits(); const actual_len = @max(buf.len + @sizeOf(usize), buf_align); const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len); const class = math.log2(slot_size) - min_class; From b14a350430aef737250b92520fa805201decf4d5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 01:04:44 -0800 Subject: [PATCH 23/39] std.heap.GeneralPurposeAllocator: reduce page size to 512K and fix compilation on 32-bit targets --- lib/std/heap/general_purpose_allocator.zig | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 2464db11989e..2c06a9ccbcb6 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -104,12 +104,13 @@ const StackTrace = std.builtin.StackTrace; const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. - else => 2 * 1024 * 1024, // Avoids too many active mappings when `page_size_max` is low. + else => 512 * 1024, // Avoids too many active mappings when `page_size_max` is low. }); const page_align: mem.Alignment = .fromByteUnits(page_size); /// Integer type for pointing to slots in a small allocation const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); +const Log2USize = std.math.Log2Int(usize); const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6; const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0; @@ -383,7 +384,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { /// This is executed only at compile-time to prepopulate a lookup table. fn calculateSlotCount(size_class_index: usize) SlotIndex { - const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); var lower: usize = 8; var upper: usize = (page_size - bucketSize(lower)) / size_class; while (upper > lower) { @@ -408,7 +409,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool { - const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); const slot_count = slot_counts[size_class_index]; var leaks = false; var used_bits_byte: usize = 0; @@ -745,7 +746,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const used_bits_byte = bucket.usedBits(slot_index / 8); const used_bit_index: u3 = @intCast(slot_index % 8); used_bits_byte.* |= (@as(u8, 1) << used_bit_index); - const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); if (config.stack_trace_frames > 0) { bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc); } @@ -858,7 +859,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const bucket: *BucketHeader = .fromPage(page_addr, slot_count); if (bucket.canary != config.canary) @panic("Invalid free"); const page_offset = freed_addr - page_addr; - const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); const slot_index: SlotIndex = @intCast(page_offset / size_class); const used_byte_index = slot_index / 8; const used_bit_index: u3 = @intCast(slot_index % 8); @@ -953,7 +954,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const bucket: *BucketHeader = .fromPage(page_addr, slot_count); if (bucket.canary != config.canary) @panic("Invalid free"); const page_offset = memory_addr - page_addr; - const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); + const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); const slot_index: SlotIndex = @intCast(page_offset / size_class); const used_byte_index = slot_index / 8; const used_bit_index: u3 = @intCast(slot_index % 8); From c8e807c44ed8c96b31a6cbe0b7f5f01a02ced270 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 13:21:15 -0800 Subject: [PATCH 24/39] std.heap.GeneralPurposeAllocator: use for loops in leak check --- lib/std/heap/general_purpose_allocator.zig | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 2c06a9ccbcb6..6ff18dfe7ef8 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -412,12 +412,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); const slot_count = slot_counts[size_class_index]; var leaks = false; - var used_bits_byte: usize = 0; - while (used_bits_byte < used_bits_count) : (used_bits_byte += 1) { + for (0..used_bits_count) |used_bits_byte| { const used_byte = bucket.usedBits(used_bits_byte).*; if (used_byte != 0) { - var bit_index: u3 = 0; - while (true) : (bit_index += 1) { + for (0..8) |bit_index_usize| { + const bit_index: u3 = @intCast(bit_index_usize); const is_used = @as(u1, @truncate(used_byte >> bit_index)) != 0; if (is_used) { const slot_index: SlotIndex = @intCast(used_bits_byte * 8 + bit_index); @@ -427,8 +426,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { log.err("memory address 0x{x} leaked: {}", .{ addr, stack_trace }); leaks = true; } - if (bit_index == math.maxInt(u3)) - break; } } } From def36f2e4460ec9dd500772daac100a655f019b7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 13:31:01 -0800 Subject: [PATCH 25/39] std.heap.GeneralPurposeAllocator: usize for used_bits improves leak checking performance. --- lib/std/heap/general_purpose_allocator.zig | 51 ++++++++++++---------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 6ff18dfe7ef8..27c6dc0c8bbf 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -265,7 +265,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { /// Bucket: In memory, in order: /// * BucketHeader - /// * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots + /// * bucket_used_bits: [N]usize, // 1 bit for every slot /// -- below only exists when config.safety is true -- /// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot /// * log2_ptr_aligns: [N]u8 // 1 byte for every slot @@ -282,10 +282,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1)); } - // TODO use usize instead of u8 - fn usedBits(bucket: *BucketHeader, index: usize) *u8 { - // TODO avoid ptr to int - return @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index); + fn usedBits(bucket: *BucketHeader, index: usize) *usize { + const ptr: [*]u8 = @ptrCast(bucket); + const bits: [*]usize = @alignCast(@ptrCast(ptr + @sizeOf(BucketHeader))); + return &bits[index]; } fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt { @@ -360,7 +360,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); return mem.alignForward( usize, - @sizeOf(BucketHeader) + usedBitsCount(slot_count), + @sizeOf(BucketHeader) + usedBitsSize(slot_count), @alignOf(LargestSizeClassInt), ); } @@ -374,7 +374,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const unaligned_start = if (config.safety) bucketAlignsStart(slot_count) + slot_count else - @sizeOf(BucketHeader) + usedBitsCount(slot_count); + @sizeOf(BucketHeader) + usedBitsSize(slot_count); return mem.alignForward(usize, unaligned_start, @alignOf(usize)); } @@ -404,8 +404,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn usedBitsCount(slot_count: usize) usize { - assert(slot_count >= 8); - return (slot_count + 7) / 8; + return (slot_count + (@bitSizeOf(usize) - 1)) / @bitSizeOf(usize); + } + + fn usedBitsSize(slot_count: usize) usize { + return usedBitsCount(slot_count) * @sizeOf(usize); } fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool { @@ -413,13 +416,13 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const slot_count = slot_counts[size_class_index]; var leaks = false; for (0..used_bits_count) |used_bits_byte| { - const used_byte = bucket.usedBits(used_bits_byte).*; - if (used_byte != 0) { - for (0..8) |bit_index_usize| { - const bit_index: u3 = @intCast(bit_index_usize); - const is_used = @as(u1, @truncate(used_byte >> bit_index)) != 0; + const used_int = bucket.usedBits(used_bits_byte).*; + if (used_int != 0) { + for (0..@bitSizeOf(usize)) |bit_index_usize| { + const bit_index: Log2USize = @intCast(bit_index_usize); + const is_used = @as(u1, @truncate(used_int >> bit_index)) != 0; if (is_used) { - const slot_index: SlotIndex = @intCast(used_bits_byte * 8 + bit_index); + const slot_index: SlotIndex = @intCast(used_bits_byte * @bitSizeOf(usize) + bit_index); const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc); const page_addr = @intFromPtr(bucket) & ~(page_size - 1); const addr = page_addr + slot_index * size_class; @@ -740,9 +743,9 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (slot_index < slot_count) { @branchHint(.likely); bucket.allocated_count = slot_index + 1; - const used_bits_byte = bucket.usedBits(slot_index / 8); - const used_bit_index: u3 = @intCast(slot_index % 8); - used_bits_byte.* |= (@as(u8, 1) << used_bit_index); + const used_bits_byte = bucket.usedBits(slot_index / @bitSizeOf(usize)); + const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); + used_bits_byte.* |= (@as(usize, 1) << used_bit_index); const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); if (config.stack_trace_frames > 0) { bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc); @@ -771,7 +774,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { self.buckets[size_class_index] = bucket; if (!config.backing_allocator_zeroes) { - @memset(@as([*]u8, @as(*[1]u8, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0); + @memset(@as([*]usize, @as(*[1]usize, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0); if (config.safety) @memset(bucket.requestedSizes(slot_count), 0); } @@ -858,8 +861,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const page_offset = freed_addr - page_addr; const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); const slot_index: SlotIndex = @intCast(page_offset / size_class); - const used_byte_index = slot_index / 8; - const used_bit_index: u3 = @intCast(slot_index % 8); + const used_byte_index = slot_index / @bitSizeOf(usize); + const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); const used_byte = bucket.usedBits(used_byte_index); const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; if (!is_used) { @@ -916,7 +919,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { bucket.captureStackTrace(return_address, slot_count, slot_index, .free); } - used_byte.* &= ~(@as(u8, 1) << used_bit_index); + used_byte.* &= ~(@as(usize, 1) << used_bit_index); if (config.safety) { bucket.requestedSizes(slot_count)[slot_index] = 0; } @@ -953,8 +956,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const page_offset = memory_addr - page_addr; const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); const slot_index: SlotIndex = @intCast(page_offset / size_class); - const used_byte_index = slot_index / 8; - const used_bit_index: u3 = @intCast(slot_index % 8); + const used_byte_index = slot_index / @bitSizeOf(usize); + const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); const used_byte = bucket.usedBits(used_byte_index); const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; if (!is_used) { From d20d934a8a096607efda7e956a2131726036b09e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 13:54:56 -0800 Subject: [PATCH 26/39] std: fix compilation under -lc --- lib/std/c.zig | 6 ++ lib/std/heap.zig | 102 ++++++++++++++++++++------------- lib/std/heap/PageAllocator.zig | 3 +- 3 files changed, 70 insertions(+), 41 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 6e1e9beb9f43..4627f4103e21 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -7867,6 +7867,11 @@ pub const MAP = switch (native_os) { else => void, }; +pub const MREMAP = switch (native_os) { + .linux => linux.MREMAP, + else => void, +}; + /// Used by libc to communicate failure. Not actually part of the underlying syscall. pub const MAP_FAILED: *anyopaque = @ptrFromInt(maxInt(usize)); @@ -9508,6 +9513,7 @@ pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize; pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize; pub extern "c" fn mmap(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque; pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int; +pub extern "c" fn mremap(addr: ?*align(page_size) const anyopaque, old_len: usize, new_len: usize, flags: MREMAP, ...) *anyopaque; pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int; pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8) c_int; pub extern "c" fn linkat(oldfd: fd_t, oldpath: [*:0]const u8, newfd: fd_t, newpath: [*:0]const u8, flags: c_int) c_int; diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 156fbdeab478..2b7f17ac69f1 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -124,6 +124,13 @@ const CAllocator = struct { } } + const vtable: Allocator.VTable = .{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }; + pub const supports_malloc_size = @TypeOf(malloc_size) != void; pub const malloc_size = if (@TypeOf(c.malloc_size) != void) c.malloc_size @@ -139,7 +146,7 @@ const CAllocator = struct { }; fn getHeader(ptr: [*]u8) *[*]u8 { - return @as(*[*]u8, @ptrFromInt(@intFromPtr(ptr) - @sizeOf(usize))); + return @ptrCast(ptr - @sizeOf(usize)); } fn alignedAlloc(len: usize, alignment: mem.Alignment) ?[*]u8 { @@ -147,13 +154,13 @@ const CAllocator = struct { if (supports_posix_memalign) { // The posix_memalign only accepts alignment values that are a // multiple of the pointer size - const eff_alignment = @max(alignment_bytes, @sizeOf(usize)); + const effective_alignment = @max(alignment_bytes, @sizeOf(usize)); var aligned_ptr: ?*anyopaque = undefined; - if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0) + if (c.posix_memalign(&aligned_ptr, effective_alignment, len) != 0) return null; - return @as([*]u8, @ptrCast(aligned_ptr)); + return @ptrCast(aligned_ptr); } // Thin wrapper around regular malloc, overallocate to account for @@ -219,6 +226,18 @@ const CAllocator = struct { return false; } + fn remap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, + ) ?[*]u8 { + // realloc would potentially return a new allocation that does not + // respect the original alignment. + return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null; + } + fn free( _: *anyopaque, buf: []u8, @@ -234,39 +253,36 @@ const CAllocator = struct { /// Supports the full Allocator interface, including alignment, and exploiting /// `malloc_usable_size` if available. For an allocator that directly calls /// `malloc`/`free`, see `raw_c_allocator`. -pub const c_allocator = Allocator{ +pub const c_allocator: Allocator = .{ .ptr = undefined, - .vtable = &c_allocator_vtable, -}; -const c_allocator_vtable = Allocator.VTable{ - .alloc = CAllocator.alloc, - .resize = CAllocator.resize, - .free = CAllocator.free, + .vtable = &CAllocator.vtable, }; -/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly calls -/// `malloc`/`free`. Does not attempt to utilize `malloc_usable_size`. +/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly +/// calls `malloc`/`free`. Does not attempt to utilize `malloc_usable_size`. /// This allocator is safe to use as the backing allocator with -/// `ArenaAllocator` for example and is more optimal in such a case -/// than `c_allocator`. -pub const raw_c_allocator = Allocator{ +/// `ArenaAllocator` for example and is more optimal in such a case than +/// `c_allocator`. +pub const raw_c_allocator: Allocator = .{ .ptr = undefined, .vtable = &raw_c_allocator_vtable, }; -const raw_c_allocator_vtable = Allocator.VTable{ +const raw_c_allocator_vtable: Allocator.VTable = .{ .alloc = rawCAlloc, .resize = rawCResize, + .remap = rawCRemap, .free = rawCFree, }; fn rawCAlloc( - _: *anyopaque, + context: *anyopaque, len: usize, alignment: mem.Alignment, - ret_addr: usize, + return_address: usize, ) ?[*]u8 { - _ = ret_addr; - assert(alignment.order(.le, comptime .fromByteUnits(@alignOf(std.c.max_align_t)))); + _ = context; + _ = return_address; + assert(alignment.compare(.lte, comptime .fromByteUnits(@alignOf(std.c.max_align_t)))); // Note that this pointer cannot be aligncasted to max_align_t because if // len is < max_align_t then the alignment can be smaller. For example, if // max_align_t is 16, but the user requests 8 bytes, there is no built-in @@ -277,35 +293,43 @@ fn rawCAlloc( } fn rawCResize( - _: *anyopaque, - buf: []u8, + context: *anyopaque, + memory: []u8, alignment: mem.Alignment, new_len: usize, - ret_addr: usize, + return_address: usize, ) bool { + _ = context; + _ = memory; _ = alignment; - _ = ret_addr; - - if (new_len <= buf.len) - return true; - - if (CAllocator.supports_malloc_size) { - const full_len = CAllocator.malloc_size(buf.ptr); - if (new_len <= full_len) return true; - } - + _ = new_len; + _ = return_address; return false; } +fn rawCRemap( + context: *anyopaque, + memory: []u8, + alignment: mem.Alignment, + new_len: usize, + return_address: usize, +) ?[*]u8 { + _ = context; + _ = alignment; + _ = return_address; + return @ptrCast(c.realloc(memory.ptr, new_len)); +} + fn rawCFree( - _: *anyopaque, - buf: []u8, + context: *anyopaque, + memory: []u8, alignment: mem.Alignment, - ret_addr: usize, + return_address: usize, ) void { + _ = context; _ = alignment; - _ = ret_addr; - c.free(buf.ptr); + _ = return_address; + c.free(memory.ptr); } /// On operating systems that support memory mapping, this allocator makes a diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index c0c0c146956d..433e0f121819 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -172,8 +172,7 @@ fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { if (new_size_aligned == page_aligned_len) return memory.ptr; - const mremap_available = native_os == .linux; - if (mremap_available) { + if (posix.MREMAP != void) { // TODO: if the next_mmap_addr_hint is within the remapped range, update it const new_memory = posix.mremap(memory.ptr, memory.len, new_len, .{ .MAYMOVE = may_move }, null) catch return null; return new_memory.ptr; From 7320e8b3cd7c8f482fd99793f2ed29cf610319d7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 14:24:41 -0800 Subject: [PATCH 27/39] std.testing: make some things not pub this looks like it was an accident to expose these --- lib/std/testing.zig | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 3387f258b802..9b56a2ad511f 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -9,18 +9,20 @@ pub var random_seed: u32 = 0; pub const FailingAllocator = @import("testing/FailingAllocator.zig"); pub const failing_allocator = failing_allocator_instance.allocator(); -pub var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.allocator(), .{ .fail_index = 0 }); +var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.allocator(), .{ + .fail_index = 0, +}); +var base_allocator_instance = std.heap.FixedBufferAllocator.init(""); /// This should only be used in temporary test programs. pub const allocator = allocator_instance.allocator(); -pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{}) = b: { - if (!builtin.is_test) - @compileError("Cannot use testing allocator outside of test block"); +pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{ + .stack_trace_frames = 10, +}) = b: { + if (!builtin.is_test) @compileError("testing allocator used when not testing"); break :b .init; }; -pub var base_allocator_instance = std.heap.FixedBufferAllocator.init(""); - /// TODO https://github.com/ziglang/zig/issues/5738 pub var log_level = std.log.Level.warn; From 86261915497fe6b9e0802b2d1e23b5f49c1d2757 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 14:25:29 -0800 Subject: [PATCH 28/39] std.heap.GeneralPurposeAllocator: fix slot_counts calculation In larger small buckets, the comptime logic that computed slot count did not verify that the number it produced was valid. Now it verifies it, which made this bug into a compile error. Then I fixed the bug by introducing a "minimum slots per bucket" declaration. --- lib/std/heap/general_purpose_allocator.zig | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 27c6dc0c8bbf..bb038e0bf583 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -104,7 +104,7 @@ const StackTrace = std.builtin.StackTrace; const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. - else => 512 * 1024, // Avoids too many active mappings when `page_size_max` is low. + else => 128 * 1024, // Avoids too many active mappings when `page_size_max` is low. }); const page_align: mem.Alignment = .fromByteUnits(page_size); @@ -112,8 +112,7 @@ const page_align: mem.Alignment = .fromByteUnits(page_size); const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); const Log2USize = std.math.Log2Int(usize); -const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6; -const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0; +const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) 6 else 0; const default_stack_trace_frames: usize = switch (builtin.mode) { .Debug => default_sys_stack_trace_frames, else => 0, @@ -219,7 +218,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { pub const Error = mem.Allocator.Error; - const small_bucket_count = math.log2(page_size); + /// Avoids creating buckets that would only be able to store a small + /// number of slots. Value of 1 means 2 is the minimum slot count. + const minimum_slots_per_bucket_log2 = 1; + const small_bucket_count = math.log2(page_size) - minimum_slots_per_bucket_log2; const largest_bucket_object_size = 1 << (small_bucket_count - 1); const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); @@ -385,21 +387,24 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { /// This is executed only at compile-time to prepopulate a lookup table. fn calculateSlotCount(size_class_index: usize) SlotIndex { const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); - var lower: usize = 8; + var lower: usize = 1 << minimum_slots_per_bucket_log2; var upper: usize = (page_size - bucketSize(lower)) / size_class; while (upper > lower) { const proposed: usize = lower + (upper - lower) / 2; if (proposed == lower) return lower; const slots_end = proposed * size_class; const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); - const bucket_size = bucketSize(proposed); - const end = header_begin + bucket_size; + const end = header_begin + bucketSize(proposed); if (end > page_size) { upper = proposed - 1; } else { lower = proposed; } } + const slots_end = lower * size_class; + const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); + const end = header_begin + bucketSize(lower); + assert(end <= page_size); return lower; } From 6aab1ea2562806124d4add8d95fb758edf914893 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 16:21:21 -0800 Subject: [PATCH 29/39] std.heap: update Windows HeapAllocator --- lib/std/heap.zig | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 2b7f17ac69f1..7d239a0ccedb 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -146,7 +146,7 @@ const CAllocator = struct { }; fn getHeader(ptr: [*]u8) *[*]u8 { - return @ptrCast(ptr - @sizeOf(usize)); + return @alignCast(@ptrCast(ptr - @sizeOf(usize))); } fn alignedAlloc(len: usize, alignment: mem.Alignment) ?[*]u8 { @@ -381,6 +381,7 @@ pub const HeapAllocator = switch (builtin.os.tag) { .vtable = &.{ .alloc = alloc, .resize = resize, + .remap = remap, .free = free, }, }; @@ -448,6 +449,26 @@ pub const HeapAllocator = switch (builtin.os.tag) { return true; } + fn remap( + ctx: *anyopaque, + buf: []u8, + alignment: mem.Alignment, + new_size: usize, + return_address: usize, + ) ?[*]u8 { + _ = alignment; + _ = return_address; + const self: *HeapAllocator = @ptrCast(@alignCast(ctx)); + + const root_addr = getRecordPtr(buf).*; + const align_offset = @intFromPtr(buf.ptr) - root_addr; + const amt = align_offset + new_size + @sizeOf(usize); + const new_ptr = windows.kernel32.HeapReAlloc(self.heap_handle.?, 0, @ptrFromInt(root_addr), amt) orelse return null; + assert(new_ptr == @as(*anyopaque, @ptrFromInt(root_addr))); + getRecordPtr(buf.ptr[0..new_size]).* = root_addr; + return @ptrCast(new_ptr); + } + fn free( ctx: *anyopaque, buf: []u8, From 8282565ce524c14e62e8c30e1feb8afc5c2ab961 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 16:30:46 -0800 Subject: [PATCH 30/39] std.heap.GeneralPurposeAllocator: fix UAF in resizeLarge There was an ensureUnusedCapacity() call that invalidated a looked-up hash table entry. Move it earlier. --- lib/std/heap/general_purpose_allocator.zig | 35 ++++++++++++---------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index bb038e0bf583..0f21830fbf99 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -532,6 +532,13 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { ret_addr: usize, may_move: bool, ) ?[*]u8 { + if (config.retain_metadata and may_move) { + // Before looking up the entry (since this could invalidate + // it), we must reserve space for the new entry in case the + // allocation is relocated. + self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; + } + const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { if (config.safety) { @panic("Invalid free"); @@ -584,15 +591,9 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { self.total_requested_bytes = new_req_bytes; } - const opt_resized_ptr = if (may_move) b: { - // So that if the allocation moves, we can memcpy the - // `LargeAlloc` value directly from old to new location. - // It's also not clear to me whether removing one item from std - // lib hash map guarantees that unused capacity increases by - // one. - self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; - break :b self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr); - } else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) + const opt_resized_ptr = if (may_move) + self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr) + else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) old_mem.ptr else null; @@ -619,6 +620,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { // Update the key of the hash map if the memory was relocated. if (resized_ptr != old_mem.ptr) { + const large_alloc = entry.value_ptr.*; + if (config.retain_metadata) { + entry.value_ptr.freed = true; + entry.value_ptr.captureStackTrace(ret_addr, .free); + } else { + self.large_allocations.removeByPtr(entry.key_ptr); + } + const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(resized_ptr)); if (config.retain_metadata and !config.never_unmap) { // Backing allocator may be reusing memory that we're retaining metadata for @@ -626,13 +635,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } else { assert(!gop.found_existing); // This would mean the kernel double-mapped pages. } - gop.value_ptr.* = entry.value_ptr.*; - if (!config.retain_metadata) { - self.large_allocations.removeByPtr(entry.key_ptr); - } else { - entry.value_ptr.freed = true; - entry.value_ptr.captureStackTrace(ret_addr, .free); - } + gop.value_ptr.* = large_alloc; } return resized_ptr; From f82ec3f02af68ca25870f8ae3861a416225af554 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 16:39:14 -0800 Subject: [PATCH 31/39] std.testing.allocator: different canary + enable resize traces Accept a slight performance degradation when unit testing for better debuggability when a leak or double-free is detected. --- lib/std/testing.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 9b56a2ad511f..616a138cbd60 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -18,6 +18,11 @@ var base_allocator_instance = std.heap.FixedBufferAllocator.init(""); pub const allocator = allocator_instance.allocator(); pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{ .stack_trace_frames = 10, + .resize_stack_traces = true, + // A unique value so that when a default-constructed + // GeneralPurposeAllocator is incorrectly passed to testing allocator, or + // vice versa, panic occurs. + .canary = @truncate(0x2731e675c3a701ba), }) = b: { if (!builtin.is_test) @compileError("testing allocator used when not testing"); break :b .init; From 5e9b8c38d360a254bf951674f4b39ea7a602c515 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 18:03:14 -0800 Subject: [PATCH 32/39] std.heap: remove HeapAllocator Windows-only, depends on kernel32 in violation of zig std lib policy, and redundant with other cross-platform APIs that perform the same functionality. --- lib/std/heap.zig | 137 -------------------------------- lib/std/os/windows.zig | 12 --- lib/std/os/windows/kernel32.zig | 9 --- 3 files changed, 158 deletions(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 7d239a0ccedb..23b19233e814 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -363,127 +363,6 @@ pub const wasm_allocator: Allocator = .{ .vtable = &WasmAllocator.vtable, }; -pub const HeapAllocator = switch (builtin.os.tag) { - .windows => struct { - heap_handle: ?HeapHandle, - - const HeapHandle = windows.HANDLE; - - pub fn init() HeapAllocator { - return HeapAllocator{ - .heap_handle = null, - }; - } - - pub fn allocator(self: *HeapAllocator) Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - pub fn deinit(self: *HeapAllocator) void { - if (self.heap_handle) |heap_handle| { - windows.HeapDestroy(heap_handle); - } - } - - fn getRecordPtr(buf: []u8) *align(1) usize { - return @as(*align(1) usize, @ptrFromInt(@intFromPtr(buf.ptr) + buf.len)); - } - - fn alloc( - ctx: *anyopaque, - n: usize, - alignment: mem.Alignment, - return_address: usize, - ) ?[*]u8 { - _ = return_address; - const self: *HeapAllocator = @ptrCast(@alignCast(ctx)); - - const ptr_align = alignment.toByteUnits(); - const amt = n + ptr_align - 1 + @sizeOf(usize); - const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, .seq_cst); - const heap_handle = optional_heap_handle orelse blk: { - const options = if (builtin.single_threaded) windows.HEAP_NO_SERIALIZE else 0; - const hh = windows.kernel32.HeapCreate(options, amt, 0) orelse return null; - const other_hh = @cmpxchgStrong(?HeapHandle, &self.heap_handle, null, hh, .seq_cst, .seq_cst) orelse break :blk hh; - windows.HeapDestroy(hh); - break :blk other_hh.?; // can't be null because of the cmpxchg - }; - const ptr = windows.kernel32.HeapAlloc(heap_handle, 0, amt) orelse return null; - const root_addr = @intFromPtr(ptr); - const aligned_addr = mem.alignForward(usize, root_addr, ptr_align); - const buf = @as([*]u8, @ptrFromInt(aligned_addr))[0..n]; - getRecordPtr(buf).* = root_addr; - return buf.ptr; - } - - fn resize( - ctx: *anyopaque, - buf: []u8, - alignment: mem.Alignment, - new_size: usize, - return_address: usize, - ) bool { - _ = alignment; - _ = return_address; - const self: *HeapAllocator = @ptrCast(@alignCast(ctx)); - - const root_addr = getRecordPtr(buf).*; - const align_offset = @intFromPtr(buf.ptr) - root_addr; - const amt = align_offset + new_size + @sizeOf(usize); - const new_ptr = windows.kernel32.HeapReAlloc( - self.heap_handle.?, - windows.HEAP_REALLOC_IN_PLACE_ONLY, - @as(*anyopaque, @ptrFromInt(root_addr)), - amt, - ) orelse return false; - assert(new_ptr == @as(*anyopaque, @ptrFromInt(root_addr))); - getRecordPtr(buf.ptr[0..new_size]).* = root_addr; - return true; - } - - fn remap( - ctx: *anyopaque, - buf: []u8, - alignment: mem.Alignment, - new_size: usize, - return_address: usize, - ) ?[*]u8 { - _ = alignment; - _ = return_address; - const self: *HeapAllocator = @ptrCast(@alignCast(ctx)); - - const root_addr = getRecordPtr(buf).*; - const align_offset = @intFromPtr(buf.ptr) - root_addr; - const amt = align_offset + new_size + @sizeOf(usize); - const new_ptr = windows.kernel32.HeapReAlloc(self.heap_handle.?, 0, @ptrFromInt(root_addr), amt) orelse return null; - assert(new_ptr == @as(*anyopaque, @ptrFromInt(root_addr))); - getRecordPtr(buf.ptr[0..new_size]).* = root_addr; - return @ptrCast(new_ptr); - } - - fn free( - ctx: *anyopaque, - buf: []u8, - alignment: mem.Alignment, - return_address: usize, - ) void { - _ = alignment; - _ = return_address; - const self: *HeapAllocator = @ptrCast(@alignCast(ctx)); - windows.HeapFree(self.heap_handle.?, 0, @as(*anyopaque, @ptrFromInt(getRecordPtr(buf).*))); - } - }, - else => @compileError("Unsupported OS"), -}; - /// Returns a `StackFallbackAllocator` allocating using either a /// `FixedBufferAllocator` on an array of size `size` and falling back to /// `fallback_allocator` if that fails. @@ -628,22 +507,6 @@ test PageAllocator { } } -test HeapAllocator { - if (builtin.os.tag == .windows) { - // https://github.com/ziglang/zig/issues/13702 - if (builtin.cpu.arch == .aarch64) return error.SkipZigTest; - - var heap_allocator = HeapAllocator.init(); - defer heap_allocator.deinit(); - const allocator = heap_allocator.allocator(); - - try testAllocator(allocator); - try testAllocatorAligned(allocator); - try testAllocatorLargeAlignment(allocator); - try testAllocatorAlignedShrink(allocator); - } -} - test ArenaAllocator { var arena_allocator = ArenaAllocator.init(page_allocator); defer arena_allocator.deinit(); diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index d3db40bac235..563b24cf83c3 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -2016,18 +2016,6 @@ pub fn InitOnceExecuteOnce(InitOnce: *INIT_ONCE, InitFn: INIT_ONCE_FN, Parameter assert(kernel32.InitOnceExecuteOnce(InitOnce, InitFn, Parameter, Context) != 0); } -pub fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: *anyopaque) void { - assert(kernel32.HeapFree(hHeap, dwFlags, lpMem) != 0); -} - -pub fn HeapDestroy(hHeap: HANDLE) void { - assert(kernel32.HeapDestroy(hHeap) != 0); -} - -pub fn LocalFree(hMem: HLOCAL) void { - assert(kernel32.LocalFree(hMem) == null); -} - pub const SetFileTimeError = error{Unexpected}; pub fn SetFileTime( diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index a8ff5fb6d32b..611085960830 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -528,11 +528,6 @@ pub extern "kernel32" fn HeapCreate( dwMaximumSize: SIZE_T, ) callconv(.winapi) ?HANDLE; -// TODO: Wrapper around RtlDestroyHeap (BOOLEAN -> BOOL). -pub extern "kernel32" fn HeapDestroy( - hHeap: HANDLE, -) callconv(.winapi) BOOL; - // TODO: Forwarder to RtlReAllocateHeap. pub extern "kernel32" fn HeapReAlloc( hHeap: HANDLE, @@ -585,10 +580,6 @@ pub extern "kernel32" fn VirtualQuery( dwLength: SIZE_T, ) callconv(.winapi) SIZE_T; -pub extern "kernel32" fn LocalFree( - hMem: HLOCAL, -) callconv(.winapi) ?HLOCAL; - // TODO: Getter for peb.ProcessHeap pub extern "kernel32" fn GetProcessHeap() callconv(.winapi) ?HANDLE; From cd99ab32294a3c22f09615c93d611593a2887cc3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 19:18:22 -0800 Subject: [PATCH 33/39] std.heap: rename GeneralPurposeAllocator to DebugAllocator --- lib/std/heap.zig | 11 +- ...pose_allocator.zig => debug_allocator.zig} | 167 +++++++++--------- 2 files changed, 88 insertions(+), 90 deletions(-) rename lib/std/heap/{general_purpose_allocator.zig => debug_allocator.zig} (92%) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 23b19233e814..f691c31e2af8 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -9,15 +9,20 @@ const Allocator = std.mem.Allocator; const windows = std.os.windows; pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; -pub const GeneralPurposeAllocatorConfig = @import("heap/general_purpose_allocator.zig").Config; -pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator; -pub const Check = @import("heap/general_purpose_allocator.zig").Check; pub const WasmAllocator = @import("heap/WasmAllocator.zig"); pub const PageAllocator = @import("heap/PageAllocator.zig"); pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); +pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; +pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; +pub const Check = enum { ok, leak }; +/// Deprecated; to be removed after 0.15.0 is tagged. +pub const GeneralPurposeAllocatorConfig = DebugAllocatorConfig; +/// Deprecated; to be removed after 0.15.0 is tagged. +pub const GeneralPurposeAllocator = DebugAllocator; + const memory_pool = @import("heap/memory_pool.zig"); pub const MemoryPool = memory_pool.MemoryPool; pub const MemoryPoolAligned = memory_pool.MemoryPoolAligned; diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/debug_allocator.zig similarity index 92% rename from lib/std/heap/general_purpose_allocator.zig rename to lib/std/heap/debug_allocator.zig index 0f21830fbf99..9a76a7020221 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/debug_allocator.zig @@ -1,54 +1,32 @@ -//! # General Purpose Allocator +//! An allocator that is intended to be used in Debug mode. //! -//! ## Design Priorities +//! ## Features //! -//! ### `OptimizationMode.debug` and `OptimizationMode.release_safe`: +//! * Captures stack traces on allocation, free, and optionally resize. +//! * Double free detection, which prints all three traces (first alloc, first +//! free, second free). +//! * Leak detection, with stack traces. +//! * Never reuses memory addresses, making it easier for Zig to detect branch +//! on undefined values in case of dangling pointers. This relies on +//! the backing allocator to also not reuse addresses. +//! * Uses a minimum backing allocation size to avoid operating system errors +//! from having too many active memory mappings. +//! * When a page of memory is no longer needed, give it back to resident +//! memory as soon as possible, so that it causes page faults when used. +//! * Cross platform. Operates based on a backing allocator which makes it work +//! everywhere, even freestanding. +//! * Compile-time configuration. //! -//! * Detect double free, and emit stack trace of: -//! - Where it was first allocated -//! - Where it was freed the first time -//! - Where it was freed the second time +//! These features require the allocator to be quite slow and wasteful. For +//! example, when allocating a single byte, the efficiency is less than 1%; +//! it requires more than 100 bytes of overhead to manage the allocation for +//! one byte. The efficiency gets better with larger allocations. //! -//! * Detect leaks and emit stack trace of: -//! - Where it was allocated +//! ## Basic Design //! -//! * When a page of memory is no longer needed, give it back to resident memory -//! as soon as possible, so that it causes page faults when used. +//! Allocations are divided into two categories, small and large. //! -//! * Do not re-use memory slots, so that memory safety is upheld. For small -//! allocations, this is handled here; for larger ones it is handled in the -//! backing allocator (by default `std.heap.page_allocator`). -//! -//! * Make pointer math errors unlikely to harm memory from -//! unrelated allocations. -//! -//! * It's OK for these mechanisms to cost some extra overhead bytes. -//! -//! * It's OK for performance cost for these mechanisms. -//! -//! * Rogue memory writes should not harm the allocator's state. -//! -//! * Cross platform. Operates based on a backing allocator which makes it work -//! everywhere, even freestanding. -//! -//! * Compile-time configuration. -//! -//! ### `OptimizationMode.release_fast` (note: not much work has gone into this use case yet): -//! -//! * Low fragmentation is primary concern -//! * Performance of worst-case latency is secondary concern -//! * Performance of average-case latency is next -//! * Finally, having freed memory unmapped, and pointer math errors unlikely to -//! harm memory from unrelated allocations are nice-to-haves. -//! -//! ### `OptimizationMode.release_small` (note: not much work has gone into this use case yet): -//! -//! * Small binary code size of the executable is the primary concern. -//! * Next, defer to the `.release_fast` priority list. -//! -//! ## Basic Design: -//! -//! Small allocations are divided into buckets: +//! Small allocations are divided into buckets based on `page_size`: //! //! ``` //! index obj_size @@ -64,33 +42,44 @@ //! 9 512 //! 10 1024 //! 11 2048 +//! ... //! ``` //! +//! This goes on for `small_bucket_count` indexes. +//! +//! Allocations are grouped into an object size based on max(len, alignment), +//! rounded up to the next power of two. +//! //! The main allocator state has an array of all the "current" buckets for each //! size class. Each slot in the array can be null, meaning the bucket for that //! size class is not allocated. When the first object is allocated for a given -//! size class, it allocates 1 page of memory from the OS. This page is -//! divided into "slots" - one per allocated object. Along with the page of memory -//! for object slots, as many pages as necessary are allocated to store the -//! BucketHeader, followed by "used bits", and two stack traces for each slot -//! (allocation trace and free trace). +//! size class, it makes one `page_size` allocation from the backing allocator. +//! This allocation is divided into "slots" - one per allocated object, leaving +//! room for the allocation metadata (starting with `BucketHeader`), which is +//! located at the very end of the "page". +//! +//! The allocation metadata includes "used bits" - 1 bit per slot representing +//! whether the slot is used. Allocations always take the next available slot +//! from the current bucket, setting the corresponding used bit, as well as +//! incrementing `allocated_count`. //! -//! The "used bits" are 1 bit per slot representing whether the slot is used. -//! Allocations use the data to iterate to find a free slot. Frees assert that the -//! corresponding bit is 1 and set it to 0. +//! Frees recover the allocation metadata based on the address, length, and +//! alignment, relying on the backing allocation's large alignment, combined +//! with the fact that allocations are never moved from small to large, or vice +//! versa. //! -//! Buckets have prev and next pointers. When there is only one bucket for a given -//! size class, both prev and next point to itself. When all slots of a bucket are -//! used, a new bucket is allocated, and enters the doubly linked list. The main -//! allocator state tracks the "current" bucket for each size class. Leak detection -//! currently only checks the current bucket. +//! When a bucket is full, a new one is allocated, containing a pointer to the +//! previous one. This singly-linked list is iterated during leak detection. //! -//! Resizing detects if the size class is unchanged or smaller, in which case the same -//! pointer is returned unmodified. If a larger size class is required, -//! `error.OutOfMemory` is returned. +//! Resizing and remapping work the same on small allocations: if the size +//! class would not change, then the operation succeeds, and the address is +//! unchanged. Otherwise, the request is rejected. //! -//! Large objects are allocated directly using the backing allocator and their metadata is stored -//! in a `std.HashMap` using the backing allocator. +//! Large objects are allocated directly using the backing allocator. Metadata +//! is stored separately in a `std.HashMap` using the backing allocator. +//! +//! Resizing and remapping are forwarded directly to the backing allocator, +//! except where such operations would change the category from large to small. const std = @import("std"); const builtin = @import("builtin"); @@ -172,10 +161,8 @@ pub const Config = struct { canary: usize = @truncate(0x9232a6ff85dff10f), }; -pub const Check = enum { ok, leak }; - /// Default initialization of this struct is deprecated; use `.init` instead. -pub fn GeneralPurposeAllocator(comptime config: Config) type { +pub fn DebugAllocator(comptime config: Config) type { return struct { backing_allocator: Allocator = std.heap.page_allocator, /// Tracks the active bucket, which is the one that has free slots in it. @@ -491,8 +478,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } } - /// Returns `Check.leak` if there were leaks; `Check.ok` otherwise. - pub fn deinit(self: *Self) Check { + /// Returns `std.heap.Check.leak` if there were leaks; `std.heap.Check.ok` otherwise. + pub fn deinit(self: *Self) std.heap.Check { const leaks = if (config.safety) self.detectLeaks() else false; if (config.retain_metadata) self.freeRetainedMetadata(); self.large_allocations.deinit(self.backing_allocator); @@ -1041,7 +1028,7 @@ const TraceKind = enum { const test_config = Config{}; test "small allocations - free in same order" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1060,7 +1047,7 @@ test "small allocations - free in same order" { } test "small allocations - free in reverse order" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1079,7 +1066,7 @@ test "small allocations - free in reverse order" { } test "large allocations" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1092,7 +1079,7 @@ test "large allocations" { } test "very large allocation" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1100,7 +1087,7 @@ test "very large allocation" { } test "realloc" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1122,7 +1109,7 @@ test "realloc" { } test "shrink" { - var gpa: GeneralPurposeAllocator(test_config) = .{}; + var gpa: DebugAllocator(test_config) = .{}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1147,7 +1134,7 @@ test "large object - grow" { // Not expected to pass on targets that do not have memory mapping. return error.SkipZigTest; } - var gpa: GeneralPurposeAllocator(test_config) = .{}; + var gpa: DebugAllocator(test_config) = .{}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1165,7 +1152,7 @@ test "large object - grow" { } test "realloc small object to large object" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1182,7 +1169,7 @@ test "realloc small object to large object" { } test "shrink large object to large object" { - var gpa: GeneralPurposeAllocator(test_config) = .{}; + var gpa: DebugAllocator(test_config) = .{}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1209,7 +1196,7 @@ test "shrink large object to large object" { test "shrink large object to large object with larger alignment" { if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1245,7 +1232,7 @@ test "shrink large object to large object with larger alignment" { } test "realloc large object to small object" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1260,7 +1247,7 @@ test "realloc large object to small object" { } test "overridable mutexes" { - var gpa = GeneralPurposeAllocator(.{ .MutexType = std.Thread.Mutex }){ + var gpa = DebugAllocator(.{ .MutexType = std.Thread.Mutex }){ .backing_allocator = std.testing.allocator, .mutex = std.Thread.Mutex{}, }; @@ -1272,7 +1259,11 @@ test "overridable mutexes" { } test "non-page-allocator backing allocator" { - var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = std.testing.allocator }; + var gpa: DebugAllocator(.{ + .backing_allocator_zeroes = false, + }) = .{ + .backing_allocator = std.testing.allocator, + }; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1283,7 +1274,7 @@ test "non-page-allocator backing allocator" { test "realloc large object to larger alignment" { if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1330,7 +1321,9 @@ test "large object rejects shrinking to small" { } var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); - var gpa: GeneralPurposeAllocator(.{}) = .{ .backing_allocator = failing_allocator.allocator() }; + var gpa: DebugAllocator(.{}) = .{ + .backing_allocator = failing_allocator.allocator(), + }; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1345,7 +1338,7 @@ test "large object rejects shrinking to small" { } test "objects of size 1024 and 2048" { - var gpa = GeneralPurposeAllocator(test_config){}; + var gpa = DebugAllocator(test_config){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1357,7 +1350,7 @@ test "objects of size 1024 and 2048" { } test "setting a memory cap" { - var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; + var gpa = DebugAllocator(.{ .enable_memory_limit = true }){}; defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); @@ -1383,7 +1376,7 @@ test "setting a memory cap" { } test "large allocations count requested size not backing size" { - var gpa: GeneralPurposeAllocator(.{ .enable_memory_limit = true }) = .{}; + var gpa: DebugAllocator(.{ .enable_memory_limit = true }) = .{}; const allocator = gpa.allocator(); var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); @@ -1395,7 +1388,7 @@ test "large allocations count requested size not backing size" { } test "retain metadata and never unmap" { - var gpa = std.heap.GeneralPurposeAllocator(.{ + var gpa = std.heap.DebugAllocator(.{ .safety = true, .never_unmap = true, .retain_metadata = true, From 960190643ab4fe2227cbc7ab69a3907bf6722a6e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 19:28:48 -0800 Subject: [PATCH 34/39] std.heap.DebugAllocator: make page size configurable --- lib/std/heap/debug_allocator.zig | 70 +++++++++++++++++++------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/lib/std/heap/debug_allocator.zig b/lib/std/heap/debug_allocator.zig index 9a76a7020221..a4ac906703b7 100644 --- a/lib/std/heap/debug_allocator.zig +++ b/lib/std/heap/debug_allocator.zig @@ -90,15 +90,12 @@ const mem = std.mem; const Allocator = std.mem.Allocator; const StackTrace = std.builtin.StackTrace; -const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { +const default_page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. else => 128 * 1024, // Avoids too many active mappings when `page_size_max` is low. }); -const page_align: mem.Alignment = .fromByteUnits(page_size); -/// Integer type for pointing to slots in a small allocation -const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); const Log2USize = std.math.Log2Int(usize); const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) 6 else 0; @@ -159,6 +156,12 @@ pub const Config = struct { /// Magic value that distinguishes allocations owned by this allocator from /// other regions of memory. canary: usize = @truncate(0x9232a6ff85dff10f), + + /// The size of allocations requested from the backing allocator for + /// subdividing into slots for small allocations. + /// + /// Must be a power of two. + page_size: usize = default_page_size, }; /// Default initialization of this struct is deprecated; use `.init` instead. @@ -184,6 +187,15 @@ pub fn DebugAllocator(comptime config: Config) type { break :init result; }; + comptime { + assert(math.isPowerOfTwo(page_size)); + } + + const page_size = config.page_size; + const page_align: mem.Alignment = .fromByteUnits(page_size); + /// Integer type for pointing to slots in a small allocation + const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); + const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {}; const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {}; @@ -1138,17 +1150,17 @@ test "large object - grow" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice1 = try allocator.alloc(u8, page_size * 2 - 20); + var slice1 = try allocator.alloc(u8, default_page_size * 2 - 20); defer allocator.free(slice1); const old = slice1; - slice1 = try allocator.realloc(slice1, page_size * 2 - 10); + slice1 = try allocator.realloc(slice1, default_page_size * 2 - 10); try std.testing.expect(slice1.ptr == old.ptr); - slice1 = try allocator.realloc(slice1, page_size * 2); + slice1 = try allocator.realloc(slice1, default_page_size * 2); try std.testing.expect(slice1.ptr == old.ptr); - slice1 = try allocator.realloc(slice1, page_size * 2 + 1); + slice1 = try allocator.realloc(slice1, default_page_size * 2 + 1); } test "realloc small object to large object" { @@ -1162,7 +1174,7 @@ test "realloc small object to large object" { slice[60] = 0x34; // This requires upgrading to a large object - const large_object_size = page_size * 2 + 50; + const large_object_size = default_page_size * 2 + 50; slice = try allocator.realloc(slice, large_object_size); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); @@ -1173,22 +1185,22 @@ test "shrink large object to large object" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, page_size * 2 + 50); + var slice = try allocator.alloc(u8, default_page_size * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[60] = 0x34; - if (!allocator.resize(slice, page_size * 2 + 1)) return; - slice = slice.ptr[0 .. page_size * 2 + 1]; + if (!allocator.resize(slice, default_page_size * 2 + 1)) return; + slice = slice.ptr[0 .. default_page_size * 2 + 1]; try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); - try std.testing.expect(allocator.resize(slice, page_size * 2 + 1)); - slice = slice[0 .. page_size * 2 + 1]; + try std.testing.expect(allocator.resize(slice, default_page_size * 2 + 1)); + slice = slice[0 .. default_page_size * 2 + 1]; try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); - slice = try allocator.realloc(slice, page_size * 2); + slice = try allocator.realloc(slice, default_page_size * 2); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[60] == 0x34); } @@ -1204,13 +1216,13 @@ test "shrink large object to large object with larger alignment" { var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); const debug_allocator = fba.allocator(); - const alloc_size = page_size * 2 + 50; + const alloc_size = default_page_size * 2 + 50; var slice = try allocator.alignedAlloc(u8, 16, alloc_size); defer allocator.free(slice); const big_alignment: usize = switch (builtin.os.tag) { - .windows => page_size * 32, // Windows aligns to 64K. - else => page_size * 2, + .windows => default_page_size * 32, // Windows aligns to 64K. + else => default_page_size * 2, }; // This loop allocates until we find a page that is not aligned to the big // alignment. Then we shrink the allocation after the loop, but increase the @@ -1236,7 +1248,7 @@ test "realloc large object to small object" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, page_size * 2 + 50); + var slice = try allocator.alloc(u8, default_page_size * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[16] = 0x34; @@ -1282,18 +1294,18 @@ test "realloc large object to larger alignment" { var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); const debug_allocator = fba.allocator(); - var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); + var slice = try allocator.alignedAlloc(u8, 16, default_page_size * 2 + 50); defer allocator.free(slice); const big_alignment: usize = switch (builtin.os.tag) { - .windows => page_size * 32, // Windows aligns to 64K. - else => page_size * 2, + .windows => default_page_size * 32, // Windows aligns to 64K. + else => default_page_size * 2, }; // This loop allocates until we find a page that is not aligned to the big alignment. var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { try stuff_to_free.append(slice); - slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); + slice = try allocator.alignedAlloc(u8, 16, default_page_size * 2 + 50); } while (stuff_to_free.popOrNull()) |item| { allocator.free(item); @@ -1301,15 +1313,15 @@ test "realloc large object to larger alignment" { slice[0] = 0x12; slice[16] = 0x34; - slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100); + slice = try allocator.reallocAdvanced(slice, 32, default_page_size * 2 + 100); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); - slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25); + slice = try allocator.reallocAdvanced(slice, 32, default_page_size * 2 + 25); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); - slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100); + slice = try allocator.reallocAdvanced(slice, big_alignment, default_page_size * 2 + 100); try std.testing.expect(slice[0] == 0x12); try std.testing.expect(slice[16] == 0x34); } @@ -1327,7 +1339,7 @@ test "large object rejects shrinking to small" { defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); const allocator = gpa.allocator(); - var slice = try allocator.alloc(u8, page_size * 2 + 50); + var slice = try allocator.alloc(u8, default_page_size * 2 + 50); defer allocator.free(slice); slice[0] = 0x12; slice[3] = 0x34; @@ -1379,8 +1391,8 @@ test "large allocations count requested size not backing size" { var gpa: DebugAllocator(.{ .enable_memory_limit = true }) = .{}; const allocator = gpa.allocator(); - var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); - try std.testing.expectEqual(page_size + 1, gpa.total_requested_bytes); + var buf = try allocator.alignedAlloc(u8, 1, default_page_size + 1); + try std.testing.expectEqual(default_page_size + 1, gpa.total_requested_bytes); buf = try allocator.realloc(buf, 1); try std.testing.expectEqual(1, gpa.total_requested_bytes); buf = try allocator.realloc(buf, 2); From d0e1a6a23de8620a42829a95343b297cefc1c7a4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Feb 2025 20:23:21 -0800 Subject: [PATCH 35/39] std.heap.DebugAllocator: update unit tests for new impl No longer need this windows-specific behavior. --- lib/std/heap/debug_allocator.zig | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/std/heap/debug_allocator.zig b/lib/std/heap/debug_allocator.zig index a4ac906703b7..8abf6133bfdc 100644 --- a/lib/std/heap/debug_allocator.zig +++ b/lib/std/heap/debug_allocator.zig @@ -1220,10 +1220,7 @@ test "shrink large object to large object with larger alignment" { var slice = try allocator.alignedAlloc(u8, 16, alloc_size); defer allocator.free(slice); - const big_alignment: usize = switch (builtin.os.tag) { - .windows => default_page_size * 32, // Windows aligns to 64K. - else => default_page_size * 2, - }; + const big_alignment: usize = default_page_size * 2; // This loop allocates until we find a page that is not aligned to the big // alignment. Then we shrink the allocation after the loop, but increase the // alignment to the higher one, that we know will force it to realloc. @@ -1297,10 +1294,7 @@ test "realloc large object to larger alignment" { var slice = try allocator.alignedAlloc(u8, 16, default_page_size * 2 + 50); defer allocator.free(slice); - const big_alignment: usize = switch (builtin.os.tag) { - .windows => default_page_size * 32, // Windows aligns to 64K. - else => default_page_size * 2, - }; + const big_alignment: usize = default_page_size * 2; // This loop allocates until we find a page that is not aligned to the big alignment. var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { From ff8e7597056aebc67213fff0b4942a66d0a4e1dd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 6 Feb 2025 14:46:16 -0800 Subject: [PATCH 36/39] std.testing: don't ask wasm to stack trace --- lib/std/testing.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 616a138cbd60..288c6cdc8955 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -17,7 +17,7 @@ var base_allocator_instance = std.heap.FixedBufferAllocator.init(""); /// This should only be used in temporary test programs. pub const allocator = allocator_instance.allocator(); pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{ - .stack_trace_frames = 10, + .stack_trace_frames = if (std.debug.sys_can_stack_trace) 10 else 0, .resize_stack_traces = true, // A unique value so that when a default-constructed // GeneralPurposeAllocator is incorrectly passed to testing allocator, or From 1bb8b4ad61659da8b4f53faf959bfdc8415bf41e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 6 Feb 2025 14:47:22 -0800 Subject: [PATCH 37/39] std.heap: fix wrong deprecation date --- lib/std/heap.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index f691c31e2af8..d1874c5b006b 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -18,9 +18,9 @@ pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; pub const Check = enum { ok, leak }; -/// Deprecated; to be removed after 0.15.0 is tagged. +/// Deprecated; to be removed after 0.14.0 is tagged. pub const GeneralPurposeAllocatorConfig = DebugAllocatorConfig; -/// Deprecated; to be removed after 0.15.0 is tagged. +/// Deprecated; to be removed after 0.14.0 is tagged. pub const GeneralPurposeAllocator = DebugAllocator; const memory_pool = @import("heap/memory_pool.zig"); From e630b20c6228aa3c8b50841af5ce7bc102202162 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 6 Feb 2025 14:50:20 -0800 Subject: [PATCH 38/39] std.mem.Allocator.VTable: improve doc comment wording --- lib/std/mem/Allocator.zig | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index 1a0108f3be38..1ad953311604 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -20,23 +20,20 @@ ptr: *anyopaque, vtable: *const VTable, pub const VTable = struct { - /// Allocate exactly `len` bytes aligned to `alignment`, or return `null` - /// indicating the allocation failed. + /// Return a pointer to `len` bytes with specified `alignment`, or return + /// `null` indicating the allocation failed. /// /// `ret_addr` is optionally provided as the first return address of the /// allocation call stack. If the value is `0` it means no return address /// has been provided. - /// - /// The returned slice of memory must have been `@memset` to `undefined` - /// by the allocator implementation. alloc: *const fn (*anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8, /// Attempt to expand or shrink memory in place. /// /// `memory.len` must equal the length requested from the most recent - /// successful call to `alloc` or `resize`. `alignment` must equal the same - /// value that was passed as the `alignment` parameter to the original - /// `alloc` call. + /// successful call to `alloc`, `resize`, or `remap`. `alignment` must + /// equal the same value that was passed as the `alignment` parameter to + /// the original `alloc` call. /// /// A result of `true` indicates the resize was successful and the /// allocation now has the same address but a size of `new_len`. `false` @@ -53,9 +50,9 @@ pub const VTable = struct { /// Attempt to expand or shrink memory, allowing relocation. /// /// `memory.len` must equal the length requested from the most recent - /// successful call to `alloc` or `resize`. `alignment` must equal the same - /// value that was passed as the `alignment` parameter to the original - /// `alloc` call. + /// successful call to `alloc`, `resize`, or `remap`. `alignment` must + /// equal the same value that was passed as the `alignment` parameter to + /// the original `alloc` call. /// /// A non-`null` return value indicates the resize was successful. The /// allocation may have same address, or may have been relocated. In either @@ -73,11 +70,10 @@ pub const VTable = struct { /// Free and invalidate a region of memory. /// - /// `memory.len` must equal the most recent length returned by `alloc` or - /// given to a successful `resize` call. - /// - /// `alignment` must equal the same value that was passed as the - /// `alignment` parameter to the original `alloc` call. + /// `memory.len` must equal the length requested from the most recent + /// successful call to `alloc`, `resize`, or `remap`. `alignment` must + /// equal the same value that was passed as the `alignment` parameter to + /// the original `alloc` call. /// /// `ret_addr` is optionally provided as the first return address of the /// allocation call stack. If the value is `0` it means no return address From b8f5cfed457726a77082b7ffe6672b6066c0a66e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 6 Feb 2025 14:50:55 -0800 Subject: [PATCH 39/39] std.heap.SbrkAllocator: fix typo --- lib/std/heap/sbrk_allocator.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/heap/sbrk_allocator.zig b/lib/std/heap/sbrk_allocator.zig index b39f8ac344f5..7cae161bea98 100644 --- a/lib/std/heap/sbrk_allocator.zig +++ b/lib/std/heap/sbrk_allocator.zig @@ -59,7 +59,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type { } const next_addr = next_addrs[class]; - if (next_addr % heap.pageSize == 0) { + if (next_addr % heap.pageSize() == 0) { const addr = allocBigPages(1); if (addr == 0) return null; //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{