From 8e903586ac453d2b31f3cb911866f0156b688343 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Fri, 26 Jan 2024 17:52:56 -0800 Subject: [PATCH 01/32] draft impl of windows watcher --- src/watcher.zig | 99 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/src/watcher.zig b/src/watcher.zig index 55d160ae9eeea6..b8750659400070 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -979,3 +979,102 @@ pub fn NewWatcher(comptime ContextType: type) type { } }; } + +pub const WindowsWatcher = struct { + iocp: w.HANDLE, + allocator: std.mem.Allocator, + watchers: Map, + rng: std.rand.DefaultPrng, + running: bool = true, + + const Map = std.ArrayHashMap(w.ULONG_PTR, *DirWatcher, false); + const w = std.os.windows; + const DirWatcher = struct { + buf: [64 * 1024]u8 align(@alignOf(w.DWORD)) = undefined, + dirHandle: w.HANDLE, + + fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { + if (nbytes == 0) return; + var offset = 0; + while (true) { + const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(&this.buf[offset .. offset + @sizeOf(w.FILE_NOTIFY_INFORMATION)])); + const name_offset = offset + @sizeOf(w.FILE_NOTIFY_INFORMATION); + const filename: []u16 = @alignCast(@ptrCast(&this.buf[name_offset .. name_offset + info.FileNameLength / @sizeOf(u16)])); + + _ = filename; + _ = info.Action; + + if (info.NextEntryOffset == 0) break; + offset += info.NextEntryOffset; + } + } + }; + + pub fn init(allocator: std.mem.Allocator) !*WindowsWatcher { + const watcher = try allocator.create(WindowsWatcher); + errdefer allocator.destroy(watcher); + + const iocp = try w.CreateIoCompletionPort(w.INVALID_HANDLE_VALUE, null, 0, 1); + watcher.* = .{ .iocp = iocp, .allocator = allocator, .watchers = Map.init(allocator), .rng = std.rand.DefaultPrng.init(0) }; + return watcher; + } + + pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: bun.FileDescriptor, path: [:0]u16) !void { + _ = dirFd; + // TODO respect dirFd + const handle = w.kernel32.CreateFileW( + path, + w.FILE_LIST_DIRECTORY, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + null, + w.OPEN_EXISTING, + w.FILE_FLAG_BACKUP_SEMANTICS | w.FILE_FLAG_OVERLAPPED, + null, + ); + if (handle == w.INVALID_HANDLE_VALUE) { + @panic("failed to open directory for watching"); + } + // TODO check if this is a directory + + errdefer _ = w.kernel32.CloseHandle(handle); + + const key = this.rng.next(); + + this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, key, 1); + + const watcher = try this.allocator.create(DirWatcher); + errdefer this.allocator.destroy(watcher); + watcher.* = .{ .dirHandle = handle }; + try this.watchers.put(key, watcher); + + const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_SIZE | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION | w.FILE_NOTIFY_CHANGE_SECURITY; + if (w.kernel32.ReadDirectoryChangesW(handle, &watcher.buf, watcher.buf.len, 1, filter, null, null, null) == 0) { + @panic("failed to start watching directory"); + } + } + + pub fn stop(this: *WindowsWatcher) void { + @atomicStore(bool, &this.running, false, .Unordered); + } + + pub fn run(this: *WindowsWatcher) void { + var nbytes: w.DWORD = 0; + var key: w.ULONG_PTR = 0; + var overlapped: ?*w.OVERLAPPED = null; + while (w.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, w.INFINITE) != 0) { + if (nbytes == 0) { + // exit notification? + break; + } + if (this.watchers.get(key)) |watcher| { + watcher.handleEvent(nbytes); + } else { + // not really an error: the watcher with this key has already been closed and we're just receiving the remaining events + Output.prettyErrorln("no watcher with key {d}", .{key}); + } + if (@atomicLoad(bool, &this.running, .Unordered) == false) { + break; + } + } + } +}; From 62a110cbc9402d2f905e380fac4920cadbb19c08 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Fri, 26 Jan 2024 19:46:59 -0800 Subject: [PATCH 02/32] synchronous watcher --- watch.zig | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 watch.zig diff --git a/watch.zig b/watch.zig new file mode 100644 index 00000000000000..739c7f515fc3f5 --- /dev/null +++ b/watch.zig @@ -0,0 +1,125 @@ +const std = @import("std"); + +pub const WindowsWatcher = struct { + iocp: w.HANDLE, + allocator: std.mem.Allocator, + // watchers: Map, + // rng: std.rand.DefaultPrng, + running: bool = true, + + // const Map = std.AutoArrayHashMap(*w.OVERLAPPED, *DirWatcher); + const w = std.os.windows; + const DirWatcher = extern struct { + // this must be the first field + overlapped: w.OVERLAPPED = undefined, + buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, + dirHandle: w.HANDLE, + + fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { + if (nbytes == 0) return; + var offset: usize = 0; + while (true) { + const info_size = @sizeOf(w.FILE_NOTIFY_INFORMATION); + const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(this.buf[offset..].ptr)); + const name_ptr: [*]u16 = @alignCast(@ptrCast(this.buf[offset + info_size ..])); + const filename: []u16 = name_ptr[0..info.FileNameLength]; + + std.debug.print("filename: {}, action: {}\n", .{ std.unicode.fmtUtf16le(filename), info.Action }); + + if (info.NextEntryOffset == 0) break; + offset += @as(usize, info.NextEntryOffset); + } + } + }; + + pub fn init(allocator: std.mem.Allocator) !*WindowsWatcher { + const watcher = try allocator.create(WindowsWatcher); + errdefer allocator.destroy(watcher); + + const iocp = try w.CreateIoCompletionPort(w.INVALID_HANDLE_VALUE, null, 0, 1); + watcher.* = .{ + .iocp = iocp, + .allocator = allocator, + }; + return watcher; + } + + pub fn addWatchedDirectory(this: *WindowsWatcher, path: [:0]u16) !void { + // TODO respect dirFd + const handle = w.kernel32.CreateFileW( + path, + w.FILE_LIST_DIRECTORY, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + null, + w.OPEN_EXISTING, + w.FILE_FLAG_BACKUP_SEMANTICS | w.FILE_FLAG_OVERLAPPED, + null, + ); + if (handle == w.INVALID_HANDLE_VALUE) { + @panic("failed to open directory for watching"); + } + // TODO check if this is a directory + {} + + errdefer _ = w.kernel32.CloseHandle(handle); + + // const key = this.rng.next(); + + this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); + + const watcher = try this.allocator.create(DirWatcher); + errdefer this.allocator.destroy(watcher); + watcher.* = .{ .dirHandle = handle }; + // try this.watchers.put(key, watcher); + + const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_SIZE | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION | w.FILE_NOTIFY_CHANGE_SECURITY; + if (w.kernel32.ReadDirectoryChangesW(handle, &watcher.buf, watcher.buf.len, 0, filter, null, null, null) == 0) { + @panic("failed to start watching directory"); + } + } + + pub fn stop(this: *WindowsWatcher) void { + @atomicStore(bool, &this.running, false, .Unordered); + } + + pub fn run(this: *WindowsWatcher) !void { + var nbytes: w.DWORD = 0; + var key: w.ULONG_PTR = 0; + var overlapped: ?*w.OVERLAPPED = null; + while (true) { + switch (w.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, w.INFINITE)) { + .Normal => {}, + .Aborted => @panic("aborted"), + .Cancelled => @panic("cancelled"), + .EOF => @panic("eof"), + } + if (nbytes == 0) { + // exit notification? + break; + } + + const watcher: *DirWatcher = @ptrCast(overlapped); + watcher.handleEvent(nbytes); + + if (@atomicLoad(bool, &this.running, .Unordered) == false) { + break; + } + } + } +}; + +pub fn main() !void { + const allocator = std.heap.page_allocator; + + var buf: [256]u16 = undefined; + const idx = try std.unicode.utf8ToUtf16Le(&buf, "C:\\bun"); + buf[idx] = 0; + const path = buf[0..idx :0]; + + std.debug.print("watching {}\n", .{std.unicode.fmtUtf16le(path)}); + + const watcher = try WindowsWatcher.init(allocator); + try watcher.addWatchedDirectory(path); + + try watcher.run(); +} From d94a6023769edacd70e60703adfeda609440fe20 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Fri, 26 Jan 2024 21:06:56 -0800 Subject: [PATCH 03/32] working standalone watcher --- watch.zig | 188 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 166 insertions(+), 22 deletions(-) diff --git a/watch.zig b/watch.zig index 739c7f515fc3f5..8957f1dcc5a14a 100644 --- a/watch.zig +++ b/watch.zig @@ -1,5 +1,93 @@ const std = @import("std"); +pub fn toNTPath(wbuf: []u16, utf8: []const u8) [:0]const u16 { + if (!std.fs.path.isAbsoluteWindows(utf8)) { + return toWPathNormalized(wbuf, utf8); + } + + wbuf[0..4].* = [_]u16{ '\\', '?', '?', '\\' }; + return wbuf[0 .. toWPathNormalized(wbuf[4..], utf8).len + 4 :0]; +} + +// These are the same because they don't have rules like needing a trailing slash +pub const toNTDir = toNTPath; + +pub fn toExtendedPathNormalized(wbuf: []u16, utf8: []const u8) [:0]const u16 { + std.debug.assert(wbuf.len > 4); + wbuf[0..4].* = [_]u16{ '\\', '\\', '?', '\\' }; + return wbuf[0 .. toWPathNormalized(wbuf[4..], utf8).len + 4 :0]; +} + +pub fn toWPathNormalizeAutoExtend(wbuf: []u16, utf8: []const u8) [:0]const u16 { + if (std.fs.path.isAbsoluteWindows(utf8)) { + return toExtendedPathNormalized(wbuf, utf8); + } + + return toWPathNormalized(wbuf, utf8); +} + +pub fn toWPathNormalized(wbuf: []u16, utf8: []const u8) [:0]const u16 { + var renormalized: []u8 = undefined; + var path_to_use = utf8; + + if (std.mem.indexOfScalar(u8, utf8, '/') != null) { + @memcpy(renormalized[0..utf8.len], utf8); + for (renormalized[0..utf8.len]) |*c| { + if (c.* == '/') { + c.* = '\\'; + } + } + path_to_use = renormalized[0..utf8.len]; + } + + // is there a trailing slash? Let's remove it before converting to UTF-16 + if (path_to_use.len > 3 and path_to_use[path_to_use.len - 1] == '\\') { + path_to_use = path_to_use[0 .. path_to_use.len - 1]; + } + + return toWPath(wbuf, path_to_use); +} + +pub fn toWDirNormalized(wbuf: []u16, utf8: []const u8) [:0]const u16 { + var renormalized: [std.fs.MAX_PATH_BYTES]u8 = undefined; + var path_to_use = utf8; + + if (std.mem.indexOfScalar(u8, utf8, '.') != null) { + @memcpy(renormalized[0..utf8.len], utf8); + for (renormalized[0..utf8.len]) |*c| { + if (c.* == '/') { + c.* = '\\'; + } + } + path_to_use = renormalized[0..utf8.len]; + } + + return toWDirPath(wbuf, path_to_use); +} + +pub fn toWPath(wbuf: []u16, utf8: []const u8) [:0]const u16 { + return toWPathMaybeDir(wbuf, utf8, false); +} + +pub fn toWDirPath(wbuf: []u16, utf8: []const u8) [:0]const u16 { + return toWPathMaybeDir(wbuf, utf8, true); +} + +pub fn toWPathMaybeDir(wbuf: []u16, utf8: []const u8, comptime add_trailing_lash: bool) [:0]const u16 { + std.debug.assert(wbuf.len > 0); + + const count = std.unicode.utf8ToUtf16Le(wbuf[0..wbuf.len -| (1 + @as(usize, @intFromBool(add_trailing_lash)))], utf8) catch unreachable; + + if (add_trailing_lash and count > 0 and wbuf[count - 1] != '\\') { + wbuf[count] = '\\'; + count += 1; + } + + wbuf[count] = 0; + + return wbuf[0..count :0]; +} + pub const WindowsWatcher = struct { iocp: w.HANDLE, allocator: std.mem.Allocator, @@ -14,6 +102,7 @@ pub const WindowsWatcher = struct { overlapped: w.OVERLAPPED = undefined, buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, dirHandle: w.HANDLE, + watch_subtree: w.BOOLEAN = 1, fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { if (nbytes == 0) return; @@ -22,7 +111,7 @@ pub const WindowsWatcher = struct { const info_size = @sizeOf(w.FILE_NOTIFY_INFORMATION); const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(this.buf[offset..].ptr)); const name_ptr: [*]u16 = @alignCast(@ptrCast(this.buf[offset + info_size ..])); - const filename: []u16 = name_ptr[0..info.FileNameLength]; + const filename: []u16 = name_ptr[0 .. info.FileNameLength / @sizeOf(u16)]; std.debug.print("filename: {}, action: {}\n", .{ std.unicode.fmtUtf16le(filename), info.Action }); @@ -30,6 +119,15 @@ pub const WindowsWatcher = struct { offset += @as(usize, info.NextEntryOffset); } } + + fn listen(this: *DirWatcher) !void { + const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_SIZE | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION | w.FILE_NOTIFY_CHANGE_SECURITY; + if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, this.watch_subtree, filter, null, &this.overlapped, null) == 0) { + const err = w.kernel32.GetLastError(); + std.debug.print("failed to start watching directory: {s}\n", .{@tagName(err)}); + @panic("failed to start watching directory"); + } + } }; pub fn init(allocator: std.mem.Allocator) !*WindowsWatcher { @@ -44,22 +142,68 @@ pub const WindowsWatcher = struct { return watcher; } - pub fn addWatchedDirectory(this: *WindowsWatcher, path: [:0]u16) !void { + pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !void { + _ = dirFd; // autofix // TODO respect dirFd - const handle = w.kernel32.CreateFileW( - path, - w.FILE_LIST_DIRECTORY, - w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + // const flags = w.STANDARD_RIGHTS_READ | w.FILE_READ_ATTRIBUTES | w.FILE_READ_EA | + // w.FILE_TRAVERSE | w.FILE_LIST_DIRECTORY; + const flags = w.FILE_LIST_DIRECTORY; + + const path_len_bytes: u16 = @truncate(path.len * 2); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @constCast(path.ptr), + }; + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = null, + // if (std.fs.path.isAbsoluteWindowsW(path)) + // null + // else if (dirFd == w.INVALID_HANDLE_VALUE) + // std.fs.cwd().fd + // else + // dirFd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + var handle: w.HANDLE = w.INVALID_HANDLE_VALUE; + var io: w.IO_STATUS_BLOCK = undefined; + const rc = w.ntdll.NtCreateFile( + &handle, + flags, + &attr, + &io, null, - w.OPEN_EXISTING, - w.FILE_FLAG_BACKUP_SEMANTICS | w.FILE_FLAG_OVERLAPPED, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + w.FILE_OPEN, + w.FILE_DIRECTORY_FILE | w.FILE_OPEN_FOR_BACKUP_INTENT, null, + 0, ); - if (handle == w.INVALID_HANDLE_VALUE) { + + if (rc != .SUCCESS) { + std.debug.print("failed to open directory for watching: {s}\n", .{@tagName(rc)}); @panic("failed to open directory for watching"); } - // TODO check if this is a directory - {} + + // const handle = w.kernel32.CreateFileW( + // path, + // w.FILE_LIST_DIRECTORY, + // w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + // null, + // w.OPEN_EXISTING, + // w.FILE_FLAG_BACKUP_SEMANTICS | w.FILE_FLAG_OVERLAPPED, + // null, + // ); + // if (handle == w.INVALID_HANDLE_VALUE) { + // @panic("failed to open directory for watching"); + // } + // // TODO check if this is a directory + // {} errdefer _ = w.kernel32.CloseHandle(handle); @@ -71,11 +215,7 @@ pub const WindowsWatcher = struct { errdefer this.allocator.destroy(watcher); watcher.* = .{ .dirHandle = handle }; // try this.watchers.put(key, watcher); - - const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_SIZE | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION | w.FILE_NOTIFY_CHANGE_SECURITY; - if (w.kernel32.ReadDirectoryChangesW(handle, &watcher.buf, watcher.buf.len, 0, filter, null, null, null) == 0) { - @panic("failed to start watching directory"); - } + try watcher.listen(); } pub fn stop(this: *WindowsWatcher) void { @@ -100,6 +240,7 @@ pub const WindowsWatcher = struct { const watcher: *DirWatcher = @ptrCast(overlapped); watcher.handleEvent(nbytes); + try watcher.listen(); if (@atomicLoad(bool, &this.running, .Unordered) == false) { break; @@ -111,15 +252,18 @@ pub const WindowsWatcher = struct { pub fn main() !void { const allocator = std.heap.page_allocator; - var buf: [256]u16 = undefined; - const idx = try std.unicode.utf8ToUtf16Le(&buf, "C:\\bun"); - buf[idx] = 0; - const path = buf[0..idx :0]; + var buf: [std.fs.MAX_PATH_BYTES]u16 = undefined; + const path = toNTPath(&buf, "C:\\bun\\"); + // const idx = try std.unicode.utf8ToUtf16Le(&buf, "C:\\bun\\"); + // buf[idx] = 0; + // const path = buf[0..idx :0]; - std.debug.print("watching {}\n", .{std.unicode.fmtUtf16le(path)}); + std.debug.print("directory: {}\n", .{std.unicode.fmtUtf16le(path)}); const watcher = try WindowsWatcher.init(allocator); - try watcher.addWatchedDirectory(path); + try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, path); + + std.debug.print("watching...\n", .{}); try watcher.run(); } From d45213d3e1b8a316e7dc0eb9b8d14cf8ed6894f3 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 10:25:41 -0800 Subject: [PATCH 04/32] in progress changes to watcher --- src/__global.zig | 2 + src/watcher.zig | 99 ------------------------------------------------ watch.zig | 72 ++++++++++++++++------------------- 3 files changed, 34 insertions(+), 139 deletions(-) diff --git a/src/__global.zig b/src/__global.zig index fcb03d76d7b464..4c20917964d5e2 100644 --- a/src/__global.zig +++ b/src/__global.zig @@ -69,6 +69,8 @@ pub fn setThreadName(name: StringTypes.stringZ) void { _ = std.os.prctl(.SET_NAME, .{@intFromPtr(name.ptr)}) catch 0; } else if (Environment.isMac) { _ = std.c.pthread_setname_np(name); + } else if (Environment.isWindows) { + // _ = std.os.SetThreadDescription(std.os.GetCurrentThread(), name); } } diff --git a/src/watcher.zig b/src/watcher.zig index b8750659400070..55d160ae9eeea6 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -979,102 +979,3 @@ pub fn NewWatcher(comptime ContextType: type) type { } }; } - -pub const WindowsWatcher = struct { - iocp: w.HANDLE, - allocator: std.mem.Allocator, - watchers: Map, - rng: std.rand.DefaultPrng, - running: bool = true, - - const Map = std.ArrayHashMap(w.ULONG_PTR, *DirWatcher, false); - const w = std.os.windows; - const DirWatcher = struct { - buf: [64 * 1024]u8 align(@alignOf(w.DWORD)) = undefined, - dirHandle: w.HANDLE, - - fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { - if (nbytes == 0) return; - var offset = 0; - while (true) { - const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(&this.buf[offset .. offset + @sizeOf(w.FILE_NOTIFY_INFORMATION)])); - const name_offset = offset + @sizeOf(w.FILE_NOTIFY_INFORMATION); - const filename: []u16 = @alignCast(@ptrCast(&this.buf[name_offset .. name_offset + info.FileNameLength / @sizeOf(u16)])); - - _ = filename; - _ = info.Action; - - if (info.NextEntryOffset == 0) break; - offset += info.NextEntryOffset; - } - } - }; - - pub fn init(allocator: std.mem.Allocator) !*WindowsWatcher { - const watcher = try allocator.create(WindowsWatcher); - errdefer allocator.destroy(watcher); - - const iocp = try w.CreateIoCompletionPort(w.INVALID_HANDLE_VALUE, null, 0, 1); - watcher.* = .{ .iocp = iocp, .allocator = allocator, .watchers = Map.init(allocator), .rng = std.rand.DefaultPrng.init(0) }; - return watcher; - } - - pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: bun.FileDescriptor, path: [:0]u16) !void { - _ = dirFd; - // TODO respect dirFd - const handle = w.kernel32.CreateFileW( - path, - w.FILE_LIST_DIRECTORY, - w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, - null, - w.OPEN_EXISTING, - w.FILE_FLAG_BACKUP_SEMANTICS | w.FILE_FLAG_OVERLAPPED, - null, - ); - if (handle == w.INVALID_HANDLE_VALUE) { - @panic("failed to open directory for watching"); - } - // TODO check if this is a directory - - errdefer _ = w.kernel32.CloseHandle(handle); - - const key = this.rng.next(); - - this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, key, 1); - - const watcher = try this.allocator.create(DirWatcher); - errdefer this.allocator.destroy(watcher); - watcher.* = .{ .dirHandle = handle }; - try this.watchers.put(key, watcher); - - const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_SIZE | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION | w.FILE_NOTIFY_CHANGE_SECURITY; - if (w.kernel32.ReadDirectoryChangesW(handle, &watcher.buf, watcher.buf.len, 1, filter, null, null, null) == 0) { - @panic("failed to start watching directory"); - } - } - - pub fn stop(this: *WindowsWatcher) void { - @atomicStore(bool, &this.running, false, .Unordered); - } - - pub fn run(this: *WindowsWatcher) void { - var nbytes: w.DWORD = 0; - var key: w.ULONG_PTR = 0; - var overlapped: ?*w.OVERLAPPED = null; - while (w.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, w.INFINITE) != 0) { - if (nbytes == 0) { - // exit notification? - break; - } - if (this.watchers.get(key)) |watcher| { - watcher.handleEvent(nbytes); - } else { - // not really an error: the watcher with this key has already been closed and we're just receiving the remaining events - Output.prettyErrorln("no watcher with key {d}", .{key}); - } - if (@atomicLoad(bool, &this.running, .Unordered) == false) { - break; - } - } - } -}; diff --git a/watch.zig b/watch.zig index 8957f1dcc5a14a..d990a157cf3497 100644 --- a/watch.zig +++ b/watch.zig @@ -92,11 +92,19 @@ pub const WindowsWatcher = struct { iocp: w.HANDLE, allocator: std.mem.Allocator, // watchers: Map, - // rng: std.rand.DefaultPrng, running: bool = true, // const Map = std.AutoArrayHashMap(*w.OVERLAPPED, *DirWatcher); const w = std.os.windows; + + const Action = enum(w.DWORD) { + Added = 1, + Removed, + Modified, + RenamedOld, + RenamedNew, + }; + const DirWatcher = extern struct { // this must be the first field overlapped: w.OVERLAPPED = undefined, @@ -113,7 +121,8 @@ pub const WindowsWatcher = struct { const name_ptr: [*]u16 = @alignCast(@ptrCast(this.buf[offset + info_size ..])); const filename: []u16 = name_ptr[0 .. info.FileNameLength / @sizeOf(u16)]; - std.debug.print("filename: {}, action: {}\n", .{ std.unicode.fmtUtf16le(filename), info.Action }); + const action: Action = @enumFromInt(info.Action); + std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(filename), @tagName(action) }); if (info.NextEntryOffset == 0) break; offset += @as(usize, info.NextEntryOffset); @@ -121,7 +130,7 @@ pub const WindowsWatcher = struct { } fn listen(this: *DirWatcher) !void { - const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_SIZE | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION | w.FILE_NOTIFY_CHANGE_SECURITY; + const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION; if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, this.watch_subtree, filter, null, &this.overlapped, null) == 0) { const err = w.kernel32.GetLastError(); std.debug.print("failed to start watching directory: {s}\n", .{@tagName(err)}); @@ -142,11 +151,14 @@ pub const WindowsWatcher = struct { return watcher; } + pub fn deinit(this: *WindowsWatcher) void { + // get all the directory watchers and close their handles + // TODO + // close the io completion port handle + w.kernel32.CloseHandle(this.iocp); + } + pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !void { - _ = dirFd; // autofix - // TODO respect dirFd - // const flags = w.STANDARD_RIGHTS_READ | w.FILE_READ_ATTRIBUTES | w.FILE_READ_EA | - // w.FILE_TRAVERSE | w.FILE_LIST_DIRECTORY; const flags = w.FILE_LIST_DIRECTORY; const path_len_bytes: u16 = @truncate(path.len * 2); @@ -157,13 +169,12 @@ pub const WindowsWatcher = struct { }; var attr = w.OBJECT_ATTRIBUTES{ .Length = @sizeOf(w.OBJECT_ATTRIBUTES), - .RootDirectory = null, - // if (std.fs.path.isAbsoluteWindowsW(path)) - // null - // else if (dirFd == w.INVALID_HANDLE_VALUE) - // std.fs.cwd().fd - // else - // dirFd, + .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(path)) + null + else if (dirFd == w.INVALID_HANDLE_VALUE) + std.fs.cwd().fd + else + dirFd, .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. .ObjectName = &nt_name, .SecurityDescriptor = null, @@ -190,25 +201,8 @@ pub const WindowsWatcher = struct { @panic("failed to open directory for watching"); } - // const handle = w.kernel32.CreateFileW( - // path, - // w.FILE_LIST_DIRECTORY, - // w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, - // null, - // w.OPEN_EXISTING, - // w.FILE_FLAG_BACKUP_SEMANTICS | w.FILE_FLAG_OVERLAPPED, - // null, - // ); - // if (handle == w.INVALID_HANDLE_VALUE) { - // @panic("failed to open directory for watching"); - // } - // // TODO check if this is a directory - // {} - errdefer _ = w.kernel32.CloseHandle(handle); - // const key = this.rng.next(); - this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); const watcher = try this.allocator.create(DirWatcher); @@ -219,6 +213,8 @@ pub const WindowsWatcher = struct { } pub fn stop(this: *WindowsWatcher) void { + // close all the handles + // w.kernel32.PostQueuedCompletionStatus(this.iocp, 0, 1, ) @atomicStore(bool, &this.running, false, .Unordered); } @@ -234,8 +230,8 @@ pub const WindowsWatcher = struct { .EOF => @panic("eof"), } if (nbytes == 0) { - // exit notification? - break; + // exit notification for this watcher - we should probably deallocate it here + continue; } const watcher: *DirWatcher = @ptrCast(overlapped); @@ -253,15 +249,11 @@ pub fn main() !void { const allocator = std.heap.page_allocator; var buf: [std.fs.MAX_PATH_BYTES]u16 = undefined; - const path = toNTPath(&buf, "C:\\bun\\"); - // const idx = try std.unicode.utf8ToUtf16Le(&buf, "C:\\bun\\"); - // buf[idx] = 0; - // const path = buf[0..idx :0]; - - std.debug.print("directory: {}\n", .{std.unicode.fmtUtf16le(path)}); const watcher = try WindowsWatcher.init(allocator); - try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, path); + try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\src")); + try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\test")); + try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\testdir")); std.debug.print("watching...\n", .{}); From ce043e49d433e599d68be7637992b3e4048c1ce0 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 11:15:02 -0800 Subject: [PATCH 05/32] make watcher non-global --- src/watcher.zig | 183 ++++++++++++++++++++---------------------------- 1 file changed, 74 insertions(+), 109 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index 55d160ae9eeea6..9f4e03fb1e9d23 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -27,38 +27,17 @@ const PackageJSON = @import("./resolver/package_json.zig").PackageJSON; const WATCHER_MAX_LIST = 8096; pub const INotify = struct { - pub const IN_CLOEXEC = std.os.O.CLOEXEC; - pub const IN_NONBLOCK = std.os.O.NONBLOCK; - - pub const IN_ACCESS = 0x00000001; - pub const IN_MODIFY = 0x00000002; - pub const IN_ATTRIB = 0x00000004; - pub const IN_CLOSE_WRITE = 0x00000008; - pub const IN_CLOSE_NOWRITE = 0x00000010; - pub const IN_CLOSE = IN_CLOSE_WRITE | IN_CLOSE_NOWRITE; - pub const IN_OPEN = 0x00000020; - pub const IN_MOVED_FROM = 0x00000040; - pub const IN_MOVED_TO = 0x00000080; - pub const IN_MOVE = IN_MOVED_FROM | IN_MOVED_TO; - pub const IN_CREATE = 0x00000100; - pub const IN_DELETE = 0x00000200; - pub const IN_DELETE_SELF = 0x00000400; - pub const IN_MOVE_SELF = 0x00000800; - pub const IN_ALL_EVENTS = 0x00000fff; - - pub const IN_UNMOUNT = 0x00002000; - pub const IN_Q_OVERFLOW = 0x00004000; - pub const IN_IGNORED = 0x00008000; - - pub const IN_ONLYDIR = 0x01000000; - pub const IN_DONT_FOLLOW = 0x02000000; - pub const IN_EXCL_UNLINK = 0x04000000; - pub const IN_MASK_ADD = 0x20000000; - - pub const IN_ISDIR = 0x40000000; - pub const IN_ONESHOT = 0x80000000; + loaded_inotify: bool = false, + inotify_fd: EventListIndex = 0, + + eventlist: EventListBuffer = undefined, + eventlist_ptrs: [128]*const INotifyEvent = undefined, + + watch_count: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), + coalesce_interval: isize = 100_000, pub const EventListIndex = c_int; + const EventListBuffer = [@sizeOf([128]INotifyEvent) + (128 * bun.MAX_PATH_BYTES + (128 * @alignOf(INotifyEvent)))]u8; pub const INotifyEvent = extern struct { watch_descriptor: c_int, @@ -76,62 +55,52 @@ pub const INotify = struct { return bun.sliceTo(@as([*:0]u8, @ptrFromInt(@intFromPtr(&this.name_len) + @sizeOf(u32))), 0)[0.. :0]; } }; - pub var inotify_fd: EventListIndex = 0; - pub var loaded_inotify = false; - - const EventListBuffer = [@sizeOf([128]INotifyEvent) + (128 * bun.MAX_PATH_BYTES + (128 * @alignOf(INotifyEvent)))]u8; - var eventlist: EventListBuffer = undefined; - var eventlist_ptrs: [128]*const INotifyEvent = undefined; - - var watch_count: std.atomic.Value(u32) = std.atomic.Value(u32).init(0); - const watch_file_mask = std.os.linux.IN.EXCL_UNLINK | std.os.linux.IN.MOVE_SELF | std.os.linux.IN.DELETE_SELF | std.os.linux.IN.MOVED_TO | std.os.linux.IN.MODIFY; - const watch_dir_mask = std.os.linux.IN.EXCL_UNLINK | std.os.linux.IN.DELETE | std.os.linux.IN.DELETE_SELF | std.os.linux.IN.CREATE | std.os.linux.IN.MOVE_SELF | std.os.linux.IN.ONLYDIR | std.os.linux.IN.MOVED_TO; - - pub fn watchPath(pathname: [:0]const u8) !EventListIndex { - std.debug.assert(loaded_inotify); - const old_count = watch_count.fetchAdd(1, .Release); - defer if (old_count == 0) Futex.wake(&watch_count, 10); - return std.os.inotify_add_watchZ(inotify_fd, pathname, watch_file_mask); + pub fn watchPath(this: *INotify, pathname: [:0]const u8) !EventListIndex { + std.debug.assert(this.loaded_inotify); + const old_count = this.watch_count.fetchAdd(1, .Release); + defer if (old_count == 0) Futex.wake(&this.watch_count, 10); + const watch_file_mask = std.os.linux.IN.EXCL_UNLINK | std.os.linux.IN.MOVE_SELF | std.os.linux.IN.DELETE_SELF | std.os.linux.IN.MOVED_TO | std.os.linux.IN.MODIFY; + return std.os.inotify_add_watchZ(this.inotify_fd, pathname, watch_file_mask); } - pub fn watchDir(pathname: [:0]const u8) !EventListIndex { - std.debug.assert(loaded_inotify); - const old_count = watch_count.fetchAdd(1, .Release); - defer if (old_count == 0) Futex.wake(&watch_count, 10); - return std.os.inotify_add_watchZ(inotify_fd, pathname, watch_dir_mask); + pub fn watchDir(this: *INotify, pathname: [:0]const u8) !EventListIndex { + std.debug.assert(this.loaded_inotify); + const old_count = this.watch_count.fetchAdd(1, .Release); + defer if (old_count == 0) Futex.wake(&this.watch_count, 10); + const watch_dir_mask = std.os.linux.IN.EXCL_UNLINK | std.os.linux.IN.DELETE | std.os.linux.IN.DELETE_SELF | std.os.linux.IN.CREATE | std.os.linux.IN.MOVE_SELF | std.os.linux.IN.ONLYDIR | std.os.linux.IN.MOVED_TO; + return std.os.inotify_add_watchZ(this.inotify_fd, pathname, watch_dir_mask); } - pub fn unwatch(wd: EventListIndex) void { - std.debug.assert(loaded_inotify); - _ = watch_count.fetchSub(1, .Release); - std.os.inotify_rm_watch(inotify_fd, wd); + pub fn unwatch(this: *INotify, wd: EventListIndex) void { + std.debug.assert(this.loaded_inotify); + _ = this.watch_count.fetchSub(1, .Release); + std.os.inotify_rm_watch(this.inotify_fd, wd); } - pub fn isRunning() bool { - return loaded_inotify; + pub fn isRunning(this: *INotify) bool { + return this.loaded_inotify; } - var coalesce_interval: isize = 100_000; - pub fn init() !void { - std.debug.assert(!loaded_inotify); - loaded_inotify = true; + pub fn init(this: *INotify) !void { + std.debug.assert(!this.loaded_inotify); + this.loaded_inotify = true; if (bun.getenvZ("BUN_INOTIFY_COALESCE_INTERVAL")) |env| { - coalesce_interval = std.fmt.parseInt(isize, env, 10) catch 100_000; + this.coalesce_interval = std.fmt.parseInt(isize, env, 10) catch 100_000; } - inotify_fd = try std.os.inotify_init1(IN_CLOEXEC); + this.inotify_fd = try std.os.inotify_init1(std.os.linux.IN.CLOEXEC); } - pub fn read() ![]*const INotifyEvent { - std.debug.assert(loaded_inotify); + pub fn read(this: *INotify) ![]*const INotifyEvent { + std.debug.assert(this.loaded_inotify); restart: while (true) { - Futex.wait(&watch_count, 0, null) catch unreachable; + Futex.wait(&this.watch_count, 0, null) catch unreachable; const rc = std.os.system.read( - inotify_fd, - @as([*]u8, @ptrCast(@alignCast(&eventlist))), + this.inotify_fd, + @as([*]u8, @ptrCast(@alignCast(&this.eventlist))), @sizeOf(EventListBuffer), ); @@ -145,16 +114,16 @@ pub const INotify = struct { // we do a 0.1ms sleep to try to coalesce events better if (len < (@sizeOf(EventListBuffer) / 2)) { var fds = [_]std.os.pollfd{.{ - .fd = inotify_fd, + .fd = this.inotify_fd, .events = std.os.POLL.IN | std.os.POLL.ERR, .revents = 0, }}; - var timespec = std.os.timespec{ .tv_sec = 0, .tv_nsec = coalesce_interval }; + var timespec = std.os.timespec{ .tv_sec = 0, .tv_nsec = this.coalesce_interval }; if ((std.os.ppoll(&fds, ×pec, null) catch 0) > 0) { while (true) { const new_rc = std.os.system.read( - inotify_fd, - @as([*]u8, @ptrCast(@alignCast(&eventlist))) + len, + this.inotify_fd, + @as([*]u8, @ptrCast(@alignCast(&this.eventlist))) + len, @sizeOf(EventListBuffer) - len, ); switch (std.os.errno(new_rc)) { @@ -186,14 +155,14 @@ pub const INotify = struct { var i: u32 = 0; while (i < len) : (i += @sizeOf(INotifyEvent)) { @setRuntimeSafety(false); - const event = @as(*INotifyEvent, @ptrCast(@alignCast(eventlist[i..][0..@sizeOf(INotifyEvent)]))); + const event = @as(*INotifyEvent, @ptrCast(@alignCast(this.eventlist[i..][0..@sizeOf(INotifyEvent)]))); i += event.name_len; - eventlist_ptrs[count] = event; + this.eventlist_ptrs[count] = event; count += 1; } - return eventlist_ptrs[0..count]; + return this.eventlist_ptrs[0..count]; }, .AGAIN => continue :restart, .INVAL => return error.ShortRead, @@ -205,10 +174,10 @@ pub const INotify = struct { unreachable; } - pub fn stop() void { - if (inotify_fd != 0) { - _ = bun.sys.close(bun.toFD(inotify_fd)); - inotify_fd = 0; + pub fn stop(this: *INotify) void { + if (this.inotify_fd != 0) { + _ = bun.sys.close(bun.toFD(this.inotify_fd)); + this.inotify_fd = 0; } } }; @@ -217,32 +186,30 @@ const DarwinWatcher = struct { pub const EventListIndex = u32; const KEvent = std.c.Kevent; + // Internal - pub var changelist: [128]KEvent = undefined; + changelist: [128]KEvent = undefined, // Everything being watched - pub var eventlist: [WATCHER_MAX_LIST]KEvent = undefined; - pub var eventlist_index: EventListIndex = 0; + eventlist: [WATCHER_MAX_LIST]KEvent = undefined, + eventlist_index: EventListIndex = 0, - pub var fd: i32 = 0; + fd: i32 = 0, - pub fn init() !void { - std.debug.assert(fd == 0); - - fd = try std.os.kqueue(); - if (fd == 0) return error.KQueueError; + pub fn init(this: *DarwinWatcher) !void { + this.fd = try std.os.kqueue(); + if (this.fd == 0) return error.KQueueError; } - pub fn isRunning() bool { - return fd != 0; + pub fn isRunning(this: *DarwinWatcher) bool { + return this.fd != 0; } - pub fn stop() void { - if (fd != 0) { - _ = bun.sys.close(fd); + pub fn stop(this: *DarwinWatcher) void { + if (this.fd != 0) { + _ = bun.sys.close(this.fd); } - - fd = 0; + this.fd = 0; } }; @@ -332,11 +299,11 @@ pub const WatchEvent = struct { pub fn fromINotify(this: *WatchEvent, event: INotify.INotifyEvent, index: WatchItemIndex) void { this.* = WatchEvent{ .op = Op{ - .delete = (event.mask & INotify.IN_DELETE_SELF) > 0 or (event.mask & INotify.IN_DELETE) > 0, + .delete = (event.mask & std.os.linux.IN.DELETE_SELF) > 0 or (event.mask & std.os.linux.IN.DELETE) > 0, .metadata = false, - .rename = (event.mask & INotify.IN_MOVE_SELF) > 0, - .move_to = (event.mask & INotify.IN_MOVED_TO) > 0, - .write = (event.mask & INotify.IN_MODIFY) > 0, + .rename = (event.mask & std.os.linux.IN.MOVE_SELF) > 0, + .move_to = (event.mask & std.os.linux.IN.MOVED_TO) > 0, + .write = (event.mask & std.os.linux.IN.MODIFY) > 0, }, .index = index, }; @@ -391,10 +358,6 @@ pub fn NewWatcher(comptime ContextType: type) type { const watcher = try allocator.create(Watcher); errdefer allocator.destroy(watcher); - if (!PlatformWatcher.isRunning()) { - try PlatformWatcher.init(); - } - watcher.* = Watcher{ .fs = fs, .fd = .zero, @@ -406,6 +369,8 @@ pub fn NewWatcher(comptime ContextType: type) type { .cwd = fs.top_level_dir, }; + try PlatformWatcher.init(&watcher.platform); + return watcher; } @@ -452,7 +417,7 @@ pub fn NewWatcher(comptime ContextType: type) type { this._watchLoop() catch |err| { this.watchloop_handle = null; - PlatformWatcher.stop(); + this.platform.stop(); if (this.running) { this.ctx.onError(err); } @@ -543,7 +508,7 @@ pub fn NewWatcher(comptime ContextType: type) type { fn _watchLoop(this: *Watcher) !void { if (Environment.isMac) { - std.debug.assert(DarwinWatcher.fd > 0); + std.debug.assert(this.platform.fd > 0); const KEvent = std.c.Kevent; var changelist_array: [128]KEvent = std.mem.zeroes([128]KEvent); @@ -552,7 +517,7 @@ pub fn NewWatcher(comptime ContextType: type) type { defer Output.flush(); var count_ = std.os.system.kevent( - DarwinWatcher.fd, + this.platform.fd, @as([*]KEvent, changelist), 0, @as([*]KEvent, changelist), @@ -566,7 +531,7 @@ pub fn NewWatcher(comptime ContextType: type) type { const remain = 128 - count_; var timespec = std.os.timespec{ .tv_sec = 0, .tv_nsec = 100_000 }; const extra = std.os.system.kevent( - DarwinWatcher.fd, + this.platform.fd, @as([*]KEvent, changelist[@as(usize, @intCast(count_))..].ptr), 0, @as([*]KEvent, changelist[@as(usize, @intCast(count_))..].ptr), @@ -768,7 +733,7 @@ pub fn NewWatcher(comptime ContextType: type) type { // - We register the event here. // our while(true) loop above receives notification of changes to any of the events created here. _ = std.os.system.kevent( - DarwinWatcher.fd, + this.platform.fd, @as([]KEvent, events[0..1]).ptr, 1, @as([]KEvent, events[0..1]).ptr, @@ -849,7 +814,7 @@ pub fn NewWatcher(comptime ContextType: type) type { // - We register the event here. // our while(true) loop above receives notification of changes to any of the events created here. _ = std.os.system.kevent( - DarwinWatcher.fd, + this.platform.fd, @as([]KEvent, events[0..1]).ptr, 1, @as([]KEvent, events[0..1]).ptr, From 04c749547698b9a4c19990a73a36d16cbc088a8f Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 11:34:00 -0800 Subject: [PATCH 06/32] prepare watcher for windows impl --- src/watcher.zig | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index 9f4e03fb1e9d23..922c6d4ca9fcc7 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -332,12 +332,10 @@ pub fn NewWatcher(comptime ContextType: type) type { // User-facing watch_events: [128]WatchEvent = undefined, - changed_filepaths: [128]?[:0]u8 = std.mem.zeroes([128]?[:0]u8), + changed_filepaths: [128]?[:0]u8 = [_]?[:0]u8{null} ** 128, - fs: *Fs.FileSystem, - // this is what kqueue knows about - fd: StoredFileDescriptorType, ctx: ContextType, + fs: *Fs.FileSystem, allocator: std.mem.Allocator, watchloop_handle: ?std.Thread.Id = null, cwd: string, @@ -345,11 +343,12 @@ pub fn NewWatcher(comptime ContextType: type) type { running: bool = true, close_descriptors: bool = false, + evict_list: [WATCHER_MAX_LIST]WatchItemIndex = undefined, + evict_list_i: WatchItemIndex = 0, + pub const HashType = u32; pub const WatchListArray = Watchlist; - var evict_list: [WATCHER_MAX_LIST]WatchItemIndex = undefined; - pub fn getHash(filepath: string) HashType { return @as(HashType, @truncate(bun.hash(filepath))); } @@ -360,7 +359,6 @@ pub fn NewWatcher(comptime ContextType: type) type { watcher.* = Watcher{ .fs = fs, - .fd = .zero, .allocator = allocator, .watched_count = 0, .ctx = ctx, @@ -447,34 +445,32 @@ pub fn NewWatcher(comptime ContextType: type) type { } } - var evict_list_i: WatchItemIndex = 0; - - pub fn removeAtIndex(_: *Watcher, index: WatchItemIndex, hash: HashType, parents: []HashType, comptime kind: WatchItem.Kind) void { + pub fn removeAtIndex(this: *Watcher, index: WatchItemIndex, hash: HashType, parents: []HashType, comptime kind: WatchItem.Kind) void { std.debug.assert(index != NoWatchItem); - evict_list[evict_list_i] = index; - evict_list_i += 1; + this.evict_list[this.evict_list_i] = index; + this.evict_list_i += 1; if (comptime kind == .directory) { for (parents) |parent| { if (parent == hash) { - evict_list[evict_list_i] = @as(WatchItemIndex, @truncate(parent)); - evict_list_i += 1; + this.evict_list[this.evict_list_i] = @as(WatchItemIndex, @truncate(parent)); + this.evict_list_i += 1; } } } } pub fn flushEvictions(this: *Watcher) void { - if (evict_list_i == 0) return; - defer evict_list_i = 0; + if (this.evict_list_i == 0) return; + defer this.evict_list_i = 0; // swapRemove messes up the order // But, it only messes up the order if any elements in the list appear after the item being removed // So if we just sort the list by the biggest index first, that should be fine std.sort.pdq( WatchItemIndex, - evict_list[0..evict_list_i], + this.evict_list[0..this.evict_list_i], {}, comptime std.sort.desc(WatchItemIndex), ); @@ -483,7 +479,7 @@ pub fn NewWatcher(comptime ContextType: type) type { const fds = slice.items(.fd); var last_item = NoWatchItem; - for (evict_list[0..evict_list_i]) |item| { + for (this.evict_list[0..this.evict_list_i]) |item| { // catch duplicates, since the list is sorted, duplicates will appear right after each other if (item == last_item) continue; @@ -499,7 +495,7 @@ pub fn NewWatcher(comptime ContextType: type) type { last_item = NoWatchItem; // This is split into two passes because reading the slice while modified is potentially unsafe. - for (evict_list[0..evict_list_i]) |item| { + for (this.evict_list[0..this.evict_list_i]) |item| { if (item == last_item) continue; this.watchlist.swapRemove(item); last_item = item; From 102fe00854ec3cc81cf9bd2c81e8d8ce22f52ef6 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 13:23:55 -0800 Subject: [PATCH 07/32] add windows watcher scaffold and clean up imports --- src/watcher.zig | 218 ++++++++++++++++++++++++++++++++++++------------ watch.zig | 31 +++++-- 2 files changed, 189 insertions(+), 60 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index 922c6d4ca9fcc7..c25a74ebc0a3a2 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -1,4 +1,3 @@ -const Fs = @import("./fs.zig"); const std = @import("std"); const bun = @import("root").bun; const string = bun.string; @@ -6,22 +5,13 @@ const Output = bun.Output; const Global = bun.Global; const Environment = bun.Environment; const strings = bun.strings; -const MutableString = bun.MutableString; const stringZ = bun.stringZ; -const StoredFileDescriptorType = bun.StoredFileDescriptorType; const FeatureFlags = bun.FeatureFlags; -const default_allocator = bun.default_allocator; -const C = bun.C; -const c = std.c; const options = @import("./options.zig"); -const IndexType = @import("./allocators.zig").IndexType; - -const os = std.os; const Mutex = @import("./lock.zig").Lock; const Futex = @import("./futex.zig"); pub const WatchItemIndex = u16; -const NoWatchItem: WatchItemIndex = std.math.maxInt(WatchItemIndex); const PackageJSON = @import("./resolver/package_json.zig").PackageJSON; const WATCHER_MAX_LIST = 8096; @@ -78,11 +68,7 @@ pub const INotify = struct { std.os.inotify_rm_watch(this.inotify_fd, wd); } - pub fn isRunning(this: *INotify) bool { - return this.loaded_inotify; - } - - pub fn init(this: *INotify) !void { + pub fn init(this: *INotify, _: std.mem.Allocator) !void { std.debug.assert(!this.loaded_inotify); this.loaded_inotify = true; @@ -196,15 +182,11 @@ const DarwinWatcher = struct { fd: i32 = 0, - pub fn init(this: *DarwinWatcher) !void { + pub fn init(this: *DarwinWatcher, _: std.mem.Allocator) !void { this.fd = try std.os.kqueue(); if (this.fd == 0) return error.KQueueError; } - pub fn isRunning(this: *DarwinWatcher) bool { - return this.fd != 0; - } - pub fn stop(this: *DarwinWatcher) void { if (this.fd != 0) { _ = bun.sys.close(this.fd); @@ -213,25 +195,156 @@ const DarwinWatcher = struct { } }; -pub const Placeholder = struct { - pub const EventListIndex = u32; +pub const WindowsWatcher = struct { + iocp: w.HANDLE = undefined, + allocator: std.mem.Allocator = undefined, + // watchers: Map, + + // const Map = std.AutoArrayHashMap(*w.OVERLAPPED, *DirWatcher); + const w = std.os.windows; + pub const EventListIndex = c_int; + + const Action = enum(w.DWORD) { + Added = 1, + Removed, + Modified, + RenamedOld, + RenamedNew, + }; + + const DirWatcher = extern struct { + // this must be the first field + overlapped: w.OVERLAPPED = undefined, + buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, + dirHandle: w.HANDLE, + watch_subtree: w.BOOLEAN = 1, + + fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { + if (nbytes == 0) return; + var offset: usize = 0; + while (true) { + const info_size = @sizeOf(w.FILE_NOTIFY_INFORMATION); + const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(this.buf[offset..].ptr)); + const name_ptr: [*]u16 = @alignCast(@ptrCast(this.buf[offset + info_size ..])); + const filename: []u16 = name_ptr[0 .. info.FileNameLength / @sizeOf(u16)]; + + const action: Action = @enumFromInt(info.Action); + std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(filename), @tagName(action) }); + + if (info.NextEntryOffset == 0) break; + offset += @as(usize, info.NextEntryOffset); + } + } + + fn listen(this: *DirWatcher) !void { + const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION; + if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, this.watch_subtree, filter, null, &this.overlapped, null) == 0) { + const err = w.kernel32.GetLastError(); + std.debug.print("failed to start watching directory: {s}\n", .{@tagName(err)}); + @panic("failed to start watching directory"); + } + } + }; + + pub fn init(this: *WindowsWatcher, allocator: std.mem.Allocator) !void { + // const watcher = try allocator.create(WindowsWatcher); + // errdefer allocator.destroy(watcher); - pub var eventlist: [WATCHER_MAX_LIST]EventListIndex = undefined; - pub var eventlist_index: EventListIndex = 0; + const iocp = try w.CreateIoCompletionPort(w.INVALID_HANDLE_VALUE, null, 0, 1); + this.* = .{ + .iocp = iocp, + .allocator = allocator, + }; + } + + pub fn deinit(this: *WindowsWatcher) void { + // get all the directory watchers and close their handles + // TODO + // close the io completion port handle + w.kernel32.CloseHandle(this.iocp); + } + + pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !void { + const path_len_bytes: u16 = @truncate(path.len * 2); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @constCast(path.ptr), + }; + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(path)) + null + else if (dirFd == w.INVALID_HANDLE_VALUE) + std.fs.cwd().fd + else + dirFd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + var handle: w.HANDLE = w.INVALID_HANDLE_VALUE; + var io: w.IO_STATUS_BLOCK = undefined; + const rc = w.ntdll.NtCreateFile( + &handle, + w.FILE_LIST_DIRECTORY, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + w.FILE_OPEN, + w.FILE_DIRECTORY_FILE | w.FILE_OPEN_FOR_BACKUP_INTENT, + null, + 0, + ); + + if (rc != .SUCCESS) { + std.debug.print("failed to open directory for watching: {s}\n", .{@tagName(rc)}); + @panic("failed to open directory for watching"); + } - pub fn isRunning() bool { - return true; + errdefer _ = w.kernel32.CloseHandle(handle); + + this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); + + const watcher = try this.allocator.create(DirWatcher); + errdefer this.allocator.destroy(watcher); + watcher.* = .{ .dirHandle = handle }; + // try this.watchers.put(key, watcher); + try watcher.listen(); } - pub fn init() !void {} + pub fn read(this: *WindowsWatcher) !void { + var nbytes: w.DWORD = 0; + var key: w.ULONG_PTR = 0; + var overlapped: ?*w.OVERLAPPED = null; + switch (w.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, w.INFINITE)) { + .Normal => {}, + .Aborted => @panic("aborted"), + .Cancelled => @panic("cancelled"), + .EOF => @panic("eof"), + } + if (nbytes == 0) { + // exit notification for this watcher - we should probably deallocate it here + return; + } + + const watcher: *DirWatcher = @ptrCast(overlapped); + watcher.handleEvent(nbytes); + try watcher.listen(); + } }; const PlatformWatcher = if (Environment.isMac) DarwinWatcher else if (Environment.isLinux) INotify +else if (Environment.isWindows) + WindowsWatcher else - Placeholder; + @compileError("Unsupported platform"); pub const WatchItem = struct { file_path: string, @@ -239,7 +352,7 @@ pub const WatchItem = struct { hash: u32, eventlist_index: PlatformWatcher.EventListIndex, loader: options.Loader, - fd: StoredFileDescriptorType, + fd: bun.FileDescriptor, count: u32, parent_hash: u32, kind: Kind, @@ -335,7 +448,7 @@ pub fn NewWatcher(comptime ContextType: type) type { changed_filepaths: [128]?[:0]u8 = [_]?[:0]u8{null} ** 128, ctx: ContextType, - fs: *Fs.FileSystem, + fs: *bun.fs.FileSystem, allocator: std.mem.Allocator, watchloop_handle: ?std.Thread.Id = null, cwd: string, @@ -348,12 +461,13 @@ pub fn NewWatcher(comptime ContextType: type) type { pub const HashType = u32; pub const WatchListArray = Watchlist; + const NoWatchItem: WatchItemIndex = std.math.maxInt(WatchItemIndex); pub fn getHash(filepath: string) HashType { return @as(HashType, @truncate(bun.hash(filepath))); } - pub fn init(ctx: ContextType, fs: *Fs.FileSystem, allocator: std.mem.Allocator) !*Watcher { + pub fn init(ctx: ContextType, fs: *bun.fs.FileSystem, allocator: std.mem.Allocator) !*Watcher { const watcher = try allocator.create(Watcher); errdefer allocator.destroy(watcher); @@ -367,16 +481,14 @@ pub fn NewWatcher(comptime ContextType: type) type { .cwd = fs.top_level_dir, }; - try PlatformWatcher.init(&watcher.platform); + try PlatformWatcher.init(&watcher.platform, allocator); return watcher; } pub fn start(this: *Watcher) !void { - if (!Environment.isWindows) { - std.debug.assert(this.watchloop_handle == null); - this.thread = try std.Thread.spawn(.{}, Watcher.watchLoop, .{this}); - } + std.debug.assert(this.watchloop_handle == null); + this.thread = try std.Thread.spawn(.{}, Watcher.watchLoop, .{this}); } pub fn deinit(this: *Watcher, close_descriptors: bool) void { @@ -403,10 +515,6 @@ pub fn NewWatcher(comptime ContextType: type) type { // This must only be called from the watcher thread pub fn watchLoop(this: *Watcher) !void { - if (Environment.isWindows) { - @compileError("watchLoop should not be used on Windows"); - } - this.watchloop_handle = std.Thread.getCurrentId(); Output.Source.configureNamedThread("File Watcher"); @@ -646,7 +754,7 @@ pub fn NewWatcher(comptime ContextType: type) type { } } } else if (Environment.isWindows) { - @compileError("watchLoop should not be used on Windows"); + @panic("todo windows watchloop"); } } @@ -661,11 +769,11 @@ pub fn NewWatcher(comptime ContextType: type) type { pub fn addFile( this: *Watcher, - fd: StoredFileDescriptorType, + fd: bun.FileDescriptor, file_path: string, hash: HashType, loader: options.Loader, - dir_fd: StoredFileDescriptorType, + dir_fd: bun.FileDescriptor, package_json: ?*PackageJSON, comptime copy_file_path: bool, ) !void { @@ -689,7 +797,7 @@ pub fn NewWatcher(comptime ContextType: type) type { fn appendFileAssumeCapacity( this: *Watcher, - fd: StoredFileDescriptorType, + fd: bun.FileDescriptor, file_path: string, hash: HashType, loader: options.Loader, @@ -711,7 +819,7 @@ pub fn NewWatcher(comptime ContextType: type) type { // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/kqueue.2.html var event = std.mem.zeroes(KEvent); - event.flags = c.EV_ADD | c.EV_CLEAR | c.EV_ENABLE; + event.flags = std.c.EV_ADD | std.c.EV_CLEAR | std.c.EV_ENABLE; // we want to know about the vnode event.filter = std.c.EVFILT_VNODE; @@ -761,7 +869,7 @@ pub fn NewWatcher(comptime ContextType: type) type { fn appendDirectoryAssumeCapacity( this: *Watcher, - stored_fd: StoredFileDescriptorType, + stored_fd: bun.FileDescriptor, file_path: string, hash: HashType, comptime copy_file_path: bool, @@ -772,7 +880,7 @@ pub fn NewWatcher(comptime ContextType: type) type { break :brk bun.toFD(dir.fd); }; - const parent_hash = Watcher.getHash(Fs.PathName.init(file_path).dirWithTrailingSlash()); + const parent_hash = Watcher.getHash(bun.fs.PathName.init(file_path).dirWithTrailingSlash()); var index: PlatformWatcher.EventListIndex = std.math.maxInt(PlatformWatcher.EventListIndex); const file_path_: string = if (comptime copy_file_path) @@ -788,7 +896,7 @@ pub fn NewWatcher(comptime ContextType: type) type { // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/kqueue.2.html var event = std.mem.zeroes(KEvent); - event.flags = c.EV_ADD | c.EV_CLEAR | c.EV_ENABLE; + event.flags = std.c.EV_ADD | std.c.EV_CLEAR | std.c.EV_ENABLE; // we want to know about the vnode event.filter = std.c.EVFILT_VNODE; @@ -846,7 +954,7 @@ pub fn NewWatcher(comptime ContextType: type) type { pub fn addDirectory( this: *Watcher, - fd: StoredFileDescriptorType, + fd: bun.FileDescriptor, file_path: string, hash: HashType, comptime copy_file_path: bool, @@ -865,11 +973,11 @@ pub fn NewWatcher(comptime ContextType: type) type { pub fn appendFileMaybeLock( this: *Watcher, - fd: StoredFileDescriptorType, + fd: bun.FileDescriptor, file_path: string, hash: HashType, loader: options.Loader, - dir_fd: StoredFileDescriptorType, + dir_fd: bun.FileDescriptor, package_json: ?*PackageJSON, comptime copy_file_path: bool, comptime lock: bool, @@ -877,7 +985,7 @@ pub fn NewWatcher(comptime ContextType: type) type { if (comptime lock) this.mutex.lock(); defer if (comptime lock) this.mutex.unlock(); std.debug.assert(file_path.len > 1); - const pathname = Fs.PathName.init(file_path); + const pathname = bun.fs.PathName.init(file_path); const parent_dir = pathname.dirWithTrailingSlash(); const parent_dir_hash: HashType = Watcher.getHash(parent_dir); @@ -889,7 +997,7 @@ pub fn NewWatcher(comptime ContextType: type) type { if (dir_fd.int() > 0) { const fds = watchlist_slice.items(.fd); - if (std.mem.indexOfScalar(StoredFileDescriptorType, fds, dir_fd)) |i| { + if (std.mem.indexOfScalar(bun.FileDescriptor, fds, dir_fd)) |i| { parent_watch_item = @as(WatchItemIndex, @truncate(i)); } } @@ -928,11 +1036,11 @@ pub fn NewWatcher(comptime ContextType: type) type { pub fn appendFile( this: *Watcher, - fd: StoredFileDescriptorType, + fd: bun.FileDescriptor, file_path: string, hash: HashType, loader: options.Loader, - dir_fd: StoredFileDescriptorType, + dir_fd: bun.FileDescriptor, package_json: ?*PackageJSON, comptime copy_file_path: bool, ) !void { diff --git a/watch.zig b/watch.zig index d990a157cf3497..c31a82d347147b 100644 --- a/watch.zig +++ b/watch.zig @@ -113,6 +113,8 @@ pub const WindowsWatcher = struct { watch_subtree: w.BOOLEAN = 1, fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { + const elapsed = clock1.read(); + std.debug.print("elapsed: {}\n", .{std.fmt.fmtDuration(elapsed)}); if (nbytes == 0) return; var offset: usize = 0; while (true) { @@ -245,17 +247,36 @@ pub const WindowsWatcher = struct { } }; +var clock1: std.time.Timer = undefined; +const data: [1 << 10]u8 = std.mem.zeroes([1 << 10]u8); + pub fn main() !void { const allocator = std.heap.page_allocator; var buf: [std.fs.MAX_PATH_BYTES]u16 = undefined; + const watchdir = "C:\\test\\node_modules"; + const iconsdir = "C:\\test\\node_modules\\@mui\\icons-material"; + _ = iconsdir; // autofix + const testdir = "C:\\test\\node_modules\\@mui\\icons-material\\mydir"; + _ = testdir; // autofix + const testfile = "C:\\test\\node_modules\\@mui\\icons-material\\myfile.txt"; + + // try std.fs.deleteDirAbsolute(dir); + const watcher = try WindowsWatcher.init(allocator); - try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\src")); - try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\test")); - try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\testdir")); + try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, watchdir)); + // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\src")); + // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\test")); + // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\testdir")); + + const file = try std.fs.createFileAbsolute(testfile, .{}); + var handle = try std.Thread.spawn(.{}, WindowsWatcher.run, .{watcher}); + std.debug.print("watcher started\n", .{}); - std.debug.print("watching...\n", .{}); + clock1 = try std.time.Timer.start(); + // try std.fs.makeDirAbsolute(dir); + try file.writeAll(&data); - try watcher.run(); + handle.join(); } From e1f0905748b96074460975e7a73f04836d6f8bd2 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 13:26:32 -0800 Subject: [PATCH 08/32] fix inotify --- src/watcher.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index c25a74ebc0a3a2..00c7f1b1e934b3 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -682,7 +682,7 @@ pub fn NewWatcher(comptime ContextType: type) type { restart: while (true) { defer Output.flush(); - var events = try INotify.read(); + var events = try this.platform.read(); if (events.len == 0) continue :restart; // TODO: is this thread safe? @@ -851,7 +851,7 @@ pub fn NewWatcher(comptime ContextType: type) type { // buf[file_path_to_use_.len] = 0; var buf = file_path_.ptr; const slice: [:0]const u8 = buf[0..file_path_.len :0]; - index = try INotify.watchPath(slice); + index = try this.platform.watchPath(slice); } this.watchlist.appendAssumeCapacity(.{ @@ -931,7 +931,7 @@ pub fn NewWatcher(comptime ContextType: type) type { bun.copy(u8, &buf, file_path_to_use_); buf[file_path_to_use_.len] = 0; const slice: [:0]u8 = buf[0..file_path_to_use_.len :0]; - index = try INotify.watchDir(slice); + index = try this.platform.watchDir(slice); } this.watchlist.appendAssumeCapacity(.{ From 502440c65a0ef330e03de4b2374ef607a9c25450 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 18:08:05 -0800 Subject: [PATCH 09/32] make watch code more generic over platforms --- src/watcher.zig | 450 ++++++++++++++++++++++++++++-------------------- watch.zig | 27 ++- 2 files changed, 287 insertions(+), 190 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index 00c7f1b1e934b3..ad3857c7cff797 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -14,6 +14,9 @@ const Futex = @import("./futex.zig"); pub const WatchItemIndex = u16; const PackageJSON = @import("./resolver/package_json.zig").PackageJSON; +// TODO @gvilums +// This entire file is a mess - rework it to be more maintainable + const WATCHER_MAX_LIST = 8096; pub const INotify = struct { @@ -196,47 +199,67 @@ const DarwinWatcher = struct { }; pub const WindowsWatcher = struct { + // TODO iocp needs to be updated atomically to allow the main thread to register new watchers while the watcher thread is running iocp: w.HANDLE = undefined, allocator: std.mem.Allocator = undefined, - // watchers: Map, - // const Map = std.AutoArrayHashMap(*w.OVERLAPPED, *DirWatcher); const w = std.os.windows; pub const EventListIndex = c_int; const Action = enum(w.DWORD) { - Added = 1, - Removed, - Modified, - RenamedOld, - RenamedNew, + Added = w.FILE_ACTION_ADDED, + Removed = w.FILE_ACTION_REMOVED, + Modified = w.FILE_ACTION_MODIFIED, + RenamedOld = w.FILE_ACTION_RENAMED_OLD_NAME, + RenamedNew = w.FILE_ACTION_RENAMED_NEW_NAME, + }; + + const FileEvent = struct { + action: Action, + filename: []u16 = undefined, }; + // each directory being watched has an associated DirWatcher const DirWatcher = extern struct { - // this must be the first field + // this must be the first field because we retrieve the DirWatcher from a pointer to its overlapped field overlapped: w.OVERLAPPED = undefined, buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, dirHandle: w.HANDLE, watch_subtree: w.BOOLEAN = 1, - fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { - if (nbytes == 0) return; - var offset: usize = 0; - while (true) { + const EventIterator = struct { + watcher: *DirWatcher, + offset: usize = 0, + hasNext: bool = true, + + pub fn next(this: *EventIterator) ?FileEvent { + if (!this.hasNext) return null; const info_size = @sizeOf(w.FILE_NOTIFY_INFORMATION); - const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(this.buf[offset..].ptr)); - const name_ptr: [*]u16 = @alignCast(@ptrCast(this.buf[offset + info_size ..])); + const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(this.watcher.buf[this.offset..].ptr)); + const name_ptr: [*]u16 = @alignCast(@ptrCast(this.watcher.buf[this.offset + info_size ..])); const filename: []u16 = name_ptr[0 .. info.FileNameLength / @sizeOf(u16)]; const action: Action = @enumFromInt(info.Action); - std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(filename), @tagName(action) }); - if (info.NextEntryOffset == 0) break; - offset += @as(usize, info.NextEntryOffset); + if (info.NextEntryOffset == 0) { + this.hasNext = false; + } else { + this.offset += @as(usize, info.NextEntryOffset); + } + + return FileEvent{ + .action = action, + .filename = filename, + }; } + }; + + fn events(this: *DirWatcher) EventIterator { + return EventIterator{ .watcher = this }; } - fn listen(this: *DirWatcher) !void { + // invalidates any EventIterators derived from this DirWatcher + fn prepare(this: *DirWatcher) !void { const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION; if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, this.watch_subtree, filter, null, &this.overlapped, null) == 0) { const err = w.kernel32.GetLastError(); @@ -247,9 +270,6 @@ pub const WindowsWatcher = struct { }; pub fn init(this: *WindowsWatcher, allocator: std.mem.Allocator) !void { - // const watcher = try allocator.create(WindowsWatcher); - // errdefer allocator.destroy(watcher); - const iocp = try w.CreateIoCompletionPort(w.INVALID_HANDLE_VALUE, null, 0, 1); this.* = .{ .iocp = iocp, @@ -264,7 +284,9 @@ pub const WindowsWatcher = struct { w.kernel32.CloseHandle(this.iocp); } - pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !void { + pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !*DirWatcher { + _ = dirFd; + std.debug.print("adding directory to watch: {s}\n", .{std.unicode.fmtUtf16le(path)}); const path_len_bytes: u16 = @truncate(path.len * 2); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, @@ -273,12 +295,13 @@ pub const WindowsWatcher = struct { }; var attr = w.OBJECT_ATTRIBUTES{ .Length = @sizeOf(w.OBJECT_ATTRIBUTES), - .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(path)) - null - else if (dirFd == w.INVALID_HANDLE_VALUE) - std.fs.cwd().fd - else - dirFd, + .RootDirectory = null, + // .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(path)) + // null + // else if (dirFd == w.INVALID_HANDLE_VALUE) + // std.fs.cwd().fd + // else + // dirFd, .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. .ObjectName = &nt_name, .SecurityDescriptor = null, @@ -312,28 +335,33 @@ pub const WindowsWatcher = struct { const watcher = try this.allocator.create(DirWatcher); errdefer this.allocator.destroy(watcher); watcher.* = .{ .dirHandle = handle }; - // try this.watchers.put(key, watcher); - try watcher.listen(); + + std.debug.print("handle: {d}\n", .{@intFromPtr(handle)}); + + try watcher.prepare(); + return watcher; } - pub fn read(this: *WindowsWatcher) !void { + // get the next dirwatcher that has events + pub fn next(this: *WindowsWatcher) !*DirWatcher { var nbytes: w.DWORD = 0; var key: w.ULONG_PTR = 0; var overlapped: ?*w.OVERLAPPED = null; - switch (w.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, w.INFINITE)) { - .Normal => {}, - .Aborted => @panic("aborted"), - .Cancelled => @panic("cancelled"), - .EOF => @panic("eof"), - } - if (nbytes == 0) { - // exit notification for this watcher - we should probably deallocate it here - return; - } + while (true) { + switch (w.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, w.INFINITE)) { + .Normal => {}, + .Aborted => @panic("aborted"), + .Cancelled => @panic("cancelled"), + .EOF => @panic("eof"), + } + if (nbytes == 0) { + // exit notification for this watcher - we should probably deallocate it here + continue; + } - const watcher: *DirWatcher = @ptrCast(overlapped); - watcher.handleEvent(nbytes); - try watcher.listen(); + const watcher: *DirWatcher = @ptrCast(overlapped); + return watcher; + } } }; @@ -346,21 +374,6 @@ else if (Environment.isWindows) else @compileError("Unsupported platform"); -pub const WatchItem = struct { - file_path: string, - // filepath hash for quick comparison - hash: u32, - eventlist_index: PlatformWatcher.EventListIndex, - loader: options.Loader, - fd: bun.FileDescriptor, - count: u32, - parent_hash: u32, - kind: Kind, - package_json: ?*PackageJSON, - - pub const Kind = enum { file, directory }; -}; - pub const WatchEvent = struct { index: WatchItemIndex, op: Op, @@ -413,7 +426,6 @@ pub const WatchEvent = struct { this.* = WatchEvent{ .op = Op{ .delete = (event.mask & std.os.linux.IN.DELETE_SELF) > 0 or (event.mask & std.os.linux.IN.DELETE) > 0, - .metadata = false, .rename = (event.mask & std.os.linux.IN.MOVE_SELF) > 0, .move_to = (event.mask & std.os.linux.IN.MOVED_TO) > 0, .write = (event.mask & std.os.linux.IN.MODIFY) > 0, @@ -422,6 +434,18 @@ pub const WatchEvent = struct { }; } + pub fn fromFileNotify(this: *WatchEvent, event: *WindowsWatcher.FileEvent, index: WatchItemIndex) void { + const w = std.os.windows; + this.* = WatchEvent{ + .op = Op{ + .delete = event.Action == w.FILE_ACTION_REMOVED, + .rename = event.Action == w.FILE_ACTION_RENAMED_OLD_NAME, + .write = event.Action == w.FILE_ACTION_MODIFIED, + }, + .index = index, + }; + } + pub const Op = packed struct { delete: bool = false, metadata: bool = false, @@ -431,7 +455,40 @@ pub const WatchEvent = struct { }; }; -pub const Watchlist = std.MultiArrayList(WatchItem); +const WatchItem = struct { + file_path: string, + // filepath hash for quick comparison + hash: u32, + loader: options.Loader, + fd: bun.FileDescriptor, + count: u32, + parent_hash: u32, + kind: Kind, + package_json: ?*PackageJSON, + platform: Platform.Data = Platform.Data{}, + + const Platform = struct { + const Linux = struct { + eventlist_index: PlatformWatcher.EventListIndex = 0, + }; + const Windows = struct { + dir_watcher: ?*WindowsWatcher.DirWatcher = null, + }; + const Darwin = struct {}; + + const Data = if (Environment.isMac) + Darwin + else if (Environment.isLinux) + Linux + else if (Environment.isWindows) + Windows + else + @compileError("Unsupported platform"); + }; + + pub const Kind = enum { file, directory }; +}; +const Watchlist = std.MultiArrayList(WatchItem); pub fn NewWatcher(comptime ContextType: type) type { return struct { @@ -461,7 +518,7 @@ pub fn NewWatcher(comptime ContextType: type) type { pub const HashType = u32; pub const WatchListArray = Watchlist; - const NoWatchItem: WatchItemIndex = std.math.maxInt(WatchItemIndex); + const no_watch_item: WatchItemIndex = std.math.maxInt(WatchItemIndex); pub fn getHash(filepath: string) HashType { return @as(HashType, @truncate(bun.hash(filepath))); @@ -542,33 +599,6 @@ pub fn NewWatcher(comptime ContextType: type) type { allocator.destroy(this); } - pub fn remove(this: *Watcher, hash: HashType) void { - this.mutex.lock(); - defer this.mutex.unlock(); - if (this.indexOf(hash)) |index| { - const fds = this.watchlist.items(.fd); - const fd = fds[index]; - _ = bun.sys.close(fd); - this.watchlist.swapRemove(index); - } - } - - pub fn removeAtIndex(this: *Watcher, index: WatchItemIndex, hash: HashType, parents: []HashType, comptime kind: WatchItem.Kind) void { - std.debug.assert(index != NoWatchItem); - - this.evict_list[this.evict_list_i] = index; - this.evict_list_i += 1; - - if (comptime kind == .directory) { - for (parents) |parent| { - if (parent == hash) { - this.evict_list[this.evict_list_i] = @as(WatchItemIndex, @truncate(parent)); - this.evict_list_i += 1; - } - } - } - } - pub fn flushEvictions(this: *Watcher) void { if (this.evict_list_i == 0) return; defer this.evict_list_i = 0; @@ -585,23 +615,24 @@ pub fn NewWatcher(comptime ContextType: type) type { var slice = this.watchlist.slice(); const fds = slice.items(.fd); - var last_item = NoWatchItem; + var last_item = no_watch_item; for (this.evict_list[0..this.evict_list_i]) |item| { // catch duplicates, since the list is sorted, duplicates will appear right after each other if (item == last_item) continue; - // close the file descriptors here. this should automatically remove it from being watched too. - _ = bun.sys.close(fds[item]); - - // if (Environment.isLinux) { - // INotify.unwatch(event_list_ids[item]); - // } - + if (Environment.isWindows) { + // on windows we need to deallocate the watcher instance + // TODO implement this + } else { + // on mac and linux we can just close the file descriptor + // TODO do we need to call inotify_rm_watch on linux? + _ = bun.sys.close(fds[item]); + } last_item = item; } - last_item = NoWatchItem; + last_item = no_watch_item; // This is split into two passes because reading the slice while modified is potentially unsafe. for (this.evict_list[0..this.evict_list_i]) |item| { if (item == last_item) continue; @@ -754,45 +785,18 @@ pub fn NewWatcher(comptime ContextType: type) type { } } } else if (Environment.isWindows) { - @panic("todo windows watchloop"); - } - } - - pub fn indexOf(this: *Watcher, hash: HashType) ?u32 { - for (this.watchlist.items(.hash), 0..) |other, i| { - if (hash == other) { - return @as(u32, @truncate(i)); - } - } - return null; - } - - pub fn addFile( - this: *Watcher, - fd: bun.FileDescriptor, - file_path: string, - hash: HashType, - loader: options.Loader, - dir_fd: bun.FileDescriptor, - package_json: ?*PackageJSON, - comptime copy_file_path: bool, - ) !void { - // This must lock due to concurrent transpiler - this.mutex.lock(); - defer this.mutex.unlock(); - - if (this.indexOf(hash)) |index| { - if (comptime FeatureFlags.atomic_file_watcher) { - // On Linux, the file descriptor might be out of date. - if (fd.int() > 0) { - var fds = this.watchlist.items(.fd); - fds[index] = fd; + while (true) { + const watcher = try this.platform.next(); + // after handling the watcher's events, it explicitly needs to start reading directory changes again + defer watcher.prepare() catch |err| { + Output.prettyErrorln("Failed to (re-)start listening to directory changes: {s}", .{@errorName(err)}); + }; + var iter = watcher.events(); + while (iter.next()) |event| { + std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); } } - return; } - - try this.appendFileMaybeLock(fd, file_path, hash, loader, dir_fd, package_json, copy_file_path, false); } fn appendFileAssumeCapacity( @@ -805,7 +809,6 @@ pub fn NewWatcher(comptime ContextType: type) type { package_json: ?*PackageJSON, comptime copy_file_path: bool, ) !void { - var index: PlatformWatcher.EventListIndex = std.math.maxInt(PlatformWatcher.EventListIndex); const watchlist_id = this.watchlist.len; const file_path_: string = if (comptime copy_file_path) @@ -813,6 +816,17 @@ pub fn NewWatcher(comptime ContextType: type) type { else file_path; + var item = WatchItem{ + .file_path = file_path_, + .fd = fd, + .hash = hash, + .count = 0, + .loader = loader, + .parent_hash = parent_hash, + .package_json = package_json, + .kind = .file, + }; + if (comptime Environment.isMac) { const KEvent = std.c.Kevent; @@ -851,20 +865,18 @@ pub fn NewWatcher(comptime ContextType: type) type { // buf[file_path_to_use_.len] = 0; var buf = file_path_.ptr; const slice: [:0]const u8 = buf[0..file_path_.len :0]; - index = try this.platform.watchPath(slice); + item.platform.index = try this.platform.watchPath(slice); + } else if (comptime Environment.isWindows) { + // TODO check if we're already watching a parent directory of this file + var pathbuf: bun.WPathBuffer = undefined; + const dirpath = std.fs.path.dirnameWindows(file_path) orelse unreachable; + const wpath = bun.strings.toNTPath(&pathbuf, dirpath); + const watcher = try this.platform.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, wpath); + item.platform.dir_watcher = watcher; + std.debug.print("watching file: {s}\n", .{file_path_}); } - this.watchlist.appendAssumeCapacity(.{ - .file_path = file_path_, - .fd = fd, - .hash = hash, - .count = 0, - .eventlist_index = index, - .loader = loader, - .parent_hash = parent_hash, - .package_json = package_json, - .kind = .file, - }); + this.watchlist.appendAssumeCapacity(item); } fn appendDirectoryAssumeCapacity( @@ -881,7 +893,6 @@ pub fn NewWatcher(comptime ContextType: type) type { }; const parent_hash = Watcher.getHash(bun.fs.PathName.init(file_path).dirWithTrailingSlash()); - var index: PlatformWatcher.EventListIndex = std.math.maxInt(PlatformWatcher.EventListIndex); const file_path_: string = if (comptime copy_file_path) bun.asByteSlice(try this.allocator.dupeZ(u8, file_path)) @@ -890,6 +901,17 @@ pub fn NewWatcher(comptime ContextType: type) type { const watchlist_id = this.watchlist.len; + var item = WatchItem{ + .file_path = file_path_, + .fd = fd, + .hash = hash, + .count = 0, + .loader = options.Loader.file, + .parent_hash = parent_hash, + .kind = .directory, + .package_json = null, + }; + if (Environment.isMac) { const KEvent = std.c.Kevent; @@ -931,46 +953,18 @@ pub fn NewWatcher(comptime ContextType: type) type { bun.copy(u8, &buf, file_path_to_use_); buf[file_path_to_use_.len] = 0; const slice: [:0]u8 = buf[0..file_path_to_use_.len :0]; - index = try this.platform.watchDir(slice); + item.platform.eventlist_index = try this.platform.watchDir(slice); + } else if (Environment.isWindows) { + var pathbuf: bun.WPathBuffer = undefined; + const wpath = bun.strings.toNTPath(&pathbuf, file_path_); + const watcher = try this.platform.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, wpath); + item.platform.dir_watcher = watcher; } - this.watchlist.appendAssumeCapacity(.{ - .file_path = file_path_, - .fd = fd, - .hash = hash, - .count = 0, - .eventlist_index = index, - .loader = options.Loader.file, - .parent_hash = parent_hash, - .kind = .directory, - .package_json = null, - }); + this.watchlist.appendAssumeCapacity(item); return @as(WatchItemIndex, @truncate(this.watchlist.len - 1)); } - pub inline fn isEligibleDirectory(this: *Watcher, dir: string) bool { - return strings.indexOf(dir, this.fs.top_level_dir) != null and strings.indexOf(dir, "node_modules") == null; - } - - pub fn addDirectory( - this: *Watcher, - fd: bun.FileDescriptor, - file_path: string, - hash: HashType, - comptime copy_file_path: bool, - ) !void { - this.mutex.lock(); - defer this.mutex.unlock(); - - if (this.indexOf(hash) != null) { - return; - } - - try this.watchlist.ensureUnusedCapacity(this.allocator, 1); - - _ = try this.appendDirectoryAssumeCapacity(fd, file_path, hash, copy_file_path); - } - pub fn appendFileMaybeLock( this: *Watcher, fd: bun.FileDescriptor, @@ -1034,6 +1028,12 @@ pub fn NewWatcher(comptime ContextType: type) type { } } + // Below is platform-independent + + inline fn isEligibleDirectory(this: *Watcher, dir: string) bool { + return strings.indexOf(dir, this.fs.top_level_dir) != null and strings.indexOf(dir, "node_modules") == null; + } + pub fn appendFile( this: *Watcher, fd: bun.FileDescriptor, @@ -1046,5 +1046,89 @@ pub fn NewWatcher(comptime ContextType: type) type { ) !void { return appendFileMaybeLock(this, fd, file_path, hash, loader, dir_fd, package_json, copy_file_path, true); } + + pub fn addDirectory( + this: *Watcher, + fd: bun.FileDescriptor, + file_path: string, + hash: HashType, + comptime copy_file_path: bool, + ) !void { + this.mutex.lock(); + defer this.mutex.unlock(); + + if (this.indexOf(hash) != null) { + return; + } + + try this.watchlist.ensureUnusedCapacity(this.allocator, 1); + + _ = try this.appendDirectoryAssumeCapacity(fd, file_path, hash, copy_file_path); + } + + pub fn addFile( + this: *Watcher, + fd: bun.FileDescriptor, + file_path: string, + hash: HashType, + loader: options.Loader, + dir_fd: bun.FileDescriptor, + package_json: ?*PackageJSON, + comptime copy_file_path: bool, + ) !void { + // This must lock due to concurrent transpiler + this.mutex.lock(); + defer this.mutex.unlock(); + + if (this.indexOf(hash)) |index| { + if (comptime FeatureFlags.atomic_file_watcher) { + // On Linux, the file descriptor might be out of date. + if (fd.int() > 0) { + var fds = this.watchlist.items(.fd); + fds[index] = fd; + } + } + return; + } + + try this.appendFileMaybeLock(fd, file_path, hash, loader, dir_fd, package_json, copy_file_path, false); + } + + pub fn indexOf(this: *Watcher, hash: HashType) ?u32 { + for (this.watchlist.items(.hash), 0..) |other, i| { + if (hash == other) { + return @as(u32, @truncate(i)); + } + } + return null; + } + + pub fn remove(this: *Watcher, hash: HashType) void { + this.mutex.lock(); + defer this.mutex.unlock(); + if (this.indexOf(hash)) |index| { + this.removeAtIndex(@truncate(index), hash, &[_]HashType{}, .file); + // const fds = this.watchlist.items(.fd); + // const fd = fds[index]; + // _ = bun.sys.close(fd); + // this.watchlist.swapRemove(index); + } + } + + pub fn removeAtIndex(this: *Watcher, index: WatchItemIndex, hash: HashType, parents: []HashType, comptime kind: WatchItem.Kind) void { + std.debug.assert(index != no_watch_item); + + this.evict_list[this.evict_list_i] = index; + this.evict_list_i += 1; + + if (comptime kind == .directory) { + for (parents) |parent| { + if (parent == hash) { + this.evict_list[this.evict_list_i] = @as(WatchItemIndex, @truncate(parent)); + this.evict_list_i += 1; + } + } + } + } }; } diff --git a/watch.zig b/watch.zig index c31a82d347147b..81f543dae58e0e 100644 --- a/watch.zig +++ b/watch.zig @@ -115,7 +115,10 @@ pub const WindowsWatcher = struct { fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { const elapsed = clock1.read(); std.debug.print("elapsed: {}\n", .{std.fmt.fmtDuration(elapsed)}); - if (nbytes == 0) return; + if (nbytes == 0) { + std.debug.print("nbytes == 0\n", .{}); + return; + } var offset: usize = 0; while (true) { const info_size = @sizeOf(w.FILE_NOTIFY_INFORMATION); @@ -160,7 +163,8 @@ pub const WindowsWatcher = struct { w.kernel32.CloseHandle(this.iocp); } - pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !void { + pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !*DirWatcher { + std.debug.print("adding directory to watch: {s}\n", .{std.unicode.fmtUtf16le(path)}); const flags = w.FILE_LIST_DIRECTORY; const path_len_bytes: u16 = @truncate(path.len * 2); @@ -207,11 +211,15 @@ pub const WindowsWatcher = struct { this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); + std.debug.print("handle: {d}\n", .{@intFromPtr(handle)}); + const watcher = try this.allocator.create(DirWatcher); errdefer this.allocator.destroy(watcher); watcher.* = .{ .dirHandle = handle }; // try this.watchers.put(key, watcher); try watcher.listen(); + + return watcher; } pub fn stop(this: *WindowsWatcher) void { @@ -255,28 +263,33 @@ pub fn main() !void { var buf: [std.fs.MAX_PATH_BYTES]u16 = undefined; - const watchdir = "C:\\test\\node_modules"; + const watchdir = "C:\\bun"; const iconsdir = "C:\\test\\node_modules\\@mui\\icons-material"; _ = iconsdir; // autofix const testdir = "C:\\test\\node_modules\\@mui\\icons-material\\mydir"; _ = testdir; // autofix const testfile = "C:\\test\\node_modules\\@mui\\icons-material\\myfile.txt"; + _ = testfile; // autofix // try std.fs.deleteDirAbsolute(dir); const watcher = try WindowsWatcher.init(allocator); - try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, watchdir)); + var handle = try std.Thread.spawn(.{}, WindowsWatcher.run, .{watcher}); + std.time.sleep(100_000_000); + const watched = try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, watchdir)); // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\src")); // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\test")); // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\testdir")); - const file = try std.fs.createFileAbsolute(testfile, .{}); - var handle = try std.Thread.spawn(.{}, WindowsWatcher.run, .{watcher}); + // const file = try std.fs.createFileAbsolute(testfile, .{}); std.debug.print("watcher started\n", .{}); clock1 = try std.time.Timer.start(); // try std.fs.makeDirAbsolute(dir); - try file.writeAll(&data); + // try file.writeAll(&data); + + // _ = std.os.windows.ntdll.NtClose(watched.dirHandle); + _ = watched; handle.join(); } From c039678b94ff7bf96a728dfa22283e9c176dc5ea Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 18:10:06 -0800 Subject: [PATCH 10/32] fix visibility --- src/watcher.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index ad3857c7cff797..c783f6809acbcf 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -455,7 +455,7 @@ pub const WatchEvent = struct { }; }; -const WatchItem = struct { +pub const WatchItem = struct { file_path: string, // filepath hash for quick comparison hash: u32, @@ -488,7 +488,8 @@ const WatchItem = struct { pub const Kind = enum { file, directory }; }; -const Watchlist = std.MultiArrayList(WatchItem); + +pub const Watchlist = std.MultiArrayList(WatchItem); pub fn NewWatcher(comptime ContextType: type) type { return struct { From 81db5177279028dc89c861829d9b6439785cb033 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 18:47:19 -0800 Subject: [PATCH 11/32] watcher starts without error --- src/watcher.zig | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index c783f6809acbcf..0eccaebf909483 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -222,7 +222,8 @@ pub const WindowsWatcher = struct { // each directory being watched has an associated DirWatcher const DirWatcher = extern struct { // this must be the first field because we retrieve the DirWatcher from a pointer to its overlapped field - overlapped: w.OVERLAPPED = undefined, + // also, even though it is never read or written, it must be initialized to zero, otherwise ReadDirectoryChangesW will fail with INVALID_HANDLE + overlapped: w.OVERLAPPED = std.mem.zeroes(w.OVERLAPPED), buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, dirHandle: w.HANDLE, watch_subtree: w.BOOLEAN = 1, @@ -328,8 +329,9 @@ pub const WindowsWatcher = struct { @panic("failed to open directory for watching"); } - errdefer _ = w.kernel32.CloseHandle(handle); + // errdefer _ = w.kernel32.CloseHandle(handle); + // TODO atomic update this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); const watcher = try this.allocator.create(DirWatcher); @@ -339,6 +341,8 @@ pub const WindowsWatcher = struct { std.debug.print("handle: {d}\n", .{@intFromPtr(handle)}); try watcher.prepare(); + + // TODO signal the watcher loop thread to start reading from the new iocp return watcher; } @@ -786,17 +790,18 @@ pub fn NewWatcher(comptime ContextType: type) type { } } } else if (Environment.isWindows) { - while (true) { - const watcher = try this.platform.next(); - // after handling the watcher's events, it explicitly needs to start reading directory changes again - defer watcher.prepare() catch |err| { - Output.prettyErrorln("Failed to (re-)start listening to directory changes: {s}", .{@errorName(err)}); - }; - var iter = watcher.events(); - while (iter.next()) |event| { - std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); - } - } + while (true) {} + // while (true) { + // const watcher = try this.platform.next(); + // // after handling the watcher's events, it explicitly needs to start reading directory changes again + // defer watcher.prepare() catch |err| { + // Output.prettyErrorln("Failed to (re-)start listening to directory changes: {s}", .{@errorName(err)}); + // }; + // var iter = watcher.events(); + // while (iter.next()) |event| { + // std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); + // } + // } } } From b1d097a46ed472250fca05118866279ef41050b8 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Mon, 29 Jan 2024 18:49:34 -0800 Subject: [PATCH 12/32] printing changes works --- src/watcher.zig | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index 0eccaebf909483..502554a211c322 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -329,7 +329,7 @@ pub const WindowsWatcher = struct { @panic("failed to open directory for watching"); } - // errdefer _ = w.kernel32.CloseHandle(handle); + errdefer _ = w.kernel32.CloseHandle(handle); // TODO atomic update this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); @@ -790,18 +790,17 @@ pub fn NewWatcher(comptime ContextType: type) type { } } } else if (Environment.isWindows) { - while (true) {} - // while (true) { - // const watcher = try this.platform.next(); - // // after handling the watcher's events, it explicitly needs to start reading directory changes again - // defer watcher.prepare() catch |err| { - // Output.prettyErrorln("Failed to (re-)start listening to directory changes: {s}", .{@errorName(err)}); - // }; - // var iter = watcher.events(); - // while (iter.next()) |event| { - // std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); - // } - // } + while (true) { + const watcher = try this.platform.next(); + // after handling the watcher's events, it explicitly needs to start reading directory changes again + defer watcher.prepare() catch |err| { + Output.prettyErrorln("Failed to (re-)start listening to directory changes: {s}", .{@errorName(err)}); + }; + var iter = watcher.events(); + while (iter.next()) |event| { + std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); + } + } } } From 04501a3f29a71e0722e46138fb79da21afeeef3b Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Tue, 30 Jan 2024 17:42:40 -0800 Subject: [PATCH 13/32] basic windows watching works --- src/bun.js/javascript.zig | 4 + src/bun.js/node/win_watcher.zig | 13 ++ src/bun.zig | 143 ++++++++++++++++++-- src/cli.zig | 8 ++ src/string_immutable.zig | 16 +++ src/watcher.zig | 227 +++++++++++++++++++++++++------- src/windows.zig | 16 ++- watch.zig | 4 +- 8 files changed, 369 insertions(+), 62 deletions(-) diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index d06d8b0701e555..59d7c56e58bbcb 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -3314,6 +3314,10 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime } }, .directory => { + if (comptime Environment.isWindows) { + // for now ignore directory updates on windows + continue; + } var affected_buf: [128][]const u8 = undefined; var entries_option: ?*Fs.FileSystem.RealFS.EntriesOption = null; diff --git a/src/bun.js/node/win_watcher.zig b/src/bun.js/node/win_watcher.zig index c94608d3efaadc..14284baf5a1661 100644 --- a/src/bun.js/node/win_watcher.zig +++ b/src/bun.js/node/win_watcher.zig @@ -223,6 +223,19 @@ pub const PathWatcherManager = struct { this.destroy(); } + + // TODO figure out what win_watcher even does... + pub fn onFileUpdate( + this: *@This(), + events: []GenericWatcher.WatchEvent, + changed_files: []?[:0]u8, + watchlist: GenericWatcher.Watchlist, + ) void { + _ = this; // autofix + _ = events; // autofix + _ = changed_files; // autofix + _ = watchlist; // autofix + } }; pub const PathWatcher = struct { diff --git a/src/bun.zig b/src/bun.zig index b67ca73bb80004..78398599b28697 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1369,9 +1369,26 @@ pub const failing_allocator = std.mem.Allocator{ .ptr = undefined, .vtable = &.{ pub fn reloadProcess( allocator: std.mem.Allocator, clear_terminal: bool, -) void { - const PosixSpawn = posix.spawn; +) noreturn { + if (clear_terminal) { + Output.flush(); + Output.disableBuffering(); + Output.resetTerminalAll(); + } const bun = @This(); + + if (comptime Environment.isWindows) { + // this assumes that our parent process assigned us to a job object (see runWatcherManager) + var procinfo: std.os.windows.PROCESS_INFORMATION = undefined; + win32.spawnProcessCopy(allocator, &procinfo, false, false) catch @panic("Unexpected error while reloading process\n"); + + std.debug.print("exiting\n", .{}); + + // terminate the current process + _ = bun.windows.TerminateProcess(@ptrFromInt(std.math.maxInt(usize)), 0); + Output.panic("Unexpected error while reloading process\n", .{}); + } + const PosixSpawn = posix.spawn; const dupe_argv = allocator.allocSentinel(?[*:0]const u8, bun.argv().len, null) catch unreachable; for (bun.argv(), dupe_argv) |src, *dest| { dest.* = (allocator.dupeZ(u8, src) catch unreachable).ptr; @@ -1396,13 +1413,6 @@ pub fn reloadProcess( // we clone envp so that the memory address of environment variables isn't the same as the libc one const envp = @as([*:null]?[*:0]const u8, @ptrCast(environ.ptr)); - // Clear the terminal - if (clear_terminal) { - Output.flush(); - Output.disableBuffering(); - Output.resetTerminalAll(); - } - // macOS doesn't have CLOEXEC, so we must go through posix_spawn if (comptime Environment.isMac) { var actions = PosixSpawn.Actions.init() catch unreachable; @@ -1839,10 +1849,13 @@ pub const posix = struct { }; pub const win32 = struct { + const w = std.os.windows; pub var STDOUT_FD: FileDescriptor = undefined; pub var STDERR_FD: FileDescriptor = undefined; pub var STDIN_FD: FileDescriptor = undefined; + const watcherChildEnv: [:0]const u16 = strings.toUTF16LiteralZ("_BUN_WATCHER_CHILD"); + pub fn stdio(i: anytype) FileDescriptor { return switch (i) { 0 => STDIN_FD, @@ -1851,6 +1864,118 @@ pub const win32 = struct { else => @panic("Invalid stdio fd"), }; } + + pub fn isWatcherChild() bool { + var buf: [1024]u16 = undefined; + return w.kernel32.GetEnvironmentVariableW(@constCast(watcherChildEnv.ptr), &buf, 1024) > 0; + } + + pub fn becomeWatcherManager(allocator: std.mem.Allocator) noreturn { + // this process will be the parent of the child process that actually runs the script + const job = windows.CreateJobObjectA(null, null); + var procinfo: std.os.windows.PROCESS_INFORMATION = undefined; + spawnProcessCopy(allocator, &procinfo, true, true) catch @panic("Failed to spawn process"); + if (windows.AssignProcessToJobObject(job, procinfo.hProcess) == 0) { + @panic("Failed to assign process to job object"); + } + if (windows.ResumeThread(procinfo.hThread) == 0) { + @panic("Failed to resume thread"); + } + std.debug.print("waiting for job object\n", .{}); + _ = windows.WaitForSingleObject(job, std.os.windows.INFINITE); + @panic("waitforsingleobject returned"); + } + + pub fn spawnProcessCopy( + allocator: std.mem.Allocator, + procinfo: *std.os.windows.PROCESS_INFORMATION, + suspended: bool, + setChild: bool, + ) !void { + var flags: std.os.windows.DWORD = w.CREATE_UNICODE_ENVIRONMENT; + if (suspended) { + // see CREATE_SUSPENDED at + // https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + flags |= 0x00000004; + } + + const image_path = &w.peb().ProcessParameters.ImagePathName; + var wbuf: WPathBuffer = undefined; + @memcpy(wbuf[0..image_path.Length], image_path.Buffer); + wbuf[image_path.Length] = 0; + + const image_pathZ = wbuf[0..image_path.Length :0]; + + // TODO environment variables + + const kernelenv = w.kernel32.GetEnvironmentStringsW(); + var newenv: ?[]u16 = null; + defer { + if (kernelenv) |envptr| { + _ = w.kernel32.FreeEnvironmentStringsW(envptr); + } + if (newenv) |ptr| { + allocator.free(ptr); + } + } + + if (setChild) { + if (kernelenv) |ptr| { + var size: usize = 0; + // array is terminated by two nulls + while (ptr[size] != 0 or ptr[size + 1] != 0) size += 1; + size += 1; + // now ptr + size points to the first null + + const buf = try allocator.alloc(u16, size + watcherChildEnv.len + 4); + @memcpy(buf[0..size], ptr); + @memcpy(buf[size .. size + watcherChildEnv.len], watcherChildEnv); + buf[size + watcherChildEnv.len] = '='; + buf[size + watcherChildEnv.len + 1] = '1'; + buf[size + watcherChildEnv.len + 2] = 0; + buf[size + watcherChildEnv.len + 3] = 0; + newenv = buf; + } + } + + const env: ?[*]u16 = if (newenv) |e| e.ptr else kernelenv; + + var startupinfo = w.STARTUPINFOW{ + .cb = @sizeOf(w.STARTUPINFOW), + .lpReserved = null, + .lpDesktop = null, + .lpTitle = null, + .dwX = 0, + .dwY = 0, + .dwXSize = 0, + .dwYSize = 0, + .dwXCountChars = 0, + .dwYCountChars = 0, + .dwFillAttribute = 0, + .dwFlags = w.STARTF_USESTDHANDLES, + .wShowWindow = 0, + .cbReserved2 = 0, + .lpReserved2 = null, + .hStdInput = std.io.getStdIn().handle, + .hStdOutput = std.io.getStdOut().handle, + .hStdError = std.io.getStdErr().handle, + }; + const rc = w.kernel32.CreateProcessW( + image_pathZ, + w.kernel32.GetCommandLineW(), + null, + null, + 1, + flags, + env, + null, + &startupinfo, + procinfo, + ); + if (rc == 0) { + Output.panic("Unexpected error while reloading process\n", .{}); + } + } }; pub usingnamespace if (@import("builtin").target.os.tag != .windows) posix else win32; diff --git a/src/cli.zig b/src/cli.zig index ecb1cbed3a020d..c7dd9f537fb822 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -1125,6 +1125,14 @@ pub const Command = struct { if (comptime Command.Tag.uses_global_options.get(command)) { ctx.args = try Arguments.parse(allocator, &ctx, command); } + + if (comptime Environment.isWindows) { + if (ctx.debug.hot_reload == .watch and !bun.isWatcherChild()) { + // this is noreturn + bun.becomeWatcherManager(allocator); + } + } + return ctx; } }; diff --git a/src/string_immutable.zig b/src/string_immutable.zig index 3b55d41bebce90..20af0c604fd86a 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -54,6 +54,22 @@ pub fn toUTF16Literal(comptime str: []const u8) []const u16 { }; } +pub fn toUTF16LiteralZ(comptime str: []const u8) [:0]const u16 { + return comptime brk: { + comptime var output: [str.len + 1]u16 = undefined; + + for (str, 0..) |c, i| { + output[i] = c; + } + output[str.len] = 0; + + const Static = struct { + pub const literal: [:0]const u16 = output[0..str.len :0]; + }; + break :brk Static.literal; + }; +} + pub const OptionalUsize = std.meta.Int(.unsigned, @bitSizeOf(usize) - 1); pub fn indexOfAny(slice: string, comptime str: anytype) ?OptionalUsize { switch (comptime str.len) { diff --git a/src/watcher.zig b/src/watcher.zig index 502554a211c322..5071787f059cf3 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -199,9 +199,9 @@ const DarwinWatcher = struct { }; pub const WindowsWatcher = struct { - // TODO iocp needs to be updated atomically to allow the main thread to register new watchers while the watcher thread is running iocp: w.HANDLE = undefined, allocator: std.mem.Allocator = undefined, + watchers: std.ArrayListUnmanaged(*DirWatcher) = std.ArrayListUnmanaged(*DirWatcher){}, const w = std.os.windows; pub const EventListIndex = c_int; @@ -220,13 +220,15 @@ pub const WindowsWatcher = struct { }; // each directory being watched has an associated DirWatcher - const DirWatcher = extern struct { - // this must be the first field because we retrieve the DirWatcher from a pointer to its overlapped field - // also, even though it is never read or written, it must be initialized to zero, otherwise ReadDirectoryChangesW will fail with INVALID_HANDLE + const DirWatcher = struct { + // must be initialized to zero (even though it's never read or written in our code), + // otherwise ReadDirectoryChangesW will fail with INVALID_HANDLE overlapped: w.OVERLAPPED = std.mem.zeroes(w.OVERLAPPED), buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, dirHandle: w.HANDLE, - watch_subtree: w.BOOLEAN = 1, + path: [:0]u16, + path_buf: bun.WPathBuffer = undefined, + refcount: usize = 1, const EventIterator = struct { watcher: *DirWatcher, @@ -255,6 +257,12 @@ pub const WindowsWatcher = struct { } }; + fn fromOverlapped(overlapped: *w.OVERLAPPED) *DirWatcher { + const offset = @offsetOf(DirWatcher, "overlapped"); + const overlapped_byteptr: [*]u8 = @ptrCast(overlapped); + return @alignCast(@ptrCast(overlapped_byteptr - offset)); + } + fn events(this: *DirWatcher) EventIterator { return EventIterator{ .watcher = this }; } @@ -262,12 +270,28 @@ pub const WindowsWatcher = struct { // invalidates any EventIterators derived from this DirWatcher fn prepare(this: *DirWatcher) !void { const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION; - if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, this.watch_subtree, filter, null, &this.overlapped, null) == 0) { + if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, 1, filter, null, &this.overlapped, null) == 0) { const err = w.kernel32.GetLastError(); std.debug.print("failed to start watching directory: {s}\n", .{@tagName(err)}); @panic("failed to start watching directory"); } } + + fn ref(this: *DirWatcher) void { + std.debug.assert(this.refcount > 0); + this.refcount += 1; + } + + fn unref(this: *DirWatcher) void { + std.debug.assert(this.refcount > 0); + this.refcount -= 1; + // TODO if refcount reaches 0 we should deallocate + // But we can't deallocate right away because we might be in the middle of iterating over the events of this watcher + // we probably need some sort of queue that can be emptied by the watcher thread. + if (this.refcount == 0) { + std.debug.print("TODO: deallocate watcher\n", .{}); + } + } }; pub fn init(this: *WindowsWatcher, allocator: std.mem.Allocator) !void { @@ -285,7 +309,7 @@ pub const WindowsWatcher = struct { w.kernel32.CloseHandle(this.iocp); } - pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !*DirWatcher { + fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !*DirWatcher { _ = dirFd; std.debug.print("adding directory to watch: {s}\n", .{std.unicode.fmtUtf16le(path)}); const path_len_bytes: u16 = @truncate(path.len * 2); @@ -331,40 +355,81 @@ pub const WindowsWatcher = struct { errdefer _ = w.kernel32.CloseHandle(handle); - // TODO atomic update - this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); + // on success we receive the same iocp handle back that we put in - no need to update it + _ = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); const watcher = try this.allocator.create(DirWatcher); errdefer this.allocator.destroy(watcher); - watcher.* = .{ .dirHandle = handle }; - - std.debug.print("handle: {d}\n", .{@intFromPtr(handle)}); + watcher.* = .{ .dirHandle = handle, .path = undefined }; + // init path + @memcpy(watcher.path_buf[0..path.len], path); + watcher.path_buf[path.len] = 0; + watcher.path = watcher.path_buf[0..path.len :0]; + // TODO think about the different sequences of errors try watcher.prepare(); + try this.watchers.append(this.allocator, watcher); - // TODO signal the watcher loop thread to start reading from the new iocp return watcher; } + pub fn watchFile(this: *WindowsWatcher, path: []const u8) !*DirWatcher { + const dirpath = std.fs.path.dirnameWindows(path) orelse @panic("get dir from file"); + std.debug.print("path: {s}, dirpath: {s}\n", .{ path, dirpath }); + return this.watchDir(dirpath); + } + + pub fn watchDir(this: *WindowsWatcher, path_: []const u8) !*DirWatcher { + // strip the trailing slash if it exists + var path = path_; + if (path.len > 0 and bun.strings.charIsAnySlash(path[path.len - 1])) { + path = path[0 .. path.len - 1]; + } + var pathbuf: bun.WPathBuffer = undefined; + const wpath = bun.strings.toNTPath(&pathbuf, path); + // check if one of the existing watchers covers this path + for (this.watchers.items) |watcher| { + if (std.mem.indexOf(u16, watcher.path, wpath) == 0) { + std.debug.print("found existing watcher\n", .{}); + watcher.ref(); + return watcher; + } + } + return this.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, wpath); + } + + const Timeout = enum(w.DWORD) { + infinite = w.INFINITE, + minimal = 1, + none = 0, + }; + // get the next dirwatcher that has events - pub fn next(this: *WindowsWatcher) !*DirWatcher { + pub fn next(this: *WindowsWatcher, timeout: Timeout) !?*DirWatcher { var nbytes: w.DWORD = 0; var key: w.ULONG_PTR = 0; var overlapped: ?*w.OVERLAPPED = null; while (true) { - switch (w.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, w.INFINITE)) { - .Normal => {}, - .Aborted => @panic("aborted"), - .Cancelled => @panic("cancelled"), - .EOF => @panic("eof"), + const rc = w.kernel32.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, @intFromEnum(timeout)); + if (rc == 0) { + const err = w.kernel32.GetLastError(); + if (err == w.Win32Error.IMEOUT) { + return null; + } else { + @panic("GetQueuedCompletionStatus failed"); + } } + + // exit notification for this watcher - we should probably deallocate it here if (nbytes == 0) { - // exit notification for this watcher - we should probably deallocate it here continue; } - - const watcher: *DirWatcher = @ptrCast(overlapped); - return watcher; + if (overlapped) |ptr| { + return DirWatcher.fromOverlapped(ptr); + } else { + // this would be an error which we should probaby signal + continue; + } } } }; @@ -438,13 +503,12 @@ pub const WatchEvent = struct { }; } - pub fn fromFileNotify(this: *WatchEvent, event: *WindowsWatcher.FileEvent, index: WatchItemIndex) void { - const w = std.os.windows; + pub fn fromFileNotify(this: *WatchEvent, event: WindowsWatcher.FileEvent, index: WatchItemIndex) void { this.* = WatchEvent{ .op = Op{ - .delete = event.Action == w.FILE_ACTION_REMOVED, - .rename = event.Action == w.FILE_ACTION_RENAMED_OLD_NAME, - .write = event.Action == w.FILE_ACTION_MODIFIED, + .delete = event.action == .Removed, + .rename = event.action == .RenamedOld, + .write = event.action == .Modified, }, .index = index, }; @@ -620,6 +684,7 @@ pub fn NewWatcher(comptime ContextType: type) type { var slice = this.watchlist.slice(); const fds = slice.items(.fd); + const platform_data = slice.items(.platform); var last_item = no_watch_item; for (this.evict_list[0..this.evict_list_i]) |item| { @@ -629,6 +694,9 @@ pub fn NewWatcher(comptime ContextType: type) type { if (Environment.isWindows) { // on windows we need to deallocate the watcher instance // TODO implement this + if (platform_data[item].dir_watcher) |watcher| { + watcher.unref(); + } } else { // on mac and linux we can just close the file descriptor // TODO do we need to call inotify_rm_watch on linux? @@ -790,16 +858,84 @@ pub fn NewWatcher(comptime ContextType: type) type { } } } else if (Environment.isWindows) { - while (true) { - const watcher = try this.platform.next(); - // after handling the watcher's events, it explicitly needs to start reading directory changes again - defer watcher.prepare() catch |err| { - Output.prettyErrorln("Failed to (re-)start listening to directory changes: {s}", .{@errorName(err)}); - }; - var iter = watcher.events(); - while (iter.next()) |event| { - std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); + restart: while (true) { + var buf: bun.PathBuffer = undefined; + var event_id: usize = 0; + + // first wait has infinite timeout - we're waiting for the next event and don't want to spin + var timeout = WindowsWatcher.Timeout.infinite; + while (true) { + // std.debug.print("waiting with timeout: {s}\n", .{@tagName(timeout)}); + const watcher = try this.platform.next(timeout) orelse break; + // after handling the watcher's events, it explicitly needs to start reading directory changes again + defer watcher.prepare() catch |err| { + Output.prettyErrorln("Failed to (re-)start listening to directory changes: {s}", .{@errorName(err)}); + }; + + // after the first wait, we want to start coalescing events, so we wait for a minimal amount of time + timeout = WindowsWatcher.Timeout.minimal; + + const item_paths = this.watchlist.items(.file_path); + + std.debug.print("event from watcher: {s}\n", .{std.unicode.fmtUtf16le(watcher.path)}); + var iter = watcher.events(); + while (iter.next()) |event| { + std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); + // convert the current event file path to utf-8 + // skip the \??\ prefix + var idx = bun.simdutf.convert.utf16.to.utf8.le(watcher.path[4..], &buf); + buf[idx] = '\\'; + idx += 1; + idx += bun.simdutf.convert.utf16.to.utf8.le(event.filename, buf[idx..]); + const eventpath = buf[0..idx]; + + std.debug.print("eventpath: {s}\n", .{eventpath}); + + // TODO this really needs a more sophisticated search algorithm + for (item_paths, 0..) |path, item_idx| { + std.debug.print("path: {s}\n", .{path}); + // check if the current change applies to this item + // if so, add it to the eventlist + if (std.mem.indexOf(u8, path, eventpath) == 0) { + // this.changed_filepaths[event_id] = path; + this.watch_events[event_id].fromFileNotify(event, @truncate(item_idx)); + event_id += 1; + } + } + } } + if (event_id == 0) { + continue :restart; + } + + std.debug.print("event_id: {d}\n", .{event_id}); + + var all_events = this.watch_events[0..event_id]; + std.sort.pdq(WatchEvent, all_events, {}, WatchEvent.sortByIndex); + + var last_event_index: usize = 0; + var last_event_id: INotify.EventListIndex = std.math.maxInt(INotify.EventListIndex); + + for (all_events, 0..) |_, i| { + // if (all_events[i].name_len > 0) { + // this.changed_filepaths[name_off] = temp_name_list[all_events[i].name_off]; + // all_events[i].name_off = name_off; + // name_off += 1; + // } + + if (all_events[i].index == last_event_id) { + all_events[last_event_index].merge(all_events[i]); + continue; + } + last_event_index = i; + last_event_id = all_events[i].index; + } + if (all_events.len == 0) continue :restart; + all_events = all_events[0 .. last_event_index + 1]; + + std.debug.print("all_events.len: {d}\n", .{all_events.len}); + + this.ctx.onFileUpdate(all_events, this.changed_filepaths[0 .. last_event_index + 1], this.watchlist); } } } @@ -872,13 +1008,7 @@ pub fn NewWatcher(comptime ContextType: type) type { const slice: [:0]const u8 = buf[0..file_path_.len :0]; item.platform.index = try this.platform.watchPath(slice); } else if (comptime Environment.isWindows) { - // TODO check if we're already watching a parent directory of this file - var pathbuf: bun.WPathBuffer = undefined; - const dirpath = std.fs.path.dirnameWindows(file_path) orelse unreachable; - const wpath = bun.strings.toNTPath(&pathbuf, dirpath); - const watcher = try this.platform.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, wpath); - item.platform.dir_watcher = watcher; - std.debug.print("watching file: {s}\n", .{file_path_}); + item.platform.dir_watcher = try this.platform.watchFile(file_path_); } this.watchlist.appendAssumeCapacity(item); @@ -960,16 +1090,15 @@ pub fn NewWatcher(comptime ContextType: type) type { const slice: [:0]u8 = buf[0..file_path_to_use_.len :0]; item.platform.eventlist_index = try this.platform.watchDir(slice); } else if (Environment.isWindows) { - var pathbuf: bun.WPathBuffer = undefined; - const wpath = bun.strings.toNTPath(&pathbuf, file_path_); - const watcher = try this.platform.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, wpath); - item.platform.dir_watcher = watcher; + item.platform.dir_watcher = try this.platform.watchDir(file_path_); } this.watchlist.appendAssumeCapacity(item); return @as(WatchItemIndex, @truncate(this.watchlist.len - 1)); } + // Below is platform-independent + pub fn appendFileMaybeLock( this: *Watcher, fd: bun.FileDescriptor, @@ -1033,8 +1162,6 @@ pub fn NewWatcher(comptime ContextType: type) type { } } - // Below is platform-independent - inline fn isEligibleDirectory(this: *Watcher, dir: string) bool { return strings.indexOf(dir, this.fs.top_level_dir) != null and strings.indexOf(dir, "node_modules") == null; } diff --git a/src/windows.zig b/src/windows.zig index b830321186c61e..8ab27489a62d40 100644 --- a/src/windows.zig +++ b/src/windows.zig @@ -3009,10 +3009,24 @@ pub fn translateWinErrorToErrno(err: win32.Win32Error) bun.C.E { pub extern "kernel32" fn GetHostNameW( lpBuffer: PWSTR, nSize: c_int, -) BOOL; +) callconv(windows.WINAPI) BOOL; /// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw pub extern "kernel32" fn GetTempPathW( nBufferLength: DWORD, // [in] lpBuffer: LPCWSTR, // [out] ) DWORD; + +pub extern "kernel32" fn CreateJobObjectA( + lpJobAttributes: ?*anyopaque, // [in, optional] + lpName: ?LPCSTR, // [in, optional] +) callconv(windows.WINAPI) HANDLE; + +pub extern "kernel32" fn AssignProcessToJobObject( + hJob: HANDLE, // [in] + hProcess: HANDLE, // [in] +) callconv(windows.WINAPI) BOOL; + +pub extern "kernel32" fn ResumeThread( + hJob: HANDLE, // [in] +) callconv(windows.WINAPI) DWORD; diff --git a/watch.zig b/watch.zig index 81f543dae58e0e..ac317ab66cb735 100644 --- a/watch.zig +++ b/watch.zig @@ -288,8 +288,8 @@ pub fn main() !void { // try std.fs.makeDirAbsolute(dir); // try file.writeAll(&data); - // _ = std.os.windows.ntdll.NtClose(watched.dirHandle); - _ = watched; + _ = std.os.windows.ntdll.NtClose(watched.dirHandle); + // _ = watched; handle.join(); } From 497aa7ba6cb57a666d1a3e7795cf8a39dfd292c7 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Tue, 30 Jan 2024 18:28:24 -0800 Subject: [PATCH 14/32] handle process exit from watcher --- src/bun.zig | 29 +++++++++++++++++++++++++---- src/windows.zig | 25 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/bun.zig b/src/bun.zig index 78398599b28697..fdaabd5b95a39d 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1866,13 +1866,23 @@ pub const win32 = struct { } pub fn isWatcherChild() bool { - var buf: [1024]u16 = undefined; - return w.kernel32.GetEnvironmentVariableW(@constCast(watcherChildEnv.ptr), &buf, 1024) > 0; + var buf: [1]u16 = undefined; + return windows.GetEnvironmentVariableW(@constCast(watcherChildEnv.ptr), &buf, 1) > 0; } pub fn becomeWatcherManager(allocator: std.mem.Allocator) noreturn { // this process will be the parent of the child process that actually runs the script + // based on https://devblogs.microsoft.com/oldnewthing/20130405-00/?p=4743 const job = windows.CreateJobObjectA(null, null); + const iocp = windows.CreateIoCompletionPort(windows.INVALID_HANDLE_VALUE, null, 0, 1) orelse @panic("Failed to create iocp"); + var assoc = windows.JOBOBJECT_ASSOCIATE_COMPLETION_PORT{ + .CompletionKey = job, + .CompletionPort = iocp, + }; + if (windows.SetInformationJobObject(job, windows.JobObjectAssociateCompletionPortInformation, &assoc, @sizeOf(windows.JOBOBJECT_ASSOCIATE_COMPLETION_PORT)) == 0) { + @panic("Failed to associate completion port"); + } + var procinfo: std.os.windows.PROCESS_INFORMATION = undefined; spawnProcessCopy(allocator, &procinfo, true, true) catch @panic("Failed to spawn process"); if (windows.AssignProcessToJobObject(job, procinfo.hProcess) == 0) { @@ -1882,8 +1892,19 @@ pub const win32 = struct { @panic("Failed to resume thread"); } std.debug.print("waiting for job object\n", .{}); - _ = windows.WaitForSingleObject(job, std.os.windows.INFINITE); - @panic("waitforsingleobject returned"); + + var completion_code: w.DWORD = 0; + var completion_key: w.ULONG_PTR = 0; + var overlapped: ?*w.OVERLAPPED = null; + while (true) { + if (w.kernel32.GetQueuedCompletionStatus(iocp, &completion_code, &completion_key, &overlapped, w.INFINITE) == 0) { + @panic("GetQueuedCompletionStatus failed"); + } + if (completion_code == windows.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO and completion_key == @intFromPtr(job)) { + break; + } + } + Global.exit(0); } pub fn spawnProcessCopy( diff --git a/src/windows.zig b/src/windows.zig index 8ab27489a62d40..00c6fbb1b2f003 100644 --- a/src/windows.zig +++ b/src/windows.zig @@ -3030,3 +3030,28 @@ pub extern "kernel32" fn AssignProcessToJobObject( pub extern "kernel32" fn ResumeThread( hJob: HANDLE, // [in] ) callconv(windows.WINAPI) DWORD; + +pub const JOBOBJECT_ASSOCIATE_COMPLETION_PORT = extern struct { + CompletionKey: windows.PVOID, + CompletionPort: HANDLE, +}; + +pub const JobObjectAssociateCompletionPortInformation: DWORD = 7; + +pub extern "kernel32" fn SetInformationJobObject( + hJob: HANDLE, + JobObjectInformationClass: DWORD, + lpJobObjectInformation: LPVOID, + cbJobObjectInformationLength: DWORD, +) callconv(windows.WINAPI) BOOL; + +// Found experimentally: +// #include +// #include +// +// int main() { +// printf("%ld\n", JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO); +// } +// +// Output: 4 +pub const JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO = 4; From 835c89093077fca338d8c3c1785c05043b8f0a95 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 10:19:19 -0800 Subject: [PATCH 15/32] cleanup in process cloning --- src/bun.zig | 42 ++++++++++++++++++++++++++++-------------- src/windows.zig | 18 ++++++++++++++++-- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/bun.zig b/src/bun.zig index fdaabd5b95a39d..91dc95958515b5 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1896,14 +1896,24 @@ pub const win32 = struct { var completion_code: w.DWORD = 0; var completion_key: w.ULONG_PTR = 0; var overlapped: ?*w.OVERLAPPED = null; + var last_pid: w.DWORD = 0; while (true) { if (w.kernel32.GetQueuedCompletionStatus(iocp, &completion_code, &completion_key, &overlapped, w.INFINITE) == 0) { @panic("GetQueuedCompletionStatus failed"); } - if (completion_code == windows.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO and completion_key == @intFromPtr(job)) { + // only care about events concerning our job object (theoretically unnecessary) + if (completion_key != @intFromPtr(job)) { + continue; + } + if (completion_code == windows.JOB_OBJECT_MSG_EXIT_PROCESS) { + last_pid = @truncate(@intFromPtr(overlapped)); + } else if (completion_code == windows.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO) { break; } } + // NOTE: for now we always exit with a zero exit code. + // This is because there's no straightforward way to communicate the exit code + // of subsequently spawned child processes to the original parent process. Global.exit(0); } @@ -1941,22 +1951,26 @@ pub const win32 = struct { } if (setChild) { + var size: usize = 0; + if (kernelenv) |ptr| { + // check that env is non-empty + if (ptr[0] != 0 or ptr[1] != 0) { + // array is terminated by two nulls + while (ptr[size] != 0 or ptr[size + 1] != 0) size += 1; + size += 1; + } + } + // now ptr[size] is the first null + const buf = try allocator.alloc(u16, size + watcherChildEnv.len + 4); if (kernelenv) |ptr| { - var size: usize = 0; - // array is terminated by two nulls - while (ptr[size] != 0 or ptr[size + 1] != 0) size += 1; - size += 1; - // now ptr + size points to the first null - - const buf = try allocator.alloc(u16, size + watcherChildEnv.len + 4); @memcpy(buf[0..size], ptr); - @memcpy(buf[size .. size + watcherChildEnv.len], watcherChildEnv); - buf[size + watcherChildEnv.len] = '='; - buf[size + watcherChildEnv.len + 1] = '1'; - buf[size + watcherChildEnv.len + 2] = 0; - buf[size + watcherChildEnv.len + 3] = 0; - newenv = buf; } + @memcpy(buf[size .. size + watcherChildEnv.len], watcherChildEnv); + buf[size + watcherChildEnv.len] = '='; + buf[size + watcherChildEnv.len + 1] = '1'; + buf[size + watcherChildEnv.len + 2] = 0; + buf[size + watcherChildEnv.len + 3] = 0; + newenv = buf; } const env: ?[*]u16 = if (newenv) |e| e.ptr else kernelenv; diff --git a/src/windows.zig b/src/windows.zig index 00c6fbb1b2f003..fa0dabe5d9136a 100644 --- a/src/windows.zig +++ b/src/windows.zig @@ -3048,10 +3048,24 @@ pub extern "kernel32" fn SetInformationJobObject( // Found experimentally: // #include // #include -// +// // int main() { // printf("%ld\n", JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO); +// printf("%ld\n", JOB_OBJECT_MSG_EXIT_PROCESS); // } // -// Output: 4 +// Output: +// 4 +// 7 pub const JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO = 4; +pub const JOB_OBJECT_MSG_EXIT_PROCESS = 7; + +pub extern "kernel32" fn OpenProcess( + dwDesiredAccess: DWORD, + bInheritHandle: BOOL, + dwProcessId: DWORD, +) callconv(windows.WINAPI) ?HANDLE; + +// https://learn.microsoft.com/en-us/windows/win32/procthread/process-security-and-access-rights +pub const PROCESS_QUERY_LIMITED_INFORMATION: DWORD = 0x1000; + From 2dd75e44229caea10eb74f68a2691a7750cbeae4 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 12:48:15 -0800 Subject: [PATCH 16/32] clean up logging and panic handling --- src/bun.js/javascript.zig | 5 ++- src/bun.zig | 42 ++++++++++++------- src/resolver/resolve_path.zig | 13 ++++++ src/watcher.zig | 79 +++++++++++++++++++---------------- 4 files changed, 88 insertions(+), 51 deletions(-) diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 59d7c56e58bbcb..1865c6c615b9a1 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -3315,7 +3315,10 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime }, .directory => { if (comptime Environment.isWindows) { - // for now ignore directory updates on windows + // on windows we receive file events for all items affected by a directory change + // so we only need to clear the directory cache. all other effects will be handled + // by the file events + resolver.bustDirCache(file_path); continue; } var affected_buf: [128][]const u8 = undefined; diff --git a/src/bun.zig b/src/bun.zig index 91dc95958515b5..09497e854bf400 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1371,22 +1371,27 @@ pub fn reloadProcess( clear_terminal: bool, ) noreturn { if (clear_terminal) { - Output.flush(); - Output.disableBuffering(); - Output.resetTerminalAll(); + // Output.flush(); + // Output.disableBuffering(); + // Output.resetTerminalAll(); } const bun = @This(); if (comptime Environment.isWindows) { // this assumes that our parent process assigned us to a job object (see runWatcherManager) var procinfo: std.os.windows.PROCESS_INFORMATION = undefined; - win32.spawnProcessCopy(allocator, &procinfo, false, false) catch @panic("Unexpected error while reloading process\n"); - - std.debug.print("exiting\n", .{}); + win32.spawnProcessCopy(allocator, &procinfo, false, false) catch |err| { + Output.panic("Error while reloading process: {s}", .{@errorName(err)}); + }; // terminate the current process - _ = bun.windows.TerminateProcess(@ptrFromInt(std.math.maxInt(usize)), 0); - Output.panic("Unexpected error while reloading process\n", .{}); + const rc = bun.windows.TerminateProcess(@ptrFromInt(std.math.maxInt(usize)), 0); + if (rc == 0) { + const err = bun.windows.GetLastError(); + Output.panic("Error while reloading process: {s}", .{@tagName(err)}); + } else { + Output.panic("Unexpected error while reloading process\n", .{}); + } } const PosixSpawn = posix.spawn; const dupe_argv = allocator.allocSentinel(?[*:0]const u8, bun.argv().len, null) catch unreachable; @@ -1874,24 +1879,30 @@ pub const win32 = struct { // this process will be the parent of the child process that actually runs the script // based on https://devblogs.microsoft.com/oldnewthing/20130405-00/?p=4743 const job = windows.CreateJobObjectA(null, null); - const iocp = windows.CreateIoCompletionPort(windows.INVALID_HANDLE_VALUE, null, 0, 1) orelse @panic("Failed to create iocp"); + const iocp = windows.CreateIoCompletionPort(windows.INVALID_HANDLE_VALUE, null, 0, 1) orelse { + Output.panic("Failed to create IOCP\n", .{}); + }; var assoc = windows.JOBOBJECT_ASSOCIATE_COMPLETION_PORT{ .CompletionKey = job, .CompletionPort = iocp, }; if (windows.SetInformationJobObject(job, windows.JobObjectAssociateCompletionPortInformation, &assoc, @sizeOf(windows.JOBOBJECT_ASSOCIATE_COMPLETION_PORT)) == 0) { - @panic("Failed to associate completion port"); + const err = windows.GetLastError(); + Output.panic("Failed to associate completion port: {s}\n", .{@tagName(err)}); } var procinfo: std.os.windows.PROCESS_INFORMATION = undefined; - spawnProcessCopy(allocator, &procinfo, true, true) catch @panic("Failed to spawn process"); + spawnProcessCopy(allocator, &procinfo, true, true) catch |err| { + Output.panic("Failed to spawn process: {s}\n", .{@errorName(err)}); + }; if (windows.AssignProcessToJobObject(job, procinfo.hProcess) == 0) { - @panic("Failed to assign process to job object"); + const err = windows.GetLastError(); + Output.panic("Failed to assign process to job object: {s}\n", .{@tagName(err)}); } if (windows.ResumeThread(procinfo.hThread) == 0) { - @panic("Failed to resume thread"); + const err = windows.GetLastError(); + Output.panic("Failed to resume child process: {s}\n", .{@tagName(err)}); } - std.debug.print("waiting for job object\n", .{}); var completion_code: w.DWORD = 0; var completion_key: w.ULONG_PTR = 0; @@ -1899,7 +1910,8 @@ pub const win32 = struct { var last_pid: w.DWORD = 0; while (true) { if (w.kernel32.GetQueuedCompletionStatus(iocp, &completion_code, &completion_key, &overlapped, w.INFINITE) == 0) { - @panic("GetQueuedCompletionStatus failed"); + const err = windows.GetLastError(); + Output.panic("Failed to query completion status: {s}\n", .{@tagName(err)}); } // only care about events concerning our job object (theoretically unnecessary) if (completion_key != @intFromPtr(job)) { diff --git a/src/resolver/resolve_path.zig b/src/resolver/resolve_path.zig index f8859c317249f0..933ad2a1782f31 100644 --- a/src/resolver/resolve_path.zig +++ b/src/resolver/resolve_path.zig @@ -48,6 +48,19 @@ inline fn @"is ../"(slice: []const u8) bool { return strings.hasPrefixComptime(slice, "../"); } +const ParentEqual = enum { + parent, + equal, + unrelated, +}; + +pub fn isParentOrEqual(parent: []const u8, child: []const u8) ParentEqual { + if (std.mem.indexOf(u8, child, parent) != 0) return .unrelated; + if (child.len == parent.len) return .equal; + if (isSepAny(child[parent.len])) return .parent; + return .unrelated; +} + pub fn getIfExistsLongestCommonPathGeneric(input: []const []const u8, comptime platform: Platform) ?[]const u8 { const separator = comptime platform.separator(); const isPathSeparator = comptime platform.getSeparatorFunc(); diff --git a/src/watcher.zig b/src/watcher.zig index 5071787f059cf3..a2a72376a620db 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -14,6 +14,8 @@ const Futex = @import("./futex.zig"); pub const WatchItemIndex = u16; const PackageJSON = @import("./resolver/package_json.zig").PackageJSON; +const log = bun.Output.scoped(.watcher, false); + // TODO @gvilums // This entire file is a mess - rework it to be more maintainable @@ -226,8 +228,8 @@ pub const WindowsWatcher = struct { overlapped: w.OVERLAPPED = std.mem.zeroes(w.OVERLAPPED), buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, dirHandle: w.HANDLE, - path: [:0]u16, - path_buf: bun.WPathBuffer = undefined, + path: [:0]u8, + path_buf: bun.PathBuffer = undefined, refcount: usize = 1, const EventIterator = struct { @@ -272,7 +274,7 @@ pub const WindowsWatcher = struct { const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION; if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, 1, filter, null, &this.overlapped, null) == 0) { const err = w.kernel32.GetLastError(); - std.debug.print("failed to start watching directory: {s}\n", .{@tagName(err)}); + log("failed to start watching directory: {s}", .{@tagName(err)}); @panic("failed to start watching directory"); } } @@ -289,7 +291,7 @@ pub const WindowsWatcher = struct { // But we can't deallocate right away because we might be in the middle of iterating over the events of this watcher // we probably need some sort of queue that can be emptied by the watcher thread. if (this.refcount == 0) { - std.debug.print("TODO: deallocate watcher\n", .{}); + log("TODO: deallocate watcher", .{}); } } }; @@ -309,14 +311,15 @@ pub const WindowsWatcher = struct { w.kernel32.CloseHandle(this.iocp); } - fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !*DirWatcher { + fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: []const u8) !*DirWatcher { _ = dirFd; - std.debug.print("adding directory to watch: {s}\n", .{std.unicode.fmtUtf16le(path)}); - const path_len_bytes: u16 = @truncate(path.len * 2); + var pathbuf: bun.WPathBuffer = undefined; + const wpath = bun.strings.toNTPath(&pathbuf, path); + const path_len_bytes: u16 = @truncate(wpath.len * 2); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, - .Buffer = @constCast(path.ptr), + .Buffer = @constCast(wpath.ptr), }; var attr = w.OBJECT_ATTRIBUTES{ .Length = @sizeOf(w.OBJECT_ATTRIBUTES), @@ -349,7 +352,7 @@ pub const WindowsWatcher = struct { ); if (rc != .SUCCESS) { - std.debug.print("failed to open directory for watching: {s}\n", .{@tagName(rc)}); + log("failed to open directory for watching: {s}", .{@tagName(rc)}); @panic("failed to open directory for watching"); } @@ -375,7 +378,6 @@ pub const WindowsWatcher = struct { pub fn watchFile(this: *WindowsWatcher, path: []const u8) !*DirWatcher { const dirpath = std.fs.path.dirnameWindows(path) orelse @panic("get dir from file"); - std.debug.print("path: {s}, dirpath: {s}\n", .{ path, dirpath }); return this.watchDir(dirpath); } @@ -385,17 +387,20 @@ pub const WindowsWatcher = struct { if (path.len > 0 and bun.strings.charIsAnySlash(path[path.len - 1])) { path = path[0 .. path.len - 1]; } - var pathbuf: bun.WPathBuffer = undefined; - const wpath = bun.strings.toNTPath(&pathbuf, path); // check if one of the existing watchers covers this path for (this.watchers.items) |watcher| { - if (std.mem.indexOf(u16, watcher.path, wpath) == 0) { - std.debug.print("found existing watcher\n", .{}); + if (bun.path.isParentOrEqual(watcher.path, path) != .unrelated) { + // there is an existing watcher that covers this path + log("found existing watcher with path: {s}", .{watcher.path}); watcher.ref(); return watcher; + } else if (bun.path.isParentOrEqual(path, watcher.path) != .unrelated) { + // the new watcher would cover this existing watcher + // TODO there could be multiple existing watchers that are covered by the new watcher + // we should deactivate all existing ones and activate the new one } } - return this.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, wpath); + return this.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, path); } const Timeout = enum(w.DWORD) { @@ -865,11 +870,11 @@ pub fn NewWatcher(comptime ContextType: type) type { // first wait has infinite timeout - we're waiting for the next event and don't want to spin var timeout = WindowsWatcher.Timeout.infinite; while (true) { - // std.debug.print("waiting with timeout: {s}\n", .{@tagName(timeout)}); + // log("waiting with timeout: {s}\n", .{@tagName(timeout)}); const watcher = try this.platform.next(timeout) orelse break; // after handling the watcher's events, it explicitly needs to start reading directory changes again defer watcher.prepare() catch |err| { - Output.prettyErrorln("Failed to (re-)start listening to directory changes: {s}", .{@errorName(err)}); + Output.prettyErrorln("Failed to start listening to directory changes: {s} - Future updates may be missed", .{@errorName(err)}); }; // after the first wait, we want to start coalescing events, so we wait for a minimal amount of time @@ -877,30 +882,34 @@ pub fn NewWatcher(comptime ContextType: type) type { const item_paths = this.watchlist.items(.file_path); - std.debug.print("event from watcher: {s}\n", .{std.unicode.fmtUtf16le(watcher.path)}); + log("event from watcher: {s}", .{watcher.path}); var iter = watcher.events(); + @memcpy(buf[0..watcher.path.len], watcher.path); + buf[watcher.path.len] = '\\'; while (iter.next()) |event| { - std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); - // convert the current event file path to utf-8 - // skip the \??\ prefix - var idx = bun.simdutf.convert.utf16.to.utf8.le(watcher.path[4..], &buf); - buf[idx] = '\\'; - idx += 1; + log("raw event (filename: {}, action: {s})", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); + var idx = watcher.path.len + 1; idx += bun.simdutf.convert.utf16.to.utf8.le(event.filename, buf[idx..]); const eventpath = buf[0..idx]; - std.debug.print("eventpath: {s}\n", .{eventpath}); + log("received event at full path: {s}\n", .{eventpath}); - // TODO this really needs a more sophisticated search algorithm - for (item_paths, 0..) |path, item_idx| { - std.debug.print("path: {s}\n", .{path}); + // TODO this probably needs a more sophisticated search algorithm + for (item_paths, 0..) |path_, item_idx| { + var path = path_; + if (path.len > 0 and bun.strings.charIsAnySlash(path[path.len - 1])) { + path = path[0 .. path.len - 1]; + } + log("checking path: {s}\n", .{path}); // check if the current change applies to this item // if so, add it to the eventlist - if (std.mem.indexOf(u8, path, eventpath) == 0) { - // this.changed_filepaths[event_id] = path; - this.watch_events[event_id].fromFileNotify(event, @truncate(item_idx)); - event_id += 1; - } + const rel = bun.path.isParentOrEqual(eventpath, path); + // skip unrelated items + if (rel == .unrelated) continue; + // if the event is for a parent dir of the item, only emit it if it's a delete or rename + if (rel == .parent and (event.action != .Removed or event.action != .RenamedOld)) continue; + this.watch_events[event_id].fromFileNotify(event, @truncate(item_idx)); + event_id += 1; } } } @@ -908,7 +917,7 @@ pub fn NewWatcher(comptime ContextType: type) type { continue :restart; } - std.debug.print("event_id: {d}\n", .{event_id}); + log("event_id: {d}\n", .{event_id}); var all_events = this.watch_events[0..event_id]; std.sort.pdq(WatchEvent, all_events, {}, WatchEvent.sortByIndex); @@ -933,7 +942,7 @@ pub fn NewWatcher(comptime ContextType: type) type { if (all_events.len == 0) continue :restart; all_events = all_events[0 .. last_event_index + 1]; - std.debug.print("all_events.len: {d}\n", .{all_events.len}); + log("calling onFileUpdate (all_events.len = {d})", .{all_events.len}); this.ctx.onFileUpdate(all_events, this.changed_filepaths[0 .. last_event_index + 1], this.watchlist); } From 65a1dfb81eafda2424b8ca4d9f9d962e3514ca94 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 13:04:28 -0800 Subject: [PATCH 17/32] fix hot reload test on windows --- test/cli/hot/hot.test.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index facffa042b0dd6..497695d4c0b7a1 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -2,10 +2,10 @@ import { spawn } from "bun"; import { expect, it } from "bun:test"; import { bunExe, bunEnv, tempDirWithFiles, bunRun, bunRunAsScript } from "harness"; -import { readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from "fs"; +import { readFileSync, renameSync, rmSync, unlinkSync, writeFileSync, copyFileSync } from "fs"; import { join } from "path"; -const hotRunnerRoot = join(import.meta.dir, "/hot-runner-root.js"); +const hotRunnerRoot = join(import.meta.dir, "hot-runner-root.js"); it("should hot reload when file is overwritten", async () => { const root = hotRunnerRoot; @@ -169,7 +169,8 @@ it("should not hot reload when a random file is written", async () => { }); it("should hot reload when a file is deleted and rewritten", async () => { - const root = hotRunnerRoot; + const root = hotRunnerRoot + '.tmp.js'; + copyFileSync(hotRunnerRoot, root); const runner = spawn({ cmd: [bunExe(), "--hot", "run", root], env: bunEnv, @@ -182,7 +183,7 @@ it("should hot reload when a file is deleted and rewritten", async () => { async function onReload() { const contents = readFileSync(root, "utf-8"); - unlinkSync(root); + rmSync(root); writeFileSync(root, contents); } @@ -205,12 +206,13 @@ it("should hot reload when a file is deleted and rewritten", async () => { if (any) await onReload(); } - + rmSync(root); expect(reloadCounter).toBe(3); }); it("should hot reload when a file is renamed() into place", async () => { - const root = hotRunnerRoot; + const root = hotRunnerRoot + '.tmp.js'; + copyFileSync(hotRunnerRoot, root); const runner = spawn({ cmd: [bunExe(), "--hot", "run", root], env: bunEnv, @@ -227,6 +229,8 @@ it("should hot reload when a file is renamed() into place", async () => { await 1; writeFileSync(root + ".tmpfile", contents); await 1; + rmSync(root); + await 1; renameSync(root + ".tmpfile", root); await 1; } @@ -250,6 +254,6 @@ it("should hot reload when a file is renamed() into place", async () => { if (any) await onReload(); } - + rmSync(root); expect(reloadCounter).toBe(3); }); From 200c6a5a8395322fed87ea27ecd3ed277adcbd2c Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 14:54:42 -0800 Subject: [PATCH 18/32] misc cleanup around watcher --- src/bun.js/javascript.zig | 34 +++++++-------- src/bun.js/module_loader.zig | 4 +- src/bun.js/node/path_watcher.zig | 18 ++++---- src/bun.js/node/win_watcher.zig | 74 +++++++------------------------- src/sys.zig | 1 + src/watcher.zig | 65 ++++++++++++++++------------ 6 files changed, 82 insertions(+), 114 deletions(-) diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 1865c6c615b9a1..ded13f8a35c81e 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -83,6 +83,7 @@ const PendingResolution = @import("../resolver/resolver.zig").PendingResolution; const ThreadSafeFunction = JSC.napi.ThreadSafeFunction; const PackageManager = @import("../install/install.zig").PackageManager; const IPC = @import("ipc.zig"); +pub const GenericWatcher = @import("../watcher.zig"); const ModuleLoader = JSC.ModuleLoader; const FetchFlags = JSC.FetchFlags; @@ -430,22 +431,22 @@ pub const ImportWatcher = union(enum) { pub fn start(this: ImportWatcher) !void { switch (this) { - inline .hot => |watcher| try watcher.start(), - inline .watch => |watcher| try watcher.start(), + inline .hot => |w| try w.start(), + inline .watch => |w| try w.start(), else => {}, } } - pub inline fn watchlist(this: ImportWatcher) Watcher.WatchListArray { + pub inline fn watchlist(this: ImportWatcher) GenericWatcher.WatchList { return switch (this) { - inline .hot, .watch => |wacher| wacher.watchlist, + inline .hot, .watch => |w| w.watchlist, else => .{}, }; } - pub inline fn indexOf(this: ImportWatcher, hash: Watcher.HashType) ?u32 { + pub inline fn indexOf(this: ImportWatcher, hash: GenericWatcher.HashType) ?u32 { return switch (this) { - inline .hot, .watch => |wacher| wacher.indexOf(hash), + inline .hot, .watch => |w| w.indexOf(hash), else => null, }; } @@ -454,7 +455,7 @@ pub const ImportWatcher = union(enum) { this: ImportWatcher, fd: StoredFileDescriptorType, file_path: string, - hash: Watcher.HashType, + hash: GenericWatcher.HashType, loader: options.Loader, dir_fd: StoredFileDescriptorType, package_json: ?*PackageJSON, @@ -2142,7 +2143,7 @@ pub const VirtualMachine = struct { pub fn reloadEntryPoint(this: *VirtualMachine, entry_path: []const u8) !*JSInternalPromise { this.has_loaded = false; this.main = entry_path; - this.main_hash = bun.JSC.Watcher.getHash(entry_path); + this.main_hash = GenericWatcher.getHash(entry_path); try this.entry_point.generate( this.allocator, @@ -2180,7 +2181,7 @@ pub const VirtualMachine = struct { pub fn reloadEntryPointForTestRunner(this: *VirtualMachine, entry_path: []const u8) !*JSInternalPromise { this.has_loaded = false; this.main = entry_path; - this.main_hash = bun.JSC.Watcher.getHash(entry_path); + this.main_hash = GenericWatcher.getHash(entry_path); this.eventLoop().ensureWaker(); @@ -3074,11 +3075,10 @@ extern fn BunDebugger__willHotReload() void; pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime reload_immediately: bool) type { return struct { - const watcher = @import("../watcher.zig"); - pub const Watcher = watcher.NewWatcher(*@This()); + pub const Watcher = GenericWatcher.NewWatcher(*@This()); const Reloader = @This(); - onAccept: std.ArrayHashMapUnmanaged(@This().Watcher.HashType, bun.BabyList(OnAcceptCallback), bun.ArrayIdentityContext, false) = .{}, + onAccept: std.ArrayHashMapUnmanaged(GenericWatcher.HashType, bun.BabyList(OnAcceptCallback), bun.ArrayIdentityContext, false) = .{}, ctx: *Ctx, verbose: bool = false, @@ -3217,7 +3217,7 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime // - Directories outside the root directory // - Directories inside node_modules if (std.mem.indexOf(u8, file_path, "node_modules") == null and std.mem.indexOf(u8, file_path, watch.fs.top_level_dir) != null) { - watch.addDirectory(dir_fd, file_path, @This().Watcher.getHash(file_path), false) catch {}; + watch.addDirectory(dir_fd, file_path, GenericWatcher.getHash(file_path), false) catch {}; } } @@ -3250,9 +3250,9 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime pub fn onFileUpdate( this: *@This(), - events: []watcher.WatchEvent, + events: []GenericWatcher.WatchEvent, changed_files: []?[:0]u8, - watchlist: watcher.Watchlist, + watchlist: GenericWatcher.WatchList, ) void { var slice = watchlist.slice(); const file_paths = slice.items(.file_path); @@ -3371,7 +3371,7 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime resolver.bustDirCache(file_path); if (entries_option) |dir_ent| { - var last_file_hash: @This().Watcher.HashType = std.math.maxInt(@This().Watcher.HashType); + var last_file_hash: GenericWatcher.HashType = std.math.maxInt(GenericWatcher.HashType); for (affected) |changed_name_| { const changed_name: []const u8 = if (comptime Environment.isMac) @@ -3384,7 +3384,7 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime var prev_entry_id: usize = std.math.maxInt(usize); if (loader != .file) { var path_string: bun.PathString = undefined; - var file_hash: @This().Watcher.HashType = last_file_hash; + var file_hash: GenericWatcher.HashType = last_file_hash; const abs_path: string = brk: { if (dir_ent.entries.get(@as([]const u8, @ptrCast(changed_name)))) |file_ent| { // reset the file descriptor diff --git a/src/bun.js/module_loader.zig b/src/bun.js/module_loader.zig index 9ffc358cfe9e47..e1f7f21d56117f 100644 --- a/src/bun.js/module_loader.zig +++ b/src/bun.js/module_loader.zig @@ -401,7 +401,7 @@ pub const RuntimeTranspilerStore = struct { var fd: ?StoredFileDescriptorType = null; var package_json: ?*PackageJSON = null; - const hash = JSC.Watcher.getHash(path.text); + const hash = JSC.GenericWatcher.getHash(path.text); switch (vm.bun_watcher) { .hot, .watch => { @@ -1447,7 +1447,7 @@ pub const ModuleLoader = struct { .js, .jsx, .ts, .tsx, .json, .toml, .text => { jsc_vm.transpiled_count += 1; jsc_vm.bundler.resetStore(); - const hash = JSC.Watcher.getHash(path.text); + const hash = JSC.GenericWatcher.getHash(path.text); const is_main = jsc_vm.main.len == path.text.len and jsc_vm.main_hash == hash and strings.eqlLong(jsc_vm.main, path.text, false); diff --git a/src/bun.js/node/path_watcher.zig b/src/bun.js/node/path_watcher.zig index c2740a753d4ab1..63cd1d5f278ab1 100644 --- a/src/bun.js/node/path_watcher.zig +++ b/src/bun.js/node/path_watcher.zig @@ -13,6 +13,7 @@ const StoredFileDescriptorType = bun.StoredFileDescriptorType; const string = bun.string; const JSC = bun.JSC; const VirtualMachine = JSC.VirtualMachine; +const GenericWatcher = @import("../../watcher.zig"); const sync = @import("../../sync.zig"); const Semaphore = sync.Semaphore; @@ -21,7 +22,6 @@ var default_manager_mutex: Mutex = Mutex.init(); var default_manager: ?*PathWatcherManager = null; pub const PathWatcherManager = struct { - const GenericWatcher = @import("../../watcher.zig"); const options = @import("../../options.zig"); pub const Watcher = GenericWatcher.NewWatcher(*PathWatcherManager); const log = Output.scoped(.PathWatcherManager, false); @@ -43,7 +43,7 @@ pub const PathWatcherManager = struct { path: [:0]const u8, dirname: string, refs: u32 = 0, - hash: Watcher.HashType, + hash: GenericWatcher.HashType, }; fn refPendingTask(this: *PathWatcherManager) bool { @@ -96,7 +96,7 @@ pub const PathWatcherManager = struct { .is_file = false, .path = cloned_path, .dirname = cloned_path, - .hash = Watcher.getHash(cloned_path), + .hash = GenericWatcher.getHash(cloned_path), .refs = 1, }; _ = try this.file_paths.put(cloned_path, result); @@ -110,7 +110,7 @@ pub const PathWatcherManager = struct { .path = cloned_path, // if is really a file we need to get the dirname .dirname = std.fs.path.dirname(cloned_path) orelse cloned_path, - .hash = Watcher.getHash(cloned_path), + .hash = GenericWatcher.getHash(cloned_path), .refs = 1, }; _ = try this.file_paths.put(cloned_path, result); @@ -154,7 +154,7 @@ pub const PathWatcherManager = struct { this: *PathWatcherManager, events: []GenericWatcher.WatchEvent, changed_files: []?[:0]u8, - watchlist: GenericWatcher.Watchlist, + watchlist: GenericWatcher.WatchList, ) void { var slice = watchlist.slice(); const file_paths = slice.items(.file_path); @@ -197,7 +197,7 @@ pub const PathWatcherManager = struct { if (event.op.write or event.op.delete or event.op.rename) { const event_type: PathWatcher.EventType = if (event.op.delete or event.op.rename or event.op.move_to) .rename else .change; - const hash = Watcher.getHash(file_path); + const hash = GenericWatcher.getHash(file_path); for (watchers) |w| { if (w) |watcher| { @@ -268,7 +268,7 @@ pub const PathWatcherManager = struct { const len = file_path_without_trailing_slash.len + changed_name.len; const path_slice = _on_file_update_path_buf[0 .. len + 1]; - const hash = Watcher.getHash(path_slice); + const hash = GenericWatcher.getHash(path_slice); // skip consecutive duplicates const event_type: PathWatcher.EventType = .rename; // renaming folders, creating folder or files will be always be rename @@ -688,7 +688,7 @@ pub const PathWatcher = struct { has_pending_directories: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), closed: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), pub const ChangeEvent = struct { - hash: PathWatcherManager.Watcher.HashType = 0, + hash: GenericWatcher.HashType = 0, event_type: EventType = .change, time_stamp: i64 = 0, }; @@ -805,7 +805,7 @@ pub const PathWatcher = struct { } } - pub fn emit(this: *PathWatcher, path: string, hash: PathWatcherManager.Watcher.HashType, time_stamp: i64, is_file: bool, event_type: EventType) void { + pub fn emit(this: *PathWatcher, path: string, hash: GenericWatcher.HashType, time_stamp: i64, is_file: bool, event_type: EventType) void { const time_diff = time_stamp - this.last_change_event.time_stamp; // skip consecutive duplicates if ((this.last_change_event.time_stamp == 0 or time_diff > 1) or this.last_change_event.event_type != event_type and this.last_change_event.hash != hash) { diff --git a/src/bun.js/node/win_watcher.zig b/src/bun.js/node/win_watcher.zig index 14284baf5a1661..bcff289446d352 100644 --- a/src/bun.js/node/win_watcher.zig +++ b/src/bun.js/node/win_watcher.zig @@ -9,17 +9,15 @@ const JSC = bun.JSC; const VirtualMachine = JSC.VirtualMachine; const StoredFileDescriptorType = bun.StoredFileDescriptorType; const Output = bun.Output; +const Watcher = @import("../../watcher.zig"); var default_manager: ?*PathWatcherManager = null; // TODO: make this a generic so we can reuse code with path_watcher // TODO: we probably should use native instead of libuv abstraction here for better performance pub const PathWatcherManager = struct { - const GenericWatcher = @import("../../watcher.zig"); const options = @import("../../options.zig"); - pub const Watcher = GenericWatcher.NewWatcher(*PathWatcherManager); const log = Output.scoped(.PathWatcherManager, false); - main_watcher: *Watcher, watchers: bun.BabyList(?*PathWatcher) = .{}, watcher_count: u32 = 0, @@ -85,55 +83,31 @@ pub const PathWatcherManager = struct { var this = PathWatcherManager.new(.{ .file_paths = bun.StringHashMap(PathInfo).init(bun.default_allocator), .watchers = watchers, - .main_watcher = undefined, .vm = vm, .watcher_count = 0, }); errdefer this.destroy(); - this.main_watcher = try Watcher.init( - this, - vm.bundler.fs, - bun.default_allocator, - ); - - errdefer this.main_watcher.deinit(false); - - try this.main_watcher.start(); return this; } - fn _addDirectory(this: *PathWatcherManager, _: *PathWatcher, path: PathInfo) !void { - const fd = path.fd; - try this.main_watcher.addDirectory(fd, path.path, path.hash, false); - } - fn registerWatcher(this: *PathWatcherManager, watcher: *PathWatcher) !void { - { - if (this.watcher_count == this.watchers.len) { - this.watcher_count += 1; - this.watchers.push(bun.default_allocator, watcher) catch |err| { - this.watcher_count -= 1; - return err; - }; - } else { - var watchers = this.watchers.slice(); - for (watchers, 0..) |w, i| { - if (w == null) { - watchers[i] = watcher; - this.watcher_count += 1; - break; - } + if (this.watcher_count == this.watchers.len) { + this.watcher_count += 1; + this.watchers.push(bun.default_allocator, watcher) catch |err| { + this.watcher_count -= 1; + return err; + }; + } else { + var watchers = this.watchers.slice(); + for (watchers, 0..) |w, i| { + if (w == null) { + watchers[i] = watcher; + this.watcher_count += 1; + break; } } } - - const path = watcher.path; - if (path.is_file) { - try this.main_watcher.addFile(path.fd, path.path, path.hash, options.Loader.file, .zero, null, false); - } else { - try this._addDirectory(watcher, path); - } } fn _incrementPathRef(this: *PathWatcherManager, file_path: [:0]const u8) void { @@ -152,7 +126,6 @@ pub const PathWatcherManager = struct { path.refs -= 1; if (path.refs == 0) { const path_ = path.path; - this.main_watcher.remove(path.hash); _ = this.file_paths.remove(path_); bun.default_allocator.free(path_); } @@ -198,8 +171,6 @@ pub const PathWatcherManager = struct { return; } - this.main_watcher.deinit(false); - if (this.watcher_count > 0) { while (this.watchers.popOrNull()) |watcher| { if (watcher) |w| { @@ -223,19 +194,6 @@ pub const PathWatcherManager = struct { this.destroy(); } - - // TODO figure out what win_watcher even does... - pub fn onFileUpdate( - this: *@This(), - events: []GenericWatcher.WatchEvent, - changed_files: []?[:0]u8, - watchlist: GenericWatcher.Watchlist, - ) void { - _ = this; // autofix - _ = events; // autofix - _ = changed_files; // autofix - _ = watchlist; // autofix - } }; pub const PathWatcher = struct { @@ -255,7 +213,7 @@ pub const PathWatcher = struct { const log = Output.scoped(.PathWatcher, false); pub const ChangeEvent = struct { - hash: PathWatcherManager.Watcher.HashType = 0, + hash: Watcher.HashType = 0, event_type: EventType = .change, time_stamp: i64 = 0, }; @@ -343,7 +301,7 @@ pub const PathWatcher = struct { return this; } - pub fn emit(this: *PathWatcher, path: string, hash: PathWatcherManager.Watcher.HashType, time_stamp: i64, is_file: bool, event_type: EventType) void { + pub fn emit(this: *PathWatcher, path: string, hash: Watcher.HashType, time_stamp: i64, is_file: bool, event_type: EventType) void { const time_diff = time_stamp - this.last_change_event.time_stamp; // skip consecutive duplicates if ((this.last_change_event.time_stamp == 0 or time_diff > 1) or this.last_change_event.event_type != event_type and this.last_change_event.hash != hash) { diff --git a/src/sys.zig b/src/sys.zig index 99c3274dca74f3..1542832885c079 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -470,6 +470,7 @@ pub noinline fn openDirAtWindowsA( var wbuf: bun.WPathBuffer = undefined; return openDirAtWindows(dirFd, bun.strings.toNTDir(&wbuf, path), iterable, no_follow); } + pub fn openatWindows(dir: bun.FileDescriptor, path: []const u16, flags: bun.Mode) Maybe(bun.FileDescriptor) { const nonblock = flags & O.NONBLOCK != 0; const overwrite = flags & O.WRONLY != 0 and flags & O.APPEND == 0; diff --git a/src/watcher.zig b/src/watcher.zig index a2a72376a620db..d4e819115a9461 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -21,7 +21,7 @@ const log = bun.Output.scoped(.watcher, false); const WATCHER_MAX_LIST = 8096; -pub const INotify = struct { +const INotify = struct { loaded_inotify: bool = false, inotify_fd: EventListIndex = 0, @@ -200,7 +200,7 @@ const DarwinWatcher = struct { } }; -pub const WindowsWatcher = struct { +const WindowsWatcher = struct { iocp: w.HANDLE = undefined, allocator: std.mem.Allocator = undefined, watchers: std.ArrayListUnmanaged(*DirWatcher) = std.ArrayListUnmanaged(*DirWatcher){}, @@ -208,6 +208,13 @@ pub const WindowsWatcher = struct { const w = std.os.windows; pub const EventListIndex = c_int; + const Error = error{ + IocpFailed, + ReadDirectoryChangesFailed, + CreateFileFailed, + InvalidPath, + }; + const Action = enum(w.DWORD) { Added = w.FILE_ACTION_ADDED, Removed = w.FILE_ACTION_REMOVED, @@ -270,12 +277,12 @@ pub const WindowsWatcher = struct { } // invalidates any EventIterators derived from this DirWatcher - fn prepare(this: *DirWatcher) !void { + fn prepare(this: *DirWatcher) Error!void { const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION; if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, 1, filter, null, &this.overlapped, null) == 0) { const err = w.kernel32.GetLastError(); log("failed to start watching directory: {s}", .{@tagName(err)}); - @panic("failed to start watching directory"); + return Error.ReadDirectoryChangesFailed; } } @@ -304,13 +311,6 @@ pub const WindowsWatcher = struct { }; } - pub fn deinit(this: *WindowsWatcher) void { - // get all the directory watchers and close their handles - // TODO - // close the io completion port handle - w.kernel32.CloseHandle(this.iocp); - } - fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: []const u8) !*DirWatcher { _ = dirFd; var pathbuf: bun.WPathBuffer = undefined; @@ -352,8 +352,9 @@ pub const WindowsWatcher = struct { ); if (rc != .SUCCESS) { - log("failed to open directory for watching: {s}", .{@tagName(rc)}); - @panic("failed to open directory for watching"); + const err = bun.windows.Win32Error.fromNTStatus(rc); + log("failed to open directory for watching: {s}", .{@tagName(err)}); + return Error.CreateFileFailed; } errdefer _ = w.kernel32.CloseHandle(handle); @@ -377,7 +378,7 @@ pub const WindowsWatcher = struct { } pub fn watchFile(this: *WindowsWatcher, path: []const u8) !*DirWatcher { - const dirpath = std.fs.path.dirnameWindows(path) orelse @panic("get dir from file"); + const dirpath = std.fs.path.dirnameWindows(path) orelse return Error.InvalidPath; return this.watchDir(dirpath); } @@ -421,7 +422,8 @@ pub const WindowsWatcher = struct { if (err == w.Win32Error.IMEOUT) { return null; } else { - @panic("GetQueuedCompletionStatus failed"); + log("GetQueuedCompletionStatus failed: {s}", .{@tagName(err)}); + return Error.IocpFailed; } } @@ -432,11 +434,19 @@ pub const WindowsWatcher = struct { if (overlapped) |ptr| { return DirWatcher.fromOverlapped(ptr); } else { - // this would be an error which we should probaby signal - continue; + log("GetQueuedCompletionStatus returned no overlapped event", .{}); + return Error.IocpFailed; } } } + + pub fn stop(this: *WindowsWatcher) void { + for (this.watchers.items) |watcher| { + w.CloseHandle(watcher.dirHandle); + this.allocator.destroy(watcher); + } + w.CloseHandle(this.iocp); + } }; const PlatformWatcher = if (Environment.isMac) @@ -562,13 +572,18 @@ pub const WatchItem = struct { pub const Kind = enum { file, directory }; }; -pub const Watchlist = std.MultiArrayList(WatchItem); +pub const WatchList = std.MultiArrayList(WatchItem); +pub const HashType = u32; + +pub fn getHash(filepath: string) HashType { + return @as(HashType, @truncate(bun.hash(filepath))); +} pub fn NewWatcher(comptime ContextType: type) type { return struct { const Watcher = @This(); - watchlist: Watchlist, + watchlist: WatchList, watched_count: usize = 0, mutex: Mutex, @@ -590,14 +605,8 @@ pub fn NewWatcher(comptime ContextType: type) type { evict_list: [WATCHER_MAX_LIST]WatchItemIndex = undefined, evict_list_i: WatchItemIndex = 0, - pub const HashType = u32; - pub const WatchListArray = Watchlist; const no_watch_item: WatchItemIndex = std.math.maxInt(WatchItemIndex); - pub fn getHash(filepath: string) HashType { - return @as(HashType, @truncate(bun.hash(filepath))); - } - pub fn init(ctx: ContextType, fs: *bun.fs.FileSystem, allocator: std.mem.Allocator) !*Watcher { const watcher = try allocator.create(Watcher); errdefer allocator.destroy(watcher); @@ -607,7 +616,7 @@ pub fn NewWatcher(comptime ContextType: type) type { .allocator = allocator, .watched_count = 0, .ctx = ctx, - .watchlist = Watchlist{}, + .watchlist = WatchList{}, .mutex = Mutex.init(), .cwd = fs.top_level_dir, }; @@ -1036,7 +1045,7 @@ pub fn NewWatcher(comptime ContextType: type) type { break :brk bun.toFD(dir.fd); }; - const parent_hash = Watcher.getHash(bun.fs.PathName.init(file_path).dirWithTrailingSlash()); + const parent_hash = getHash(bun.fs.PathName.init(file_path).dirWithTrailingSlash()); const file_path_: string = if (comptime copy_file_path) bun.asByteSlice(try this.allocator.dupeZ(u8, file_path)) @@ -1125,7 +1134,7 @@ pub fn NewWatcher(comptime ContextType: type) type { const pathname = bun.fs.PathName.init(file_path); const parent_dir = pathname.dirWithTrailingSlash(); - const parent_dir_hash: HashType = Watcher.getHash(parent_dir); + const parent_dir_hash: HashType = getHash(parent_dir); var parent_watch_item: ?WatchItemIndex = null; const autowatch_parent_dir = (comptime FeatureFlags.watch_directories) and this.isEligibleDirectory(parent_dir); From 297863b1bd9bcc5bf523c09b5ed02e97d9caa048 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 14:55:00 -0800 Subject: [PATCH 19/32] make watch test actually useful --- test/cli/watch/watch.test.ts | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/test/cli/watch/watch.test.ts b/test/cli/watch/watch.test.ts index fc4d65a1dadd19..a4cf98aac5e9ff 100644 --- a/test/cli/watch/watch.test.ts +++ b/test/cli/watch/watch.test.ts @@ -1,14 +1,14 @@ -import { describe, test, expect, afterEach } from "bun:test"; +import { it, expect, afterEach } from "bun:test"; import type { Subprocess } from "bun"; import { spawn } from "bun"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { mkdtempSync, writeFileSync } from "node:fs"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { bunExe, bunEnv } from "harness"; let watchee: Subprocess; -describe("bun --watch", () => { +it("should watch files", async () => { const cwd = mkdtempSync(join(tmpdir(), "bun-test-")); const path = join(cwd, "watchee.js"); @@ -16,16 +16,25 @@ describe("bun --watch", () => { writeFileSync(path, `console.log(${i});`); }; - test("should watch files", async () => { - watchee = spawn({ - cwd, - cmd: [bunExe(), "--watch", "watchee.js"], - env: bunEnv, - stdout: "inherit", - stderr: "inherit", - }); - await Bun.sleep(2000); + let i = 0; + updateFile(i); + watchee = spawn({ + cwd, + cmd: [bunExe(), "--watch", "watchee.js"], + env: bunEnv, + stdout: "pipe", + stderr: "inherit", + stdin: "ignore", }); + + for await (const line of watchee.stdout) { + if (i == 10) break; + var str = new TextDecoder().decode(line); + expect(str).toContain(`${i}`); + i++; + updateFile(i); + } + rmSync(path); }); afterEach(() => { From 3d477abcdf12c036eaf55f1cb36735734715a70f Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 22:57:34 +0000 Subject: [PATCH 20/32] [autofix.ci] apply automated fixes --- src/windows.zig | 19 ++++++++++--------- test/cli/hot/hot.test.ts | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/windows.zig b/src/windows.zig index fa0dabe5d9136a..ab4da2601b96f0 100644 --- a/src/windows.zig +++ b/src/windows.zig @@ -17,6 +17,7 @@ pub const FALSE = windows.FALSE; pub const TRUE = windows.TRUE; pub const INVALID_HANDLE_VALUE = windows.INVALID_HANDLE_VALUE; pub const FILE_BEGIN = windows.FILE_BEGIN; +pub const FILE_END = windows.FILE_END; pub const FILE_CURRENT = windows.FILE_CURRENT; pub const ULONG = windows.ULONG; pub const LARGE_INTEGER = windows.LARGE_INTEGER; @@ -62,6 +63,8 @@ pub const advapi32 = windows.advapi32; pub const INVALID_FILE_ATTRIBUTES: u32 = std.math.maxInt(u32); +pub const nt_object_prefix = [4]u16{ '\\', '?', '?', '\\' }; + const std = @import("std"); pub const HANDLE = win32.HANDLE; @@ -2982,24 +2985,23 @@ pub extern "kernel32" fn SetFileInformationByHandle( ) BOOL; pub fn getLastErrno() bun.C.E { - return translateWinErrorToErrno(bun.windows.kernel32.GetLastError()); + return (bun.C.SystemErrno.init(bun.windows.kernel32.GetLastError()) orelse SystemErrno.EUNKNOWN).toE(); } -pub fn translateWinErrorToErrno(err: win32.Win32Error) bun.C.E { +pub fn translateNTStatusToErrno(err: win32.NTSTATUS) bun.C.E { return switch (err) { .SUCCESS => .SUCCESS, - .FILE_NOT_FOUND => .NOENT, - .PATH_NOT_FOUND => .NOENT, - .TOO_MANY_OPEN_FILES => .NOMEM, .ACCESS_DENIED => .PERM, .INVALID_HANDLE => .BADF, - .NOT_ENOUGH_MEMORY => .NOMEM, - .OUTOFMEMORY => .NOMEM, .INVALID_PARAMETER => .INVAL, + .OBJECT_NAME_COLLISION => .EXIST, + .FILE_IS_A_DIRECTORY => .ISDIR, + .OBJECT_PATH_NOT_FOUND => .NOENT, + .OBJECT_NAME_NOT_FOUND => .NOENT, else => |t| { // if (bun.Environment.isDebug) { - bun.Output.warn("Called getLastErrno with {s} which does not have a mapping to errno.", .{@tagName(t)}); + bun.Output.warn("Called translateNTStatusToErrno with {s} which does not have a mapping to errno.", .{@tagName(t)}); // } return .UNKNOWN; }, @@ -3068,4 +3070,3 @@ pub extern "kernel32" fn OpenProcess( // https://learn.microsoft.com/en-us/windows/win32/procthread/process-security-and-access-rights pub const PROCESS_QUERY_LIMITED_INFORMATION: DWORD = 0x1000; - diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index 497695d4c0b7a1..c3c97fc782066b 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -169,7 +169,7 @@ it("should not hot reload when a random file is written", async () => { }); it("should hot reload when a file is deleted and rewritten", async () => { - const root = hotRunnerRoot + '.tmp.js'; + const root = hotRunnerRoot + ".tmp.js"; copyFileSync(hotRunnerRoot, root); const runner = spawn({ cmd: [bunExe(), "--hot", "run", root], @@ -211,7 +211,7 @@ it("should hot reload when a file is deleted and rewritten", async () => { }); it("should hot reload when a file is renamed() into place", async () => { - const root = hotRunnerRoot + '.tmp.js'; + const root = hotRunnerRoot + ".tmp.js"; copyFileSync(hotRunnerRoot, root); const runner = spawn({ cmd: [bunExe(), "--hot", "run", root], From 5ee49a355228a739551bb7b57e3d5c9e448a6f98 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 15:19:19 -0800 Subject: [PATCH 21/32] remove old files --- test/js/node/fs/fs-stream.link.js | 1 - watch.zig | 295 ------------------------------ 2 files changed, 296 deletions(-) delete mode 120000 test/js/node/fs/fs-stream.link.js delete mode 100644 watch.zig diff --git a/test/js/node/fs/fs-stream.link.js b/test/js/node/fs/fs-stream.link.js deleted file mode 120000 index 0cadae0e54f349..00000000000000 --- a/test/js/node/fs/fs-stream.link.js +++ /dev/null @@ -1 +0,0 @@ -./test/bun.js/fs-stream.js \ No newline at end of file diff --git a/watch.zig b/watch.zig deleted file mode 100644 index ac317ab66cb735..00000000000000 --- a/watch.zig +++ /dev/null @@ -1,295 +0,0 @@ -const std = @import("std"); - -pub fn toNTPath(wbuf: []u16, utf8: []const u8) [:0]const u16 { - if (!std.fs.path.isAbsoluteWindows(utf8)) { - return toWPathNormalized(wbuf, utf8); - } - - wbuf[0..4].* = [_]u16{ '\\', '?', '?', '\\' }; - return wbuf[0 .. toWPathNormalized(wbuf[4..], utf8).len + 4 :0]; -} - -// These are the same because they don't have rules like needing a trailing slash -pub const toNTDir = toNTPath; - -pub fn toExtendedPathNormalized(wbuf: []u16, utf8: []const u8) [:0]const u16 { - std.debug.assert(wbuf.len > 4); - wbuf[0..4].* = [_]u16{ '\\', '\\', '?', '\\' }; - return wbuf[0 .. toWPathNormalized(wbuf[4..], utf8).len + 4 :0]; -} - -pub fn toWPathNormalizeAutoExtend(wbuf: []u16, utf8: []const u8) [:0]const u16 { - if (std.fs.path.isAbsoluteWindows(utf8)) { - return toExtendedPathNormalized(wbuf, utf8); - } - - return toWPathNormalized(wbuf, utf8); -} - -pub fn toWPathNormalized(wbuf: []u16, utf8: []const u8) [:0]const u16 { - var renormalized: []u8 = undefined; - var path_to_use = utf8; - - if (std.mem.indexOfScalar(u8, utf8, '/') != null) { - @memcpy(renormalized[0..utf8.len], utf8); - for (renormalized[0..utf8.len]) |*c| { - if (c.* == '/') { - c.* = '\\'; - } - } - path_to_use = renormalized[0..utf8.len]; - } - - // is there a trailing slash? Let's remove it before converting to UTF-16 - if (path_to_use.len > 3 and path_to_use[path_to_use.len - 1] == '\\') { - path_to_use = path_to_use[0 .. path_to_use.len - 1]; - } - - return toWPath(wbuf, path_to_use); -} - -pub fn toWDirNormalized(wbuf: []u16, utf8: []const u8) [:0]const u16 { - var renormalized: [std.fs.MAX_PATH_BYTES]u8 = undefined; - var path_to_use = utf8; - - if (std.mem.indexOfScalar(u8, utf8, '.') != null) { - @memcpy(renormalized[0..utf8.len], utf8); - for (renormalized[0..utf8.len]) |*c| { - if (c.* == '/') { - c.* = '\\'; - } - } - path_to_use = renormalized[0..utf8.len]; - } - - return toWDirPath(wbuf, path_to_use); -} - -pub fn toWPath(wbuf: []u16, utf8: []const u8) [:0]const u16 { - return toWPathMaybeDir(wbuf, utf8, false); -} - -pub fn toWDirPath(wbuf: []u16, utf8: []const u8) [:0]const u16 { - return toWPathMaybeDir(wbuf, utf8, true); -} - -pub fn toWPathMaybeDir(wbuf: []u16, utf8: []const u8, comptime add_trailing_lash: bool) [:0]const u16 { - std.debug.assert(wbuf.len > 0); - - const count = std.unicode.utf8ToUtf16Le(wbuf[0..wbuf.len -| (1 + @as(usize, @intFromBool(add_trailing_lash)))], utf8) catch unreachable; - - if (add_trailing_lash and count > 0 and wbuf[count - 1] != '\\') { - wbuf[count] = '\\'; - count += 1; - } - - wbuf[count] = 0; - - return wbuf[0..count :0]; -} - -pub const WindowsWatcher = struct { - iocp: w.HANDLE, - allocator: std.mem.Allocator, - // watchers: Map, - running: bool = true, - - // const Map = std.AutoArrayHashMap(*w.OVERLAPPED, *DirWatcher); - const w = std.os.windows; - - const Action = enum(w.DWORD) { - Added = 1, - Removed, - Modified, - RenamedOld, - RenamedNew, - }; - - const DirWatcher = extern struct { - // this must be the first field - overlapped: w.OVERLAPPED = undefined, - buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, - dirHandle: w.HANDLE, - watch_subtree: w.BOOLEAN = 1, - - fn handleEvent(this: *DirWatcher, nbytes: w.DWORD) void { - const elapsed = clock1.read(); - std.debug.print("elapsed: {}\n", .{std.fmt.fmtDuration(elapsed)}); - if (nbytes == 0) { - std.debug.print("nbytes == 0\n", .{}); - return; - } - var offset: usize = 0; - while (true) { - const info_size = @sizeOf(w.FILE_NOTIFY_INFORMATION); - const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(this.buf[offset..].ptr)); - const name_ptr: [*]u16 = @alignCast(@ptrCast(this.buf[offset + info_size ..])); - const filename: []u16 = name_ptr[0 .. info.FileNameLength / @sizeOf(u16)]; - - const action: Action = @enumFromInt(info.Action); - std.debug.print("filename: {}, action: {s}\n", .{ std.unicode.fmtUtf16le(filename), @tagName(action) }); - - if (info.NextEntryOffset == 0) break; - offset += @as(usize, info.NextEntryOffset); - } - } - - fn listen(this: *DirWatcher) !void { - const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION; - if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, this.watch_subtree, filter, null, &this.overlapped, null) == 0) { - const err = w.kernel32.GetLastError(); - std.debug.print("failed to start watching directory: {s}\n", .{@tagName(err)}); - @panic("failed to start watching directory"); - } - } - }; - - pub fn init(allocator: std.mem.Allocator) !*WindowsWatcher { - const watcher = try allocator.create(WindowsWatcher); - errdefer allocator.destroy(watcher); - - const iocp = try w.CreateIoCompletionPort(w.INVALID_HANDLE_VALUE, null, 0, 1); - watcher.* = .{ - .iocp = iocp, - .allocator = allocator, - }; - return watcher; - } - - pub fn deinit(this: *WindowsWatcher) void { - // get all the directory watchers and close their handles - // TODO - // close the io completion port handle - w.kernel32.CloseHandle(this.iocp); - } - - pub fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: [:0]const u16) !*DirWatcher { - std.debug.print("adding directory to watch: {s}\n", .{std.unicode.fmtUtf16le(path)}); - const flags = w.FILE_LIST_DIRECTORY; - - const path_len_bytes: u16 = @truncate(path.len * 2); - var nt_name = w.UNICODE_STRING{ - .Length = path_len_bytes, - .MaximumLength = path_len_bytes, - .Buffer = @constCast(path.ptr), - }; - var attr = w.OBJECT_ATTRIBUTES{ - .Length = @sizeOf(w.OBJECT_ATTRIBUTES), - .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(path)) - null - else if (dirFd == w.INVALID_HANDLE_VALUE) - std.fs.cwd().fd - else - dirFd, - .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. - .ObjectName = &nt_name, - .SecurityDescriptor = null, - .SecurityQualityOfService = null, - }; - var handle: w.HANDLE = w.INVALID_HANDLE_VALUE; - var io: w.IO_STATUS_BLOCK = undefined; - const rc = w.ntdll.NtCreateFile( - &handle, - flags, - &attr, - &io, - null, - 0, - w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, - w.FILE_OPEN, - w.FILE_DIRECTORY_FILE | w.FILE_OPEN_FOR_BACKUP_INTENT, - null, - 0, - ); - - if (rc != .SUCCESS) { - std.debug.print("failed to open directory for watching: {s}\n", .{@tagName(rc)}); - @panic("failed to open directory for watching"); - } - - errdefer _ = w.kernel32.CloseHandle(handle); - - this.iocp = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); - - std.debug.print("handle: {d}\n", .{@intFromPtr(handle)}); - - const watcher = try this.allocator.create(DirWatcher); - errdefer this.allocator.destroy(watcher); - watcher.* = .{ .dirHandle = handle }; - // try this.watchers.put(key, watcher); - try watcher.listen(); - - return watcher; - } - - pub fn stop(this: *WindowsWatcher) void { - // close all the handles - // w.kernel32.PostQueuedCompletionStatus(this.iocp, 0, 1, ) - @atomicStore(bool, &this.running, false, .Unordered); - } - - pub fn run(this: *WindowsWatcher) !void { - var nbytes: w.DWORD = 0; - var key: w.ULONG_PTR = 0; - var overlapped: ?*w.OVERLAPPED = null; - while (true) { - switch (w.GetQueuedCompletionStatus(this.iocp, &nbytes, &key, &overlapped, w.INFINITE)) { - .Normal => {}, - .Aborted => @panic("aborted"), - .Cancelled => @panic("cancelled"), - .EOF => @panic("eof"), - } - if (nbytes == 0) { - // exit notification for this watcher - we should probably deallocate it here - continue; - } - - const watcher: *DirWatcher = @ptrCast(overlapped); - watcher.handleEvent(nbytes); - try watcher.listen(); - - if (@atomicLoad(bool, &this.running, .Unordered) == false) { - break; - } - } - } -}; - -var clock1: std.time.Timer = undefined; -const data: [1 << 10]u8 = std.mem.zeroes([1 << 10]u8); - -pub fn main() !void { - const allocator = std.heap.page_allocator; - - var buf: [std.fs.MAX_PATH_BYTES]u16 = undefined; - - const watchdir = "C:\\bun"; - const iconsdir = "C:\\test\\node_modules\\@mui\\icons-material"; - _ = iconsdir; // autofix - const testdir = "C:\\test\\node_modules\\@mui\\icons-material\\mydir"; - _ = testdir; // autofix - const testfile = "C:\\test\\node_modules\\@mui\\icons-material\\myfile.txt"; - _ = testfile; // autofix - - // try std.fs.deleteDirAbsolute(dir); - - const watcher = try WindowsWatcher.init(allocator); - var handle = try std.Thread.spawn(.{}, WindowsWatcher.run, .{watcher}); - std.time.sleep(100_000_000); - const watched = try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, watchdir)); - // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\src")); - // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\test")); - // try watcher.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, toNTPath(&buf, "C:\\bun\\testdir")); - - // const file = try std.fs.createFileAbsolute(testfile, .{}); - std.debug.print("watcher started\n", .{}); - - clock1 = try std.time.Timer.start(); - // try std.fs.makeDirAbsolute(dir); - // try file.writeAll(&data); - - _ = std.os.windows.ntdll.NtClose(watched.dirHandle); - // _ = watched; - - handle.join(); -} From ba3a262ca4b4add0ef9c2eefc16f6ac1b21117fb Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 16:00:35 -0800 Subject: [PATCH 22/32] clean up watchers --- src/watcher.zig | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/watcher.zig b/src/watcher.zig index d4e819115a9461..2c92e5508572f7 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -16,9 +16,6 @@ const PackageJSON = @import("./resolver/package_json.zig").PackageJSON; const log = bun.Output.scoped(.watcher, false); -// TODO @gvilums -// This entire file is a mess - rework it to be more maintainable - const WATCHER_MAX_LIST = 8096; const INotify = struct { @@ -201,6 +198,7 @@ const DarwinWatcher = struct { }; const WindowsWatcher = struct { + mutex: Mutex = Mutex.init(), iocp: w.HANDLE = undefined, allocator: std.mem.Allocator = undefined, watchers: std.ArrayListUnmanaged(*DirWatcher) = std.ArrayListUnmanaged(*DirWatcher){}, @@ -238,6 +236,7 @@ const WindowsWatcher = struct { path: [:0]u8, path_buf: bun.PathBuffer = undefined, refcount: usize = 1, + parent: *WindowsWatcher, const EventIterator = struct { watcher: *DirWatcher, @@ -298,7 +297,8 @@ const WindowsWatcher = struct { // But we can't deallocate right away because we might be in the middle of iterating over the events of this watcher // we probably need some sort of queue that can be emptied by the watcher thread. if (this.refcount == 0) { - log("TODO: deallocate watcher", .{}); + // closing the handle will send a 0-length notification to the iocp which can then deallocate the watcher + w.CloseHandle(this.dirHandle); } } }; @@ -311,8 +311,7 @@ const WindowsWatcher = struct { }; } - fn addWatchedDirectory(this: *WindowsWatcher, dirFd: w.HANDLE, path: []const u8) !*DirWatcher { - _ = dirFd; + fn addWatchedDirectory(this: *WindowsWatcher, path: []const u8) !*DirWatcher { var pathbuf: bun.WPathBuffer = undefined; const wpath = bun.strings.toNTPath(&pathbuf, path); const path_len_bytes: u16 = @truncate(wpath.len * 2); @@ -324,12 +323,6 @@ const WindowsWatcher = struct { var attr = w.OBJECT_ATTRIBUTES{ .Length = @sizeOf(w.OBJECT_ATTRIBUTES), .RootDirectory = null, - // .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(path)) - // null - // else if (dirFd == w.INVALID_HANDLE_VALUE) - // std.fs.cwd().fd - // else - // dirFd, .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. .ObjectName = &nt_name, .SecurityDescriptor = null, @@ -364,13 +357,12 @@ const WindowsWatcher = struct { const watcher = try this.allocator.create(DirWatcher); errdefer this.allocator.destroy(watcher); - watcher.* = .{ .dirHandle = handle, .path = undefined }; + watcher.* = .{ .dirHandle = handle, .parent = this, .path = undefined }; // init path @memcpy(watcher.path_buf[0..path.len], path); watcher.path_buf[path.len] = 0; watcher.path = watcher.path_buf[0..path.len :0]; - // TODO think about the different sequences of errors try watcher.prepare(); try this.watchers.append(this.allocator, watcher); @@ -383,6 +375,8 @@ const WindowsWatcher = struct { } pub fn watchDir(this: *WindowsWatcher, path_: []const u8) !*DirWatcher { + this.mutex.lock(); + defer this.mutex.unlock(); // strip the trailing slash if it exists var path = path_; if (path.len > 0 and bun.strings.charIsAnySlash(path[path.len - 1])) { @@ -401,7 +395,7 @@ const WindowsWatcher = struct { // we should deactivate all existing ones and activate the new one } } - return this.addWatchedDirectory(std.os.windows.INVALID_HANDLE_VALUE, path); + return this.addWatchedDirectory(path); } const Timeout = enum(w.DWORD) { @@ -427,12 +421,22 @@ const WindowsWatcher = struct { } } - // exit notification for this watcher - we should probably deallocate it here - if (nbytes == 0) { - continue; - } if (overlapped) |ptr| { - return DirWatcher.fromOverlapped(ptr); + const watcher = DirWatcher.fromOverlapped(ptr); + // exit notification for this watcher + if (nbytes == 0) { + this.mutex.lock(); + defer this.mutex.unlock(); + this.allocator.destroy(watcher); + for (this.watchers.items, 0..) |_watcher, i| { + if (_watcher == watcher) { + _ = this.watchers.swapRemove(i); + break; + } + } + } else { + return watcher; + } } else { log("GetQueuedCompletionStatus returned no overlapped event", .{}); return Error.IocpFailed; @@ -443,6 +447,9 @@ const WindowsWatcher = struct { pub fn stop(this: *WindowsWatcher) void { for (this.watchers.items) |watcher| { w.CloseHandle(watcher.dirHandle); + // this may not be safe, as windows might be writing into the buffer while we deallocate it + // the correct way to do this would be to continue running GetQueuedCompletionStatus until we get an exit notification + // for all of the watchers, but that might block indefinitely if the shutdown notification is not delivered this.allocator.destroy(watcher); } w.CloseHandle(this.iocp); From edcdcfa9c3f632e8a4bd9842f63bb837bfa36e9e Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 16:03:08 -0800 Subject: [PATCH 23/32] update .gitignore --- .gitignore | 331 +++++++++++++++++++++++++++-------------------------- 1 file changed, 166 insertions(+), 165 deletions(-) diff --git a/.gitignore b/.gitignore index 8721435d6b7681..8373c489d1cf4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,166 +1,167 @@ -.DS_Store -zig-cache -packages/*/*.wasm -*.o -*.a -profile.json - -node_modules -.envrc -.swcrc -yarn.lock -dist -*.tmp -*.log -*.out.js -*.out.refresh.js -**/package-lock.json -build -*.wat -zig-out -pnpm-lock.yaml -README.md.template -src/deps/zig-clap/example -src/deps/zig-clap/README.md -src/deps/zig-clap/.github -src/deps/zig-clap/.gitattributes -out -outdir - -.trace -cover -coverage -coverv -*.trace -github -out.* -out -.parcel-cache -esbuilddir -*.bun -parceldist -esbuilddir -outdir/ -outcss -.next -txt.js -.idea -.vscode/cpp* -.vscode/clang* - -node_modules_* -*.jsb -*.zip -bun-zigld -bun-singlehtreaded -bun-nomimalloc -bun-mimalloc -examples/lotta-modules/bun-yday -examples/lotta-modules/bun-old -examples/lotta-modules/bun-nofscache - -src/node-fallbacks/out/* -src/node-fallbacks/node_modules -sign.json -release/ -*.dmg -sign.*.json -packages/debug-* -packages/bun-cli/postinstall.js -packages/bun-*/bun -packages/bun-*/bun-profile -packages/bun-*/debug-bun -packages/bun-*/*.o -packages/bun-cli/postinstall.js - -packages/bun-cli/bin/* -bun-test-scratch -misctools/fetch - -src/deps/libiconv -src/deps/openssl -src/tests.zig -*.blob -src/deps/s2n-tls -.npm -.npm.gz - -bun-binary - -src/deps/PLCrashReporter/ - -*.dSYM -*.crash -misctools/sha -packages/bun-wasm/*.mjs -packages/bun-wasm/*.cjs -packages/bun-wasm/*.map -packages/bun-wasm/*.js -packages/bun-wasm/*.d.ts -packages/bun-wasm/*.d.cts -packages/bun-wasm/*.d.mts -*.bc - -src/fallback.version -src/runtime.version -*.sqlite -*.database -*.db -misctools/machbench -*.big -.eslintcache - -/bun-webkit - -src/deps/c-ares/build -src/bun.js/bindings-obj -src/bun.js/debug-bindings-obj - -failing-tests.txt -test.txt -myscript.sh - -cold-jsc-start -cold-jsc-start.d - -/test.ts -/test.js - -src/js/out/modules* -src/js/out/functions* -src/js/out/tmp -src/js/out/DebugPath.h - -make-dev-stats.csv - -.uuid -tsconfig.tsbuildinfo - -test/js/bun/glob/fixtures -*.lib -*.pdb -CMakeFiles -build.ninja -.ninja_deps -.ninja_log -CMakeCache.txt -cmake_install.cmake -compile_commands.json - -*.lib -x64 -**/*.vcxproj* -**/*.sln* -**/*.dir -**/*.pdb - -/.webkit-cache -/.cache -/src/deps/libuv -/build-*/ - -.vs - -**/.verdaccio-db.json -/test-report.md +.DS_Store +zig-cache +packages/*/*.wasm +*.o +*.a +profile.json + +node_modules +.envrc +.swcrc +yarn.lock +dist +*.tmp +*.log +*.out.js +*.out.refresh.js +**/package-lock.json +build +*.wat +zig-out +pnpm-lock.yaml +README.md.template +src/deps/zig-clap/example +src/deps/zig-clap/README.md +src/deps/zig-clap/.github +src/deps/zig-clap/.gitattributes +out +outdir + +.trace +cover +coverage +coverv +*.trace +github +out.* +out +.parcel-cache +esbuilddir +*.bun +parceldist +esbuilddir +outdir/ +outcss +.next +txt.js +.idea +.vscode/cpp* +.vscode/clang* + +node_modules_* +*.jsb +*.zip +bun-zigld +bun-singlehtreaded +bun-nomimalloc +bun-mimalloc +examples/lotta-modules/bun-yday +examples/lotta-modules/bun-old +examples/lotta-modules/bun-nofscache + +src/node-fallbacks/out/* +src/node-fallbacks/node_modules +sign.json +release/ +*.dmg +sign.*.json +packages/debug-* +packages/bun-cli/postinstall.js +packages/bun-*/bun +packages/bun-*/bun-profile +packages/bun-*/debug-bun +packages/bun-*/*.o +packages/bun-cli/postinstall.js + +packages/bun-cli/bin/* +bun-test-scratch +misctools/fetch + +src/deps/libiconv +src/deps/openssl +src/tests.zig +*.blob +src/deps/s2n-tls +.npm +.npm.gz + +bun-binary + +src/deps/PLCrashReporter/ + +*.dSYM +*.crash +misctools/sha +packages/bun-wasm/*.mjs +packages/bun-wasm/*.cjs +packages/bun-wasm/*.map +packages/bun-wasm/*.js +packages/bun-wasm/*.d.ts +packages/bun-wasm/*.d.cts +packages/bun-wasm/*.d.mts +*.bc + +src/fallback.version +src/runtime.version +*.sqlite +*.database +*.db +misctools/machbench +*.big +.eslintcache + +/bun-webkit + +src/deps/c-ares/build +src/bun.js/bindings-obj +src/bun.js/debug-bindings-obj + +failing-tests.txt +test.txt +myscript.sh + +cold-jsc-start +cold-jsc-start.d + +/testdir +/test.ts +/test.js + +src/js/out/modules* +src/js/out/functions* +src/js/out/tmp +src/js/out/DebugPath.h + +make-dev-stats.csv + +.uuid +tsconfig.tsbuildinfo + +test/js/bun/glob/fixtures +*.lib +*.pdb +CMakeFiles +build.ninja +.ninja_deps +.ninja_log +CMakeCache.txt +cmake_install.cmake +compile_commands.json + +*.lib +x64 +**/*.vcxproj* +**/*.sln* +**/*.dir +**/*.pdb + +/.webkit-cache +/.cache +/src/deps/libuv +/build-*/ + +.vs + +**/.verdaccio-db.json +/test-report.md /test-report.json \ No newline at end of file From afb9204f5f091b9ab713fc85d730e68d0f3ab0ee Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 18:10:27 -0800 Subject: [PATCH 24/32] rework windows watcher into single watcher instance watching top level project dir --- src/bun.js/event_loop.zig | 12 +- src/resolver/resolve_path.zig | 10 +- src/watcher.zig | 264 ++++++++++------------------------ 3 files changed, 92 insertions(+), 194 deletions(-) diff --git a/src/bun.js/event_loop.zig b/src/bun.js/event_loop.zig index fee9a9f27dc402..04308920282dd3 100644 --- a/src/bun.js/event_loop.zig +++ b/src/bun.js/event_loop.zig @@ -1178,14 +1178,10 @@ pub const EventLoop = struct { } if (!loop.isActive()) { - if (comptime Environment.isWindows) { - bun.todo(@src(), {}); - } else { - if (this.forever_timer == null) { - var t = uws.Timer.create(loop, this); - t.set(this, &noopForeverTimer, 1000 * 60 * 4, 1000 * 60 * 4); - this.forever_timer = t; - } + if (this.forever_timer == null) { + var t = uws.Timer.create(loop, this); + t.set(this, &noopForeverTimer, 1000 * 60 * 4, 1000 * 60 * 4); + this.forever_timer = t; } } diff --git a/src/resolver/resolve_path.zig b/src/resolver/resolve_path.zig index 933ad2a1782f31..37b270eb58df45 100644 --- a/src/resolver/resolve_path.zig +++ b/src/resolver/resolve_path.zig @@ -54,7 +54,15 @@ const ParentEqual = enum { unrelated, }; -pub fn isParentOrEqual(parent: []const u8, child: []const u8) ParentEqual { +pub fn isParentOrEqual(parent_: []const u8, child_: []const u8) ParentEqual { + var parent = parent_; + if (parent.len > 0 and isSepAny(parent[parent.len - 1])) { + parent = parent[0 .. parent.len - 1]; + } + var child = child_; + if (child.len > 0 and isSepAny(child[child.len - 1])) { + child = child[0 .. child.len - 1]; + } if (std.mem.indexOf(u8, child, parent) != 0) return .unrelated; if (child.len == parent.len) return .equal; if (isSepAny(child[parent.len])) return .parent; diff --git a/src/watcher.zig b/src/watcher.zig index 2c92e5508572f7..c2e44efede815c 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -70,7 +70,7 @@ const INotify = struct { std.os.inotify_rm_watch(this.inotify_fd, wd); } - pub fn init(this: *INotify, _: std.mem.Allocator) !void { + pub fn init(this: *INotify, _: []const u8) !void { std.debug.assert(!this.loaded_inotify); this.loaded_inotify = true; @@ -184,7 +184,7 @@ const DarwinWatcher = struct { fd: i32 = 0, - pub fn init(this: *DarwinWatcher, _: std.mem.Allocator) !void { + pub fn init(this: *DarwinWatcher, _: []const u8) !void { this.fd = try std.os.kqueue(); if (this.fd == 0) return error.KQueueError; } @@ -200,8 +200,7 @@ const DarwinWatcher = struct { const WindowsWatcher = struct { mutex: Mutex = Mutex.init(), iocp: w.HANDLE = undefined, - allocator: std.mem.Allocator = undefined, - watchers: std.ArrayListUnmanaged(*DirWatcher) = std.ArrayListUnmanaged(*DirWatcher){}, + watcher: DirWatcher = undefined, const w = std.os.windows; pub const EventListIndex = c_int; @@ -226,56 +225,14 @@ const WindowsWatcher = struct { filename: []u16 = undefined, }; - // each directory being watched has an associated DirWatcher const DirWatcher = struct { // must be initialized to zero (even though it's never read or written in our code), // otherwise ReadDirectoryChangesW will fail with INVALID_HANDLE overlapped: w.OVERLAPPED = std.mem.zeroes(w.OVERLAPPED), buf: [64 * 1024]u8 align(@alignOf(w.FILE_NOTIFY_INFORMATION)) = undefined, dirHandle: w.HANDLE, - path: [:0]u8, - path_buf: bun.PathBuffer = undefined, - refcount: usize = 1, - parent: *WindowsWatcher, - - const EventIterator = struct { - watcher: *DirWatcher, - offset: usize = 0, - hasNext: bool = true, - - pub fn next(this: *EventIterator) ?FileEvent { - if (!this.hasNext) return null; - const info_size = @sizeOf(w.FILE_NOTIFY_INFORMATION); - const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(this.watcher.buf[this.offset..].ptr)); - const name_ptr: [*]u16 = @alignCast(@ptrCast(this.watcher.buf[this.offset + info_size ..])); - const filename: []u16 = name_ptr[0 .. info.FileNameLength / @sizeOf(u16)]; - - const action: Action = @enumFromInt(info.Action); - - if (info.NextEntryOffset == 0) { - this.hasNext = false; - } else { - this.offset += @as(usize, info.NextEntryOffset); - } - - return FileEvent{ - .action = action, - .filename = filename, - }; - } - }; - - fn fromOverlapped(overlapped: *w.OVERLAPPED) *DirWatcher { - const offset = @offsetOf(DirWatcher, "overlapped"); - const overlapped_byteptr: [*]u8 = @ptrCast(overlapped); - return @alignCast(@ptrCast(overlapped_byteptr - offset)); - } - fn events(this: *DirWatcher) EventIterator { - return EventIterator{ .watcher = this }; - } - - // invalidates any EventIterators derived from this DirWatcher + // invalidates any EventIterators fn prepare(this: *DirWatcher) Error!void { const filter = w.FILE_NOTIFY_CHANGE_FILE_NAME | w.FILE_NOTIFY_CHANGE_DIR_NAME | w.FILE_NOTIFY_CHANGE_LAST_WRITE | w.FILE_NOTIFY_CHANGE_CREATION; if (w.kernel32.ReadDirectoryChangesW(this.dirHandle, &this.buf, this.buf.len, 1, filter, null, &this.overlapped, null) == 0) { @@ -284,36 +241,38 @@ const WindowsWatcher = struct { return Error.ReadDirectoryChangesFailed; } } + }; - fn ref(this: *DirWatcher) void { - std.debug.assert(this.refcount > 0); - this.refcount += 1; - } + const EventIterator = struct { + watcher: *DirWatcher, + offset: usize = 0, + hasNext: bool = true, - fn unref(this: *DirWatcher) void { - std.debug.assert(this.refcount > 0); - this.refcount -= 1; - // TODO if refcount reaches 0 we should deallocate - // But we can't deallocate right away because we might be in the middle of iterating over the events of this watcher - // we probably need some sort of queue that can be emptied by the watcher thread. - if (this.refcount == 0) { - // closing the handle will send a 0-length notification to the iocp which can then deallocate the watcher - w.CloseHandle(this.dirHandle); + pub fn next(this: *EventIterator) ?FileEvent { + if (!this.hasNext) return null; + const info_size = @sizeOf(w.FILE_NOTIFY_INFORMATION); + const info: *w.FILE_NOTIFY_INFORMATION = @alignCast(@ptrCast(this.watcher.buf[this.offset..].ptr)); + const name_ptr: [*]u16 = @alignCast(@ptrCast(this.watcher.buf[this.offset + info_size ..])); + const filename: []u16 = name_ptr[0 .. info.FileNameLength / @sizeOf(u16)]; + + const action: Action = @enumFromInt(info.Action); + + if (info.NextEntryOffset == 0) { + this.hasNext = false; + } else { + this.offset += @as(usize, info.NextEntryOffset); } + + return FileEvent{ + .action = action, + .filename = filename, + }; } }; - pub fn init(this: *WindowsWatcher, allocator: std.mem.Allocator) !void { - const iocp = try w.CreateIoCompletionPort(w.INVALID_HANDLE_VALUE, null, 0, 1); - this.* = .{ - .iocp = iocp, - .allocator = allocator, - }; - } - - fn addWatchedDirectory(this: *WindowsWatcher, path: []const u8) !*DirWatcher { + pub fn init(this: *WindowsWatcher, root: []const u8) !void { var pathbuf: bun.WPathBuffer = undefined; - const wpath = bun.strings.toNTPath(&pathbuf, path); + const wpath = bun.strings.toNTPath(&pathbuf, root); const path_len_bytes: u16 = @truncate(wpath.len * 2); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, @@ -349,53 +308,12 @@ const WindowsWatcher = struct { log("failed to open directory for watching: {s}", .{@tagName(err)}); return Error.CreateFileFailed; } - errdefer _ = w.kernel32.CloseHandle(handle); - // on success we receive the same iocp handle back that we put in - no need to update it - _ = try w.CreateIoCompletionPort(handle, this.iocp, 0, 1); - - const watcher = try this.allocator.create(DirWatcher); - errdefer this.allocator.destroy(watcher); - watcher.* = .{ .dirHandle = handle, .parent = this, .path = undefined }; - // init path - @memcpy(watcher.path_buf[0..path.len], path); - watcher.path_buf[path.len] = 0; - watcher.path = watcher.path_buf[0..path.len :0]; + this.iocp = try w.CreateIoCompletionPort(handle, null, 0, 1); + errdefer _ = w.kernel32.CloseHandle(this.iocp); - try watcher.prepare(); - try this.watchers.append(this.allocator, watcher); - - return watcher; - } - - pub fn watchFile(this: *WindowsWatcher, path: []const u8) !*DirWatcher { - const dirpath = std.fs.path.dirnameWindows(path) orelse return Error.InvalidPath; - return this.watchDir(dirpath); - } - - pub fn watchDir(this: *WindowsWatcher, path_: []const u8) !*DirWatcher { - this.mutex.lock(); - defer this.mutex.unlock(); - // strip the trailing slash if it exists - var path = path_; - if (path.len > 0 and bun.strings.charIsAnySlash(path[path.len - 1])) { - path = path[0 .. path.len - 1]; - } - // check if one of the existing watchers covers this path - for (this.watchers.items) |watcher| { - if (bun.path.isParentOrEqual(watcher.path, path) != .unrelated) { - // there is an existing watcher that covers this path - log("found existing watcher with path: {s}", .{watcher.path}); - watcher.ref(); - return watcher; - } else if (bun.path.isParentOrEqual(path, watcher.path) != .unrelated) { - // the new watcher would cover this existing watcher - // TODO there could be multiple existing watchers that are covered by the new watcher - // we should deactivate all existing ones and activate the new one - } - } - return this.addWatchedDirectory(path); + this.watcher = .{ .dirHandle = handle }; } const Timeout = enum(w.DWORD) { @@ -405,7 +323,9 @@ const WindowsWatcher = struct { }; // get the next dirwatcher that has events - pub fn next(this: *WindowsWatcher, timeout: Timeout) !?*DirWatcher { + pub fn next(this: *WindowsWatcher, timeout: Timeout) !?EventIterator { + try this.watcher.prepare(); + var nbytes: w.DWORD = 0; var key: w.ULONG_PTR = 0; var overlapped: ?*w.OVERLAPPED = null; @@ -422,21 +342,16 @@ const WindowsWatcher = struct { } if (overlapped) |ptr| { - const watcher = DirWatcher.fromOverlapped(ptr); - // exit notification for this watcher + // ignore possible spurious events + if (ptr != &this.watcher.overlapped) { + continue; + } if (nbytes == 0) { - this.mutex.lock(); - defer this.mutex.unlock(); - this.allocator.destroy(watcher); - for (this.watchers.items, 0..) |_watcher, i| { - if (_watcher == watcher) { - _ = this.watchers.swapRemove(i); - break; - } - } - } else { - return watcher; + // shutdown notification + // TODO close handles? + return Error.IocpFailed; } + return EventIterator{ .watcher = &this.watcher }; } else { log("GetQueuedCompletionStatus returned no overlapped event", .{}); return Error.IocpFailed; @@ -445,13 +360,7 @@ const WindowsWatcher = struct { } pub fn stop(this: *WindowsWatcher) void { - for (this.watchers.items) |watcher| { - w.CloseHandle(watcher.dirHandle); - // this may not be safe, as windows might be writing into the buffer while we deallocate it - // the correct way to do this would be to continue running GetQueuedCompletionStatus until we get an exit notification - // for all of the watchers, but that might block indefinitely if the shutdown notification is not delivered - this.allocator.destroy(watcher); - } + w.CloseHandle(this.watcher.dirHandle); w.CloseHandle(this.iocp); } }; @@ -555,26 +464,7 @@ pub const WatchItem = struct { parent_hash: u32, kind: Kind, package_json: ?*PackageJSON, - platform: Platform.Data = Platform.Data{}, - - const Platform = struct { - const Linux = struct { - eventlist_index: PlatformWatcher.EventListIndex = 0, - }; - const Windows = struct { - dir_watcher: ?*WindowsWatcher.DirWatcher = null, - }; - const Darwin = struct {}; - - const Data = if (Environment.isMac) - Darwin - else if (Environment.isLinux) - Linux - else if (Environment.isWindows) - Windows - else - @compileError("Unsupported platform"); - }; + eventlist_index: if (Environment.isLinux) PlatformWatcher.EventListIndex else u0 = 0, pub const Kind = enum { file, directory }; }; @@ -628,7 +518,7 @@ pub fn NewWatcher(comptime ContextType: type) type { .cwd = fs.top_level_dir, }; - try PlatformWatcher.init(&watcher.platform, allocator); + try PlatformWatcher.init(&watcher.platform, fs.top_level_dir); return watcher; } @@ -705,20 +595,13 @@ pub fn NewWatcher(comptime ContextType: type) type { var slice = this.watchlist.slice(); const fds = slice.items(.fd); - const platform_data = slice.items(.platform); var last_item = no_watch_item; for (this.evict_list[0..this.evict_list_i]) |item| { // catch duplicates, since the list is sorted, duplicates will appear right after each other if (item == last_item) continue; - if (Environment.isWindows) { - // on windows we need to deallocate the watcher instance - // TODO implement this - if (platform_data[item].dir_watcher) |watcher| { - watcher.unref(); - } - } else { + if (!Environment.isWindows) { // on mac and linux we can just close the file descriptor // TODO do we need to call inotify_rm_watch on linux? _ = bun.sys.close(fds[item]); @@ -879,32 +762,29 @@ pub fn NewWatcher(comptime ContextType: type) type { } } } else if (Environment.isWindows) { + var buf: bun.PathBuffer = undefined; + const root = this.fs.top_level_dir; + @memcpy(buf[0..root.len], root); + const needs_slash = root.len == 0 or !bun.strings.charIsAnySlash(root[root.len - 1]); + if (needs_slash) { + buf[root.len] = '\\'; + } + const baseidx = if (needs_slash) root.len + 1 else root.len; restart: while (true) { - var buf: bun.PathBuffer = undefined; var event_id: usize = 0; // first wait has infinite timeout - we're waiting for the next event and don't want to spin var timeout = WindowsWatcher.Timeout.infinite; while (true) { - // log("waiting with timeout: {s}\n", .{@tagName(timeout)}); - const watcher = try this.platform.next(timeout) orelse break; - // after handling the watcher's events, it explicitly needs to start reading directory changes again - defer watcher.prepare() catch |err| { - Output.prettyErrorln("Failed to start listening to directory changes: {s} - Future updates may be missed", .{@errorName(err)}); - }; - + log("waiting with timeout: {s}\n", .{@tagName(timeout)}); + var iter = try this.platform.next(timeout) orelse break; // after the first wait, we want to start coalescing events, so we wait for a minimal amount of time - timeout = WindowsWatcher.Timeout.minimal; - + timeout = WindowsWatcher.Timeout.none; const item_paths = this.watchlist.items(.file_path); - - log("event from watcher: {s}", .{watcher.path}); - var iter = watcher.events(); - @memcpy(buf[0..watcher.path.len], watcher.path); - buf[watcher.path.len] = '\\'; + log("number of items: {d}\n", .{item_paths.len}); while (iter.next()) |event| { log("raw event (filename: {}, action: {s})", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); - var idx = watcher.path.len + 1; + var idx = baseidx; idx += bun.simdutf.convert.utf16.to.utf8.le(event.filename, buf[idx..]); const eventpath = buf[0..idx]; @@ -975,6 +855,15 @@ pub fn NewWatcher(comptime ContextType: type) type { package_json: ?*PackageJSON, comptime copy_file_path: bool, ) !void { + if (comptime Environment.isWindows) { + // on windows we can only watch items that are in the directory tree of the top level dir + const rel = bun.path.isParentOrEqual(this.fs.top_level_dir, file_path); + if (rel == .unrelated) { + Output.warn("File {s} is not in the project directory and will not be watched\n", .{file_path}); + return; + } + } + const watchlist_id = this.watchlist.len; const file_path_: string = if (comptime copy_file_path) @@ -1032,8 +921,6 @@ pub fn NewWatcher(comptime ContextType: type) type { var buf = file_path_.ptr; const slice: [:0]const u8 = buf[0..file_path_.len :0]; item.platform.index = try this.platform.watchPath(slice); - } else if (comptime Environment.isWindows) { - item.platform.dir_watcher = try this.platform.watchFile(file_path_); } this.watchlist.appendAssumeCapacity(item); @@ -1046,6 +933,15 @@ pub fn NewWatcher(comptime ContextType: type) type { hash: HashType, comptime copy_file_path: bool, ) !WatchItemIndex { + if (comptime Environment.isWindows) { + // on windows we can only watch items that are in the directory tree of the top level dir + const rel = bun.path.isParentOrEqual(this.fs.top_level_dir, file_path); + if (rel == .unrelated) { + Output.warn("Directory {s} is not in the project directory and will not be watched\n", .{file_path}); + return no_watch_item; + } + } + const fd = brk: { if (stored_fd.int() > 0) break :brk stored_fd; const dir = try std.fs.cwd().openDir(file_path, .{}); @@ -1113,9 +1009,7 @@ pub fn NewWatcher(comptime ContextType: type) type { bun.copy(u8, &buf, file_path_to_use_); buf[file_path_to_use_.len] = 0; const slice: [:0]u8 = buf[0..file_path_to_use_.len :0]; - item.platform.eventlist_index = try this.platform.watchDir(slice); - } else if (Environment.isWindows) { - item.platform.dir_watcher = try this.platform.watchDir(file_path_); + item.eventlist_index = try this.platform.watchDir(slice); } this.watchlist.appendAssumeCapacity(item); From d27a65a136ffcb93c0f78bbe087e7fbbc24a9e42 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 18:58:07 -0800 Subject: [PATCH 25/32] use non-strict utf16 conversion --- src/resolver/resolve_path.zig | 8 ++------ src/watcher.zig | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/resolver/resolve_path.zig b/src/resolver/resolve_path.zig index 37b270eb58df45..d644b3ad9e47fe 100644 --- a/src/resolver/resolve_path.zig +++ b/src/resolver/resolve_path.zig @@ -54,15 +54,11 @@ const ParentEqual = enum { unrelated, }; -pub fn isParentOrEqual(parent_: []const u8, child_: []const u8) ParentEqual { +pub fn isParentOrEqual(parent_: []const u8, child: []const u8) ParentEqual { var parent = parent_; - if (parent.len > 0 and isSepAny(parent[parent.len - 1])) { + while (parent.len > 0 and isSepAny(parent[parent.len - 1])) { parent = parent[0 .. parent.len - 1]; } - var child = child_; - if (child.len > 0 and isSepAny(child[child.len - 1])) { - child = child[0 .. child.len - 1]; - } if (std.mem.indexOf(u8, child, parent) != 0) return .unrelated; if (child.len == parent.len) return .equal; if (isSepAny(child[parent.len])) return .parent; diff --git a/src/watcher.zig b/src/watcher.zig index c2e44efede815c..30eb5c5605f887 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -322,7 +322,7 @@ const WindowsWatcher = struct { none = 0, }; - // get the next dirwatcher that has events + // wait until new events are available pub fn next(this: *WindowsWatcher, timeout: Timeout) !?EventIterator { try this.watcher.prepare(); @@ -776,27 +776,31 @@ pub fn NewWatcher(comptime ContextType: type) type { // first wait has infinite timeout - we're waiting for the next event and don't want to spin var timeout = WindowsWatcher.Timeout.infinite; while (true) { - log("waiting with timeout: {s}\n", .{@tagName(timeout)}); var iter = try this.platform.next(timeout) orelse break; // after the first wait, we want to start coalescing events, so we wait for a minimal amount of time - timeout = WindowsWatcher.Timeout.none; + timeout = WindowsWatcher.Timeout.minimal; const item_paths = this.watchlist.items(.file_path); - log("number of items: {d}\n", .{item_paths.len}); + log("number of watched items: {d}", .{item_paths.len}); while (iter.next()) |event| { - log("raw event (filename: {}, action: {s})", .{ std.unicode.fmtUtf16le(event.filename), @tagName(event.action) }); - var idx = baseidx; - idx += bun.simdutf.convert.utf16.to.utf8.le(event.filename, buf[idx..]); - const eventpath = buf[0..idx]; + const convert_res = bun.strings.copyUTF16IntoUTF8(buf[baseidx..], []const u16, event.filename, false); + const eventpath = buf[0 .. baseidx + convert_res.written]; - log("received event at full path: {s}\n", .{eventpath}); + log("watcher update event: (filename: {s}, action: {s}", .{ eventpath, @tagName(event.action) }); + + // TODO this probably needs a more sophisticated search algorithm in the future + // Possible approaches: + // - Keep a sorted list of the watched paths and perform a binary search. We could use a bool to keep + // track of whether the list is sorted and only sort it when we detect a change. + // - Use a prefix tree. Potentially more efficient for large numbers of watched paths, but complicated + // to implement and maintain. + // - others that i'm not thinking of - // TODO this probably needs a more sophisticated search algorithm for (item_paths, 0..) |path_, item_idx| { var path = path_; if (path.len > 0 and bun.strings.charIsAnySlash(path[path.len - 1])) { path = path[0 .. path.len - 1]; } - log("checking path: {s}\n", .{path}); + // log("checking path: {s}\n", .{path}); // check if the current change applies to this item // if so, add it to the eventlist const rel = bun.path.isParentOrEqual(eventpath, path); @@ -813,7 +817,7 @@ pub fn NewWatcher(comptime ContextType: type) type { continue :restart; } - log("event_id: {d}\n", .{event_id}); + // log("event_id: {d}\n", .{event_id}); var all_events = this.watch_events[0..event_id]; std.sort.pdq(WatchEvent, all_events, {}, WatchEvent.sortByIndex); @@ -1159,10 +1163,6 @@ pub fn NewWatcher(comptime ContextType: type) type { defer this.mutex.unlock(); if (this.indexOf(hash)) |index| { this.removeAtIndex(@truncate(index), hash, &[_]HashType{}, .file); - // const fds = this.watchlist.items(.fd); - // const fd = fds[index]; - // _ = bun.sys.close(fd); - // this.watchlist.swapRemove(index); } } From c26498ea98960e581083ceb2fa86bd29ef754758 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 19:05:53 -0800 Subject: [PATCH 26/32] change to contains --- src/watcher.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/watcher.zig b/src/watcher.zig index 30eb5c5605f887..88cf50868bd91f 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -1086,7 +1086,7 @@ pub fn NewWatcher(comptime ContextType: type) type { } inline fn isEligibleDirectory(this: *Watcher, dir: string) bool { - return strings.indexOf(dir, this.fs.top_level_dir) != null and strings.indexOf(dir, "node_modules") == null; + return strings.contains(dir, this.fs.top_level_dir) and !strings.contains(dir, "node_modules"); } pub fn appendFile( From 642b9ff64cab1283f9fed3c7029dde0b180c9ddf Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 19:12:57 -0800 Subject: [PATCH 27/32] fix mac and linux compile --- src/bun.js/javascript.zig | 4 ++-- src/bun.zig | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index ded13f8a35c81e..85be94f81352aa 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -3391,7 +3391,7 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime file_ent.entry.cache.fd = .zero; file_ent.entry.need_stat = true; path_string = file_ent.entry.abs_path; - file_hash = @This().Watcher.getHash(path_string.slice()); + file_hash = GenericWatcher.getHash(path_string.slice()); for (hashes, 0..) |hash, entry_id| { if (hash == file_hash) { if (file_descriptors[entry_id] != .zero) { @@ -3419,7 +3419,7 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime @memcpy(_on_file_update_path_buf[file_path_without_trailing_slash.len..][0..changed_name.len], changed_name); const path_slice = _on_file_update_path_buf[0 .. file_path_without_trailing_slash.len + changed_name.len + 1]; - file_hash = @This().Watcher.getHash(path_slice); + file_hash = GenericWatcher.getHash(path_slice); break :brk path_slice; } }; diff --git a/src/bun.zig b/src/bun.zig index 16fdc6daf50549..2937d2eb0368fd 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1399,9 +1399,9 @@ pub fn reloadProcess( clear_terminal: bool, ) noreturn { if (clear_terminal) { - // Output.flush(); - // Output.disableBuffering(); - // Output.resetTerminalAll(); + Output.flush(); + Output.disableBuffering(); + Output.resetTerminalAll(); } const bun = @This(); @@ -1469,7 +1469,9 @@ pub fn reloadProcess( .err => |err| { Output.panic("Unexpected error while reloading: {d} {s}", .{ err.errno, @tagName(err.getErrno()) }); }, - .result => |_| {}, + .result => |_| { + Output.panic("Unexpected error while reloading: posix_spawn returned a result", .{}); + }, } } else if (comptime Environment.isPosix) { const on_before_reload_process_linux = struct { @@ -1483,10 +1485,8 @@ pub fn reloadProcess( envp, ); Output.panic("Unexpected error while reloading: {s}", .{@errorName(err)}); - } else if (comptime Environment.isWindows) { - @panic("TODO on Windows!"); } else { - @panic("Unsupported platform"); + @compileError("unsupported platform for reloadProcess"); } } pub var auto_reload_on_crash = false; From 152bddee8f16c5806d92ab650f124c428128de96 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Wed, 31 Jan 2024 18:13:39 -0800 Subject: [PATCH 28/32] add baseline in crash report (#8606) --- src/bun.js/WebKit | 2 +- src/report.zig | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bun.js/WebKit b/src/bun.js/WebKit index 347037014ae069..54eae570cdca14 160000 --- a/src/bun.js/WebKit +++ b/src/bun.js/WebKit @@ -1 +1 @@ -Subproject commit 347037014ae069eed1c4f4687001a256949b124e +Subproject commit 54eae570cdca140cff95a1dda35ee2eb7a3523ab diff --git a/src/report.zig b/src/report.zig index d5bf3ed7d205cf..2dfb5b05137858 100644 --- a/src/report.zig +++ b/src/report.zig @@ -119,10 +119,12 @@ pub fn printMetadata() void { const analytics_platform = Platform.forOS(); + const maybe_baseline = if (Environment.baseline) " (baseline)" else ""; + crash_report_writer.print( \\ \\----- bun meta ----- - ++ "\nBun v" ++ Global.package_json_version_with_sha ++ " " ++ platform ++ " " ++ arch ++ " {s}\n" ++ + ++ "\nBun v" ++ Global.package_json_version_with_sha ++ " " ++ platform ++ " " ++ arch ++ maybe_baseline ++ " {s}\n" ++ \\{s}: {} \\ , .{ From f00556ec8efdca1f336e17af04ec73763e1af7d1 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Wed, 31 Jan 2024 19:08:50 -0800 Subject: [PATCH 29/32] allow linking bins that do not exist. (#8605) --- src/install/bin.zig | 57 ++++++++++---------- src/install/windows-shim/BinLinkingShim.zig | 5 +- src/install/windows-shim/bun_shim_impl.exe | Bin 10240 -> 10752 bytes src/install/windows-shim/bun_shim_impl.zig | 37 +++++++++---- 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/src/install/bin.zig b/src/install/bin.zig index e95ffa1525225f..1d752d9be29eaf 100644 --- a/src/install/bin.zig +++ b/src/install/bin.zig @@ -392,40 +392,41 @@ pub const Bin = extern struct { }; defer file.close(); - const first_content_chunk = contents: { - const fd = bun.sys.openatWindows( - this.package_installed_node_modules, - if (link_global) - bun.strings.toWPathNormalized( - &filename3_buf, - target_path[this.relative_path_to_bin_for_windows_global_link_offset..], - ) - else - target_wpath, - std.os.O.RDONLY, - ).unwrap() catch |err| { - this.err = err; - return; - }; - defer _ = bun.sys.close(fd); - const reader = fd.asFile().reader(); - const read = reader.read(&read_in_buf) catch |err| { - this.err = err; - return; + const shebang = shebang: { + const first_content_chunk = contents: { + const fd = bun.sys.openatWindows( + this.package_installed_node_modules, + if (link_global) + bun.strings.toWPathNormalized( + &filename3_buf, + target_path[this.relative_path_to_bin_for_windows_global_link_offset..], + ) + else + target_wpath, + std.os.O.RDONLY, + ).unwrap() catch break :contents null; + defer _ = bun.sys.close(fd); + const reader = fd.asFile().reader(); + const read = reader.read(&read_in_buf) catch break :contents null; + if (read == 0) { + break :contents null; + } + break :contents read_in_buf[0..read]; }; - if (read == 0) { - this.err = error.FileNotFound; - return; + + if (first_content_chunk) |chunk| { + break :shebang WinBinLinkingShim.Shebang.parse(chunk, target_wpath) catch { + this.err = error.InvalidBinContent; + return; + }; + } else { + break :shebang WinBinLinkingShim.Shebang.parseFromBinPath(target_wpath); } - break :contents read_in_buf[0..read]; }; const shim = WinBinLinkingShim{ .bin_path = target_wpath, - .shebang = WinBinLinkingShim.Shebang.parse(first_content_chunk, target_wpath) catch { - this.err = error.InvalidBinContent; - return; - }, + .shebang = shebang, }; const len = shim.encodedLength(); diff --git a/src/install/windows-shim/BinLinkingShim.zig b/src/install/windows-shim/BinLinkingShim.zig index 8cb4c552e9e1d4..ae8438d7e37e83 100644 --- a/src/install/windows-shim/BinLinkingShim.zig +++ b/src/install/windows-shim/BinLinkingShim.zig @@ -30,10 +30,13 @@ shebang: ?Shebang, /// These arbitrary numbers will probably not show up in the other fields. /// This will reveal off-by-one mistakes. pub const VersionFlag = enum(u13) { - pub const current = .v2; + pub const current = .v3; v1 = 5474, + // Fix bug where paths were not joined correctly v2 = 5475, + // Added an error message for when the process is not found + v3 = 5476, _, }; diff --git a/src/install/windows-shim/bun_shim_impl.exe b/src/install/windows-shim/bun_shim_impl.exe index 4ec808262ee5800acafc277363b38c57c3af7c9b..e2e560b5d03ff04d1bc2dd73e69e0bdff2fe582c 100755 GIT binary patch delta 1624 zcmbtUQEXFH7(VCPu3fwBP67)KQ|@XvEx{#aK!R*@U0S$ZTEz$pE6DW;r14% zS<+%x;X0booTvd8!M#W{#&nGmXEfE3p=OD&C*)y{g~*+du&5bxD)m3@UFez^A3Vu_ zzVrRx_n-gd+>@Kwm)L(M(J($aAs}@uA~P~FbqLjD-WIi2w;|L{F!Q2zunnQZnWjx7 zf%Q9g8o*r-Q3Ih-D^=hS>TebOUdfBlr|TgJ3Df{lPrky9b)yU{yC7&;OYUdf&M-ZC zV$t+6W<43weBS)B9-)ry2<6$~b}*;%Y-$&v{n|Zy76ibAn)E0+U-iL$UcTY9<(;Q`4~tbxy-P`<3QO< z`Hov~MmF^c6}w=md_h-JFgfI#*A|?%8C%{qbEk#fJdDVY&VH26}qr`GrQB8QHt!k}tcyzRO92a=s_+;Oh8Y*JB|#rG-C%kt=y+ z$z#3E%Rjp16^}fpSm1gC-B*T-su{Vg_rUVNllDK(xaFHXH7zTJM-cMh+le8|K{)yf zG_twnKY009enhhLyW~Y>mO9eyuPn@hsN{=xdD@ZyoG8Fhp3{=Neq&strI%b1l#|e! zl2hPPUIPNRe@Mx~g5Gh=pu9na_vsXZDbPxT*o-uBZo8ZOrKFtXpe_NR2(^|U~p-I3#iW4xwYtq(jp=48u zyk1mHC6ZC?ZWSks;>MELT`Z#qggt&QDvc#lTU5*lt)o(iO4F+&_k-~X9G`y-q1DXv zbfNi11)r(l-&gQ&{)10c=;IZ<`NN9DSOq@^oMdVvFnO?qB+JqyOEAWSicp*(*qHZ&Y%MVnob2%%}0V zWdjorh9yBfDhe=j!l2-bN#22w!1*Gvke>@jByK?90+CqQj|PI_a)&=6M2iMtFc_7% vNSN~nqeswDG2#=VQBD+&#eyPOgphFDD+MEA^l(p~D0n4-18>gS62kugKDQV} delta 1266 zcmb7DZAcVB7@pboyz5SGElT7fzuL152~RD^FVfE1mAk$ZlL#%7s3=7eE1vvr&>O_enT;g~`N5N%%_duL#Y`V#&Hkh31aQ{e&W55x141BQ*Nea1TOvEq9b7P@p zGYf5F(>}tRGCcB6O_S66WbZ2;y1La5Be~4+Z=ICYS90%Ya&4U&E)6)7t-UtQ|7TOFbg79mtq9BG_gO+|A07(`~a2dZ;8mpUx5&^PLVkQ;wi z>cw*S(1aOHbGX#+k~$&zYsE@g9j6CaB9MP>7>AuEL(9HiY{A z49$LUfv#AfcP-GR|Ioz?)^q3U<@T)khJ7t)Qe?f5cyUV8yZ}GoI-m{k1keqLOoi6P z-Cvr9;6nc%C*ZoV3!XX0s{^k&yCucu3G41#o!{w9b)MAucehxmWFt}MOr5vrT&{C8 n1vgf&|4aWY^xqqh{#DmPbz&1H$vCqkZ8;ZG)+m_7amRiE%te=* diff --git a/src/install/windows-shim/bun_shim_impl.zig b/src/install/windows-shim/bun_shim_impl.zig index 76b1cb2f250024..ef918cfc72c757 100644 --- a/src/install/windows-shim/bun_shim_impl.zig +++ b/src/install/windows-shim/bun_shim_impl.zig @@ -34,7 +34,7 @@ //! Prior Art: //! - https://github.com/ScoopInstaller/Shim/blob/master/src/shim.cs //! -//! The compiled binary is 10240 bytes and is `@embedFile`d into Bun itself. +//! The compiled binary is 10752 bytes and is `@embedFile`d into Bun itself. //! When this file is updated, the new binary should be compiled and BinLinkingShim.VersionFlag.current should be updated. const std = @import("std"); const builtin = @import("builtin"); @@ -148,6 +148,9 @@ const FailReason = enum { InvalidShimValidation, InvalidShimBounds, CouldNotDirectLaunch, + BinNotFound, + InterpreterNotFound, + ElevationRequired, pub fn render(reason: FailReason) []const u8 { return switch (reason) { @@ -159,6 +162,12 @@ const FailReason = enum { .InvalidShimDataSize => "bin metadata is corrupt (size)", .InvalidShimValidation => "bin metadata is corrupt (validate)", .InvalidShimBounds => "bin metadata is corrupt (bounds)", + // The difference between these two is that one is with a shebang (#!/usr/bin/env node) and + // the other is without. This is a helpful distinction because it can detect if something + // like node or bun is not in %path%, vs the actual executable was not installed in node_modules. + .InterpreterNotFound => "interpreter executable could not be found", + .BinNotFound => "bin executable does not exist on disk", + .ElevationRequired => "process requires elevation", .CreateProcessFailed => "could not create process", .CouldNotDirectLaunch => if (!is_standalone) @@ -678,17 +687,27 @@ fn launcher(bun_ctx: anytype) noreturn { &process, ); if (did_process_spawn == 0) { + const spawn_err = k32.GetLastError(); if (dbg) { - const spawn_err = k32.GetLastError(); printError("CreateProcessW failed: {s}\n", .{@tagName(spawn_err)}); } - // TODO: ERROR_ELEVATION_REQUIRED must take a fallback path, this path is potentially slower: - // This likely will not be an issue anyone runs into for a while, because it implies - // the shebang depends on something that requires UAC, which .... why? - // - // https://learn.microsoft.com/en-us/windows/security/application-security/application-control/user-account-control/how-it-works#user - // https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-shellexecutew - fail(.CreateProcessFailed); + switch (spawn_err) { + .FILE_NOT_FOUND => if (flags.has_shebang) + fail(.InterpreterNotFound) + else + fail(.BinNotFound), + + // TODO: ERROR_ELEVATION_REQUIRED must take a fallback path, this path is potentially slower: + // This likely will not be an issue anyone runs into for a while, because it implies + // the shebang depends on something that requires UAC, which .... why? + // + // https://learn.microsoft.com/en-us/windows/security/application-security/application-control/user-account-control/how-it-works#user + // https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-shellexecutew + .ELEVATION_REQUIRED => fail(.ElevationRequired), + + else => fail(.CreateProcessFailed), + } + comptime unreachable; } _ = k32.WaitForSingleObject(process.hProcess, w.INFINITE); From 7b00e58d70bd1971c69b1f6a348603b28e292f5c Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 19:25:14 -0800 Subject: [PATCH 30/32] fix linux compile --- src/watcher.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/watcher.zig b/src/watcher.zig index 88cf50868bd91f..9f42bf43a5d804 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -924,7 +924,7 @@ pub fn NewWatcher(comptime ContextType: type) type { // buf[file_path_to_use_.len] = 0; var buf = file_path_.ptr; const slice: [:0]const u8 = buf[0..file_path_.len :0]; - item.platform.index = try this.platform.watchPath(slice); + item.index = try this.platform.watchPath(slice); } this.watchlist.appendAssumeCapacity(item); From 73740054c9f0fbbf58253463404a49cd87f16e90 Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 19:26:44 -0800 Subject: [PATCH 31/32] fix linux compile (again) --- src/watcher.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/watcher.zig b/src/watcher.zig index 9f42bf43a5d804..360a102895b945 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -924,7 +924,7 @@ pub fn NewWatcher(comptime ContextType: type) type { // buf[file_path_to_use_.len] = 0; var buf = file_path_.ptr; const slice: [:0]const u8 = buf[0..file_path_.len :0]; - item.index = try this.platform.watchPath(slice); + item.eventlist_index = try this.platform.watchPath(slice); } this.watchlist.appendAssumeCapacity(item); From abb2a5d3221018394547659a84b56f6106b6f1da Mon Sep 17 00:00:00 2001 From: Georgijs Vilums Date: Wed, 31 Jan 2024 19:35:28 -0800 Subject: [PATCH 32/32] remove outdated todo --- src/bun.zig | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/bun.zig b/src/bun.zig index 2937d2eb0368fd..353c466b622aa0 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1989,8 +1989,6 @@ pub const win32 = struct { const image_pathZ = wbuf[0..image_path.Length :0]; - // TODO environment variables - const kernelenv = w.kernel32.GetEnvironmentStringsW(); var newenv: ?[]u16 = null; defer {